From 5825841873670368f26a492aa045a3ed744c4a5c Mon Sep 17 00:00:00 2001 From: Matt Snider Date: Sun, 10 Apr 2016 16:03:31 +0200 Subject: [PATCH 1/9] Revert Allow setting the time of a point manually This reverts commit c25ec08926e398a6e013f2d0bae34e3c40e92c7d, which this commit is part of PR #304. --- influxdb/helper.py | 28 +------------------ influxdb/tests/helper_test.py | 52 ++++++++--------------------------- test-requirements.txt | 3 +- 3 files changed, 14 insertions(+), 69 deletions(-) diff --git a/influxdb/helper.py b/influxdb/helper.py index 900df8d7..803a9bdd 100644 --- a/influxdb/helper.py +++ b/influxdb/helper.py @@ -3,7 +3,6 @@ Helper class for InfluxDB """ from collections import namedtuple, defaultdict -from datetime import datetime from warnings import warn import six @@ -17,16 +16,6 @@ class SeriesHelper(object): Each subclass can write to its own database. The time series names can also be based on one or more defined fields. - A field "time" can be used to write data points at a specific time, - rather than the default current time. The time field can take any of - the following forms: - * An integer unix timestamp in nanoseconds, assumed to be in UTC. - * A string in the ISO time format, including a timezone. - * A naive python datetime, which will be treated as UTC. - * A localized python datetime, which will use the chosen timezone. - If no time field is provided, the current UTC system time in microseconds - at the time of assembling the point data will be used. - Annotated example:: class MySeriesHelper(SeriesHelper): @@ -153,23 +142,8 @@ def _json_body_(cls): "tags": {}, } - ts = getattr(point, 'time', None) - if not ts: - # No time provided. Use current UTC time. - ts = datetime.utcnow().isoformat() + "+00:00" - elif isinstance(ts, datetime): - if ts.tzinfo is None or ts.tzinfo.utcoffset(ts) is None: - # Assuming naive datetime provided. Format with UTC tz. - ts = ts.isoformat() + "+00:00" - else: - # Assuming localized datetime provided. - ts = ts.isoformat() - # Neither of the above match. Assuming correct string or int. - json_point['time'] = ts - for field in cls._fields: - if field != 'time': - json_point['fields'][field] = getattr(point, field) + json_point['fields'][field] = getattr(point, field) for tag in cls._tags: json_point['tags'][tag] = getattr(point, tag) diff --git a/influxdb/tests/helper_test.py b/influxdb/tests/helper_test.py index ac2872f1..9721a9c9 100644 --- a/influxdb/tests/helper_test.py +++ b/influxdb/tests/helper_test.py @@ -1,7 +1,5 @@ # -*- coding: utf-8 -*- -import datetime -import pytz import sys if sys.version_info < (2, 7): import unittest2 as unittest @@ -40,18 +38,6 @@ class Meta: TestSeriesHelper.MySeriesHelper = MySeriesHelper - class MySeriesTimeHelper(SeriesHelper): - - class Meta: - client = TestSeriesHelper.client - series_name = 'events.stats.{server_name}' - fields = ['time', 'some_stat'] - tags = ['server_name', 'other_tag'] - bulk_size = 5 - autocommit = True - - TestSeriesHelper.MySeriesTimeHelper = MySeriesTimeHelper - def test_auto_commit(self): """ Tests that write_points is called after the right number of events @@ -80,20 +66,14 @@ def testSingleSeriesName(self): """ Tests JSON conversion when there is only one series name. """ - dt = datetime.datetime(2016, 1, 2, 3, 4, 5, 678912) - ts1 = dt - ts2 = "2016-10-11T01:02:03.123456789-04:00" - ts3 = 1234567890123456789 - ts4 = pytz.timezone("Europe/Berlin").localize(dt) - - TestSeriesHelper.MySeriesTimeHelper( - time=ts1, server_name='us.east-1', other_tag='ello', some_stat=159) - TestSeriesHelper.MySeriesTimeHelper( - time=ts2, server_name='us.east-1', other_tag='ello', some_stat=158) - TestSeriesHelper.MySeriesTimeHelper( - time=ts3, server_name='us.east-1', other_tag='ello', some_stat=157) - TestSeriesHelper.MySeriesTimeHelper( - time=ts4, server_name='us.east-1', other_tag='ello', some_stat=156) + TestSeriesHelper.MySeriesHelper( + server_name='us.east-1', other_tag='ello', some_stat=159) + TestSeriesHelper.MySeriesHelper( + server_name='us.east-1', other_tag='ello', some_stat=158) + TestSeriesHelper.MySeriesHelper( + server_name='us.east-1', other_tag='ello', some_stat=157) + TestSeriesHelper.MySeriesHelper( + server_name='us.east-1', other_tag='ello', some_stat=156) expectation = [ { "measurement": "events.stats.us.east-1", @@ -104,7 +84,6 @@ def testSingleSeriesName(self): "fields": { "some_stat": 159 }, - "time": "2016-01-02T03:04:05.678912+00:00", }, { "measurement": "events.stats.us.east-1", @@ -115,7 +94,6 @@ def testSingleSeriesName(self): "fields": { "some_stat": 158 }, - "time": "2016-10-11T01:02:03.123456789-04:00", }, { "measurement": "events.stats.us.east-1", @@ -126,7 +104,6 @@ def testSingleSeriesName(self): "fields": { "some_stat": 157 }, - "time": 1234567890123456789, }, { "measurement": "events.stats.us.east-1", @@ -137,24 +114,23 @@ def testSingleSeriesName(self): "fields": { "some_stat": 156 }, - "time": "2016-01-02T03:04:05.678912+01:00", } ] - rcvd = TestSeriesHelper.MySeriesTimeHelper._json_body_() + rcvd = TestSeriesHelper.MySeriesHelper._json_body_() self.assertTrue(all([el in expectation for el in rcvd]) and all([el in rcvd for el in expectation]), 'Invalid JSON body of time series returned from ' '_json_body_ for one series name: {0}.'.format(rcvd)) - TestSeriesHelper.MySeriesTimeHelper._reset_() + TestSeriesHelper.MySeriesHelper._reset_() self.assertEqual( - TestSeriesHelper.MySeriesTimeHelper._json_body_(), + TestSeriesHelper.MySeriesHelper._json_body_(), [], 'Resetting helper did not empty datapoints.') def testSeveralSeriesNames(self): ''' - Tests JSON conversion when there are multiple series names. + Tests JSON conversion when there is only one series name. ''' TestSeriesHelper.MySeriesHelper( server_name='us.east-1', some_stat=159, other_tag='ello') @@ -208,10 +184,6 @@ def testSeveralSeriesNames(self): ] rcvd = TestSeriesHelper.MySeriesHelper._json_body_() - for r in rcvd: - self.assertTrue(r.get('time'), - "No time field in received JSON body.") - del(r["time"]) self.assertTrue(all([el in expectation for el in rcvd]) and all([el in rcvd for el in expectation]), 'Invalid JSON body of time series returned from ' diff --git a/test-requirements.txt b/test-requirements.txt index 9e18b7d2..cbc6add3 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,5 +1,4 @@ nose nose-cov mock -requests-mock -pytz +requests-mock \ No newline at end of file From ab983d32f07048a72f7a8a7cb2a8a6dcef7c1d99 Mon Sep 17 00:00:00 2001 From: Matt Snider Date: Sat, 12 Mar 2016 23:55:17 +0100 Subject: [PATCH 2/9] Allow time to be specified in SeriesHelper.__init__() --- influxdb/helper.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/influxdb/helper.py b/influxdb/helper.py index 803a9bdd..5072f602 100644 --- a/influxdb/helper.py +++ b/influxdb/helper.py @@ -6,6 +6,7 @@ from warnings import warn import six +import time class SeriesHelper(object): @@ -87,8 +88,11 @@ def __new__(cls, *args, **kwargs): ' autocommit is false.'.format(cls.__name__)) cls._datapoints = defaultdict(list) - cls._type = namedtuple(cls.__name__, cls._fields + cls._tags) + if 'time' in cls._fields: + cls._fields.remove('time') + cls._type = namedtuple(cls.__name__, + cls._fields + cls._tags + ['time']) return super(SeriesHelper, cls).__new__(cls) def __init__(self, **kw): @@ -99,6 +103,7 @@ def __init__(self, **kw): :warning: Data points are *immutable* (`namedtuples`). """ cls = self.__class__ + timestamp = kw.pop('time', time.time()) if sorted(cls._fields + cls._tags) != sorted(kw.keys()): raise NameError( @@ -106,7 +111,9 @@ def __init__(self, **kw): sorted(cls._fields + cls._tags), kw.keys())) - cls._datapoints[cls._series_name.format(**kw)].append(cls._type(**kw)) + cls._datapoints[cls._series_name.format(**kw)].append( + cls._type(time=timestamp, **kw) + ) if cls._autocommit and \ sum(len(series) for series in cls._datapoints.values()) \ @@ -140,6 +147,7 @@ def _json_body_(cls): "measurement": series_name, "fields": {}, "tags": {}, + "time": getattr(point, "time") } for field in cls._fields: From 6aeaee4313237ce6f87b5e76542349b42ccf6704 Mon Sep 17 00:00:00 2001 From: Matt Snider Date: Sun, 13 Mar 2016 01:15:51 +0100 Subject: [PATCH 3/9] Extract SeriesHelper default timestamp into method for testability --- influxdb/helper.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/influxdb/helper.py b/influxdb/helper.py index 5072f602..3146f86f 100644 --- a/influxdb/helper.py +++ b/influxdb/helper.py @@ -103,7 +103,7 @@ def __init__(self, **kw): :warning: Data points are *immutable* (`namedtuples`). """ cls = self.__class__ - timestamp = kw.pop('time', time.time()) + timestamp = kw.pop('time', self._current_timestamp()) if sorted(cls._fields + cls._tags) != sorted(kw.keys()): raise NameError( @@ -165,3 +165,6 @@ def _reset_(cls): Reset data storage. """ cls._datapoints = defaultdict(list) + + def _current_timestamp(self): + return time.time() From 76826743a2ea8557c1f98a8ca1f0cf4cc76fe609 Mon Sep 17 00:00:00 2001 From: Matt Snider Date: Sun, 13 Mar 2016 01:19:04 +0100 Subject: [PATCH 4/9] Use datetime.utcnow() as default timestamp in SeriesHelper This is preferable to time.time() because _convert_timestamp() from influxdb.line_protocol will do precision handling and conversion if a datetime object is given. --- influxdb/helper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/influxdb/helper.py b/influxdb/helper.py index 3146f86f..7f35ab85 100644 --- a/influxdb/helper.py +++ b/influxdb/helper.py @@ -3,10 +3,10 @@ Helper class for InfluxDB """ from collections import namedtuple, defaultdict +from datetime import datetime from warnings import warn import six -import time class SeriesHelper(object): @@ -167,4 +167,4 @@ def _reset_(cls): cls._datapoints = defaultdict(list) def _current_timestamp(self): - return time.time() + return datetime.utcnow() From e3bce6c2a3ebad2bfd075a1e331c04157dcdb7a6 Mon Sep 17 00:00:00 2001 From: Matt Snider Date: Sun, 13 Mar 2016 01:27:07 +0100 Subject: [PATCH 5/9] Get existing tests working by mocking SeriesHelper._current_timestamp() --- influxdb/tests/helper_test.py | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/influxdb/tests/helper_test.py b/influxdb/tests/helper_test.py index 9721a9c9..3fc38992 100644 --- a/influxdb/tests/helper_test.py +++ b/influxdb/tests/helper_test.py @@ -8,6 +8,8 @@ import warnings import mock +from datetime import datetime +from unittest.mock import patch from influxdb import SeriesHelper, InfluxDBClient from requests.exceptions import ConnectionError @@ -62,10 +64,12 @@ class Meta: AutoCommitTest(server_name='us.east-1', some_stat=3443, other_tag='gg') self.assertTrue(fake_write_points.called) - def testSingleSeriesName(self): + @patch('influxdb.helper.SeriesHelper._current_timestamp') + def testSingleSeriesName(self, current_timestamp): """ Tests JSON conversion when there is only one series name. """ + current_timestamp.return_value = current_date = datetime.today() TestSeriesHelper.MySeriesHelper( server_name='us.east-1', other_tag='ello', some_stat=159) TestSeriesHelper.MySeriesHelper( @@ -84,6 +88,7 @@ def testSingleSeriesName(self): "fields": { "some_stat": 159 }, + "time": current_date, }, { "measurement": "events.stats.us.east-1", @@ -94,6 +99,7 @@ def testSingleSeriesName(self): "fields": { "some_stat": 158 }, + "time": current_date, }, { "measurement": "events.stats.us.east-1", @@ -104,6 +110,7 @@ def testSingleSeriesName(self): "fields": { "some_stat": 157 }, + "time": current_date, }, { "measurement": "events.stats.us.east-1", @@ -114,6 +121,7 @@ def testSingleSeriesName(self): "fields": { "some_stat": 156 }, + "time": current_date, } ] @@ -128,10 +136,12 @@ def testSingleSeriesName(self): [], 'Resetting helper did not empty datapoints.') - def testSeveralSeriesNames(self): - ''' - Tests JSON conversion when there is only one series name. - ''' + @patch('influxdb.helper.SeriesHelper._current_timestamp') + def testSeveralSeriesNames(self, current_timestamp): + """ + Tests JSON conversion when there are multiple series names. + """ + current_timestamp.return_value = current_date = datetime.today() TestSeriesHelper.MySeriesHelper( server_name='us.east-1', some_stat=159, other_tag='ello') TestSeriesHelper.MySeriesHelper( @@ -149,7 +159,8 @@ def testSeveralSeriesNames(self): 'tags': { 'other_tag': 'ello', 'server_name': 'lu.lux' - } + }, + "time": current_date, }, { 'fields': { @@ -159,7 +170,8 @@ def testSeveralSeriesNames(self): 'tags': { 'other_tag': 'ello', 'server_name': 'uk.london' - } + }, + "time": current_date, }, { 'fields': { @@ -169,7 +181,8 @@ def testSeveralSeriesNames(self): 'tags': { 'other_tag': 'ello', 'server_name': 'fr.paris-10' - } + }, + "time": current_date, }, { 'fields': { @@ -179,7 +192,8 @@ def testSeveralSeriesNames(self): 'tags': { 'other_tag': 'ello', 'server_name': 'us.east-1' - } + }, + "time": current_date, } ] From 309d295afee326517d104d4a618dc9561bd0f2a6 Mon Sep 17 00:00:00 2001 From: Matt Snider Date: Sun, 13 Mar 2016 17:03:44 +0100 Subject: [PATCH 6/9] Add additional tests for SeriesHelper time field --- influxdb/tests/helper_test.py | 60 ++++++++++++++++++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/influxdb/tests/helper_test.py b/influxdb/tests/helper_test.py index 3fc38992..92a2c32d 100644 --- a/influxdb/tests/helper_test.py +++ b/influxdb/tests/helper_test.py @@ -8,7 +8,7 @@ import warnings import mock -from datetime import datetime +from datetime import datetime, timedelta from unittest.mock import patch from influxdb import SeriesHelper, InfluxDBClient from requests.exceptions import ConnectionError @@ -209,6 +209,64 @@ def testSeveralSeriesNames(self, current_timestamp): [], 'Resetting helper did not empty datapoints.') + @patch('influxdb.helper.SeriesHelper._current_timestamp') + def testSeriesWithoutTimeField(self, current_timestamp): + """ + Tests that time is optional on a series without a time field. + """ + current_date = datetime.today() + yesterday = current_date - timedelta(days=1) + current_timestamp.return_value = yesterday + TestSeriesHelper.MySeriesHelper( + server_name='us.east-1', other_tag='ello', + some_stat=159, time=current_date + ) + TestSeriesHelper.MySeriesHelper( + server_name='us.east-1', other_tag='ello', + some_stat=158, + ) + point1, point2 = TestSeriesHelper.MySeriesHelper._json_body_() + self.assertTrue('time' in point1 and 'time' in point2) + self.assertEqual(point1['time'], current_date) + self.assertEqual(point2['time'], yesterday) + TestSeriesHelper.MySeriesHelper._reset_() + self.assertEqual( + TestSeriesHelper.MySeriesHelper._json_body_(), + [], + 'Resetting helper did not empty datapoints.') + + @patch('influxdb.helper.SeriesHelper._current_timestamp') + def testSeriesWithTimeField(self, current_timestamp): + """ + Test that time is optional on a series with a time field. + """ + current_date = datetime.today() + yesterday = current_date - timedelta(days=1) + current_timestamp.return_value = yesterday + + class MyTimeFieldSeriesHelper(SeriesHelper): + + class Meta: + client = TestSeriesHelper.client + series_name = 'events.stats.{server_name}' + fields = ['some_stat', 'time'] + tags = ['server_name', 'other_tag'] + bulk_size = 5 + autocommit = True + + MyTimeFieldSeriesHelper( + server_name='us.east-1', other_tag='ello', + some_stat=159, time=current_date + ) + MyTimeFieldSeriesHelper( + server_name='us.east-1', other_tag='ello', + some_stat=158, + ) + point1, point2 = MyTimeFieldSeriesHelper._json_body_() + self.assertTrue('time' in point1 and 'time' in point2) + self.assertEqual(point1['time'], current_date) + self.assertEqual(point2['time'], yesterday) + def testInvalidHelpers(self): ''' Tests errors in invalid helpers. From c6d391ae232f545e208c2cbb6b7c2a276286d87e Mon Sep 17 00:00:00 2001 From: Matt Snider Date: Sun, 13 Mar 2016 17:13:21 +0100 Subject: [PATCH 7/9] Move _reset_() calls in TestSeriesHelper to tearDown() --- influxdb/tests/helper_test.py | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/influxdb/tests/helper_test.py b/influxdb/tests/helper_test.py index 92a2c32d..1758d3c4 100644 --- a/influxdb/tests/helper_test.py +++ b/influxdb/tests/helper_test.py @@ -40,6 +40,14 @@ class Meta: TestSeriesHelper.MySeriesHelper = MySeriesHelper + def tearDown(self): + super(TestSeriesHelper, self).tearDown() + TestSeriesHelper.MySeriesHelper._reset_() + self.assertEqual( + TestSeriesHelper.MySeriesHelper._json_body_(), + [], + 'Resetting helper did not empty datapoints.') + def test_auto_commit(self): """ Tests that write_points is called after the right number of events @@ -130,11 +138,6 @@ def testSingleSeriesName(self, current_timestamp): all([el in rcvd for el in expectation]), 'Invalid JSON body of time series returned from ' '_json_body_ for one series name: {0}.'.format(rcvd)) - TestSeriesHelper.MySeriesHelper._reset_() - self.assertEqual( - TestSeriesHelper.MySeriesHelper._json_body_(), - [], - 'Resetting helper did not empty datapoints.') @patch('influxdb.helper.SeriesHelper._current_timestamp') def testSeveralSeriesNames(self, current_timestamp): @@ -203,11 +206,6 @@ def testSeveralSeriesNames(self, current_timestamp): 'Invalid JSON body of time series returned from ' '_json_body_ for several series names: {0}.' .format(rcvd)) - TestSeriesHelper.MySeriesHelper._reset_() - self.assertEqual( - TestSeriesHelper.MySeriesHelper._json_body_(), - [], - 'Resetting helper did not empty datapoints.') @patch('influxdb.helper.SeriesHelper._current_timestamp') def testSeriesWithoutTimeField(self, current_timestamp): @@ -229,11 +227,6 @@ def testSeriesWithoutTimeField(self, current_timestamp): self.assertTrue('time' in point1 and 'time' in point2) self.assertEqual(point1['time'], current_date) self.assertEqual(point2['time'], yesterday) - TestSeriesHelper.MySeriesHelper._reset_() - self.assertEqual( - TestSeriesHelper.MySeriesHelper._json_body_(), - [], - 'Resetting helper did not empty datapoints.') @patch('influxdb.helper.SeriesHelper._current_timestamp') def testSeriesWithTimeField(self, current_timestamp): From b934c7d8540467b6624041bc341c56528814aa97 Mon Sep 17 00:00:00 2001 From: Matt Snider Date: Sun, 13 Mar 2016 18:16:11 +0100 Subject: [PATCH 8/9] Use mock.patch() instead of unittest.mock.patch() for py27 --- influxdb/tests/helper_test.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/influxdb/tests/helper_test.py b/influxdb/tests/helper_test.py index 1758d3c4..405b5ed8 100644 --- a/influxdb/tests/helper_test.py +++ b/influxdb/tests/helper_test.py @@ -9,7 +9,6 @@ import mock from datetime import datetime, timedelta -from unittest.mock import patch from influxdb import SeriesHelper, InfluxDBClient from requests.exceptions import ConnectionError @@ -72,7 +71,7 @@ class Meta: AutoCommitTest(server_name='us.east-1', some_stat=3443, other_tag='gg') self.assertTrue(fake_write_points.called) - @patch('influxdb.helper.SeriesHelper._current_timestamp') + @mock.patch('influxdb.helper.SeriesHelper._current_timestamp') def testSingleSeriesName(self, current_timestamp): """ Tests JSON conversion when there is only one series name. @@ -139,7 +138,7 @@ def testSingleSeriesName(self, current_timestamp): 'Invalid JSON body of time series returned from ' '_json_body_ for one series name: {0}.'.format(rcvd)) - @patch('influxdb.helper.SeriesHelper._current_timestamp') + @mock.patch('influxdb.helper.SeriesHelper._current_timestamp') def testSeveralSeriesNames(self, current_timestamp): """ Tests JSON conversion when there are multiple series names. @@ -207,7 +206,7 @@ def testSeveralSeriesNames(self, current_timestamp): '_json_body_ for several series names: {0}.' .format(rcvd)) - @patch('influxdb.helper.SeriesHelper._current_timestamp') + @mock.patch('influxdb.helper.SeriesHelper._current_timestamp') def testSeriesWithoutTimeField(self, current_timestamp): """ Tests that time is optional on a series without a time field. @@ -228,7 +227,7 @@ def testSeriesWithoutTimeField(self, current_timestamp): self.assertEqual(point1['time'], current_date) self.assertEqual(point2['time'], yesterday) - @patch('influxdb.helper.SeriesHelper._current_timestamp') + @mock.patch('influxdb.helper.SeriesHelper._current_timestamp') def testSeriesWithTimeField(self, current_timestamp): """ Test that time is optional on a series with a time field. From 10828b52aef34ca0983bd8414df136073611c41c Mon Sep 17 00:00:00 2001 From: Matt Snider Date: Sun, 13 Mar 2016 18:32:32 +0100 Subject: [PATCH 9/9] Update SeriesHelper docstring --- influxdb/helper.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/influxdb/helper.py b/influxdb/helper.py index 7f35ab85..7f64de46 100644 --- a/influxdb/helper.py +++ b/influxdb/helper.py @@ -16,6 +16,9 @@ class SeriesHelper(object): All data points are immutable, insuring they do not get overwritten. Each subclass can write to its own database. The time series names can also be based on one or more defined fields. + The field "time" can be specified when creating a point, and may be any of + the time types supported by the client (i.e. str, datetime, int). + If the time is not specified, the current system time (utc) will be used. Annotated example::