From a66c555126bd6e6590909bc0c7f3c92ec4c0e339 Mon Sep 17 00:00:00 2001 From: Matthew McGinn Date: Wed, 27 Sep 2017 07:57:23 -0400 Subject: [PATCH 001/133] adding retry logic for all requests.exceptions adding an exponential backoff with jitter on POST calls reducing the number of retries in some of the tests to comply with the backoff feature. pass through all requests.exceptions after the permitted number of retries. --- influxdb/client.py | 30 ++++++++++++++++++++---------- influxdb/tests/client_test.py | 10 +++++----- 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/influxdb/client.py b/influxdb/client.py index 2b6dc689..5e1f0c81 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -7,6 +7,8 @@ from __future__ import unicode_literals from sys import version_info +import time +import random import json import socket @@ -237,6 +239,7 @@ def request(self, url, method='GET', params=None, data=None, _try = 0 while retry: try: + _error = False response = self._session.request( method=method, url=url, @@ -249,20 +252,27 @@ def request(self, url, method='GET', params=None, data=None, timeout=self._timeout ) break - except requests.exceptions.ConnectionError: + except (requests.exceptions.ConnectionError, + requests.exceptions.HTTPError, + requests.exceptions.Timeout) as _e: + _error = _e _try += 1 if self._retries != 0: retry = _try < self._retries - - else: - raise requests.exceptions.ConnectionError - - if 500 <= response.status_code < 600: - raise InfluxDBServerError(response.content) - elif response.status_code == expected_response_code: - return response + if method == "POST": + time.sleep((2 ** _try) * random.random() / 100.0) + if _error: + raise(_error) else: - raise InfluxDBClientError(response.content, response.status_code) + # if there's not an error, there must have been a successful + # response + if 500 <= response.status_code < 600: + raise InfluxDBServerError(response.content) + elif response.status_code == expected_response_code: + return response + else: + raise InfluxDBClientError(response.content, + response.status_code) def write(self, data, params=None, expected_response_code=204, protocol='json'): diff --git a/influxdb/tests/client_test.py b/influxdb/tests/client_test.py index 5c4f26dd..3413c0bf 100644 --- a/influxdb/tests/client_test.py +++ b/influxdb/tests/client_test.py @@ -686,7 +686,7 @@ def connection_error(self, *args, **kwargs): @mock.patch('requests.Session.request') def test_request_retry_raises(self, mock_request): - """Test that three connection errors will not be handled.""" + """Test that three requests errors will not be handled.""" class CustomMock(object): """Create custom mock object for test.""" @@ -698,7 +698,7 @@ def connection_error(self, *args, **kwargs): self.i += 1 if self.i < 4: - raise requests.exceptions.ConnectionError + raise requests.exceptions.HTTPError else: r = requests.Response() r.status_code = 200 @@ -708,7 +708,7 @@ def connection_error(self, *args, **kwargs): cli = InfluxDBClient(database='db') - with self.assertRaises(requests.exceptions.ConnectionError): + with self.assertRaises(requests.exceptions.HTTPError): cli.write_points(self.dummy_points) @mock.patch('requests.Session.request') @@ -732,7 +732,7 @@ def connection_error(self, *args, **kwargs): r.status_code = 204 return r - retries = random.randint(1, 100) + retries = random.randint(1, 5) mock_request.side_effect = CustomMock(retries).connection_error cli = InfluxDBClient(database='db', retries=retries) @@ -759,7 +759,7 @@ def connection_error(self, *args, **kwargs): r.status_code = 200 return r - retries = random.randint(1, 100) + retries = random.randint(1, 5) mock_request.side_effect = CustomMock(retries).connection_error cli = InfluxDBClient(database='db', retries=retries) From 9590a4d689bf5eb76d57ea06fda2f3bbe759fe21 Mon Sep 17 00:00:00 2001 From: Matthew McGinn Date: Thu, 26 Oct 2017 08:10:19 -0400 Subject: [PATCH 002/133] fixing broken pypy test --- .travis.yml | 1 + test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8c0093b8..03e6bc36 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,6 +31,7 @@ matrix: install: - pip install tox + - pip install setuptools==20.6.6 - pip install coveralls - mkdir influxdb_install - wget https://dl.influxdata.com/influxdb/releases/influxdb_1.2.4_amd64.deb diff --git a/test-requirements.txt b/test-requirements.txt index cbc6add3..9b31f5f1 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,4 +1,4 @@ nose nose-cov mock -requests-mock \ No newline at end of file +requests-mock From 4d9564aac34bc1d41f61bee72bd827935bd7e040 Mon Sep 17 00:00:00 2001 From: Matthew McGinn Date: Sat, 28 Oct 2017 14:47:17 -0400 Subject: [PATCH 003/133] replace dataframe.ix with dataframe.iloc. fixes #528 --- influxdb/_dataframe_client.py | 4 ++-- influxdb/influxdb08/dataframe_client.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/influxdb/_dataframe_client.py b/influxdb/_dataframe_client.py index 31ee1c32..af1decd0 100644 --- a/influxdb/_dataframe_client.py +++ b/influxdb/_dataframe_client.py @@ -86,7 +86,7 @@ def write_points(self, if protocol == 'line': points = self._convert_dataframe_to_lines( - dataframe.ix[start_index:end_index].copy(), + dataframe.iloc[start_index:end_index].copy(), measurement=measurement, global_tags=tags, time_precision=time_precision, @@ -95,7 +95,7 @@ def write_points(self, numeric_precision=numeric_precision) else: points = self._convert_dataframe_to_json( - dataframe.ix[start_index:end_index].copy(), + dataframe.iloc[start_index:end_index].copy(), measurement=measurement, tags=tags, time_precision=time_precision, diff --git a/influxdb/influxdb08/dataframe_client.py b/influxdb/influxdb08/dataframe_client.py index 71e7e881..2867125d 100644 --- a/influxdb/influxdb08/dataframe_client.py +++ b/influxdb/influxdb08/dataframe_client.py @@ -59,7 +59,7 @@ def write_points(self, data, *args, **kwargs): self._convert_dataframe_to_json( name=key, dataframe=data_frame - .ix[start_index:end_index].copy(), + .iloc[start_index:end_index].copy(), time_precision=time_precision)] InfluxDBClient.write_points(self, outdata, *args, **kwargs) return True From 56ab7203d2a2b125ab5eac50802b0ae88f2b7bca Mon Sep 17 00:00:00 2001 From: Andrew Spott Date: Tue, 14 Nov 2017 06:24:39 -0700 Subject: [PATCH 004/133] Added support for not including all fields when using the series helper (#518) * Added support for not including all fields when using the series helper * fixed flake8 check * Added tests * fixed pep error * more fixes for pep * fixed flake8 errors --- examples/tutorial_serieshelper.py | 3 ++- influxdb/helper.py | 23 ++++++++++++++++------- influxdb/tests/helper_test.py | 16 ++++++++++++++++ 3 files changed, 34 insertions(+), 8 deletions(-) diff --git a/examples/tutorial_serieshelper.py b/examples/tutorial_serieshelper.py index 13929df2..72b80bb5 100644 --- a/examples/tutorial_serieshelper.py +++ b/examples/tutorial_serieshelper.py @@ -51,7 +51,8 @@ class Meta: MySeriesHelper(server_name='us.east-1', some_stat=159, other_stat=10) MySeriesHelper(server_name='us.east-1', some_stat=158, other_stat=20) MySeriesHelper(server_name='us.east-1', some_stat=157, other_stat=30) -MySeriesHelper(server_name='us.east-1', some_stat=156, other_stat=40) +MySeriesHelper(server_name='us.east-1', some_stat=156, other_stat=30) +MySeriesHelper(server_name='us.east-1', some_stat=156) MySeriesHelper(server_name='us.east-1', some_stat=155, other_stat=50) # To manually submit data points which are not yet written, call commit: diff --git a/influxdb/helper.py b/influxdb/helper.py index c56a636a..e622526d 100644 --- a/influxdb/helper.py +++ b/influxdb/helper.py @@ -98,24 +98,31 @@ def __new__(cls, *args, **kwargs): if 'time' in cls._fields: cls._fields.remove('time') cls._type = namedtuple(cls.__name__, - cls._fields + cls._tags + ['time']) + ['time'] + cls._tags + cls._fields) + cls._type.__new__.__defaults__ = (None,) * len(cls._fields) return super(SeriesHelper, cls).__new__(cls) def __init__(self, **kw): - """Call to constructor creates a new data point. All fields must be present. + """Call to constructor creates a new data point. :note: Data points written when `bulk_size` is reached per Helper. :warning: Data points are *immutable* (`namedtuples`). """ cls = self.__class__ timestamp = kw.pop('time', self._current_timestamp()) + tags = set(cls._tags) + fields = set(cls._fields) + keys = set(kw.keys()) - if sorted(cls._fields + cls._tags) != sorted(kw.keys()): + # all tags should be passed, and keys - tags should be a subset of keys + if not(tags <= keys): raise NameError( - 'Expected {0}, got {1}.'.format( - sorted(cls._fields + cls._tags), - kw.keys())) + 'Expected arguments to contain all tags {0}, instead got {1}.' + .format(cls._tags, kw.keys())) + if not(keys - tags <= fields): + raise NameError('Got arguments not in tags or fields: {0}' + .format(keys - tags - fields)) cls._datapoints[cls._series_name.format(**kw)].append( cls._type(time=timestamp, **kw) @@ -157,7 +164,9 @@ def _json_body_(cls): } for field in cls._fields: - json_point['fields'][field] = getattr(point, field) + value = getattr(point, field) + if value is not None: + json_point['fields'][field] = value for tag in cls._tags: json_point['tags'][tag] = getattr(point, tag) diff --git a/influxdb/tests/helper_test.py b/influxdb/tests/helper_test.py index 39e5ee75..6f24e85d 100644 --- a/influxdb/tests/helper_test.py +++ b/influxdb/tests/helper_test.py @@ -231,6 +231,22 @@ def testSeriesWithoutTimeField(self, current_timestamp): self.assertEqual(point1['time'], current_date) self.assertEqual(point2['time'], yesterday) + def testSeriesWithoutAllTags(self): + """Test that creating a data point without a tag throws an error.""" + class MyTimeFieldSeriesHelper(SeriesHelper): + + class Meta: + client = TestSeriesHelper.client + series_name = 'events.stats.{server_name}' + fields = ['some_stat', 'time'] + tags = ['server_name', 'other_tag'] + bulk_size = 5 + autocommit = True + + self.assertRaises(NameError, MyTimeFieldSeriesHelper, + **{"server_name": 'us.east-1', + "some_stat": 158}) + @mock.patch('influxdb.helper.SeriesHelper._current_timestamp') def testSeriesWithTimeField(self, current_timestamp): """Test that time is optional on a series with a time field.""" From 2b95797fc36cec74dba2044122b34f6b28e4d424 Mon Sep 17 00:00:00 2001 From: Jason Swails Date: Tue, 14 Nov 2017 08:41:20 -0500 Subject: [PATCH 005/133] Add some small improvements (#536) * Add some small improvements - six is already listed as a requirement, so use six.moves instead of defining builtins based on checking sys.version_info - Fix formatting of several exceptions. * Add a blank line to appease flake8 --- influxdb/_dataframe_client.py | 8 ++++---- influxdb/client.py | 36 +++++++++++------------------------ influxdb/influxdb08/client.py | 13 ++----------- 3 files changed, 17 insertions(+), 40 deletions(-) diff --git a/influxdb/_dataframe_client.py b/influxdb/_dataframe_client.py index 31a78672..6a66558b 100644 --- a/influxdb/_dataframe_client.py +++ b/influxdb/_dataframe_client.py @@ -222,8 +222,8 @@ def _convert_dataframe_to_json(dataframe, .format(type(dataframe))) if not (isinstance(dataframe.index, pd.PeriodIndex) or isinstance(dataframe.index, pd.DatetimeIndex)): - raise TypeError('Must be DataFrame with DatetimeIndex or \ - PeriodIndex.') + raise TypeError('Must be DataFrame with DatetimeIndex or ' + 'PeriodIndex.') # Make sure tags and tag columns are correctly typed tag_columns = tag_columns if tag_columns is not None else [] @@ -279,8 +279,8 @@ def _convert_dataframe_to_lines(self, .format(type(dataframe))) if not (isinstance(dataframe.index, pd.PeriodIndex) or isinstance(dataframe.index, pd.DatetimeIndex)): - raise TypeError('Must be DataFrame with DatetimeIndex or \ - PeriodIndex.') + raise TypeError('Must be DataFrame with DatetimeIndex or ' + 'PeriodIndex.') # Create a Series of columns for easier indexing column_series = pd.Series(dataframe.columns) diff --git a/influxdb/client.py b/influxdb/client.py index 4721fa3f..608e8dbc 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -6,7 +6,6 @@ from __future__ import print_function from __future__ import unicode_literals -from sys import version_info import time import random @@ -14,22 +13,14 @@ import socket import requests import requests.exceptions +from six.moves import xrange +from six.moves.urllib.parse import urlparse from influxdb.line_protocol import make_lines, quote_ident, quote_literal from influxdb.resultset import ResultSet from .exceptions import InfluxDBClientError from .exceptions import InfluxDBServerError -try: - xrange -except NameError: - xrange = range - -if version_info[0] == 3: - from urllib.parse import urlparse -else: - from urlparse import urlparse - class InfluxDBClient(object): """InfluxDBClient primary client object to connect InfluxDB. @@ -239,7 +230,6 @@ def request(self, url, method='GET', params=None, data=None, _try = 0 while retry: try: - _error = False response = self._session.request( method=method, url=url, @@ -254,25 +244,21 @@ def request(self, url, method='GET', params=None, data=None, break except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError, - requests.exceptions.Timeout) as _e: - _error = _e + requests.exceptions.Timeout): _try += 1 if self._retries != 0: retry = _try < self._retries if method == "POST": time.sleep((2 ** _try) * random.random() / 100.0) - if _error: - raise(_error) + if not retry: + raise + # if there's not an error, there must have been a successful response + if 500 <= response.status_code < 600: + raise InfluxDBServerError(response.content) + elif response.status_code == expected_response_code: + return response else: - # if there's not an error, there must have been a successful - # response - if 500 <= response.status_code < 600: - raise InfluxDBServerError(response.content) - elif response.status_code == expected_response_code: - return response - else: - raise InfluxDBClientError(response.content, - response.status_code) + raise InfluxDBClientError(response.content, response.status_code) def write(self, data, params=None, expected_response_code=204, protocol='json'): diff --git a/influxdb/influxdb08/client.py b/influxdb/influxdb08/client.py index 9954133e..41600404 100644 --- a/influxdb/influxdb08/client.py +++ b/influxdb/influxdb08/client.py @@ -2,25 +2,16 @@ """Python client for InfluxDB v0.8.""" import warnings -from sys import version_info import json import socket import requests import requests.exceptions +from six.moves import xrange +from six.moves.urllib.parse import urlparse from influxdb import chunked_json -try: - xrange -except NameError: - xrange = range - -if version_info[0] == 3: - from urllib.parse import urlparse -else: - from urlparse import urlparse - session = requests.Session() From 16c02ecf2e9031ade91f956ea64830490b173761 Mon Sep 17 00:00:00 2001 From: Martin Englund Date: Wed, 15 Nov 2017 16:29:35 -0800 Subject: [PATCH 006/133] add ping method to the client (#409) * add ping method to the client * capitalize and pep257 compliance * one more try for pep257 compliance for the ping function * Update client.py * Update client_test.py fixing up failing CI tests * Update client_test.py fixing up failing CI tests --- influxdb/client.py | 13 +++++++++++++ influxdb/tests/client_test.py | 12 ++++++++++++ 2 files changed, 25 insertions(+) diff --git a/influxdb/client.py b/influxdb/client.py index 608e8dbc..68261ece 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -457,6 +457,19 @@ def write_points(self, retention_policy=retention_policy, tags=tags, protocol=protocol) + def ping(self): + """Check connectivity to InfluxDB. + + :returns: The version of the InfluxDB the client is connected to + """ + response = self.request( + url="ping", + method='GET', + expected_response_code=204 + ) + + return response.headers['X-Influxdb-Version'] + @staticmethod def _batches(iterable, size): for i in xrange(0, len(iterable), size): diff --git a/influxdb/tests/client_test.py b/influxdb/tests/client_test.py index 3413c0bf..ff325907 100644 --- a/influxdb/tests/client_test.py +++ b/influxdb/tests/client_test.py @@ -418,6 +418,18 @@ def test_query_fail(self): with _mocked_session(self.cli, 'get', 401): self.cli.query('select column_one from foo;') + def test_ping(self): + """Test ping querying InfluxDB version.""" + with requests_mock.Mocker() as m: + m.register_uri( + requests_mock.GET, + "http://localhost:8086/ping", + status_code=204, + headers={'X-Influxdb-Version': '1.2.3'} + ) + version = self.cli.ping() + self.assertEqual(version, '1.2.3') + def test_create_database(self): """Test create database for TestInfluxDBClient object.""" with requests_mock.Mocker() as m: From 530b4c504c114259475f46540f9ded8b6214bba2 Mon Sep 17 00:00:00 2001 From: Ivan <8692788+vaniakov@users.noreply.github.com> Date: Fri, 17 Nov 2017 04:37:30 +0200 Subject: [PATCH 007/133] Add pool size parameter to client constructor (#534) * Add pool size parameter to client constructor * come back removed newlines * fix flake8: line too long * Cast InfluxDBClient pool_size parameter to int * Move pool_size parameter to the end of args to prevent tests fail --- influxdb/client.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/influxdb/client.py b/influxdb/client.py index 68261ece..02128462 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -37,6 +37,8 @@ class InfluxDBClient(object): :type username: str :param password: password of the user, defaults to 'root' :type password: str + :param pool_size: urllib3 connection pool size, defaults to 10. + :type pool_size: int :param database: database name to connect to, defaults to None :type database: str :param ssl: use https instead of http to connect to InfluxDB, defaults to @@ -72,6 +74,7 @@ def __init__(self, use_udp=False, udp_port=4444, proxies=None, + pool_size=10, ): """Construct a new InfluxDBClient object.""" self.__host = host @@ -87,6 +90,11 @@ def __init__(self, self.__use_udp = use_udp self.__udp_port = udp_port self._session = requests.Session() + adapter = requests.adapters.HTTPAdapter( + pool_connections=int(pool_size), + pool_maxsize=int(pool_size) + ) + if use_udp: self.udp_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) @@ -95,6 +103,8 @@ def __init__(self, if ssl is True: self._scheme = "https" + self._session.mount(self._scheme, adapter) + if proxies is None: self._proxies = {} else: From 89961cf3e82d77e31c88cd6933888f591172acb6 Mon Sep 17 00:00:00 2001 From: xginn8 Date: Mon, 20 Nov 2017 23:46:21 -0500 Subject: [PATCH 008/133] Fix failing tags match in get_points() on a ResultSet (#511) * rename serie to series * pass in tags variable to perform match fix the format of the object in the ResultSet test to match what's returned from a query fix other flake8 formatting issues * meh * more serie -> series * fixing broken tags filtering * readding missing parameter that i somehow dropped --- docs/source/resultset.rst | 4 +- influxdb/influxdb08/client.py | 2 +- influxdb/resultset.py | 67 +++++------ influxdb/tests/resultset_test.py | 105 +++++++++++------- .../server_tests/client_test_with_server.py | 33 +++--- 5 files changed, 118 insertions(+), 93 deletions(-) diff --git a/docs/source/resultset.rst b/docs/source/resultset.rst index 6c36463f..b1c3206f 100644 --- a/docs/source/resultset.rst +++ b/docs/source/resultset.rst @@ -18,7 +18,7 @@ Using ``rs.get_points()`` will return a generator for all the points in the Resu Filtering by measurement ------------------------ -Using ``rs.get_points('cpu')`` will return a generator for all the points that are in a serie with measurement name ``cpu``, no matter the tags. +Using ``rs.get_points('cpu')`` will return a generator for all the points that are in a series with measurement name ``cpu``, no matter the tags. :: rs = cli.query("SELECT * from cpu") @@ -36,7 +36,7 @@ Using ``rs.get_points(tags={'host_name': 'influxdb.com'})`` will return a genera Filtering by measurement and tags --------------------------------- -Using measurement name and tags will return a generator for all the points that are in a serie with the specified measurement name AND whose tags match the given tags. +Using measurement name and tags will return a generator for all the points that are in a series with the specified measurement name AND whose tags match the given tags. :: rs = cli.query("SELECT * from cpu") diff --git a/influxdb/influxdb08/client.py b/influxdb/influxdb08/client.py index 41600404..965a91db 100644 --- a/influxdb/influxdb08/client.py +++ b/influxdb/influxdb08/client.py @@ -435,7 +435,7 @@ def _query(self, query, time_precision='s', chunked=False): else: chunked_param = 'false' - # Build the URL of the serie to query + # Build the URL of the series to query url = "db/{0}/series".format(self._database) params = { diff --git a/influxdb/resultset.py b/influxdb/resultset.py index 79d72ca8..ba4f3c13 100644 --- a/influxdb/resultset.py +++ b/influxdb/resultset.py @@ -41,12 +41,12 @@ def error(self): def __getitem__(self, key): """Retrieve the series name or specific set based on key. - :param key: Either a serie name, or a tags_dict, or - a 2-tuple(serie_name, tags_dict). - If the serie name is None (or not given) then any serie + :param key: Either a series name, or a tags_dict, or + a 2-tuple(series_name, tags_dict). + If the series name is None (or not given) then any serie matching the eventual given tags will be given its points one after the other. - To get the points of every serie in this resultset then + To get the points of every series in this resultset then you have to provide None as key. :return: A generator yielding `Point`s matching the given key. NB: @@ -93,22 +93,25 @@ def get_points(self, measurement=None, tags=None): (bytes, type(b''.decode()), type(None))): raise TypeError('measurement must be an str or None') - for serie in self._get_series(): - serie_name = serie.get('measurement', serie.get('name', 'results')) - if serie_name is None: + for series in self._get_series(): + series_name = series.get('measurement', + series.get('name', 'results')) + if series_name is None: # this is a "system" query or a query which # doesn't return a name attribute. # like 'show retention policies' .. if tags is None: - for item in self._get_points_for_serie(serie): + for item in self._get_points_for_series(series): yield item - elif measurement in (None, serie_name): + elif measurement in (None, series_name): # by default if no tags was provided then - # we will matches every returned serie - serie_tags = serie.get('tags', {}) - if tags is None or self._tag_matches(serie_tags, tags): - for item in self._get_points_for_serie(serie): + # we will matches every returned series + series_tags = series.get('tags', {}) + for item in self._get_points_for_series(series): + if tags is None or \ + self._tag_matches(item, tags) or \ + self._tag_matches(series_tags, tags): yield item def __repr__(self): @@ -121,7 +124,7 @@ def __repr__(self): return "ResultSet({%s})" % ", ".join(items) def __iter__(self): - """Yield one dict instance per serie result.""" + """Yield one dict instance per series result.""" for key in self.keys(): yield list(self.__getitem__(key)) @@ -131,10 +134,10 @@ def _tag_matches(tags, filter): for tag_name, tag_value in filter.items(): # using _sentinel as I'm not sure that "None" # could be used, because it could be a valid - # serie_tags value : when a serie has no such tag + # series_tags value : when a series has no such tag # then I think it's set to /null/None/.. TBC.. - serie_tag_value = tags.get(tag_name, _sentinel) - if serie_tag_value != tag_value: + series_tag_value = tags.get(tag_name, _sentinel) + if series_tag_value != tag_value: return False return True @@ -150,14 +153,14 @@ def __len__(self): def keys(self): """Return the list of keys in the ResultSet. - :return: List of keys. Keys are tuples (serie_name, tags) + :return: List of keys. Keys are tuples (series_name, tags) """ keys = [] - for serie in self._get_series(): + for series in self._get_series(): keys.append( - (serie.get('measurement', - serie.get('name', 'results')), - serie.get('tags', None)) + (series.get('measurement', + series.get('name', 'results')), + series.get('tags', None)) ) return keys @@ -167,24 +170,24 @@ def items(self): :return: List of tuples, (key, generator) """ items = [] - for serie in self._get_series(): - serie_key = (serie.get('measurement', - serie.get('name', 'results')), - serie.get('tags', None)) + for series in self._get_series(): + series_key = (series.get('measurement', + series.get('name', 'results')), + series.get('tags', None)) items.append( - (serie_key, self._get_points_for_serie(serie)) + (series_key, self._get_points_for_series(series)) ) return items - def _get_points_for_serie(self, serie): - """Return generator of dict from columns and values of a serie. + def _get_points_for_series(self, series): + """Return generator of dict from columns and values of a series. - :param serie: One serie + :param series: One series :return: Generator of dicts """ - for point in serie.get('values', []): + for point in series.get('values', []): yield self.point_from_cols_vals( - serie['columns'], + series['columns'], point ) diff --git a/influxdb/tests/resultset_test.py b/influxdb/tests/resultset_test.py index dd088b79..83faa4dd 100644 --- a/influxdb/tests/resultset_test.py +++ b/influxdb/tests/resultset_test.py @@ -19,26 +19,25 @@ def setUp(self): """Set up an instance of TestResultSet.""" self.query_response = { "results": [ - {"series": [{"measurement": "cpu_load_short", - "tags": {"host": "server01", - "region": "us-west"}, - "columns": ["time", "value"], + {"series": [{"name": "cpu_load_short", + "columns": ["time", "value", "host", "region"], "values": [ - ["2015-01-29T21:51:28.968422294Z", 0.64] + ["2015-01-29T21:51:28.968422294Z", + 0.64, + "server01", + "us-west"], + ["2015-01-29T21:51:28.968422294Z", + 0.65, + "server02", + "us-west"], ]}, - {"measurement": "cpu_load_short", - "tags": {"host": "server02", - "region": "us-west"}, - "columns": ["time", "value"], + {"name": "other_series", + "columns": ["time", "value", "host", "region"], "values": [ - ["2015-01-29T21:51:28.968422294Z", 0.65] - ]}, - {"measurement": "other_serie", - "tags": {"host": "server01", - "region": "us-west"}, - "columns": ["time", "value"], - "values": [ - ["2015-01-29T21:51:28.968422294Z", 0.66] + ["2015-01-29T21:51:28.968422294Z", + 0.66, + "server01", + "us-west"], ]}]} ] } @@ -48,8 +47,14 @@ def setUp(self): def test_filter_by_name(self): """Test filtering by name in TestResultSet object.""" expected = [ - {'value': 0.64, 'time': '2015-01-29T21:51:28.968422294Z'}, - {'value': 0.65, 'time': '2015-01-29T21:51:28.968422294Z'} + {'value': 0.64, + 'time': '2015-01-29T21:51:28.968422294Z', + 'host': 'server01', + 'region': 'us-west'}, + {'value': 0.65, + 'time': '2015-01-29T21:51:28.968422294Z', + 'host': 'server02', + 'region': 'us-west'}, ] self.assertEqual(expected, list(self.rs['cpu_load_short'])) @@ -60,8 +65,14 @@ def test_filter_by_name(self): def test_filter_by_tags(self): """Test filter by tags in TestResultSet object.""" expected = [ - {'time': '2015-01-29T21:51:28.968422294Z', 'value': 0.64}, - {'time': '2015-01-29T21:51:28.968422294Z', 'value': 0.66} + {'value': 0.64, + 'time': '2015-01-29T21:51:28.968422294Z', + 'host': 'server01', + 'region': 'us-west'}, + {'value': 0.66, + 'time': '2015-01-29T21:51:28.968422294Z', + 'host': 'server01', + 'region': 'us-west'}, ] self.assertEqual( @@ -78,14 +89,23 @@ def test_filter_by_name_and_tags(self): """Test filter by name and tags in TestResultSet object.""" self.assertEqual( list(self.rs[('cpu_load_short', {"host": "server01"})]), - [{'time': '2015-01-29T21:51:28.968422294Z', 'value': 0.64}] + [{'value': 0.64, + 'time': '2015-01-29T21:51:28.968422294Z', + 'host': 'server01', + 'region': 'us-west'}] ) self.assertEqual( list(self.rs[('cpu_load_short', {"region": "us-west"})]), [ - {'value': 0.64, 'time': '2015-01-29T21:51:28.968422294Z'}, - {'value': 0.65, 'time': '2015-01-29T21:51:28.968422294Z'} + {'value': 0.64, + 'time': '2015-01-29T21:51:28.968422294Z', + 'host': 'server01', + 'region': 'us-west'}, + {'value': 0.65, + 'time': '2015-01-29T21:51:28.968422294Z', + 'host': 'server02', + 'region': 'us-west'}, ] ) @@ -94,9 +114,8 @@ def test_keys(self): self.assertEqual( self.rs.keys(), [ - ('cpu_load_short', {'host': 'server01', 'region': 'us-west'}), - ('cpu_load_short', {'host': 'server02', 'region': 'us-west'}), - ('other_serie', {'host': 'server01', 'region': 'us-west'}) + ('cpu_load_short', None), + ('other_series', None), ] ) @@ -104,7 +123,7 @@ def test_len(self): """Test length in TestResultSet object.""" self.assertEqual( len(self.rs), - 3 + 2 ) def test_items(self): @@ -116,21 +135,23 @@ def test_items(self): items_lists, [ ( - ('cpu_load_short', - {'host': 'server01', 'region': 'us-west'}), - [{'value': 0.64, 'time': '2015-01-29T21:51:28.968422294Z'}] - ), + ('cpu_load_short', None), + [ + {'time': '2015-01-29T21:51:28.968422294Z', + 'value': 0.64, + 'host': 'server01', + 'region': 'us-west'}, + {'time': '2015-01-29T21:51:28.968422294Z', + 'value': 0.65, + 'host': 'server02', + 'region': 'us-west'}]), ( - ('cpu_load_short', - {'host': 'server02', 'region': 'us-west'}), - [{'value': 0.65, 'time': '2015-01-29T21:51:28.968422294Z'}] - ), - ( - ('other_serie', - {'host': 'server01', 'region': 'us-west'}), - [{'value': 0.66, 'time': '2015-01-29T21:51:28.968422294Z'}] - ) - ] + ('other_series', None), + [ + {'time': '2015-01-29T21:51:28.968422294Z', + 'value': 0.66, + 'host': 'server01', + 'region': 'us-west'}])] ) def test_point_from_cols_vals(self): diff --git a/influxdb/tests/server_tests/client_test_with_server.py b/influxdb/tests/server_tests/client_test_with_server.py index a7dedddd..701f72ac 100644 --- a/influxdb/tests/server_tests/client_test_with_server.py +++ b/influxdb/tests/server_tests/client_test_with_server.py @@ -38,9 +38,9 @@ THIS_DIR = os.path.abspath(os.path.dirname(__file__)) -def point(serie_name, timestamp=None, tags=None, **fields): +def point(series_name, timestamp=None, tags=None, **fields): """Define what a point looks like.""" - res = {'measurement': serie_name} + res = {'measurement': series_name} if timestamp: res['time'] = timestamp @@ -638,7 +638,7 @@ def test_drop_retention_policy(self): def test_issue_143(self): """Test for PR#143 from repo.""" - pt = partial(point, 'a_serie_name', timestamp='2015-03-30T16:16:37Z') + pt = partial(point, 'a_series_name', timestamp='2015-03-30T16:16:37Z') pts = [ pt(value=15), pt(tags={'tag_1': 'value1'}, value=5), @@ -646,19 +646,20 @@ def test_issue_143(self): ] self.cli.write_points(pts) time.sleep(1) - rsp = list(self.cli.query('SELECT * FROM a_serie_name GROUP BY tag_1')) + rsp = list(self.cli.query('SELECT * FROM a_series_name \ +GROUP BY tag_1').get_points()) self.assertEqual( [ - [{'value': 15, 'time': '2015-03-30T16:16:37Z'}], - [{'value': 5, 'time': '2015-03-30T16:16:37Z'}], - [{'value': 10, 'time': '2015-03-30T16:16:37Z'}] + {'time': '2015-03-30T16:16:37Z', 'value': 15}, + {'time': '2015-03-30T16:16:37Z', 'value': 5}, + {'time': '2015-03-30T16:16:37Z', 'value': 10} ], rsp ) # a slightly more complex one with 2 tags values: - pt = partial(point, 'serie2', timestamp='2015-03-30T16:16:37Z') + pt = partial(point, 'series2', timestamp='2015-03-30T16:16:37Z') pts = [ pt(tags={'tag1': 'value1', 'tag2': 'v1'}, value=0), pt(tags={'tag1': 'value1', 'tag2': 'v2'}, value=5), @@ -666,18 +667,18 @@ def test_issue_143(self): ] self.cli.write_points(pts) time.sleep(1) - rsp = self.cli.query('SELECT * FROM serie2 GROUP BY tag1,tag2') + rsp = self.cli.query('SELECT * FROM series2 GROUP BY tag1,tag2') self.assertEqual( [ - [{'value': 0, 'time': '2015-03-30T16:16:37Z'}], - [{'value': 5, 'time': '2015-03-30T16:16:37Z'}], - [{'value': 10, 'time': '2015-03-30T16:16:37Z'}] + {'value': 0, 'time': '2015-03-30T16:16:37Z'}, + {'value': 5, 'time': '2015-03-30T16:16:37Z'}, + {'value': 10, 'time': '2015-03-30T16:16:37Z'} ], - list(rsp) + list(rsp['series2']) ) - all_tag2_equal_v1 = list(rsp[None, {'tag2': 'v1'}]) + all_tag2_equal_v1 = list(rsp.get_points(tags={'tag2': 'v1'})) self.assertEqual( [{'value': 0, 'time': '2015-03-30T16:16:37Z'}, @@ -687,13 +688,13 @@ def test_issue_143(self): def test_query_multiple_series(self): """Test query for multiple series.""" - pt = partial(point, 'serie1', timestamp='2015-03-30T16:16:37Z') + pt = partial(point, 'series1', timestamp='2015-03-30T16:16:37Z') pts = [ pt(tags={'tag1': 'value1', 'tag2': 'v1'}, value=0), ] self.cli.write_points(pts) - pt = partial(point, 'serie2', timestamp='1970-03-30T16:16:37Z') + pt = partial(point, 'series2', timestamp='1970-03-30T16:16:37Z') pts = [ pt(tags={'tag1': 'value1', 'tag2': 'v1'}, value=0, data1=33, data2="bla"), From b59fe971af88d6e2a02f4f4c153c8fc4e464349c Mon Sep 17 00:00:00 2001 From: aviau Date: Mon, 20 Nov 2017 23:51:42 -0500 Subject: [PATCH 009/133] 5.0.0 release --- influxdb/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/influxdb/__init__.py b/influxdb/__init__.py index 6442e26b..33b7df4f 100644 --- a/influxdb/__init__.py +++ b/influxdb/__init__.py @@ -18,4 +18,4 @@ ] -__version__ = '4.1.1' +__version__ = '5.0.0' From 1fbba9719b2da619bc9ac73c5aa34dd4a60fb584 Mon Sep 17 00:00:00 2001 From: Alexandre Viau Date: Tue, 21 Nov 2017 00:03:46 -0500 Subject: [PATCH 010/133] README: add PyPI status --- README.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 0b7c144a..2eaf949f 100644 --- a/README.rst +++ b/README.rst @@ -12,7 +12,9 @@ InfluxDB-Python :target: https://coveralls.io/r/influxdata/influxdb-python :alt: Coverage - +.. image:: https://img.shields.io/pypi/v/influxdb.svg + :target: https://pypi.python.org/pypi/influxdb + :alt: PyPI Status InfluxDB-Python is a client for interacting with InfluxDB_. Development of this library is maintained by From dda70e51ed7da1e483f4eb059b91ae0c1008e2b0 Mon Sep 17 00:00:00 2001 From: Alexandre Viau Date: Tue, 21 Nov 2017 00:04:40 -0500 Subject: [PATCH 011/133] README: styling --- README.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 2eaf949f..c145cfc4 100644 --- a/README.rst +++ b/README.rst @@ -3,7 +3,6 @@ InfluxDB-Python .. image:: https://travis-ci.org/influxdata/influxdb-python.svg?branch=master :target: https://travis-ci.org/influxdata/influxdb-python - .. image:: https://readthedocs.org/projects/influxdb-python/badge/?version=latest&style :target: http://influxdb-python.readthedocs.org/ :alt: Documentation Status @@ -16,7 +15,9 @@ InfluxDB-Python :target: https://pypi.python.org/pypi/influxdb :alt: PyPI Status -InfluxDB-Python is a client for interacting with InfluxDB_. Development of this library is maintained by +InfluxDB-Python is a client for interacting with InfluxDB_. + +Development of this library is maintained by: +-----------+-------------------------------+ | Github ID | URL | From bf232a7aef9eb498751170ec3223f2020eadfecf Mon Sep 17 00:00:00 2001 From: Patrick Hoebeke Date: Sat, 25 Nov 2017 17:31:19 +0100 Subject: [PATCH 012/133] Fix for DataFrameClient issue - seems does not process correctly DateTimeIndex dates (issue #479) (#495) * [FIX] : compatibility with new version of pandas pd.tseries.period.PeriodIndex has been moved to pd.PeriodIndex since at least pandas 0.18.1 pd.tseries.period.DatetimeIndex has been moved to pd.DatetimeIndex since at least pandas 0.18.1 * [FIX] : Fixes #479 : DateTimeIndex not correctly converted to Unix Epoch (e.g .on (some?) Windows machines) * [FIX] : new fix for #479 : DateTimeIndex not correctly converted to Unix Epoch (e.g .on (some?) Windows machines) * [ENH] : added feature : DataFrame.write_points : NaNs and None values allowed in input DataFrame (corresponding entries are removed from the list of points to push to Influx) * [FIX] : error in unittest dataframe_client test_write_points_from_dataframe_with_all_none --- influxdb/_dataframe_client.py | 34 ++++++++--- influxdb/tests/dataframe_client_test.py | 75 +++++++++++++++++++++++++ 2 files changed, 101 insertions(+), 8 deletions(-) diff --git a/influxdb/_dataframe_client.py b/influxdb/_dataframe_client.py index 6a66558b..b8c83f59 100644 --- a/influxdb/_dataframe_client.py +++ b/influxdb/_dataframe_client.py @@ -10,6 +10,7 @@ from collections import defaultdict import pandas as pd +import numpy as np from .client import InfluxDBClient from .line_protocol import _escape_tag @@ -257,7 +258,7 @@ def _convert_dataframe_to_json(dataframe, {'measurement': measurement, 'tags': dict(list(tag.items()) + list(tags.items())), 'fields': rec, - 'time': int(ts.value / precision_factor)} + 'time': np.int64(ts.value / precision_factor)} for ts, tag, rec in zip(dataframe.index, dataframe[tag_columns].to_dict('record'), dataframe[field_columns].to_dict('record')) @@ -274,6 +275,10 @@ def _convert_dataframe_to_lines(self, time_precision=None, numeric_precision=None): + dataframe = dataframe.dropna(how='all').copy() + if len(dataframe) == 0: + return [] + if not isinstance(dataframe, pd.DataFrame): raise TypeError('Must be DataFrame, but type was: {0}.' .format(type(dataframe))) @@ -319,11 +324,11 @@ def _convert_dataframe_to_lines(self, # Make array of timestamp ints if isinstance(dataframe.index, pd.PeriodIndex): - time = ((dataframe.index.to_timestamp().values.astype(int) / - precision_factor).astype(int).astype(str)) + time = ((dataframe.index.to_timestamp().values.astype(np.int64) / + precision_factor).astype(np.int64).astype(str)) else: - time = ((pd.to_datetime(dataframe.index).values.astype(int) / - precision_factor).astype(int).astype(str)) + time = ((pd.to_datetime(dataframe.index).values.astype(np.int64) / + precision_factor).astype(np.int64).astype(str)) # If tag columns exist, make an array of formatted tag keys and values if tag_columns: @@ -357,12 +362,16 @@ def _convert_dataframe_to_lines(self, # Make an array of formatted field keys and values field_df = dataframe[field_columns] + field_df = self._stringify_dataframe(field_df, numeric_precision, datatype='field') - field_df = (field_df.columns.values + '=').tolist() + field_df - field_df[field_df.columns[1:]] = ',' + field_df[field_df.columns[1:]] - fields = field_df.sum(axis=1) + + def format_line(line): + line = line[~line.isnull()] # drop None entries + return ",".join((line.index + '=' + line.values)) + + fields = field_df.apply(format_line, axis=1) del field_df # Generate line protocol string @@ -371,6 +380,13 @@ def _convert_dataframe_to_lines(self, @staticmethod def _stringify_dataframe(dframe, numeric_precision, datatype='field'): + + # Prevent modification of input dataframe + dframe = dframe.copy() + + # Keep the positions where Null values are found + mask_null = dframe.isnull().values + # Find int and string columns for field-type data int_columns = dframe.select_dtypes(include=['integer']).columns string_columns = dframe.select_dtypes(include=['object']).columns @@ -414,6 +430,8 @@ def _stringify_dataframe(dframe, numeric_precision, datatype='field'): dframe = dframe.apply(_escape_pandas_series) dframe.columns = dframe.columns.astype(str) + + dframe = dframe.where(~mask_null, None) return dframe def _datetime_to_epoch(self, datetime, time_precision='s'): diff --git a/influxdb/tests/dataframe_client_test.py b/influxdb/tests/dataframe_client_test.py index 02aaac5f..269261d5 100644 --- a/influxdb/tests/dataframe_client_test.py +++ b/influxdb/tests/dataframe_client_test.py @@ -59,6 +59,81 @@ def test_write_points_from_dataframe(self): cli.write_points(dataframe, 'foo', tags=None) self.assertEqual(m.last_request.body, expected) + def test_write_points_from_dataframe_with_none(self): + """Test write points from df in TestDataFrameClient object.""" + now = pd.Timestamp('1970-01-01 00:00+00:00') + dataframe = pd.DataFrame(data=[["1", None, 1.0], ["2", 2.0, 2.0]], + index=[now, now + timedelta(hours=1)], + columns=["column_one", "column_two", + "column_three"]) + expected = ( + b"foo column_one=\"1\",column_three=1.0 0\n" + b"foo column_one=\"2\",column_two=2.0,column_three=2.0 " + b"3600000000000\n" + ) + + with requests_mock.Mocker() as m: + m.register_uri(requests_mock.POST, + "http://localhost:8086/write", + status_code=204) + + cli = DataFrameClient(database='db') + + cli.write_points(dataframe, 'foo') + self.assertEqual(m.last_request.body, expected) + + cli.write_points(dataframe, 'foo', tags=None) + self.assertEqual(m.last_request.body, expected) + + def test_write_points_from_dataframe_with_line_of_none(self): + """Test write points from df in TestDataFrameClient object.""" + now = pd.Timestamp('1970-01-01 00:00+00:00') + dataframe = pd.DataFrame(data=[[None, None, None], ["2", 2.0, 2.0]], + index=[now, now + timedelta(hours=1)], + columns=["column_one", "column_two", + "column_three"]) + expected = ( + b"foo column_one=\"2\",column_two=2.0,column_three=2.0 " + b"3600000000000\n" + ) + + with requests_mock.Mocker() as m: + m.register_uri(requests_mock.POST, + "http://localhost:8086/write", + status_code=204) + + cli = DataFrameClient(database='db') + + cli.write_points(dataframe, 'foo') + self.assertEqual(m.last_request.body, expected) + + cli.write_points(dataframe, 'foo', tags=None) + self.assertEqual(m.last_request.body, expected) + + def test_write_points_from_dataframe_with_all_none(self): + """Test write points from df in TestDataFrameClient object.""" + now = pd.Timestamp('1970-01-01 00:00+00:00') + dataframe = pd.DataFrame(data=[[None, None, None], [None, None, None]], + index=[now, now + timedelta(hours=1)], + columns=["column_one", "column_two", + "column_three"]) + expected = ( + b"\n" + ) + + with requests_mock.Mocker() as m: + m.register_uri(requests_mock.POST, + "http://localhost:8086/write", + status_code=204) + + cli = DataFrameClient(database='db') + + cli.write_points(dataframe, 'foo') + self.assertEqual(m.last_request.body, expected) + + cli.write_points(dataframe, 'foo', tags=None) + self.assertEqual(m.last_request.body, expected) + def test_write_points_from_dataframe_in_batches(self): """Test write points in batch from df in TestDataFrameClient object.""" now = pd.Timestamp('1970-01-01 00:00+00:00') From 12125309ca44c49d25ac4bbcfc614b749e7a9187 Mon Sep 17 00:00:00 2001 From: Matthew McGinn Date: Wed, 29 Nov 2017 08:11:00 -0500 Subject: [PATCH 013/133] adding back dropped database param fixes #539 --- influxdb/_dataframe_client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/influxdb/_dataframe_client.py b/influxdb/_dataframe_client.py index b8c83f59..d5b41d9f 100644 --- a/influxdb/_dataframe_client.py +++ b/influxdb/_dataframe_client.py @@ -175,6 +175,7 @@ def query(self, expected_response_code=expected_response_code, raise_errors=raise_errors, chunked=chunked, + database=database, chunk_size=chunk_size) results = super(DataFrameClient, self).query(query, **query_args) if query.strip().upper().startswith("SELECT"): From df70f5ce395d5650c14b9f68707220ab451b5ce8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thiago=20Figueir=C3=B3?= Date: Mon, 11 Dec 2017 08:21:39 +1100 Subject: [PATCH 014/133] doc: clarify that send_packet takes a list (#545) --- influxdb/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/influxdb/client.py b/influxdb/client.py index 02128462..01559cfc 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -869,7 +869,7 @@ def send_packet(self, packet, protocol='json'): :param packet: the packet to be sent :type packet: (if protocol is 'json') dict - (if protocol is 'line') sequence of line protocol strings + (if protocol is 'line') list of line protocol strings :param protocol: protocol of input data, either 'json' or 'line' :type protocol: str """ From 6b5db784dcb37495c1f24e3722973878eb3dd53d Mon Sep 17 00:00:00 2001 From: Ivan <8692788+vaniakov@users.noreply.github.com> Date: Mon, 11 Dec 2017 15:40:18 +0200 Subject: [PATCH 015/133] Escape tag values that ends with backslash (#537) * FAM-1163 escape tag values that ends with backslash * Add tests for _escape_tag_value func * Remove print statement --- influxdb/line_protocol.py | 9 ++++++++- influxdb/tests/test_line_protocol.py | 3 ++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/influxdb/line_protocol.py b/influxdb/line_protocol.py index c399a1d5..e8816fc0 100644 --- a/influxdb/line_protocol.py +++ b/influxdb/line_protocol.py @@ -57,6 +57,13 @@ def _escape_tag(tag): ) +def _escape_tag_value(value): + ret = _escape_tag(value) + if ret.endswith('\\'): + ret += ' ' + return ret + + def quote_ident(value): """Indent the quotes.""" return "\"{}\"".format(value @@ -135,7 +142,7 @@ def make_lines(data, precision=None): # tags should be sorted client-side to take load off server for tag_key, tag_value in sorted(iteritems(tags)): key = _escape_tag(tag_key) - value = _escape_tag(tag_value) + value = _escape_tag_value(tag_value) if key != '' and value != '': key_values.append(key + "=" + value) diff --git a/influxdb/tests/test_line_protocol.py b/influxdb/tests/test_line_protocol.py index dbee8cda..a3d84793 100644 --- a/influxdb/tests/test_line_protocol.py +++ b/influxdb/tests/test_line_protocol.py @@ -22,6 +22,7 @@ def test_make_lines(self): "tags": { "empty_tag": "", "none_tag": None, + "backslash_tag": "C:\\", "integer_tag": 2, "string_tag": "hello" }, @@ -41,7 +42,7 @@ def test_make_lines(self): self.assertEqual( line_protocol.make_lines(data), - 'test,integer_tag=2,string_tag=hello ' + 'test,backslash_tag=C:\\\\ ,integer_tag=2,string_tag=hello ' 'bool_val=True,float_val=1.1,int_val=1i,string_val="hello!"\n' ) From 35732cd7dfe5a585564999c9f881bd88e7c1531d Mon Sep 17 00:00:00 2001 From: Tzong Hao Chen Date: Tue, 12 Dec 2017 08:14:07 -0500 Subject: [PATCH 016/133] DataFrameClient should escape measurement names (#542) * Fixed: DataFrameClient should escape measurement names Issue #520 * Fix pep257 error --- influxdb/_dataframe_client.py | 1 + influxdb/tests/dataframe_client_test.py | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/influxdb/_dataframe_client.py b/influxdb/_dataframe_client.py index d5b41d9f..86b582af 100644 --- a/influxdb/_dataframe_client.py +++ b/influxdb/_dataframe_client.py @@ -376,6 +376,7 @@ def format_line(line): del field_df # Generate line protocol string + measurement = _escape_tag(measurement) points = (measurement + tags + ' ' + fields + ' ' + time).tolist() return points diff --git a/influxdb/tests/dataframe_client_test.py b/influxdb/tests/dataframe_client_test.py index 269261d5..5a717f5c 100644 --- a/influxdb/tests/dataframe_client_test.py +++ b/influxdb/tests/dataframe_client_test.py @@ -59,6 +59,28 @@ def test_write_points_from_dataframe(self): cli.write_points(dataframe, 'foo', tags=None) self.assertEqual(m.last_request.body, expected) + def test_dataframe_write_points_with_whitespace_measurement(self): + """write_points should escape white space in measurements.""" + now = pd.Timestamp('1970-01-01 00:00+00:00') + dataframe = pd.DataFrame(data=[["1", 1, 1.0], ["2", 2, 2.0]], + index=[now, now + timedelta(hours=1)], + columns=["column_one", "column_two", + "column_three"]) + expected = ( + b"meas\\ with\\ space " + b"column_one=\"1\",column_two=1i,column_three=1.0 0\n" + b"meas\\ with\\ space " + b"column_one=\"2\",column_two=2i,column_three=2.0 " + b"3600000000000\n" + ) + with requests_mock.Mocker() as m: + m.register_uri(requests_mock.POST, + "http://localhost:8086/write", + status_code=204) + cli = DataFrameClient(database='db') + cli.write_points(dataframe, 'meas with space') + self.assertEqual(m.last_request.body, expected) + def test_write_points_from_dataframe_with_none(self): """Test write points from df in TestDataFrameClient object.""" now = pd.Timestamp('1970-01-01 00:00+00:00') From a26b39fe3fbdc6471c8d21bdc581c677b3dbd09f Mon Sep 17 00:00:00 2001 From: xginn8 Date: Sat, 10 Feb 2018 18:32:16 -0500 Subject: [PATCH 017/133] specify the numpy dependency explicitly to prevent regression in test (#563) suite --- tox.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tox.ini b/tox.ini index bfa25165..5c54d680 100644 --- a/tox.ini +++ b/tox.ini @@ -7,6 +7,7 @@ setenv = INFLUXDB_PYTHON_SKIP_SERVER_TESTS=False deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt py27,py34,py35,py36: pandas==0.20.1 + py27,py34,py35,py36: numpy==1.13.3 # Only install pandas with non-pypy interpreters commands = nosetests -v --with-doctest {posargs} @@ -25,11 +26,13 @@ deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt pandas coverage + numpy==1.13.3 commands = nosetests -v --with-coverage --cover-html --cover-package=influxdb [testenv:docs] deps = -r{toxinidir}/requirements.txt pandas==0.20.1 + numpy==1.13.3 Sphinx==1.5.5 sphinx_rtd_theme commands = sphinx-build -b html docs/source docs/build From b47362c04938330237b49234d62b5dacfa8b5036 Mon Sep 17 00:00:00 2001 From: Jan Stodt Date: Sun, 11 Feb 2018 19:31:00 +0100 Subject: [PATCH 018/133] Fix pandas example (#547) Current example code raises KeyError: '[0] not in index' and can be fixed by adding columns=['0'] to the DataFrame creation. Fixed issue #497 --- examples/tutorial_pandas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/tutorial_pandas.py b/examples/tutorial_pandas.py index 9cec910d..b5fb8f79 100644 --- a/examples/tutorial_pandas.py +++ b/examples/tutorial_pandas.py @@ -20,7 +20,7 @@ def main(host='localhost', port=8086): print("Create pandas DataFrame") df = pd.DataFrame(data=list(range(30)), index=pd.date_range(start='2014-11-16', - periods=30, freq='H')) + periods=30, freq='H'), columns=['0']) print("Create database: " + dbname) client.create_database(dbname) From ed276e2187c5961ef92018c61febaa538acda644 Mon Sep 17 00:00:00 2001 From: Maura Hausman Date: Tue, 13 Feb 2018 17:48:38 -0500 Subject: [PATCH 019/133] Remove UDP Precision Restrictions (#557) - address issue #554 - UDP writes can now convert timestamps to the desired precision - add time_precision argument to `InfluxDBClient.send_packet`, defaults to None - add tests for udp precision - remove old udp precision failure tests --- influxdb/client.py | 15 ++++--- influxdb/tests/client_test.py | 75 +++++++++++++++++++++++++++-------- 2 files changed, 66 insertions(+), 24 deletions(-) diff --git a/influxdb/client.py b/influxdb/client.py index 01559cfc..e38d4b78 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -497,11 +497,6 @@ def _write_points(self, "Invalid time precision is given. " "(use 'n', 'u', 'ms', 's', 'm' or 'h')") - if self._use_udp and time_precision and time_precision != 's': - raise ValueError( - "InfluxDB only supports seconds precision for udp writes" - ) - if protocol == 'json': data = { 'points': points @@ -523,7 +518,9 @@ def _write_points(self, params['rp'] = retention_policy if self._use_udp: - self.send_packet(data, protocol=protocol) + self.send_packet( + data, protocol=protocol, time_precision=time_precision + ) else: self.write( data=data, @@ -864,7 +861,7 @@ def get_list_privileges(self, username): text = "SHOW GRANTS FOR {0}".format(quote_ident(username)) return list(self.query(text).get_points()) - def send_packet(self, packet, protocol='json'): + def send_packet(self, packet, protocol='json', time_precision=None): """Send an UDP packet. :param packet: the packet to be sent @@ -872,9 +869,11 @@ def send_packet(self, packet, protocol='json'): (if protocol is 'line') list of line protocol strings :param protocol: protocol of input data, either 'json' or 'line' :type protocol: str + :param time_precision: Either 's', 'm', 'ms' or 'u', defaults to None + :type time_precision: str """ if protocol == 'json': - data = make_lines(packet).encode('utf-8') + data = make_lines(packet, time_precision).encode('utf-8') elif protocol == 'line': data = ('\n'.join(packet) + '\n').encode('utf-8') self.udp_socket.sendto(data, (self._host, self._udp_port)) diff --git a/influxdb/tests/client_test.py b/influxdb/tests/client_test.py index ff325907..ebf5d424 100644 --- a/influxdb/tests/client_test.py +++ b/influxdb/tests/client_test.py @@ -259,22 +259,6 @@ def test_write_points_udp(self): received_data.decode() ) - def test_write_bad_precision_udp(self): - """Test write bad precision in UDP for TestInfluxDBClient object.""" - cli = InfluxDBClient( - 'localhost', 8086, 'root', 'root', - 'test', use_udp=True, udp_port=4444 - ) - - with self.assertRaisesRegexp( - Exception, - "InfluxDB only supports seconds precision for udp writes" - ): - cli.write_points( - self.dummy_points, - time_precision='ms' - ) - @raises(Exception) def test_write_points_fails(self): """Test write points fail for TestInfluxDBClient object.""" @@ -335,6 +319,65 @@ def test_write_points_with_precision(self): m.last_request.body, ) + def test_write_points_with_precision_udp(self): + """Test write points with precision for TestInfluxDBClient object.""" + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + port = random.randint(4000, 8000) + s.bind(('0.0.0.0', port)) + + cli = InfluxDBClient( + 'localhost', 8086, 'root', 'root', + 'test', use_udp=True, udp_port=port + ) + + cli.write_points(self.dummy_points, time_precision='n') + received_data, addr = s.recvfrom(1024) + self.assertEqual( + b'cpu_load_short,host=server01,region=us-west ' + b'value=0.64 1257894000123456000\n', + received_data, + ) + + cli.write_points(self.dummy_points, time_precision='u') + received_data, addr = s.recvfrom(1024) + self.assertEqual( + b'cpu_load_short,host=server01,region=us-west ' + b'value=0.64 1257894000123456\n', + received_data, + ) + + cli.write_points(self.dummy_points, time_precision='ms') + received_data, addr = s.recvfrom(1024) + self.assertEqual( + b'cpu_load_short,host=server01,region=us-west ' + b'value=0.64 1257894000123\n', + received_data, + ) + + cli.write_points(self.dummy_points, time_precision='s') + received_data, addr = s.recvfrom(1024) + self.assertEqual( + b"cpu_load_short,host=server01,region=us-west " + b"value=0.64 1257894000\n", + received_data, + ) + + cli.write_points(self.dummy_points, time_precision='m') + received_data, addr = s.recvfrom(1024) + self.assertEqual( + b'cpu_load_short,host=server01,region=us-west ' + b'value=0.64 20964900\n', + received_data, + ) + + cli.write_points(self.dummy_points, time_precision='h') + received_data, addr = s.recvfrom(1024) + self.assertEqual( + b'cpu_load_short,host=server01,region=us-west ' + b'value=0.64 349415\n', + received_data, + ) + def test_write_points_bad_precision(self): """Test write points w/bad precision TestInfluxDBClient object.""" cli = InfluxDBClient() From 13cdddb178656f3069aa806cdad2c7964b7fb167 Mon Sep 17 00:00:00 2001 From: Ivan <8692788+vaniakov@users.noreply.github.com> Date: Fri, 30 Mar 2018 20:07:43 +0300 Subject: [PATCH 020/133] Fix wrong session mount (#571) --- influxdb/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/influxdb/client.py b/influxdb/client.py index e38d4b78..f5c0b55b 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -103,7 +103,7 @@ def __init__(self, if ssl is True: self._scheme = "https" - self._session.mount(self._scheme, adapter) + self._session.mount(self._scheme + '://', adapter) if proxies is None: self._proxies = {} From fd4579c95856e0e79014204f308f02ed8a2a55a0 Mon Sep 17 00:00:00 2001 From: dennis Date: Thu, 3 May 2018 05:05:20 +0300 Subject: [PATCH 021/133] Remove comment as issues have been resolved (#581) --- examples/tutorial_pandas.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/tutorial_pandas.py b/examples/tutorial_pandas.py index b5fb8f79..67a5457d 100644 --- a/examples/tutorial_pandas.py +++ b/examples/tutorial_pandas.py @@ -12,7 +12,6 @@ def main(host='localhost', port=8086): user = 'root' password = 'root' dbname = 'demo' - # Temporarily avoid line protocol time conversion issues #412, #426, #431. protocol = 'json' client = DataFrameClient(host, port, user, password, dbname) From ed4561b8d31a8f8490660a192948527c17f55c5a Mon Sep 17 00:00:00 2001 From: dennis Date: Tue, 8 May 2018 03:48:49 +0300 Subject: [PATCH 022/133] Parse column names in a dataframe to avoid breaking the line protocol (#584) * Remove comment as issues have been resolved * Parse column names in a dataframe to handle spaces in tag of field keys * Patch for ERROR: Test failed write points from df with series * Patch for flake8 issues(E231, E501) * Test case for spaces in column names * flake8 issues * flake8 issues: trailing wspace * Testing if test case would catch a regression * Test catches a regressed build * Re-run the build --- influxdb/_dataframe_client.py | 2 ++ influxdb/tests/dataframe_client_test.py | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/influxdb/_dataframe_client.py b/influxdb/_dataframe_client.py index 86b582af..2444a77f 100644 --- a/influxdb/_dataframe_client.py +++ b/influxdb/_dataframe_client.py @@ -288,6 +288,8 @@ def _convert_dataframe_to_lines(self, raise TypeError('Must be DataFrame with DatetimeIndex or ' 'PeriodIndex.') + dataframe = dataframe.rename( + columns={item: _escape_tag(item) for item in dataframe.columns}) # Create a Series of columns for easier indexing column_series = pd.Series(dataframe.columns) diff --git a/influxdb/tests/dataframe_client_test.py b/influxdb/tests/dataframe_client_test.py index 5a717f5c..78f5437f 100644 --- a/influxdb/tests/dataframe_client_test.py +++ b/influxdb/tests/dataframe_client_test.py @@ -81,6 +81,26 @@ def test_dataframe_write_points_with_whitespace_measurement(self): cli.write_points(dataframe, 'meas with space') self.assertEqual(m.last_request.body, expected) + def test_dataframe_write_points_with_whitespace_in_column_names(self): + """write_points should escape white space in column names.""" + now = pd.Timestamp('1970-01-01 00:00+00:00') + dataframe = pd.DataFrame(data=[["1", 1, 1.0], ["2", 2, 2.0]], + index=[now, now + timedelta(hours=1)], + columns=["column one", "column two", + "column three"]) + expected = ( + b"foo column\\ one=\"1\",column\\ two=1i,column\\ three=1.0 0\n" + b"foo column\\ one=\"2\",column\\ two=2i,column\\ three=2.0 " + b"3600000000000\n" + ) + with requests_mock.Mocker() as m: + m.register_uri(requests_mock.POST, + "http://localhost:8086/write", + status_code=204) + cli = DataFrameClient(database='db') + cli.write_points(dataframe, 'foo') + self.assertEqual(m.last_request.body, expected) + def test_write_points_from_dataframe_with_none(self): """Test write points from df in TestDataFrameClient object.""" now = pd.Timestamp('1970-01-01 00:00+00:00') From f3c6acf503e1befd767c4cd70f46d9561341c05a Mon Sep 17 00:00:00 2001 From: Frederik Gladhorn Date: Tue, 8 May 2018 03:22:20 +0200 Subject: [PATCH 023/133] Allow connecting to influxdb running on a path on the server (#556) * Allow connecting to influxdb running on a path on the server Make it possible to connect to the databases on a path on servers. https://someserver.com/myinfluxdb instead of the root of the server. * Test and fix for None path --- influxdb/client.py | 19 +++++++++++++++++-- influxdb/tests/client_test.py | 18 ++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/influxdb/client.py b/influxdb/client.py index f5c0b55b..62d5a025 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -59,6 +59,8 @@ class InfluxDBClient(object): :type udp_port: int :param proxies: HTTP(S) proxy to use for Requests, defaults to {} :type proxies: dict + :param path: path of InfluxDB on the server to connect, defaults to '' + :type path: str """ def __init__(self, @@ -75,6 +77,7 @@ def __init__(self, udp_port=4444, proxies=None, pool_size=10, + path='', ): """Construct a new InfluxDBClient object.""" self.__host = host @@ -98,6 +101,13 @@ def __init__(self, if use_udp: self.udp_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + if not path: + self.__path = '' + elif path[0] == '/': + self.__path = path + else: + self.__path = '/' + path + self._scheme = "http" if ssl is True: @@ -110,10 +120,11 @@ def __init__(self, else: self._proxies = proxies - self.__baseurl = "{0}://{1}:{2}".format( + self.__baseurl = "{0}://{1}:{2}{3}".format( self._scheme, self._host, - self._port) + self._port, + self._path) self._headers = { 'Content-Type': 'application/json', @@ -132,6 +143,10 @@ def _host(self): def _port(self): return self.__port + @property + def _path(self): + return self.__path + @property def _udp_port(self): return self.__udp_port diff --git a/influxdb/tests/client_test.py b/influxdb/tests/client_test.py index ebf5d424..efdfb770 100644 --- a/influxdb/tests/client_test.py +++ b/influxdb/tests/client_test.py @@ -109,6 +109,24 @@ def test_scheme(self): ) self.assertEqual('https://host:8086', cli._baseurl) + cli = InfluxDBClient( + 'host', 8086, 'username', 'password', 'database', ssl=True, + path="somepath" + ) + self.assertEqual('https://host:8086/somepath', cli._baseurl) + + cli = InfluxDBClient( + 'host', 8086, 'username', 'password', 'database', ssl=True, + path=None + ) + self.assertEqual('https://host:8086', cli._baseurl) + + cli = InfluxDBClient( + 'host', 8086, 'username', 'password', 'database', ssl=True, + path="/somepath" + ) + self.assertEqual('https://host:8086/somepath', cli._baseurl) + def test_dsn(self): """Set up the test datasource name for TestInfluxDBClient object.""" cli = InfluxDBClient.from_dsn('influxdb://192.168.0.1:1886') From 472de65b99bb0228f24688ef3e84ed1d5263e426 Mon Sep 17 00:00:00 2001 From: dragoshenron Date: Mon, 11 Jun 2018 04:13:22 +0200 Subject: [PATCH 024/133] Update _dataframe_client.py (#593) Permanent error "NameError: name 'to_datetime' is not defined". Reason: to_datetime is a function defined in pandas not a method on a DataFrame. (see: https://stackoverflow.com/questions/48387878/attributeerror-dataframe-object-has-no-attribute-to-datetime) Tested with Python 3.5.3 and Pandas 0.23.0 --- influxdb/_dataframe_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/influxdb/_dataframe_client.py b/influxdb/_dataframe_client.py index 2444a77f..4273ef1b 100644 --- a/influxdb/_dataframe_client.py +++ b/influxdb/_dataframe_client.py @@ -236,7 +236,7 @@ def _convert_dataframe_to_json(dataframe, field_columns = list( set(dataframe.columns).difference(set(tag_columns))) - dataframe.index = dataframe.index.to_datetime() + dataframe.index = pd.to_datetime(dataframe.index) if dataframe.index.tzinfo is None: dataframe.index = dataframe.index.tz_localize('UTC') From b3ed5db34345e1763974048f700bd7ae9cf147e4 Mon Sep 17 00:00:00 2001 From: aviau Date: Tue, 26 Jun 2018 19:14:53 -0400 Subject: [PATCH 025/133] set version to 5.1.0 --- influxdb/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/influxdb/__init__.py b/influxdb/__init__.py index 33b7df4f..374fddc7 100644 --- a/influxdb/__init__.py +++ b/influxdb/__init__.py @@ -18,4 +18,4 @@ ] -__version__ = '5.0.0' +__version__ = '5.1.0' From c300105d906dd44fb9ef894fda295010e227f521 Mon Sep 17 00:00:00 2001 From: Shu Shen Date: Sat, 30 Jun 2018 09:24:06 -0700 Subject: [PATCH 026/133] Fix performance degradation with line protocol (#592) Assemble line by line in the commit bf232a7aef to remove NaN has significant performance impact. This change fixes the issue by keeping the NaN fields before stringify the dataframe, replacing the fields with empty string, and reverting back to use pd.DataFrame.sum() function to yield the lines. Fixes: #591 --- influxdb/_dataframe_client.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/influxdb/_dataframe_client.py b/influxdb/_dataframe_client.py index 4273ef1b..646f298c 100644 --- a/influxdb/_dataframe_client.py +++ b/influxdb/_dataframe_client.py @@ -365,16 +365,18 @@ def _convert_dataframe_to_lines(self, # Make an array of formatted field keys and values field_df = dataframe[field_columns] + # Keep the positions where Null values are found + mask_null = field_df.isnull().values field_df = self._stringify_dataframe(field_df, numeric_precision, datatype='field') - def format_line(line): - line = line[~line.isnull()] # drop None entries - return ",".join((line.index + '=' + line.values)) - - fields = field_df.apply(format_line, axis=1) + field_df = (field_df.columns.values + '=').tolist() + field_df + field_df[field_df.columns[1:]] = ',' + field_df[ + field_df.columns[1:]] + field_df = field_df.where(~mask_null, '') # drop Null entries + fields = field_df.sum(axis=1) del field_df # Generate line protocol string @@ -388,9 +390,6 @@ def _stringify_dataframe(dframe, numeric_precision, datatype='field'): # Prevent modification of input dataframe dframe = dframe.copy() - # Keep the positions where Null values are found - mask_null = dframe.isnull().values - # Find int and string columns for field-type data int_columns = dframe.select_dtypes(include=['integer']).columns string_columns = dframe.select_dtypes(include=['object']).columns @@ -435,7 +434,6 @@ def _stringify_dataframe(dframe, numeric_precision, datatype='field'): dframe.columns = dframe.columns.astype(str) - dframe = dframe.where(~mask_null, None) return dframe def _datetime_to_epoch(self, datetime, time_precision='s'): From 9b2d7d1c75a620f161582e9a21127b5083b64213 Mon Sep 17 00:00:00 2001 From: Matthew McGinn Date: Sat, 30 Jun 2018 18:11:18 -0400 Subject: [PATCH 027/133] Add an initial stub for a CHANGELOG.md --- CHANGELOG.md | 263 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 263 insertions(+) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..2f9cbf97 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,263 @@ +# Changelog +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) +and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). + +## [Unreleased] +### Added +### Changed +- Fix performance degradation when removing NaN values via line protocol (#592) +### Removed + +## [v5.1.0] - 2018-06-26 +### Added +- Connect to InfluxDB path running on server (#556 thx @gladhorn) +- Escape measurement names in DataFrameClient (#542 thx @tzonghao) +- Escape tags that end with a backslash (#537 thx @vaniakov) +- Add back mistakenly-dropped database parameter (#540) +- Add PyPI status to README.md +### Changed +- Fix bad session mount scheme (#571 thx @vaniakov) +- Fixed issue with DataFrameClient calling to_datetime function (#593 thx @dragoshenron) +- Escape columns in DataFrameClient for line protocol (#584 thx @dmuiruri) +- Convert DataFrameClient times from int to np.int64 (#495 thx patrickhoebeke) +- Updated pandas tutorial (#547 thx @techaddicted) +- Explicitly set numpy version for tox (#563) +### Removed +- Removed UDP precision restrictions on timestamp (#557 thx @mdhausman) + +## [v5.0.0] - 2017-11-20 +### Added +### Changed +### Removed + +## [v4.1.1] - 2017-06-06 +### Added +### Changed +### Removed + +## [v4.1.0] - 2017-04-12 +### Added +### Changed +### Removed + +## [v4.0.0] - 2016-12-07 +### Added +### Changed +### Removed + +## [v3.0.0] - 2016-06-26 +### Added +### Changed +### Removed + +## [v2.12.0] - 2016-01-29 +### Added +### Changed +### Removed + +## [v2.11.0] - 2016-01-11 +### Added +### Changed +### Removed + +## [v2.10.0] - 2015-11-13 +### Added +### Changed +### Removed + +## [v2.9.3] - 2015-10-30 +### Added +### Changed +### Removed + +## [v2.9.2] - 2015-10-07 +### Added +### Changed +### Removed + +## [v2.9.1] - 2015-08-30 +### Added +### Changed +### Removed + +## [v2.9.0] - 2015-08-28 +### Added +### Changed +### Removed + +## [v2.8.0] - 2015-08-06 +### Added +### Changed +### Removed + +## [v2.7.3] - 2015-07-31 +### Added +### Changed +### Removed + +## [v2.7.2] - 2015-07-31 +### Added +### Changed +### Removed + +## [v2.7.1] - 2015-07-26 +### Added +### Changed +### Removed + +## [v2.7.0] - 2015-07-23 +### Added +### Changed +### Removed + +## [v2.6.0] - 2015-06-16 +### Added +### Changed +### Removed + +## [v2.5.1] - 2015-06-15 +### Added +### Changed +### Removed + +## [v2.5.0] - 2015-06-15 +### Added +### Changed +### Removed + +## [v2.4.0] - 2015-06-12 +### Added +### Changed +### Removed + +## [v2.3.0] - 2015-05-13 +### Added +### Changed +### Removed + +## [v2.2.0] - 2015-05-05 +### Added +### Changed +### Removed + +## [v2.1.0] - 2015-04-24 +### Added +### Changed +### Removed + +## [v2.0.2] - 2015-04-22 +### Added +### Changed +### Removed + +## [v2.0.1] - 2015-04-17 +### Added +### Changed +### Removed + +## [v2.0.0] - 2015-04-17 +### Added +### Changed +### Removed + +## [v1.0.1] - 2015-03-30 +### Added +### Changed +### Removed + +## [v1.0.0] - 2015-03-20 +### Added +### Changed +### Removed + +## [v0.4.1] - 2015-03-18 +### Added +### Changed +### Removed + +## [v0.4.0] - 2015-03-17 +### Added +### Changed +### Removed + +## [v0.3.1] - 2015-02-23 +### Added +### Changed +### Removed + +## [v0.3.0] - 2015-02-17 +### Added +### Changed +### Removed + +## [v0.2.0] - 2015-01-23 +### Added +### Changed +### Removed + +## [v0.1.13] - 2014-11-12 +### Added +### Changed +### Removed + +## [v0.1.12] - 2014-08-22 +### Added +### Changed +### Removed + +## [v0.1.11] - 2014-06-20 +### Added +### Changed +### Removed + +## [v0.1.10] - 2014-06-09 +### Added +### Changed +### Removed + +## [v0.1.9] - 2014-06-06 +### Added +### Changed +### Removed + +## [v0.1.8] - 2014-06-06 +### Added +### Changed +### Removed + +## [v0.1.7] - 2014-05-21 +### Added +### Changed +### Removed + +## [v0.1.6] - 2014-04-02 +### Added +### Changed +### Removed + +## [v0.1.5] - 2014-03-25 +### Added +### Changed +### Removed + +## [v0.1.4] - 2014-03-03 +### Added +### Changed +### Removed + +## [v0.1.3] - 2014-02-11 +### Added +### Changed +### Removed + +## [v0.1.2] - 2013-12-09 +### Added +### Changed +### Removed + +## [v0.1.1] - 2013-11-14 +### Added +### Changed +### Removed From 6847fe4936a9f640a23f18f9f8946f7154b91d2a Mon Sep 17 00:00:00 2001 From: Matthew McGinn Date: Sun, 1 Jul 2018 12:01:25 -0400 Subject: [PATCH 028/133] Add changelog entries for 5.0.0 release --- CHANGELOG.md | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f9cbf97..326fde7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,8 +29,28 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [v5.0.0] - 2017-11-20 ### Added -### Changed -### Removed +- Add pool size parameter to client constructor (#534 thx @vaniakov) +- Add ping method to client for checking connectivity (#409 thx @pmenglund) +- Add retry logic & exponential backoff when a connection fails (#508) +- Declare which setuptools version is required in PyPy env +- Functions for drop_measurement and get_list_measurements in InfluxDBClient (#402 thx @Vic020) +- Allow single string as data argument in write (#492 thx @baftek) +- Support chunked queries in DataFrameClient (#439 thx @gusutabopb) +- Add close method to InfluxDBClient (#465 thx @Linux-oiD) +- PEP257 linting & code compliance (#473) +### Changed +- Fix broken tags filtering on a ResultSet (#511) +- Improve retry codepath for connecting to InfluxDB (#536 thx @swails) +- Clean up imports using six instead of sys.version (#536 thx @swails) +- Replace references to dataframe.ix with dataframe.iloc (#528) +- Improve performance of tag processing when converting DataFrameClient to line protocol (#503 thx @tzonghao) +- Typo in Content-Type header (#513 thx @milancermak) +- Clean up README.md formatting +- Catch TypeError when casting to float to return False with objects (#475 thx @BenHewins) +- Improve efficiency of tag appending in DataFrameClient when converting to line protocol (#486 thx @maxdolle) +### Removed +- Drop requirement for all fields in SeriesHelper (#518 thx @spott) +- use_udp and udp_port are now private properties in InfluxDBClient ## [v4.1.1] - 2017-06-06 ### Added From 25b7b8957e08ec959b5ecfa18d28fbfdf8819075 Mon Sep 17 00:00:00 2001 From: Matthew McGinn Date: Sun, 1 Jul 2018 12:12:30 -0400 Subject: [PATCH 029/133] Add changelog to CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 326fde7a..757e20e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] ### Added +- Finally add a CHANGELOG.md to communicate breaking changes (#598) ### Changed - Fix performance degradation when removing NaN values via line protocol (#592) ### Removed From 8022fb6f1db37a06c974f9e354b42152942fb35f Mon Sep 17 00:00:00 2001 From: Matthew McGinn Date: Sun, 1 Jul 2018 14:43:49 -0400 Subject: [PATCH 030/133] Mention in README.md that Python 3.3 is no longer supported --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index c145cfc4..65e000f8 100644 --- a/README.rst +++ b/README.rst @@ -60,9 +60,9 @@ On Debian/Ubuntu, you can install it with this command:: Dependencies ------------ -The influxdb-python distribution is supported and tested on Python 2.7, 3.3, 3.4, 3.5, 3.6, PyPy and PyPy3. +The influxdb-python distribution is supported and tested on Python 2.7, 3.4, 3.5, 3.6, PyPy and PyPy3. -**Note:** Python 3.2 is currently untested. See ``.travis.yml``. +**Note:** Python 3.2 and 3.3 are currently untested. See ``.travis.yml``. Main dependency is: From 4dc7e024f8c26a2109f6936d0f3cf7c70fffb8e1 Mon Sep 17 00:00:00 2001 From: Matthew McGinn Date: Sun, 1 Jul 2018 17:13:38 -0400 Subject: [PATCH 031/133] Update travis to test multiple versions of InfluxDB Comment out other InfluxDB versions for now (pending compatibility) --- .travis.yml | 42 ++++++++++++++++++++++-------------------- CHANGELOG.md | 1 + 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/.travis.yml b/.travis.yml index 03e6bc36..da25d8ea 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,27 +1,26 @@ language: python +python: + - "2.7" + - "3.5" + - "3.6" + - "pypy-5.3.1" + +env: + - INFLUXDB_VER=1.2.4 +# - INFLUXDB_VER=1.3.9 +# - INFLUXDB_VER=1.4.2 +# - INFLUXDB_VER=1.5.4 + addons: apt: packages: - wget matrix: - allow_failures: - - python: 3.4 - env: TOX_ENV=docs include: - - python: 2.7 - env: TOX_ENV=py27 - python: 2.7 env: TOX_ENV=pep257 - - python: pypy-5.3.1 - env: TOX_ENV=pypy - - python: 3.4 - env: TOX_ENV=py34 - - python: 3.5 - env: TOX_ENV=py35 - - python: 3.6 - env: TOX_ENV=py36 - python: 3.6 env: TOX_ENV=docs - python: 3.6 @@ -30,17 +29,20 @@ matrix: env: TOX_ENV=coverage install: - - pip install tox + - pip install tox-travis - pip install setuptools==20.6.6 - pip install coveralls - - mkdir influxdb_install - - wget https://dl.influxdata.com/influxdb/releases/influxdb_1.2.4_amd64.deb - - dpkg -x influxdb*.deb influxdb_install + - mkdir -p "influxdb_install/${INFLUXDB_VER}" + - if [ -n "${INFLUXDB_VER}" ] ; then wget "https://dl.influxdata.com/influxdb/releases/influxdb_${INFLUXDB_VER}_amd64.deb" ; fi + - if [ -n "${INFLUXDB_VER}" ] ; then dpkg -x influxdb*.deb "influxdb_install/${INFLUXDB_VER}" ; fi + script: - - export INFLUXDB_PYTHON_INFLUXD_PATH=$(pwd)/influxdb_install/usr/bin/influxd - - tox -e $TOX_ENV + - export "INFLUXDB_PYTHON_INFLUXD_PATH=$(pwd)/influxdb_install/${INFLUXDB_VER}/usr/bin/influxd" + - if [ -n "${TOX_ENV}" ]; then tox -e "${TOX_ENV}"; else tox; fi + after_success: - - if [ "$TOX_ENV" == "coverage" ] ; then coveralls; fi + - if [ "${TOX_ENV}" == "coverage" ] ; then coveralls; fi + notifications: email: false diff --git a/CHANGELOG.md b/CHANGELOG.md index 757e20e1..fe7c5b9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] ### Added - Finally add a CHANGELOG.md to communicate breaking changes (#598) +- Test multiple versions of InfluxDB in travis ### Changed - Fix performance degradation when removing NaN values via line protocol (#592) ### Removed From ae2a3c71c11d9b58a6538195b4282cee1d76bdb9 Mon Sep 17 00:00:00 2001 From: Matthew McGinn Date: Sun, 1 Jul 2018 16:14:40 -0400 Subject: [PATCH 032/133] Enable testing for InfluxDB v1.3.9, v1.4.2, and v1.5.4 (tsi) Swap admin_port config for global_port config --- .travis.yml | 6 +++--- CHANGELOG.md | 2 ++ README.rst | 3 +-- influxdb/tests/server_tests/influxdb.conf.template | 7 +++---- influxdb/tests/server_tests/influxdb_instance.py | 6 +++--- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index da25d8ea..a5fc1831 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,9 +8,9 @@ python: env: - INFLUXDB_VER=1.2.4 -# - INFLUXDB_VER=1.3.9 -# - INFLUXDB_VER=1.4.2 -# - INFLUXDB_VER=1.5.4 + - INFLUXDB_VER=1.3.9 + - INFLUXDB_VER=1.4.2 + - INFLUXDB_VER=1.5.4 addons: apt: diff --git a/CHANGELOG.md b/CHANGELOG.md index fe7c5b9b..0791c2ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Finally add a CHANGELOG.md to communicate breaking changes (#598) - Test multiple versions of InfluxDB in travis ### Changed +- Update test suite to support InfluxDB v1.3.9, v1.4.2, and v1.5.4 - Fix performance degradation when removing NaN values via line protocol (#592) ### Removed +- Dropped support for Python3.4 ## [v5.1.0] - 2018-06-26 ### Added diff --git a/README.rst b/README.rst index 65e000f8..af7c50d0 100644 --- a/README.rst +++ b/README.rst @@ -39,8 +39,7 @@ InfluxDB is an open-source distributed time series database, find more about Inf InfluxDB pre v1.1.0 users ------------------------- -This module is tested with InfluxDB v1.2.4, our recommended version. Though there have been v1.3 (initial TSI branch) and v1.4 releases these are not -yet supported. +This module is tested with Python {2.7,3.5,3.6} and InfluxDB v{1.2.4,1.3.9,1.4.2,1.5.4}. Those users still on InfluxDB v0.8.x users may still use the legacy client by importing ``from influxdb.influxdb08 import InfluxDBClient``. diff --git a/influxdb/tests/server_tests/influxdb.conf.template b/influxdb/tests/server_tests/influxdb.conf.template index 9a289635..efcff78a 100644 --- a/influxdb/tests/server_tests/influxdb.conf.template +++ b/influxdb/tests/server_tests/influxdb.conf.template @@ -1,3 +1,5 @@ +bind-address = ":{global_port}" + [meta] dir = "{meta_dir}" hostname = "localhost" @@ -6,10 +8,7 @@ [data] dir = "{data_dir}" wal-dir = "{wal_dir}" - -[admin] - enabled = true - bind-address = ":{admin_port}" + index-version = "tsi1" [http] enabled = true diff --git a/influxdb/tests/server_tests/influxdb_instance.py b/influxdb/tests/server_tests/influxdb_instance.py index 21e20fde..1dcd7567 100644 --- a/influxdb/tests/server_tests/influxdb_instance.py +++ b/influxdb/tests/server_tests/influxdb_instance.py @@ -80,7 +80,7 @@ def _start_server(self, conf_template, udp_enabled): # find a couple free ports : free_ports = get_free_ports(4) ports = {} - for service in 'http', 'admin', 'meta', 'udp': + for service in 'http', 'global', 'meta', 'udp': ports[service + '_port'] = free_ports.pop() if not udp_enabled: ports['udp_port'] = -1 @@ -113,7 +113,7 @@ def _start_server(self, conf_template, udp_enabled): "%s > Started influxdb bin in %r with ports %s and %s.." % ( datetime.datetime.now(), self.temp_dir_base, - self.admin_port, + self.global_port, self.http_port ) ) @@ -126,7 +126,7 @@ def _start_server(self, conf_template, udp_enabled): try: while time.time() < timeout: if (is_port_open(self.http_port) and - is_port_open(self.admin_port)): + is_port_open(self.global_port)): # it's hard to check if a UDP port is open.. if udp_enabled: # so let's just sleep 0.5 sec in this case From 4fea43ee6c84b683a06bb63e2c37cab899e04f83 Mon Sep 17 00:00:00 2001 From: Matthew McGinn Date: Tue, 3 Jul 2018 20:29:02 -0400 Subject: [PATCH 033/133] Add back PyPy3, remove references to Python 3.4 --- .travis.yml | 1 + README.rst | 6 +++--- tox.ini | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index a5fc1831..7f3d4a5d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,7 @@ python: - "3.5" - "3.6" - "pypy-5.3.1" + - "pypy3" env: - INFLUXDB_VER=1.2.4 diff --git a/README.rst b/README.rst index af7c50d0..d4f9611c 100644 --- a/README.rst +++ b/README.rst @@ -39,7 +39,7 @@ InfluxDB is an open-source distributed time series database, find more about Inf InfluxDB pre v1.1.0 users ------------------------- -This module is tested with Python {2.7,3.5,3.6} and InfluxDB v{1.2.4,1.3.9,1.4.2,1.5.4}. +This module is tested with InfluxDB versions: v1.2.4, v1.3.9, v1.4.2, and v1.5.4. Those users still on InfluxDB v0.8.x users may still use the legacy client by importing ``from influxdb.influxdb08 import InfluxDBClient``. @@ -59,9 +59,9 @@ On Debian/Ubuntu, you can install it with this command:: Dependencies ------------ -The influxdb-python distribution is supported and tested on Python 2.7, 3.4, 3.5, 3.6, PyPy and PyPy3. +The influxdb-python distribution is supported and tested on Python 2.7, 3.5, 3.6, PyPy and PyPy3. -**Note:** Python 3.2 and 3.3 are currently untested. See ``.travis.yml``. +**Note:** Python <3.5 are currently untested. See ``.travis.yml``. Main dependency is: diff --git a/tox.ini b/tox.ini index 5c54d680..d0d87fec 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27, py34, py35, py36, pypy, pypy3, flake8, pep257, coverage, docs +envlist = py27, py35, py36, pypy, pypy3, flake8, pep257, coverage, docs [testenv] passenv = INFLUXDB_PYTHON_INFLUXD_PATH From b7d75afa438a73ad0d8cee415326a6004b277b10 Mon Sep 17 00:00:00 2001 From: xginn8 Date: Fri, 6 Jul 2018 09:53:15 -0400 Subject: [PATCH 034/133] Use GET & POST appropriately according to InfluxDB documentation (#608) * Use GET & POST appropriately according to InfluxDB documentation From 'https://docs.influxdata.com/influxdb/v1.5/tools/api/#verb-usage', certain queries should be POST (where previously they have been GET) * Appease flake8 --- CHANGELOG.md | 1 + examples/tutorial_sine_wave.py | 2 +- influxdb/client.py | 44 ++++++++++------- influxdb/tests/client_test.py | 49 ++++++++++++++----- .../server_tests/client_test_with_server.py | 20 +++++++- 5 files changed, 84 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0791c2ea..c6f8761b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Finally add a CHANGELOG.md to communicate breaking changes (#598) - Test multiple versions of InfluxDB in travis ### Changed +- Update POST/GET requests to follow verb guidelines from InfluxDB documentation - Update test suite to support InfluxDB v1.3.9, v1.4.2, and v1.5.4 - Fix performance degradation when removing NaN values via line protocol (#592) ### Removed diff --git a/examples/tutorial_sine_wave.py b/examples/tutorial_sine_wave.py index 99b3d388..5dfebf3c 100644 --- a/examples/tutorial_sine_wave.py +++ b/examples/tutorial_sine_wave.py @@ -43,7 +43,7 @@ def main(host='localhost', port=8086): time.sleep(3) query = 'SELECT * FROM foobar' - print("Queying data: " + query) + print("Querying data: " + query) result = client.query(query, database=DBNAME) print("Result: {0}".format(result)) diff --git a/influxdb/client.py b/influxdb/client.py index 62d5a025..e3299fe8 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -350,7 +350,8 @@ def query(self, database=None, raise_errors=True, chunked=False, - chunk_size=0): + chunk_size=0, + method="GET"): """Send a query to InfluxDB. :param query: the actual query string @@ -384,6 +385,9 @@ def query(self, :param chunk_size: Size of each chunk to tell InfluxDB to use. :type chunk_size: int + :param method: the HTTP method for the request, defaults to GET + :type method: str + :returns: the queried data :rtype: :class:`~.ResultSet` """ @@ -401,9 +405,12 @@ def query(self, if chunk_size > 0: params['chunk_size'] = chunk_size + if query.lower().startswith("select ") and " into " in query.lower(): + method = "POST" + response = self.request( url="query", - method='GET', + method=method, params=params, data=None, expected_response_code=expected_response_code @@ -568,7 +575,8 @@ def create_database(self, dbname): :param dbname: the name of the database to create :type dbname: str """ - self.query("CREATE DATABASE {0}".format(quote_ident(dbname))) + self.query("CREATE DATABASE {0}".format(quote_ident(dbname)), + method="POST") def drop_database(self, dbname): """Drop a database from InfluxDB. @@ -576,7 +584,8 @@ def drop_database(self, dbname): :param dbname: the name of the database to drop :type dbname: str """ - self.query("DROP DATABASE {0}".format(quote_ident(dbname))) + self.query("DROP DATABASE {0}".format(quote_ident(dbname)), + method="POST") def get_list_measurements(self): """Get the list of measurements in InfluxDB. @@ -602,7 +611,8 @@ def drop_measurement(self, measurement): :param measurement: the name of the measurement to drop :type measurement: str """ - self.query("DROP MEASUREMENT {0}".format(quote_ident(measurement))) + self.query("DROP MEASUREMENT {0}".format(quote_ident(measurement)), + method="POST") def create_retention_policy(self, name, duration, replication, database=None, default=False): @@ -634,11 +644,11 @@ def create_retention_policy(self, name, duration, replication, if default is True: query_string += " DEFAULT" - self.query(query_string) + self.query(query_string, method="POST") def alter_retention_policy(self, name, database=None, duration=None, replication=None, default=None): - """Mofidy an existing retention policy for a database. + """Modify an existing retention policy for a database. :param name: the name of the retention policy to modify :type name: str @@ -671,7 +681,7 @@ def alter_retention_policy(self, name, database=None, if default is True: query_string += " DEFAULT" - self.query(query_string) + self.query(query_string, method="POST") def drop_retention_policy(self, name, database=None): """Drop an existing retention policy for a database. @@ -685,7 +695,7 @@ def drop_retention_policy(self, name, database=None): query_string = ( "DROP RETENTION POLICY {0} ON {1}" ).format(quote_ident(name), quote_ident(database or self._database)) - self.query(query_string) + self.query(query_string, method="POST") def get_list_retention_policies(self, database=None): """Get the list of retention policies for a database. @@ -751,7 +761,7 @@ def create_user(self, username, password, admin=False): quote_ident(username), quote_literal(password)) if admin: text += ' WITH ALL PRIVILEGES' - self.query(text) + self.query(text, method="POST") def drop_user(self, username): """Drop a user from InfluxDB. @@ -759,8 +769,8 @@ def drop_user(self, username): :param username: the username to drop :type username: str """ - text = "DROP USER {0}".format(quote_ident(username)) - self.query(text) + text = "DROP USER {0}".format(quote_ident(username), method="POST") + self.query(text, method="POST") def set_user_password(self, username, password): """Change the password of an existing user. @@ -796,7 +806,7 @@ def delete_series(self, database=None, measurement=None, tags=None): tag_eq_list = ["{0}={1}".format(quote_ident(k), quote_literal(v)) for k, v in tags.items()] query_str += ' WHERE ' + ' AND '.join(tag_eq_list) - self.query(query_str, database=database) + self.query(query_str, database=database, method="POST") def grant_admin_privileges(self, username): """Grant cluster administration privileges to a user. @@ -808,7 +818,7 @@ def grant_admin_privileges(self, username): and manage users. """ text = "GRANT ALL PRIVILEGES TO {0}".format(quote_ident(username)) - self.query(text) + self.query(text, method="POST") def revoke_admin_privileges(self, username): """Revoke cluster administration privileges from a user. @@ -820,7 +830,7 @@ def revoke_admin_privileges(self, username): and manage users. """ text = "REVOKE ALL PRIVILEGES FROM {0}".format(quote_ident(username)) - self.query(text) + self.query(text, method="POST") def grant_privilege(self, privilege, database, username): """Grant a privilege on a database to a user. @@ -836,7 +846,7 @@ def grant_privilege(self, privilege, database, username): text = "GRANT {0} ON {1} TO {2}".format(privilege, quote_ident(database), quote_ident(username)) - self.query(text) + self.query(text, method="POST") def revoke_privilege(self, privilege, database, username): """Revoke a privilege on a database from a user. @@ -852,7 +862,7 @@ def revoke_privilege(self, privilege, database, username): text = "REVOKE {0} ON {1} FROM {2}".format(privilege, quote_ident(database), quote_ident(username)) - self.query(text) + self.query(text, method="POST") def get_list_privileges(self, username): """Get the list of all privileges granted to given user. diff --git a/influxdb/tests/client_test.py b/influxdb/tests/client_test.py index efdfb770..859e8bc9 100644 --- a/influxdb/tests/client_test.py +++ b/influxdb/tests/client_test.py @@ -439,6 +439,29 @@ def test_query(self): [{'value': 0.64, 'time': '2009-11-10T23:00:00Z'}] ) + def test_select_into_post(self): + """Test SELECT.*INTO is POSTed.""" + example_response = ( + '{"results": [{"series": [{"measurement": "sdfsdfsdf", ' + '"columns": ["time", "value"], "values": ' + '[["2009-11-10T23:00:00Z", 0.64]]}]}, {"series": ' + '[{"measurement": "cpu_load_short", "columns": ["time", "value"], ' + '"values": [["2009-11-10T23:00:00Z", 0.64]]}]}]}' + ) + + with requests_mock.Mocker() as m: + m.register_uri( + requests_mock.POST, + "http://localhost:8086/query", + text=example_response + ) + rs = self.cli.query('select * INTO newmeas from foo') + + self.assertListEqual( + list(rs[0].get_points()), + [{'value': 0.64, 'time': '2009-11-10T23:00:00Z'}] + ) + @unittest.skip('Not implemented for 0.9') def test_query_chunked(self): """Test chunked query for TestInfluxDBClient object.""" @@ -495,7 +518,7 @@ def test_create_database(self): """Test create database for TestInfluxDBClient object.""" with requests_mock.Mocker() as m: m.register_uri( - requests_mock.GET, + requests_mock.POST, "http://localhost:8086/query", text='{"results":[{}]}' ) @@ -509,7 +532,7 @@ def test_create_numeric_named_database(self): """Test create db w/numeric name for TestInfluxDBClient object.""" with requests_mock.Mocker() as m: m.register_uri( - requests_mock.GET, + requests_mock.POST, "http://localhost:8086/query", text='{"results":[{}]}' ) @@ -529,7 +552,7 @@ def test_drop_database(self): """Test drop database for TestInfluxDBClient object.""" with requests_mock.Mocker() as m: m.register_uri( - requests_mock.GET, + requests_mock.POST, "http://localhost:8086/query", text='{"results":[{}]}' ) @@ -543,7 +566,7 @@ def test_drop_measurement(self): """Test drop measurement for TestInfluxDBClient object.""" with requests_mock.Mocker() as m: m.register_uri( - requests_mock.GET, + requests_mock.POST, "http://localhost:8086/query", text='{"results":[{}]}' ) @@ -557,7 +580,7 @@ def test_drop_numeric_named_database(self): """Test drop numeric db for TestInfluxDBClient object.""" with requests_mock.Mocker() as m: m.register_uri( - requests_mock.GET, + requests_mock.POST, "http://localhost:8086/query", text='{"results":[{}]}' ) @@ -615,7 +638,7 @@ def test_create_retention_policy_default(self): with requests_mock.Mocker() as m: m.register_uri( - requests_mock.GET, + requests_mock.POST, "http://localhost:8086/query", text=example_response ) @@ -635,7 +658,7 @@ def test_create_retention_policy(self): with requests_mock.Mocker() as m: m.register_uri( - requests_mock.GET, + requests_mock.POST, "http://localhost:8086/query", text=example_response ) @@ -655,7 +678,7 @@ def test_alter_retention_policy(self): with requests_mock.Mocker() as m: m.register_uri( - requests_mock.GET, + requests_mock.POST, "http://localhost:8086/query", text=example_response ) @@ -695,7 +718,7 @@ def test_drop_retention_policy(self): with requests_mock.Mocker() as m: m.register_uri( - requests_mock.GET, + requests_mock.POST, "http://localhost:8086/query", text=example_response ) @@ -879,7 +902,7 @@ def test_grant_admin_privileges(self): with requests_mock.Mocker() as m: m.register_uri( - requests_mock.GET, + requests_mock.POST, "http://localhost:8086/query", text=example_response ) @@ -903,7 +926,7 @@ def test_revoke_admin_privileges(self): with requests_mock.Mocker() as m: m.register_uri( - requests_mock.GET, + requests_mock.POST, "http://localhost:8086/query", text=example_response ) @@ -927,7 +950,7 @@ def test_grant_privilege(self): with requests_mock.Mocker() as m: m.register_uri( - requests_mock.GET, + requests_mock.POST, "http://localhost:8086/query", text=example_response ) @@ -951,7 +974,7 @@ def test_revoke_privilege(self): with requests_mock.Mocker() as m: m.register_uri( - requests_mock.GET, + requests_mock.POST, "http://localhost:8086/query", text=example_response ) diff --git a/influxdb/tests/server_tests/client_test_with_server.py b/influxdb/tests/server_tests/client_test_with_server.py index 701f72ac..d2370e63 100644 --- a/influxdb/tests/server_tests/client_test_with_server.py +++ b/influxdb/tests/server_tests/client_test_with_server.py @@ -211,7 +211,7 @@ def test_drop_user(self): self.assertEqual(users, []) def test_drop_user_nonexisting(self): - """Test dropping a nonexistant user.""" + """Test dropping a nonexistent user.""" with self.assertRaises(InfluxDBClientError) as ctx: self.cli.drop_user('test') self.assertIn('user not found', @@ -383,6 +383,24 @@ def test_write_multiple_points_different_series(self): ]] ) + def test_select_into_as_post(self): + """Test SELECT INTO is POSTed.""" + self.assertIs(True, self.cli.write_points(dummy_points)) + time.sleep(1) + rsp = self.cli.query('SELECT * INTO "newmeas" FROM "memory"') + rsp = self.cli.query('SELECT * FROM "newmeas"') + lrsp = list(rsp) + + self.assertEqual( + lrsp, + [[ + {'value': 33, + 'time': '2009-11-10T23:01:35Z', + "host": "server01", + "region": "us-west"} + ]] + ) + @unittest.skip("Broken as of 0.9.0") def test_write_multiple_points_different_series_DF(self): """Test write multiple points using dataframe to different series.""" From b7e43c32c9de0577e6174dd2eeaf2ee5bb92afa5 Mon Sep 17 00:00:00 2001 From: xginn8 Date: Fri, 6 Jul 2018 10:20:53 -0400 Subject: [PATCH 035/133] =?UTF-8?q?Add=20shard=5Fduration=20parameter=20wh?= =?UTF-8?q?en=20creating=20or=20altering=20retention=20poli=E2=80=A6=20(#6?= =?UTF-8?q?06)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add shard_duration parameter when creating or altering retention policies Fixes #560 * Remove debug print statement --- CHANGELOG.md | 1 + influxdb/client.py | 32 ++++++-- influxdb/tests/client_test.py | 12 ++- .../server_tests/client_test_with_server.py | 74 ++++++++++++++++++- 4 files changed, 108 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c6f8761b..9306a7ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added - Finally add a CHANGELOG.md to communicate breaking changes (#598) - Test multiple versions of InfluxDB in travis +- Add SHARD DURATION parameter to retention policy create/alter ### Changed - Update POST/GET requests to follow verb guidelines from InfluxDB documentation - Update test suite to support InfluxDB v1.3.9, v1.4.2, and v1.5.4 diff --git a/influxdb/client.py b/influxdb/client.py index e3299fe8..8f8b14ae 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -615,7 +615,8 @@ def drop_measurement(self, measurement): method="POST") def create_retention_policy(self, name, duration, replication, - database=None, default=False): + database=None, + default=False, shard_duration="0s"): """Create a retention policy for a database. :param name: the name of the new retention policy @@ -634,12 +635,21 @@ def create_retention_policy(self, name, duration, replication, :type database: str :param default: whether or not to set the policy as default :type default: bool + :param shard_duration: the shard duration of the retention policy. + Durations such as 1h, 90m, 12h, 7d, and 4w, are all supported and + mean 1 hour, 90 minutes, 12 hours, 7 day, and 4 weeks, + respectively. Infinite retention is not supported. As a workaround, + specify a "1000w" duration to achieve an extremely long shard group + duration. Defaults to "0s", which is interpreted by the database + to mean the default value given the duration. + The minimum shard group duration is 1 hour. + :type shard_duration: str """ query_string = \ "CREATE RETENTION POLICY {0} ON {1} " \ - "DURATION {2} REPLICATION {3}".format( + "DURATION {2} REPLICATION {3} SHARD DURATION {4}".format( quote_ident(name), quote_ident(database or self._database), - duration, replication) + duration, replication, shard_duration) if default is True: query_string += " DEFAULT" @@ -647,7 +657,8 @@ def create_retention_policy(self, name, duration, replication, self.query(query_string, method="POST") def alter_retention_policy(self, name, database=None, - duration=None, replication=None, default=None): + duration=None, replication=None, + default=None, shard_duration=None): """Modify an existing retention policy for a database. :param name: the name of the retention policy to modify @@ -667,15 +678,26 @@ def alter_retention_policy(self, name, database=None, :type replication: int :param default: whether or not to set the modified policy as default :type default: bool + :param shard_duration: the shard duration of the retention policy. + Durations such as 1h, 90m, 12h, 7d, and 4w, are all supported and + mean 1 hour, 90 minutes, 12 hours, 7 day, and 4 weeks, + respectively. Infinite retention is not supported. As a workaround, + specify a "1000w" duration to achieve an extremely long shard group + duration. + The minimum shard group duration is 1 hour. + :type shard_duration: str .. note:: at least one of duration, replication, or default flag should be set. Otherwise the operation will fail. """ query_string = ( "ALTER RETENTION POLICY {0} ON {1}" - ).format(quote_ident(name), quote_ident(database or self._database)) + ).format(quote_ident(name), + quote_ident(database or self._database), shard_duration) if duration: query_string += " DURATION {0}".format(duration) + if shard_duration: + query_string += " SHARD DURATION {0}".format(shard_duration) if replication: query_string += " REPLICATION {0}".format(replication) if default is True: diff --git a/influxdb/tests/client_test.py b/influxdb/tests/client_test.py index 859e8bc9..e27eef17 100644 --- a/influxdb/tests/client_test.py +++ b/influxdb/tests/client_test.py @@ -649,7 +649,7 @@ def test_create_retention_policy_default(self): self.assertEqual( m.last_request.qs['q'][0], 'create retention policy "somename" on ' - '"db" duration 1d replication 4 default' + '"db" duration 1d replication 4 shard duration 0s default' ) def test_create_retention_policy(self): @@ -669,7 +669,7 @@ def test_create_retention_policy(self): self.assertEqual( m.last_request.qs['q'][0], 'create retention policy "somename" on ' - '"db" duration 1d replication 4' + '"db" duration 1d replication 4 shard duration 0s' ) def test_alter_retention_policy(self): @@ -697,6 +697,14 @@ def test_alter_retention_policy(self): 'alter retention policy "somename" on "db" replication 4' ) + # Test alter shard duration + self.cli.alter_retention_policy('somename', 'db', + shard_duration='1h') + self.assertEqual( + m.last_request.qs['q'][0], + 'alter retention policy "somename" on "db" shard duration 1h' + ) + # Test alter default self.cli.alter_retention_policy('somename', 'db', default=True) diff --git a/influxdb/tests/server_tests/client_test_with_server.py b/influxdb/tests/server_tests/client_test_with_server.py index d2370e63..2f8a2097 100644 --- a/influxdb/tests/server_tests/client_test_with_server.py +++ b/influxdb/tests/server_tests/client_test_with_server.py @@ -544,13 +544,57 @@ def test_create_retention_policy(self): rsp ) + self.cli.drop_retention_policy('somename', 'db') + # recreate the RP + self.cli.create_retention_policy('somename', '1w', 1, + shard_duration='1h') + + rsp = self.cli.get_list_retention_policies() + self.assertEqual( + [ + {'duration': '0s', + 'default': True, + 'replicaN': 1, + 'shardGroupDuration': u'168h0m0s', + 'name': 'autogen'}, + {'duration': '168h0m0s', + 'default': False, + 'replicaN': 1, + 'shardGroupDuration': u'1h0m0s', + 'name': 'somename'} + ], + rsp + ) + + self.cli.drop_retention_policy('somename', 'db') + # recreate the RP + self.cli.create_retention_policy('somename', '1w', 1) + + rsp = self.cli.get_list_retention_policies() + self.assertEqual( + [ + {'duration': '0s', + 'default': True, + 'replicaN': 1, + 'shardGroupDuration': u'168h0m0s', + 'name': 'autogen'}, + {'duration': '168h0m0s', + 'default': False, + 'replicaN': 1, + 'shardGroupDuration': u'24h0m0s', + 'name': 'somename'} + ], + rsp + ) + def test_alter_retention_policy(self): """Test alter a retention policy, not default.""" self.cli.create_retention_policy('somename', '1d', 1) # Test alter duration self.cli.alter_retention_policy('somename', 'db', - duration='4d') + duration='4d', + shard_duration='2h') # NB: altering retention policy doesn't change shard group duration rsp = self.cli.get_list_retention_policies() self.assertEqual( @@ -563,7 +607,7 @@ def test_alter_retention_policy(self): {'duration': '96h0m0s', 'default': False, 'replicaN': 1, - 'shardGroupDuration': u'1h0m0s', + 'shardGroupDuration': u'2h0m0s', 'name': 'somename'} ], rsp @@ -572,6 +616,7 @@ def test_alter_retention_policy(self): # Test alter replication self.cli.alter_retention_policy('somename', 'db', replication=4) + # NB: altering retention policy doesn't change shard group duration rsp = self.cli.get_list_retention_policies() self.assertEqual( @@ -584,7 +629,7 @@ def test_alter_retention_policy(self): {'duration': '96h0m0s', 'default': False, 'replicaN': 4, - 'shardGroupDuration': u'1h0m0s', + 'shardGroupDuration': u'2h0m0s', 'name': 'somename'} ], rsp @@ -605,7 +650,28 @@ def test_alter_retention_policy(self): {'duration': '96h0m0s', 'default': True, 'replicaN': 4, - 'shardGroupDuration': u'1h0m0s', + 'shardGroupDuration': u'2h0m0s', + 'name': 'somename'} + ], + rsp + ) + + # Test alter shard_duration + self.cli.alter_retention_policy('somename', 'db', + shard_duration='4h') + + rsp = self.cli.get_list_retention_policies() + self.assertEqual( + [ + {'duration': '0s', + 'default': False, + 'replicaN': 1, + 'shardGroupDuration': u'168h0m0s', + 'name': 'autogen'}, + {'duration': '96h0m0s', + 'default': True, + 'replicaN': 4, + 'shardGroupDuration': u'4h0m0s', 'name': 'somename'} ], rsp From 1c96ce244869b8d4fdee3b70bd441d707f42cd2c Mon Sep 17 00:00:00 2001 From: aviau Date: Tue, 10 Jul 2018 18:07:52 -0400 Subject: [PATCH 036/133] v5.2.0 --- CHANGELOG.md | 7 +++++++ influxdb/__init__.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9306a7ed..22a5b19d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,13 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] ### Added + +### Changed + +### Removed + +## [v5.2.0] - 2018-07-10 +### Added - Finally add a CHANGELOG.md to communicate breaking changes (#598) - Test multiple versions of InfluxDB in travis - Add SHARD DURATION parameter to retention policy create/alter diff --git a/influxdb/__init__.py b/influxdb/__init__.py index 374fddc7..03f74581 100644 --- a/influxdb/__init__.py +++ b/influxdb/__init__.py @@ -18,4 +18,4 @@ ] -__version__ = '5.1.0' +__version__ = '5.2.0' From caa9e7d044f0ac344200d45c385799207c311701 Mon Sep 17 00:00:00 2001 From: xginn8 Date: Sun, 15 Jul 2018 19:31:44 -0400 Subject: [PATCH 037/133] Pass through method kwarg to DataFrameClient query method (#617) Also, add db maintenance tests from InfluxDBClient fixes #616 --- CHANGELOG.md | 1 + influxdb/_dataframe_client.py | 4 +- influxdb/tests/dataframe_client_test.py | 242 ++++++++++++++++++++++++ 3 files changed, 246 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 22a5b19d..c156b678 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added ### Changed +- Pass through the "method" kwarg to DataFrameClient queries ### Removed diff --git a/influxdb/_dataframe_client.py b/influxdb/_dataframe_client.py index 646f298c..06da7ac4 100644 --- a/influxdb/_dataframe_client.py +++ b/influxdb/_dataframe_client.py @@ -148,9 +148,10 @@ def query(self, raise_errors=True, chunked=False, chunk_size=0, + method="GET", dropna=True): """ - Quering data into a DataFrame. + Query data into a DataFrame. :param query: the actual query string :param params: additional parameters for the request, defaults to {} @@ -176,6 +177,7 @@ def query(self, raise_errors=raise_errors, chunked=chunked, database=database, + method=method, chunk_size=chunk_size) results = super(DataFrameClient, self).query(query, **query_args) if query.strip().upper().startswith("SELECT"): diff --git a/influxdb/tests/dataframe_client_test.py b/influxdb/tests/dataframe_client_test.py index 78f5437f..72447c89 100644 --- a/influxdb/tests/dataframe_client_test.py +++ b/influxdb/tests/dataframe_client_test.py @@ -552,6 +552,248 @@ def test_write_points_from_dataframe_fails_with_series(self): cli = DataFrameClient(database='db') cli.write_points(dataframe, "foo") + def test_create_database(self): + """Test create database for TestInfluxDBClient object.""" + cli = DataFrameClient(database='db') + with requests_mock.Mocker() as m: + m.register_uri( + requests_mock.POST, + "http://localhost:8086/query", + text='{"results":[{}]}' + ) + cli.create_database('new_db') + self.assertEqual( + m.last_request.qs['q'][0], + 'create database "new_db"' + ) + + def test_create_numeric_named_database(self): + """Test create db w/numeric name for TestInfluxDBClient object.""" + cli = DataFrameClient(database='db') + with requests_mock.Mocker() as m: + m.register_uri( + requests_mock.POST, + "http://localhost:8086/query", + text='{"results":[{}]}' + ) + cli.create_database('123') + self.assertEqual( + m.last_request.qs['q'][0], + 'create database "123"' + ) + + @raises(Exception) + def test_create_database_fails(self): + """Test create database fail for TestInfluxDBClient object.""" + cli = DataFrameClient(database='db') + with _mocked_session(cli, 'post', 401): + cli.create_database('new_db') + + def test_drop_database(self): + """Test drop database for TestInfluxDBClient object.""" + cli = DataFrameClient(database='db') + with requests_mock.Mocker() as m: + m.register_uri( + requests_mock.POST, + "http://localhost:8086/query", + text='{"results":[{}]}' + ) + cli.drop_database('new_db') + self.assertEqual( + m.last_request.qs['q'][0], + 'drop database "new_db"' + ) + + def test_drop_measurement(self): + """Test drop measurement for TestInfluxDBClient object.""" + cli = DataFrameClient(database='db') + with requests_mock.Mocker() as m: + m.register_uri( + requests_mock.POST, + "http://localhost:8086/query", + text='{"results":[{}]}' + ) + cli.drop_measurement('new_measurement') + self.assertEqual( + m.last_request.qs['q'][0], + 'drop measurement "new_measurement"' + ) + + def test_drop_numeric_named_database(self): + """Test drop numeric db for TestInfluxDBClient object.""" + cli = DataFrameClient(database='db') + with requests_mock.Mocker() as m: + m.register_uri( + requests_mock.POST, + "http://localhost:8086/query", + text='{"results":[{}]}' + ) + cli.drop_database('123') + self.assertEqual( + m.last_request.qs['q'][0], + 'drop database "123"' + ) + + @raises(Exception) + def test_get_list_database_fails(self): + """Test get list of dbs fail for TestInfluxDBClient object.""" + cli = DataFrameClient('host', 8086, 'username', 'password') + with _mocked_session(cli, 'get', 401): + cli.get_list_database() + + def test_get_list_measurements(self): + """Test get list of measurements for TestInfluxDBClient object.""" + cli = DataFrameClient(database='db') + data = { + "results": [{ + "series": [ + {"name": "measurements", + "columns": ["name"], + "values": [["cpu"], ["disk"] + ]}]} + ] + } + + with _mocked_session(cli, 'get', 200, json.dumps(data)): + self.assertListEqual( + cli.get_list_measurements(), + [{'name': 'cpu'}, {'name': 'disk'}] + ) + + def test_create_retention_policy_default(self): + """Test create default ret policy for TestInfluxDBClient object.""" + cli = DataFrameClient(database='db') + example_response = '{"results":[{}]}' + + with requests_mock.Mocker() as m: + m.register_uri( + requests_mock.POST, + "http://localhost:8086/query", + text=example_response + ) + cli.create_retention_policy( + 'somename', '1d', 4, default=True, database='db' + ) + + self.assertEqual( + m.last_request.qs['q'][0], + 'create retention policy "somename" on ' + '"db" duration 1d replication 4 shard duration 0s default' + ) + + def test_create_retention_policy(self): + """Test create retention policy for TestInfluxDBClient object.""" + cli = DataFrameClient(database='db') + example_response = '{"results":[{}]}' + + with requests_mock.Mocker() as m: + m.register_uri( + requests_mock.POST, + "http://localhost:8086/query", + text=example_response + ) + cli.create_retention_policy( + 'somename', '1d', 4, database='db' + ) + + self.assertEqual( + m.last_request.qs['q'][0], + 'create retention policy "somename" on ' + '"db" duration 1d replication 4 shard duration 0s' + ) + + def test_alter_retention_policy(self): + """Test alter retention policy for TestInfluxDBClient object.""" + cli = DataFrameClient(database='db') + example_response = '{"results":[{}]}' + + with requests_mock.Mocker() as m: + m.register_uri( + requests_mock.POST, + "http://localhost:8086/query", + text=example_response + ) + # Test alter duration + cli.alter_retention_policy('somename', 'db', + duration='4d') + self.assertEqual( + m.last_request.qs['q'][0], + 'alter retention policy "somename" on "db" duration 4d' + ) + # Test alter replication + cli.alter_retention_policy('somename', 'db', + replication=4) + self.assertEqual( + m.last_request.qs['q'][0], + 'alter retention policy "somename" on "db" replication 4' + ) + + # Test alter shard duration + cli.alter_retention_policy('somename', 'db', + shard_duration='1h') + self.assertEqual( + m.last_request.qs['q'][0], + 'alter retention policy "somename" on "db" shard duration 1h' + ) + + # Test alter default + cli.alter_retention_policy('somename', 'db', + default=True) + self.assertEqual( + m.last_request.qs['q'][0], + 'alter retention policy "somename" on "db" default' + ) + + @raises(Exception) + def test_alter_retention_policy_invalid(self): + """Test invalid alter ret policy for TestInfluxDBClient object.""" + cli = DataFrameClient('host', 8086, 'username', 'password') + with _mocked_session(cli, 'get', 400): + cli.alter_retention_policy('somename', 'db') + + def test_drop_retention_policy(self): + """Test drop retention policy for TestInfluxDBClient object.""" + cli = DataFrameClient(database='db') + example_response = '{"results":[{}]}' + + with requests_mock.Mocker() as m: + m.register_uri( + requests_mock.POST, + "http://localhost:8086/query", + text=example_response + ) + cli.drop_retention_policy('somename', 'db') + self.assertEqual( + m.last_request.qs['q'][0], + 'drop retention policy "somename" on "db"' + ) + + @raises(Exception) + def test_drop_retention_policy_fails(self): + """Test failed drop ret policy for TestInfluxDBClient object.""" + cli = DataFrameClient('host', 8086, 'username', 'password') + with _mocked_session(cli, 'delete', 401): + cli.drop_retention_policy('default', 'db') + + def test_get_list_retention_policies(self): + """Test get retention policies for TestInfluxDBClient object.""" + cli = DataFrameClient(database='db') + example_response = \ + '{"results": [{"series": [{"values": [["fsfdsdf", "24h0m0s", 2]],'\ + ' "columns": ["name", "duration", "replicaN"]}]}]}' + + with requests_mock.Mocker() as m: + m.register_uri( + requests_mock.GET, + "http://localhost:8086/query", + text=example_response + ) + self.assertListEqual( + cli.get_list_retention_policies("db"), + [{'duration': '24h0m0s', + 'name': 'fsfdsdf', 'replicaN': 2}] + ) + def test_query_into_dataframe(self): """Test query into df for TestDataFrameClient object.""" data = { From 7b7a719f4a61304f13f3b13ff20b5f118d6674bb Mon Sep 17 00:00:00 2001 From: aviau Date: Fri, 7 Dec 2018 15:56:57 -0500 Subject: [PATCH 038/133] 5.2.1 release --- influxdb/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/influxdb/__init__.py b/influxdb/__init__.py index 03f74581..a1eb3789 100644 --- a/influxdb/__init__.py +++ b/influxdb/__init__.py @@ -18,4 +18,4 @@ ] -__version__ = '5.2.0' +__version__ = '5.2.1' From d37089508794c4bd5c63775357a4967b832e1f31 Mon Sep 17 00:00:00 2001 From: aviau Date: Fri, 7 Dec 2018 15:59:02 -0500 Subject: [PATCH 039/133] release 5.2.1 in changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c156b678..14a9abf4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,14 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). ## [Unreleased] + +### Added + +### Changed + +### Removed + +## [v5.2.1] - 2018-12-07 ### Added ### Changed From 7d924d5faa5afc0f698297709e95b5d8209a7f52 Mon Sep 17 00:00:00 2001 From: xginn8 Date: Tue, 12 Feb 2019 12:48:49 -0300 Subject: [PATCH 040/133] Unpin setuptools to fix travis (#674) * Unpin setuptools to fix travis Signed-off-by: Matthew McGinn * Add some ignores for new flake8 tests Signed-off-by: Matthew McGinn --- .travis.yml | 2 +- tox.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7f3d4a5d..22626f40 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,7 +31,7 @@ matrix: install: - pip install tox-travis - - pip install setuptools==20.6.6 + - pip install setuptools - pip install coveralls - mkdir -p "influxdb_install/${INFLUXDB_VER}" - if [ -n "${INFLUXDB_VER}" ] ; then wget "https://dl.influxdata.com/influxdb/releases/influxdb_${INFLUXDB_VER}_amd64.deb" ; fi diff --git a/tox.ini b/tox.ini index d0d87fec..2f9c212c 100644 --- a/tox.ini +++ b/tox.ini @@ -15,7 +15,7 @@ commands = nosetests -v --with-doctest {posargs} deps = flake8 pep8-naming -commands = flake8 influxdb +commands = flake8 --ignore=W503,W504,W605,N802,F821 influxdb [testenv:pep257] deps = pydocstyle From 1cce011d6c4f5623141b3bcff65197c2db18ddc8 Mon Sep 17 00:00:00 2001 From: Colas Le Guernic Date: Thu, 14 Mar 2019 11:25:58 +0000 Subject: [PATCH 041/133] unpin pypy (#682) cryptography-2.5 is not compatible with PyPy < 5.4 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 22626f40..a1cf7b55 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ python: - "2.7" - "3.5" - "3.6" - - "pypy-5.3.1" + - "pypy" - "pypy3" env: From 02de23fb2c27abc8473d108af2751997634d906a Mon Sep 17 00:00:00 2001 From: xginn8 Date: Thu, 14 Mar 2019 09:11:52 -0400 Subject: [PATCH 042/133] Rename all mixedCase globals to snake case to appease N816 (#689) Signed-off-by: Matthew McGinn --- influxdb/tests/__init__.py | 6 ++-- influxdb/tests/dataframe_client_test.py | 4 +-- .../tests/influxdb08/dataframe_client_test.py | 4 +-- .../server_tests/client_test_with_server.py | 34 +++++++++---------- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/influxdb/tests/__init__.py b/influxdb/tests/__init__.py index adf2f20c..f7c5dfb9 100644 --- a/influxdb/tests/__init__.py +++ b/influxdb/tests/__init__.py @@ -12,10 +12,10 @@ import unittest using_pypy = hasattr(sys, "pypy_version_info") -skipIfPYpy = unittest.skipIf(using_pypy, "Skipping this test on pypy.") +skip_if_pypy = unittest.skipIf(using_pypy, "Skipping this test on pypy.") _skip_server_tests = os.environ.get( 'INFLUXDB_PYTHON_SKIP_SERVER_TESTS', None) == 'True' -skipServerTests = unittest.skipIf(_skip_server_tests, - "Skipping server tests...") +skip_server_tests = unittest.skipIf(_skip_server_tests, + "Skipping server tests...") diff --git a/influxdb/tests/dataframe_client_test.py b/influxdb/tests/dataframe_client_test.py index 72447c89..9fd6427b 100644 --- a/influxdb/tests/dataframe_client_test.py +++ b/influxdb/tests/dataframe_client_test.py @@ -13,7 +13,7 @@ import warnings import requests_mock -from influxdb.tests import skipIfPYpy, using_pypy +from influxdb.tests import skip_if_pypy, using_pypy from nose.tools import raises from .client_test import _mocked_session @@ -24,7 +24,7 @@ from influxdb import DataFrameClient -@skipIfPYpy +@skip_if_pypy class TestDataFrameClient(unittest.TestCase): """Set up a test DataFrameClient object.""" diff --git a/influxdb/tests/influxdb08/dataframe_client_test.py b/influxdb/tests/influxdb08/dataframe_client_test.py index 6e6fa2cc..0a766af0 100644 --- a/influxdb/tests/influxdb08/dataframe_client_test.py +++ b/influxdb/tests/influxdb08/dataframe_client_test.py @@ -12,7 +12,7 @@ from nose.tools import raises -from influxdb.tests import skipIfPYpy, using_pypy +from influxdb.tests import skip_if_pypy, using_pypy from .client_test import _mocked_session @@ -22,7 +22,7 @@ from influxdb.influxdb08 import DataFrameClient -@skipIfPYpy +@skip_if_pypy class TestDataFrameClient(unittest.TestCase): """Define the DataFramClient test object.""" diff --git a/influxdb/tests/server_tests/client_test_with_server.py b/influxdb/tests/server_tests/client_test_with_server.py index 2f8a2097..4dbc1b75 100644 --- a/influxdb/tests/server_tests/client_test_with_server.py +++ b/influxdb/tests/server_tests/client_test_with_server.py @@ -23,7 +23,7 @@ from influxdb import InfluxDBClient from influxdb.exceptions import InfluxDBClientError -from influxdb.tests import skipIfPYpy, using_pypy, skipServerTests +from influxdb.tests import skip_if_pypy, using_pypy, skip_server_tests from influxdb.tests.server_tests.base import ManyTestCasesWithServerMixin from influxdb.tests.server_tests.base import SingleTestCaseWithServerMixin @@ -82,7 +82,7 @@ def point(series_name, timestamp=None, tags=None, **fields): ] if not using_pypy: - dummy_pointDF = { + dummy_point_df = { "measurement": "cpu_load_short", "tags": {"host": "server01", "region": "us-west"}, @@ -90,7 +90,7 @@ def point(series_name, timestamp=None, tags=None, **fields): [[0.64]], columns=['value'], index=pd.to_datetime(["2009-11-10T23:00:00Z"])) } - dummy_pointsDF = [{ + dummy_points_df = [{ "measurement": "cpu_load_short", "tags": {"host": "server01", "region": "us-west"}, "dataframe": pd.DataFrame( @@ -120,7 +120,7 @@ def point(series_name, timestamp=None, tags=None, **fields): ] -@skipServerTests +@skip_server_tests class SimpleTests(SingleTestCaseWithServerMixin, unittest.TestCase): """Define the class of simple tests.""" @@ -267,7 +267,7 @@ def test_invalid_port_fails(self): InfluxDBClient('host', '80/redir', 'username', 'password') -@skipServerTests +@skip_server_tests class CommonTests(ManyTestCasesWithServerMixin, unittest.TestCase): """Define a class to handle common tests for the server.""" @@ -293,15 +293,15 @@ def test_write_points(self): """Test writing points to the server.""" self.assertIs(True, self.cli.write_points(dummy_point)) - @skipIfPYpy + @skip_if_pypy def test_write_points_DF(self): """Test writing points with dataframe.""" self.assertIs( True, self.cliDF.write_points( - dummy_pointDF['dataframe'], - dummy_pointDF['measurement'], - dummy_pointDF['tags'] + dummy_point_df['dataframe'], + dummy_point_df['measurement'], + dummy_point_df['tags'] ) ) @@ -342,7 +342,7 @@ def test_write_points_check_read_DF(self): rsp = self.cliDF.query('SELECT * FROM cpu_load_short') assert_frame_equal( rsp['cpu_load_short'], - dummy_pointDF['dataframe'] + dummy_point_df['dataframe'] ) # Query with Tags @@ -351,7 +351,7 @@ def test_write_points_check_read_DF(self): assert_frame_equal( rsp[('cpu_load_short', (('host', 'server01'), ('region', 'us-west')))], - dummy_pointDF['dataframe'] + dummy_point_df['dataframe'] ) def test_write_multiple_points_different_series(self): @@ -407,21 +407,21 @@ def test_write_multiple_points_different_series_DF(self): for i in range(2): self.assertIs( True, self.cliDF.write_points( - dummy_pointsDF[i]['dataframe'], - dummy_pointsDF[i]['measurement'], - dummy_pointsDF[i]['tags'])) + dummy_points_df[i]['dataframe'], + dummy_points_df[i]['measurement'], + dummy_points_df[i]['tags'])) time.sleep(1) rsp = self.cliDF.query('SELECT * FROM cpu_load_short') assert_frame_equal( rsp['cpu_load_short'], - dummy_pointsDF[0]['dataframe'] + dummy_points_df[0]['dataframe'] ) rsp = self.cliDF.query('SELECT * FROM memory') assert_frame_equal( rsp['memory'], - dummy_pointsDF[1]['dataframe'] + dummy_points_df[1]['dataframe'] ) def test_write_points_batch(self): @@ -786,7 +786,7 @@ def test_query_multiple_series(self): self.cli.write_points(pts) -@skipServerTests +@skip_server_tests class UdpTests(ManyTestCasesWithServerMixin, unittest.TestCase): """Define a class to test UDP series.""" From 78dafd7d9d58cbf0f0ab183b0268770251f1b9b6 Mon Sep 17 00:00:00 2001 From: Matthew McGinn Date: Thu, 14 Mar 2019 10:33:50 -0300 Subject: [PATCH 043/133] Fixup small test docstring typo Signed-off-by: Matthew McGinn --- influxdb/tests/dataframe_client_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/influxdb/tests/dataframe_client_test.py b/influxdb/tests/dataframe_client_test.py index 9fd6427b..aa055032 100644 --- a/influxdb/tests/dataframe_client_test.py +++ b/influxdb/tests/dataframe_client_test.py @@ -837,7 +837,7 @@ def test_query_into_dataframe(self): assert_frame_equal(expected[k], result[k]) def test_multiquery_into_dataframe(self): - """Test multiquyer into df for TestDataFrameClient object.""" + """Test multiquery into df for TestDataFrameClient object.""" data = { "results": [ { From f03f4957c70ac5429576281cbe8aaaec06e803fd Mon Sep 17 00:00:00 2001 From: Colas Le Guernic Date: Thu, 14 Mar 2019 14:04:32 +0000 Subject: [PATCH 044/133] Fix tz localize (#684) * fix already tz-aware error * fix tests tz_localize * update CHANGELOG.md --- CHANGELOG.md | 1 + influxdb/_dataframe_client.py | 3 ++- influxdb/tests/dataframe_client_test.py | 15 ++++++++++----- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 14a9abf4..590bd4f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added ### Changed +- Fix 'TypeError: Already tz-aware' introduced with recent versions of Panda (#671, #676, thx @f4bsch @clslgrnc) ### Removed diff --git a/influxdb/_dataframe_client.py b/influxdb/_dataframe_client.py index 06da7ac4..3b7a39db 100644 --- a/influxdb/_dataframe_client.py +++ b/influxdb/_dataframe_client.py @@ -202,7 +202,8 @@ def _to_dataframe(self, rs, dropna=True): df = pd.DataFrame(data) df.time = pd.to_datetime(df.time) df.set_index('time', inplace=True) - df.index = df.index.tz_localize('UTC') + if df.index.tzinfo is None: + df.index = df.index.tz_localize('UTC') df.index.name = None result[key].append(df) for key, data in result.items(): diff --git a/influxdb/tests/dataframe_client_test.py b/influxdb/tests/dataframe_client_test.py index aa055032..ad910a6d 100644 --- a/influxdb/tests/dataframe_client_test.py +++ b/influxdb/tests/dataframe_client_test.py @@ -818,13 +818,15 @@ def test_query_into_dataframe(self): pd1 = pd.DataFrame( [[23422]], columns=['value'], index=pd.to_datetime(["2009-11-10T23:00:00Z"])) - pd1.index = pd1.index.tz_localize('UTC') + if pd1.index.tzinfo is None: + pd1.index = pd1.index.tz_localize('UTC') pd2 = pd.DataFrame( [[23422], [23422], [23422]], columns=['value'], index=pd.to_datetime(["2009-11-10T23:00:00Z", "2009-11-10T23:00:00Z", "2009-11-10T23:00:00Z"])) - pd2.index = pd2.index.tz_localize('UTC') + if pd2.index.tzinfo is None: + pd2.index = pd2.index.tz_localize('UTC') expected = { ('network', (('direction', ''),)): pd1, ('network', (('direction', 'in'),)): pd2 @@ -871,11 +873,14 @@ def test_multiquery_into_dataframe(self): index=pd.to_datetime([ "2015-01-29 21:55:43.702900257+0000", "2015-01-29 21:55:43.702900257+0000", - "2015-06-11 20:46:02+0000"])).tz_localize('UTC') + "2015-06-11 20:46:02+0000"])) + if pd1.index.tzinfo is None: + pd1.index = pd1.index.tz_localize('UTC') pd2 = pd.DataFrame( [[3]], columns=['count'], - index=pd.to_datetime(["1970-01-01 00:00:00+00:00"]))\ - .tz_localize('UTC') + index=pd.to_datetime(["1970-01-01 00:00:00+00:00"])) + if pd2.index.tzinfo is None: + pd2.index = pd2.index.tz_localize('UTC') expected = [{'cpu_load_short': pd1}, {'cpu_load_short': pd2}] cli = DataFrameClient('host', 8086, 'username', 'password', 'db') From 5eda20403ca9ace10d2b5e0f76b28a703c5d5de8 Mon Sep 17 00:00:00 2001 From: Matthew McGinn Date: Thu, 14 Mar 2019 11:19:55 -0300 Subject: [PATCH 045/133] Bump version to 5.2.2 Signed-off-by: Matthew McGinn --- CHANGELOG.md | 7 ++++++- influxdb/__init__.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 590bd4f3..035476ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,10 +9,15 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added ### Changed -- Fix 'TypeError: Already tz-aware' introduced with recent versions of Panda (#671, #676, thx @f4bsch @clslgrnc) ### Removed +## [v5.2.2] - 2019-03-14 +### Added + +### Changed +- Fix 'TypeError: Already tz-aware' introduced with recent versions of Panda (#671, #676, thx @f4bsch @clslgrnc) + ## [v5.2.1] - 2018-12-07 ### Added diff --git a/influxdb/__init__.py b/influxdb/__init__.py index a1eb3789..288880b1 100644 --- a/influxdb/__init__.py +++ b/influxdb/__init__.py @@ -18,4 +18,4 @@ ] -__version__ = '5.2.1' +__version__ = '5.2.2' From bf95e0f2969fc9a07042b7419a18fc2a2c698085 Mon Sep 17 00:00:00 2001 From: Colas Le Guernic Date: Fri, 15 Mar 2019 17:13:43 +0000 Subject: [PATCH 046/133] [WIP] add py37 and recent influxdb (#692) * add py37 and recent influxdb * remove useless py34 dep * use py36 for pydocstyle (py27 soon deprecated) * ugly fix to numpy inconsistencies * py37 is not in ubuntu 14.04 * move import numpy and add noqa * get 3.7 into travis matrix * get 3.7 into travis matrix --- .travis.yml | 36 +++++++++++++++++++++---- influxdb/tests/dataframe_client_test.py | 18 ++++++++++--- tox.ini | 30 ++++++++++++++------- 3 files changed, 65 insertions(+), 19 deletions(-) diff --git a/.travis.yml b/.travis.yml index a1cf7b55..8c660b67 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,10 +8,12 @@ python: - "pypy3" env: - - INFLUXDB_VER=1.2.4 - - INFLUXDB_VER=1.3.9 - - INFLUXDB_VER=1.4.2 - - INFLUXDB_VER=1.5.4 + - INFLUXDB_VER=1.2.4 # 2017-05-08 + - INFLUXDB_VER=1.3.9 # 2018-01-19 + - INFLUXDB_VER=1.4.3 # 2018-01-30 + - INFLUXDB_VER=1.5.4 # 2018-06-22 + - INFLUXDB_VER=1.6.4 # 2018-10-24 + - INFLUXDB_VER=1.7.4 # 2019-02-14 addons: apt: @@ -20,7 +22,31 @@ addons: matrix: include: - - python: 2.7 + - python: 3.7 + dist: xenial + sudo: true + env: INFLUXDB_VER=1.2.4 + - python: 3.7 + dist: xenial + sudo: true + env: INFLUXDB_VER=1.3.9 + - python: 3.7 + dist: xenial + sudo: true + env: INFLUXDB_VER=1.4.3 + - python: 3.7 + dist: xenial + sudo: true + env: INFLUXDB_VER=1.5.4 + - python: 3.7 + dist: xenial + sudo: true + env: INFLUXDB_VER=1.6.4 + - python: 3.7 + dist: xenial + sudo: true + env: INFLUXDB_VER=1.7.4 + - python: 3.6 env: TOX_ENV=pep257 - python: 3.6 env: TOX_ENV=docs diff --git a/influxdb/tests/dataframe_client_test.py b/influxdb/tests/dataframe_client_test.py index ad910a6d..1de3a501 100644 --- a/influxdb/tests/dataframe_client_test.py +++ b/influxdb/tests/dataframe_client_test.py @@ -22,6 +22,7 @@ import pandas as pd from pandas.util.testing import assert_frame_equal from influxdb import DataFrameClient + import numpy @skip_if_pypy @@ -396,10 +397,16 @@ def test_write_points_from_dataframe_with_numeric_precision(self): ["2", 2, 2.2222222222222]], index=[now, now + timedelta(hours=1)]) - expected_default_precision = ( - b'foo,hello=there 0=\"1\",1=1i,2=1.11111111111 0\n' - b'foo,hello=there 0=\"2\",1=2i,2=2.22222222222 3600000000000\n' - ) + if tuple(map(int, numpy.version.version.split('.'))) <= (1, 13, 3): + expected_default_precision = ( + b'foo,hello=there 0=\"1\",1=1i,2=1.11111111111 0\n' + b'foo,hello=there 0=\"2\",1=2i,2=2.22222222222 3600000000000\n' + ) + else: + expected_default_precision = ( + b'foo,hello=there 0=\"1\",1=1i,2=1.1111111111111 0\n' + b'foo,hello=there 0=\"2\",1=2i,2=2.2222222222222 3600000000000\n' # noqa E501 line too long + ) expected_specified_precision = ( b'foo,hello=there 0=\"1\",1=1i,2=1.1111 0\n' @@ -419,6 +426,9 @@ def test_write_points_from_dataframe_with_numeric_precision(self): cli = DataFrameClient(database='db') cli.write_points(dataframe, "foo", {"hello": "there"}) + print(expected_default_precision) + print(m.last_request.body) + self.assertEqual(m.last_request.body, expected_default_precision) cli = DataFrameClient(database='db') diff --git a/tox.ini b/tox.ini index 2f9c212c..4a1921e2 100644 --- a/tox.ini +++ b/tox.ini @@ -1,21 +1,28 @@ [tox] -envlist = py27, py35, py36, pypy, pypy3, flake8, pep257, coverage, docs +envlist = py27, py35, py36, py37, pypy, pypy3, flake8, pep257, coverage, docs [testenv] passenv = INFLUXDB_PYTHON_INFLUXD_PATH setenv = INFLUXDB_PYTHON_SKIP_SERVER_TESTS=False deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt - py27,py34,py35,py36: pandas==0.20.1 - py27,py34,py35,py36: numpy==1.13.3 + py27: pandas==0.21.1 + py27: numpy==1.13.3 + py35: pandas==0.22.0 + py35: numpy==1.14.6 + py36: pandas==0.23.4 + py36: numpy==1.15.4 + py37: pandas==0.24.2 + py37: numpy==1.16.2 # Only install pandas with non-pypy interpreters +# Testing all combinations would be too expensive commands = nosetests -v --with-doctest {posargs} [testenv:flake8] deps = flake8 pep8-naming -commands = flake8 --ignore=W503,W504,W605,N802,F821 influxdb +commands = flake8 influxdb [testenv:pep257] deps = pydocstyle @@ -26,19 +33,22 @@ deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt pandas coverage - numpy==1.13.3 + numpy commands = nosetests -v --with-coverage --cover-html --cover-package=influxdb [testenv:docs] deps = -r{toxinidir}/requirements.txt - pandas==0.20.1 - numpy==1.13.3 - Sphinx==1.5.5 + pandas==0.24.2 + numpy==1.16.2 + Sphinx==1.8.5 sphinx_rtd_theme commands = sphinx-build -b html docs/source docs/build [flake8] -ignore = N802,F821,E402 -# E402: module level import not at top of file +ignore = W503,W504,W605,N802,F821,E402 +# W503: Line break occurred before a binary operator +# W504: Line break occurred after a binary operator +# W605: invalid escape sequence # N802: nosetests's setUp function # F821: False positive in intluxdb/dataframe_client.py +# E402: module level import not at top of file From 05a101d5e4b4554899a2a8830d50ddcf07fba747 Mon Sep 17 00:00:00 2001 From: Colas Le Guernic Date: Sat, 16 Mar 2019 17:47:41 +0000 Subject: [PATCH 047/133] Python and influxdb supported versions (#693) * numpy might use non-numerical version * update * update CHANGELOG.md --- CHANGELOG.md | 1 + README.rst | 4 ++-- influxdb/tests/dataframe_client_test.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 035476ab..d18d5bc4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added ### Changed +- Update test suite to add support for Python 3.7 and InfluxDB v1.6.4 and 1.7.4 (#692 thx @clslgrnc) ### Removed diff --git a/README.rst b/README.rst index d4f9611c..026171b2 100644 --- a/README.rst +++ b/README.rst @@ -39,7 +39,7 @@ InfluxDB is an open-source distributed time series database, find more about Inf InfluxDB pre v1.1.0 users ------------------------- -This module is tested with InfluxDB versions: v1.2.4, v1.3.9, v1.4.2, and v1.5.4. +This module is tested with InfluxDB versions: v1.2.4, v1.3.9, v1.4.3, v1.5.4, v1.6.4, and 1.7.4. Those users still on InfluxDB v0.8.x users may still use the legacy client by importing ``from influxdb.influxdb08 import InfluxDBClient``. @@ -59,7 +59,7 @@ On Debian/Ubuntu, you can install it with this command:: Dependencies ------------ -The influxdb-python distribution is supported and tested on Python 2.7, 3.5, 3.6, PyPy and PyPy3. +The influxdb-python distribution is supported and tested on Python 2.7, 3.5, 3.6, 3.7, PyPy and PyPy3. **Note:** Python <3.5 are currently untested. See ``.travis.yml``. diff --git a/influxdb/tests/dataframe_client_test.py b/influxdb/tests/dataframe_client_test.py index 1de3a501..f861cf2e 100644 --- a/influxdb/tests/dataframe_client_test.py +++ b/influxdb/tests/dataframe_client_test.py @@ -397,7 +397,7 @@ def test_write_points_from_dataframe_with_numeric_precision(self): ["2", 2, 2.2222222222222]], index=[now, now + timedelta(hours=1)]) - if tuple(map(int, numpy.version.version.split('.'))) <= (1, 13, 3): + if numpy.lib.NumpyVersion(numpy.__version__) <= '1.13.3': expected_default_precision = ( b'foo,hello=there 0=\"1\",1=1i,2=1.11111111111 0\n' b'foo,hello=there 0=\"2\",1=2i,2=2.22222222222 3600000000000\n' From 02865ae6ca184102559d8d6579e9a86f0498e363 Mon Sep 17 00:00:00 2001 From: Matthew McGinn Date: Tue, 19 Mar 2019 23:27:47 -0500 Subject: [PATCH 048/133] Add CODEOWNERS file for automatic reviewers on GitHub Signed-off-by: Matthew McGinn --- CODEOWNERS | 1 + 1 file changed, 1 insertion(+) create mode 100644 CODEOWNERS diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 00000000..0acbd7c8 --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1 @@ +* @aviau @xginn8 @sebito91 From afcfd25b21523d84a7d1088eff2abb4d08de7647 Mon Sep 17 00:00:00 2001 From: Colas Le Guernic Date: Wed, 20 Mar 2019 04:30:26 +0000 Subject: [PATCH 049/133] Parameter binding for client's `query()` method (#678) * add bind_params to query * tutorial for bind_params --- CHANGELOG.md | 1 + examples/tutorial.py | 9 ++++++++- influxdb/_dataframe_client.py | 12 ++++++++++++ influxdb/client.py | 18 ++++++++++++++++++ influxdb/tests/dataframe_client_test.py | 7 ++++--- .../server_tests/client_test_with_server.py | 4 +++- 6 files changed, 46 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d18d5bc4..a5bc07fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] ### Added +- query() now accepts a bind_params argument for parameter binding (#678 thx @clslgrnc) ### Changed - Update test suite to add support for Python 3.7 and InfluxDB v1.6.4 and 1.7.4 (#692 thx @clslgrnc) diff --git a/examples/tutorial.py b/examples/tutorial.py index 4083bfc5..12cd49c1 100644 --- a/examples/tutorial.py +++ b/examples/tutorial.py @@ -13,7 +13,9 @@ def main(host='localhost', port=8086): dbname = 'example' dbuser = 'smly' dbuser_password = 'my_secret_password' - query = 'select value from cpu_load_short;' + query = 'select Float_value from cpu_load_short;' + query_where = 'select Int_value from cpu_load_short where host=$host;' + bind_params = {'host': 'server01'} json_body = [ { "measurement": "cpu_load_short", @@ -50,6 +52,11 @@ def main(host='localhost', port=8086): print("Result: {0}".format(result)) + print("Querying data: " + query_where) + result = client.query(query_where, bind_params=bind_params) + + print("Result: {0}".format(result)) + print("Switch user: " + user) client.switch_user(user, password) diff --git a/influxdb/_dataframe_client.py b/influxdb/_dataframe_client.py index 3b7a39db..1ce6e947 100644 --- a/influxdb/_dataframe_client.py +++ b/influxdb/_dataframe_client.py @@ -142,6 +142,7 @@ def write_points(self, def query(self, query, params=None, + bind_params=None, epoch=None, expected_response_code=200, database=None, @@ -153,8 +154,18 @@ def query(self, """ Query data into a DataFrame. + .. danger:: + In order to avoid injection vulnerabilities (similar to `SQL + injection `_ + vulnerabilities), do not directly include untrusted data into the + ``query`` parameter, use ``bind_params`` instead. + :param query: the actual query string :param params: additional parameters for the request, defaults to {} + :param bind_params: bind parameters for the query: + any variable in the query written as ``'$var_name'`` will be + replaced with ``bind_params['var_name']``. Only works in the + ``WHERE`` clause and takes precedence over ``params['params']`` :param epoch: response timestamps to be in epoch format either 'h', 'm', 's', 'ms', 'u', or 'ns',defaults to `None` which is RFC3339 UTC format with nanosecond precision @@ -172,6 +183,7 @@ def query(self, :rtype: :class:`~.ResultSet` """ query_args = dict(params=params, + bind_params=bind_params, epoch=epoch, expected_response_code=expected_response_code, raise_errors=raise_errors, diff --git a/influxdb/client.py b/influxdb/client.py index 8f8b14ae..e94ae25d 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -345,6 +345,7 @@ def _read_chunked_response(response, raise_errors=True): def query(self, query, params=None, + bind_params=None, epoch=None, expected_response_code=200, database=None, @@ -354,6 +355,12 @@ def query(self, method="GET"): """Send a query to InfluxDB. + .. danger:: + In order to avoid injection vulnerabilities (similar to `SQL + injection `_ + vulnerabilities), do not directly include untrusted data into the + ``query`` parameter, use ``bind_params`` instead. + :param query: the actual query string :type query: str @@ -361,6 +368,12 @@ def query(self, defaults to {} :type params: dict + :param bind_params: bind parameters for the query: + any variable in the query written as ``'$var_name'`` will be + replaced with ``bind_params['var_name']``. Only works in the + ``WHERE`` clause and takes precedence over ``params['params']`` + :type bind_params: dict + :param epoch: response timestamps to be in epoch format either 'h', 'm', 's', 'ms', 'u', or 'ns',defaults to `None` which is RFC3339 UTC format with nanosecond precision @@ -394,6 +407,11 @@ def query(self, if params is None: params = {} + if bind_params is not None: + params_dict = json.loads(params.get('params', '{}')) + params_dict.update(bind_params) + params['params'] = json.dumps(params_dict) + params['q'] = query params['db'] = database or self._database diff --git a/influxdb/tests/dataframe_client_test.py b/influxdb/tests/dataframe_client_test.py index f861cf2e..cb380ac5 100644 --- a/influxdb/tests/dataframe_client_test.py +++ b/influxdb/tests/dataframe_client_test.py @@ -894,10 +894,11 @@ def test_multiquery_into_dataframe(self): expected = [{'cpu_load_short': pd1}, {'cpu_load_short': pd2}] cli = DataFrameClient('host', 8086, 'username', 'password', 'db') - iql = "SELECT value FROM cpu_load_short WHERE region='us-west';"\ - "SELECT count(value) FROM cpu_load_short WHERE region='us-west'" + iql = "SELECT value FROM cpu_load_short WHERE region=$region;"\ + "SELECT count(value) FROM cpu_load_short WHERE region=$region" + bind_params = {'region': 'us-west'} with _mocked_session(cli, 'GET', 200, data): - result = cli.query(iql) + result = cli.query(iql, bind_params=bind_params) for r, e in zip(result, expected): for k in e: assert_frame_equal(e[k], r[k]) diff --git a/influxdb/tests/server_tests/client_test_with_server.py b/influxdb/tests/server_tests/client_test_with_server.py index 4dbc1b75..121d2c82 100644 --- a/influxdb/tests/server_tests/client_test_with_server.py +++ b/influxdb/tests/server_tests/client_test_with_server.py @@ -440,7 +440,9 @@ def test_write_points_batch(self): batch_size=2) time.sleep(5) net_in = self.cli.query("SELECT value FROM network " - "WHERE direction='in'").raw + "WHERE direction=$dir", + bind_params={'dir': 'in'} + ).raw net_out = self.cli.query("SELECT value FROM network " "WHERE direction='out'").raw cpu = self.cli.query("SELECT value FROM cpu_usage").raw From 73503b5396ed3e9dbc115af31edb0112622b160a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Dudek?= <45991310+lukaszdudek-silvair@users.noreply.github.com> Date: Mon, 1 Apr 2019 19:05:02 +0200 Subject: [PATCH 050/133] Add CQs management methods to the client (#681) * Add CQs management methods to the client --- CHANGELOG.md | 2 + influxdb/client.py | 92 +++++++++++++++ influxdb/tests/client_test.py | 108 ++++++++++++++++++ .../server_tests/client_test_with_server.py | 30 +++++ 4 files changed, 232 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a5bc07fb..9834a5ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] ### Added +- Add `get_list_continuous_queries`, `drop_continuous_query`, and `create_continuous_query` management methods for + continuous queries (#681 thx @lukaszdudek-silvair) - query() now accepts a bind_params argument for parameter binding (#678 thx @clslgrnc) ### Changed diff --git a/influxdb/client.py b/influxdb/client.py index e94ae25d..d365643d 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -926,6 +926,98 @@ def get_list_privileges(self, username): text = "SHOW GRANTS FOR {0}".format(quote_ident(username)) return list(self.query(text).get_points()) + def get_list_continuous_queries(self): + """Get the list of continuous queries in InfluxDB. + + :return: all CQs in InfluxDB + :rtype: list of dictionaries + + :Example: + + :: + + >> cqs = client.get_list_cqs() + >> cqs + [ + { + u'db1': [] + }, + { + u'db2': [ + { + u'name': u'vampire', + u'query': u'CREATE CONTINUOUS QUERY vampire ON ' + 'mydb BEGIN SELECT count(dracula) INTO ' + 'mydb.autogen.all_of_them FROM ' + 'mydb.autogen.one GROUP BY time(5m) END' + } + ] + } + ] + """ + query_string = "SHOW CONTINUOUS QUERIES" + return [{sk[0]: list(p)} for sk, p in self.query(query_string).items()] + + def create_continuous_query(self, name, select, database=None, + resample_opts=None): + r"""Create a continuous query for a database. + + :param name: the name of continuous query to create + :type name: str + :param select: select statement for the continuous query + :type select: str + :param database: the database for which the continuous query is + created. Defaults to current client's database + :type database: str + :param resample_opts: resample options + :type resample_opts: str + + :Example: + + :: + + >> select_clause = 'SELECT mean("value") INTO "cpu_mean" ' \ + ... 'FROM "cpu" GROUP BY time(1m)' + >> client.create_continuous_query( + ... 'cpu_mean', select_clause, 'db_name', 'EVERY 10s FOR 2m' + ... ) + >> client.get_list_continuous_queries() + [ + { + 'db_name': [ + { + 'name': 'cpu_mean', + 'query': 'CREATE CONTINUOUS QUERY "cpu_mean" ' + 'ON "db_name" ' + 'RESAMPLE EVERY 10s FOR 2m ' + 'BEGIN SELECT mean("value") ' + 'INTO "cpu_mean" FROM "cpu" ' + 'GROUP BY time(1m) END' + } + ] + } + ] + """ + query_string = ( + "CREATE CONTINUOUS QUERY {0} ON {1}{2} BEGIN {3} END" + ).format(quote_ident(name), quote_ident(database or self._database), + ' RESAMPLE ' + resample_opts if resample_opts else '', select) + self.query(query_string) + + def drop_continuous_query(self, name, database=None): + """Drop an existing continuous query for a database. + + :param name: the name of continuous query to drop + :type name: str + :param database: the database for which the continuous query is + dropped. Defaults to current client's database + :type database: str + """ + query_string = ( + "DROP CONTINUOUS QUERY {0} ON {1}" + ).format(quote_ident(name), quote_ident(database or self._database)) + self.query(query_string) + def send_packet(self, packet, protocol='json', time_precision=None): """Send an UDP packet. diff --git a/influxdb/tests/client_test.py b/influxdb/tests/client_test.py index e27eef17..e1a30b81 100644 --- a/influxdb/tests/client_test.py +++ b/influxdb/tests/client_test.py @@ -1027,6 +1027,114 @@ def test_get_list_privileges_fails(self): with _mocked_session(cli, 'get', 401): cli.get_list_privileges('test') + def test_get_list_continuous_queries(self): + """Test getting a list of continuous queries.""" + data = { + "results": [ + { + "statement_id": 0, + "series": [ + { + "name": "testdb01", + "columns": ["name", "query"], + "values": [["testname01", "testquery01"], + ["testname02", "testquery02"]] + }, + { + "name": "testdb02", + "columns": ["name", "query"], + "values": [["testname03", "testquery03"]] + }, + { + "name": "testdb03", + "columns": ["name", "query"] + } + ] + } + ] + } + + with _mocked_session(self.cli, 'get', 200, json.dumps(data)): + self.assertListEqual( + self.cli.get_list_continuous_queries(), + [ + { + 'testdb01': [ + {'name': 'testname01', 'query': 'testquery01'}, + {'name': 'testname02', 'query': 'testquery02'} + ] + }, + { + 'testdb02': [ + {'name': 'testname03', 'query': 'testquery03'} + ] + }, + { + 'testdb03': [] + } + ] + ) + + @raises(Exception) + def test_get_list_continuous_queries_fails(self): + """Test failing to get a list of continuous queries.""" + with _mocked_session(self.cli, 'get', 400): + self.cli.get_list_continuous_queries() + + def test_create_continuous_query(self): + """Test continuous query creation.""" + data = {"results": [{}]} + with requests_mock.Mocker() as m: + m.register_uri( + requests_mock.GET, + "http://localhost:8086/query", + text=json.dumps(data) + ) + query = 'SELECT count("value") INTO "6_months"."events" FROM ' \ + '"events" GROUP BY time(10m)' + self.cli.create_continuous_query('cq_name', query, 'db_name') + self.assertEqual( + m.last_request.qs['q'][0], + 'create continuous query "cq_name" on "db_name" begin select ' + 'count("value") into "6_months"."events" from "events" group ' + 'by time(10m) end' + ) + self.cli.create_continuous_query('cq_name', query, 'db_name', + 'EVERY 10s FOR 2m') + self.assertEqual( + m.last_request.qs['q'][0], + 'create continuous query "cq_name" on "db_name" resample ' + 'every 10s for 2m begin select count("value") into ' + '"6_months"."events" from "events" group by time(10m) end' + ) + + @raises(Exception) + def test_create_continuous_query_fails(self): + """Test failing to create a continuous query.""" + with _mocked_session(self.cli, 'get', 400): + self.cli.create_continuous_query('cq_name', 'select', 'db_name') + + def test_drop_continuous_query(self): + """Test dropping a continuous query.""" + data = {"results": [{}]} + with requests_mock.Mocker() as m: + m.register_uri( + requests_mock.GET, + "http://localhost:8086/query", + text=json.dumps(data) + ) + self.cli.drop_continuous_query('cq_name', 'db_name') + self.assertEqual( + m.last_request.qs['q'][0], + 'drop continuous query "cq_name" on "db_name"' + ) + + @raises(Exception) + def test_drop_continuous_query_fails(self): + """Test failing to drop a continuous query.""" + with _mocked_session(self.cli, 'get', 400): + self.cli.drop_continuous_query('cq_name', 'db_name') + def test_invalid_port_fails(self): """Test invalid port fail for TestInfluxDBClient object.""" with self.assertRaises(ValueError): diff --git a/influxdb/tests/server_tests/client_test_with_server.py b/influxdb/tests/server_tests/client_test_with_server.py index 121d2c82..fda3f720 100644 --- a/influxdb/tests/server_tests/client_test_with_server.py +++ b/influxdb/tests/server_tests/client_test_with_server.py @@ -722,6 +722,36 @@ def test_drop_retention_policy(self): rsp ) + def test_create_continuous_query(self): + """Test continuous query creation.""" + self.cli.create_retention_policy('some_rp', '1d', 1) + query = 'select count("value") into "some_rp"."events" from ' \ + '"events" group by time(10m)' + self.cli.create_continuous_query('test_cq', query, 'db') + cqs = self.cli.get_list_continuous_queries() + expected_cqs = [ + { + 'db': [ + { + 'name': 'test_cq', + 'query': 'CREATE CONTINUOUS QUERY test_cq ON db ' + 'BEGIN SELECT count(value) INTO ' + 'db.some_rp.events FROM db.autogen.events ' + 'GROUP BY time(10m) END' + } + ] + } + ] + self.assertEqual(cqs, expected_cqs) + + def test_drop_continuous_query(self): + """Test continuous query drop.""" + self.test_create_continuous_query() + self.cli.drop_continuous_query('test_cq', 'db') + cqs = self.cli.get_list_continuous_queries() + expected_cqs = [{'db': []}] + self.assertEqual(cqs, expected_cqs) + def test_issue_143(self): """Test for PR#143 from repo.""" pt = partial(point, 'a_series_name', timestamp='2015-03-30T16:16:37Z') From e24b42c36269725946164dc98f2802d96df9f708 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A7=8B=E8=91=89?= Date: Sun, 7 Apr 2019 09:32:09 -0500 Subject: [PATCH 051/133] Fix a warning under Python 3.7 (#697) * Fix a warning under Python 3.7 Signed-off-by: Matthew McGinn --- CHANGELOG.md | 1 + setup.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9834a5ef..33302f16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Changed - Update test suite to add support for Python 3.7 and InfluxDB v1.6.4 and 1.7.4 (#692 thx @clslgrnc) +- Update classifiers tuple to list in setup.py (#697 thx @Hanaasagi) ### Removed diff --git a/setup.py b/setup.py index cd6e4e9b..d44875f6 100755 --- a/setup.py +++ b/setup.py @@ -42,7 +42,7 @@ tests_require=test_requires, install_requires=requires, extras_require={'test': test_requires}, - classifiers=( + classifiers=[ 'Development Status :: 3 - Alpha', 'Intended Audience :: Developers', 'License :: OSI Approved :: MIT License', @@ -55,5 +55,5 @@ 'Programming Language :: Python :: 3.6', 'Topic :: Software Development :: Libraries', 'Topic :: Software Development :: Libraries :: Python Modules', - ), + ], ) From 47aeb9bf48f7556b570b40279ca169980c141d1f Mon Sep 17 00:00:00 2001 From: xginn8 Date: Sun, 7 Apr 2019 20:09:36 -0500 Subject: [PATCH 052/133] Update delete_series docstring to differentiate between drop_database (#699) Closes #666 Signed-off-by: Matthew McGinn --- CHANGELOG.md | 1 + influxdb/client.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 33302f16..a7630a9d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Changed - Update test suite to add support for Python 3.7 and InfluxDB v1.6.4 and 1.7.4 (#692 thx @clslgrnc) - Update classifiers tuple to list in setup.py (#697 thx @Hanaasagi) +- Update documentation for empty `delete_series` confusion ### Removed diff --git a/influxdb/client.py b/influxdb/client.py index d365643d..dc77bfc1 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -827,7 +827,9 @@ def set_user_password(self, username, password): def delete_series(self, database=None, measurement=None, tags=None): """Delete series from a database. - Series can be filtered by measurement and tags. + Series must be filtered by either measurement and tags. + This method cannot be used to delete all series, use + `drop_database` instead. :param database: the database from which the series should be deleted, defaults to client's current database From dc54e5eb716d0356a10138ee2f9b7b2ca83a3796 Mon Sep 17 00:00:00 2001 From: Ron Rothman Date: Mon, 8 Apr 2019 13:13:13 -0400 Subject: [PATCH 053/133] add consistency parameter to write_points (#664) * add consistency parameter to write_points [https://github.com/influxdata/influxdb-python/issues/643] --- CHANGELOG.md | 1 + influxdb/client.py | 21 +++++++++++++++++---- influxdb/tests/client_test.py | 26 ++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a7630a9d..7f1503b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - query() now accepts a bind_params argument for parameter binding (#678 thx @clslgrnc) ### Changed +- Add consistency param to InfluxDBClient.write_points (#643 thx @RonRothman) - Update test suite to add support for Python 3.7 and InfluxDB v1.6.4 and 1.7.4 (#692 thx @clslgrnc) - Update classifiers tuple to list in setup.py (#697 thx @Hanaasagi) - Update documentation for empty `delete_series` confusion diff --git a/influxdb/client.py b/influxdb/client.py index dc77bfc1..8ac557d3 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -458,7 +458,8 @@ def write_points(self, retention_policy=None, tags=None, batch_size=None, - protocol='json' + protocol='json', + consistency=None ): """Write to multiple time series names. @@ -486,6 +487,9 @@ def write_points(self, :type batch_size: int :param protocol: Protocol for writing data. Either 'line' or 'json'. :type protocol: str + :param consistency: Consistency for the points. + One of {'any','one','quorum','all'}. + :type consistency: str :returns: True, if the operation is successful :rtype: bool @@ -498,14 +502,16 @@ def write_points(self, time_precision=time_precision, database=database, retention_policy=retention_policy, - tags=tags, protocol=protocol) + tags=tags, protocol=protocol, + consistency=consistency) return True return self._write_points(points=points, time_precision=time_precision, database=database, retention_policy=retention_policy, - tags=tags, protocol=protocol) + tags=tags, protocol=protocol, + consistency=consistency) def ping(self): """Check connectivity to InfluxDB. @@ -531,12 +537,16 @@ def _write_points(self, database, retention_policy, tags, - protocol='json'): + protocol='json', + consistency=None): if time_precision not in ['n', 'u', 'ms', 's', 'm', 'h', None]: raise ValueError( "Invalid time precision is given. " "(use 'n', 'u', 'ms', 's', 'm' or 'h')") + if consistency not in ['any', 'one', 'quorum', 'all', None]: + raise ValueError('Invalid consistency: {}'.format(consistency)) + if protocol == 'json': data = { 'points': points @@ -551,6 +561,9 @@ def _write_points(self, 'db': database or self._database } + if consistency is not None: + params['consistency'] = consistency + if time_precision is not None: params['precision'] = time_precision diff --git a/influxdb/tests/client_test.py b/influxdb/tests/client_test.py index e1a30b81..e4cc7e11 100644 --- a/influxdb/tests/client_test.py +++ b/influxdb/tests/client_test.py @@ -337,6 +337,23 @@ def test_write_points_with_precision(self): m.last_request.body, ) + def test_write_points_with_consistency(self): + """Test write points with consistency for TestInfluxDBClient object.""" + with requests_mock.Mocker() as m: + m.register_uri( + requests_mock.POST, + 'http://localhost:8086/write', + status_code=204 + ) + + cli = InfluxDBClient(database='db') + + cli.write_points(self.dummy_points, consistency='any') + self.assertEqual( + m.last_request.qs, + {'db': ['db'], 'consistency': ['any']} + ) + def test_write_points_with_precision_udp(self): """Test write points with precision for TestInfluxDBClient object.""" s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) @@ -409,6 +426,15 @@ def test_write_points_bad_precision(self): time_precision='g' ) + def test_write_points_bad_consistency(self): + """Test write points w/bad consistency value.""" + cli = InfluxDBClient() + with self.assertRaises(ValueError): + cli.write_points( + self.dummy_points, + consistency='boo' + ) + @raises(Exception) def test_write_points_with_precision_fails(self): """Test write points w/precision fail for TestInfluxDBClient object.""" From d5d12499f3755199d5eedd8b363450f1cf4073bd Mon Sep 17 00:00:00 2001 From: Shan Desai Date: Wed, 10 Apr 2019 15:30:45 +0200 Subject: [PATCH 054/133] Add Example for sending information to DB via UDP (#648) Due to lack of documentation for UDP, this example provides basic usage of sending information points via UDP. The code structure followed is similar, if not same as other examples in the `examples` directory. Signed-off-by: Shantanoo --- docs/source/examples.rst | 6 ++++ examples/tutorial_udp.py | 66 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) create mode 100644 examples/tutorial_udp.py diff --git a/docs/source/examples.rst b/docs/source/examples.rst index 2c85fbda..fdda62a9 100644 --- a/docs/source/examples.rst +++ b/docs/source/examples.rst @@ -25,3 +25,9 @@ Tutorials - SeriesHelper .. literalinclude:: ../../examples/tutorial_serieshelper.py :language: python + +Tutorials - UDP +=============== + +.. literalinclude:: ../../examples/tutorial_udp.py + :language: python diff --git a/examples/tutorial_udp.py b/examples/tutorial_udp.py new file mode 100644 index 00000000..517ae858 --- /dev/null +++ b/examples/tutorial_udp.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +"""Example for sending batch information to InfluxDB via UDP.""" + +""" +INFO: In order to use UDP, one should enable the UDP service from the +`influxdb.conf` under section + [[udp]] + enabled = true + bind-address = ":8089" # port number for sending data via UDP + database = "udp1" # name of database to be stored + [[udp]] + enabled = true + bind-address = ":8090" + database = "udp2" +""" + + +import argparse + +from influxdb import InfluxDBClient + + +def main(uport): + """Instantiate connection to the InfluxDB.""" + # NOTE: structure of the UDP packet is different than that of information + # sent via HTTP + json_body = { + "tags": { + "host": "server01", + "region": "us-west" + }, + "time": "2009-11-10T23:00:00Z", + "points": [{ + "measurement": "cpu_load_short", + "fields": { + "value": 0.64 + } + }, + { + "measurement": "cpu_load_short", + "fields": { + "value": 0.67 + } + }] + } + + # make `use_udp` True and add `udp_port` number from `influxdb.conf` file + # no need to mention the database name since it is already configured + client = InfluxDBClient(use_udp=True, udp_port=uport) + + # Instead of `write_points` use `send_packet` + client.send_packet(json_body) + + +def parse_args(): + """Parse the args.""" + parser = argparse.ArgumentParser( + description='example code to play with InfluxDB along with UDP Port') + parser.add_argument('--uport', type=int, required=True, + help=' UDP port of InfluxDB') + return parser.parse_args() + + +if __name__ == '__main__': + args = parse_args() + main(uport=args.uport) From 08e02996a17c531a87e02cf6c5e693c4038000ed Mon Sep 17 00:00:00 2001 From: Debanjan Date: Fri, 12 Jul 2019 00:33:24 +0200 Subject: [PATCH 055/133] Line protocol leading comma (#694) * [fix] typo in comment + [fix] handles leading comma for the case that the first value column is Null valued * [refactor] consolidated similar logic to a new function * [fix] covering scenario where is a string * Revert "[fix] covering scenario where is a string" This reverts commit 49af5abadda346929f728ff61625e38062adc434. * Revert "[refactor] consolidated similar logic to a new function" This reverts commit 366e7714668cc1fd28b5a5f689e93661f1b9da5f. * [tests][feature] added tests to check if first none value results in invalid line protocol * [fix] deleted debug lines * [fix] overspecified date_range args * [fix] overspecified date_range args * [fix] removed endpoint in date-range * [fix] reordered columns in test target * [fix] [test] freeze order of columns * [refactor] [test] used loc instead of dict-like invocation of columns * [fix] [test] [lint] cleared up complainsts from flake8 and pep257 --- influxdb/_dataframe_client.py | 4 +- influxdb/tests/dataframe_client_test.py | 65 +++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/influxdb/_dataframe_client.py b/influxdb/_dataframe_client.py index 1ce6e947..d16e29ca 100644 --- a/influxdb/_dataframe_client.py +++ b/influxdb/_dataframe_client.py @@ -363,7 +363,7 @@ def _convert_dataframe_to_lines(self, tag_df = self._stringify_dataframe( tag_df, numeric_precision, datatype='tag') - # join preprendded tags, leaving None values out + # join prepended tags, leaving None values out tags = tag_df.apply( lambda s: [',' + s.name + '=' + v if v else '' for v in s]) tags = tags.sum(axis=1) @@ -392,6 +392,8 @@ def _convert_dataframe_to_lines(self, field_df.columns[1:]] field_df = field_df.where(~mask_null, '') # drop Null entries fields = field_df.sum(axis=1) + # take out leading , where first column has a Null value + fields = fields.str.lstrip(",") del field_df # Generate line protocol string diff --git a/influxdb/tests/dataframe_client_test.py b/influxdb/tests/dataframe_client_test.py index cb380ac5..90312ed8 100644 --- a/influxdb/tests/dataframe_client_test.py +++ b/influxdb/tests/dataframe_client_test.py @@ -389,6 +389,71 @@ def test_write_points_from_dataframe_with_numeric_column_names(self): self.assertEqual(m.last_request.body, expected) + def test_write_points_from_dataframe_with_leading_none_column(self): + """write_points detect erroneous leading comma for null first field.""" + dataframe = pd.DataFrame( + dict( + first=[1, None, None, 8, 9], + second=[2, None, None, None, 10], + third=[3, 4.1, None, None, 11], + first_tag=["one", None, None, "eight", None], + second_tag=["two", None, None, None, None], + third_tag=["three", "four", None, None, None], + comment=[ + "All columns filled", + "First two of three empty", + "All empty", + "Last two of three empty", + "Empty tags with values", + ] + ), + index=pd.date_range( + start=pd.to_datetime('2018-01-01'), + freq='1D', + periods=5, + ) + ) + expected = ( + b'foo,first_tag=one,second_tag=two,third_tag=three' + b' comment="All columns filled",first=1.0,second=2.0,third=3.0' + b' 1514764800000000000\n' + b'foo,third_tag=four' + b' comment="First two of three empty",third=4.1' + b' 1514851200000000000\n' + b'foo comment="All empty" 1514937600000000000\n' + b'foo,first_tag=eight' + b' comment="Last two of three empty",first=8.0' + b' 1515024000000000000\n' + b'foo' + b' comment="Empty tags with values",first=9.0,second=10.0' + b',third=11.0' + b' 1515110400000000000\n' + ) + + with requests_mock.Mocker() as m: + m.register_uri(requests_mock.POST, + "http://localhost:8086/write", + status_code=204) + + cli = DataFrameClient(database='db') + + colnames = [ + "first_tag", + "second_tag", + "third_tag", + "comment", + "first", + "second", + "third" + ] + cli.write_points(dataframe.loc[:, colnames], 'foo', + tag_columns=[ + "first_tag", + "second_tag", + "third_tag"]) + + self.assertEqual(m.last_request.body, expected) + def test_write_points_from_dataframe_with_numeric_precision(self): """Test write points from df with numeric precision.""" now = pd.Timestamp('1970-01-01 00:00+00:00') From 47d24c7489ecef56b742b57ef8616f5eae0800af Mon Sep 17 00:00:00 2001 From: Lloyd Wallis Date: Thu, 11 Jul 2019 23:34:52 +0100 Subject: [PATCH 056/133] Mutual TLS authentication (#702) * Add support for providing a client certificate for mutual TLS authentication. * Be more explicit in documentation on valid values for the parameter --- influxdb/client.py | 16 ++++++++++++++++ influxdb/tests/client_test.py | 8 ++++++++ 2 files changed, 24 insertions(+) diff --git a/influxdb/client.py b/influxdb/client.py index 8ac557d3..ad4c6b66 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -61,6 +61,13 @@ class InfluxDBClient(object): :type proxies: dict :param path: path of InfluxDB on the server to connect, defaults to '' :type path: str + :param cert: Path to client certificate information to use for mutual TLS + authentication. You can specify a local cert to use + as a single file containing the private key and the certificate, or as + a tuple of both files’ paths, defaults to None + :type cert: str + + :raises ValueError: if cert is provided but ssl is disabled (set to False) """ def __init__(self, @@ -78,6 +85,7 @@ def __init__(self, proxies=None, pool_size=10, path='', + cert=None, ): """Construct a new InfluxDBClient object.""" self.__host = host @@ -120,6 +128,14 @@ def __init__(self, else: self._proxies = proxies + if cert: + if not ssl: + raise ValueError( + "Client certificate provided but ssl is disabled." + ) + else: + self._session.cert = cert + self.__baseurl = "{0}://{1}:{2}{3}".format( self._scheme, self._host, diff --git a/influxdb/tests/client_test.py b/influxdb/tests/client_test.py index e4cc7e11..b741cf7a 100644 --- a/influxdb/tests/client_test.py +++ b/influxdb/tests/client_test.py @@ -149,6 +149,14 @@ def test_dsn(self): **{'ssl': False}) self.assertEqual('http://my.host.fr:1886', cli._baseurl) + def test_cert(self): + """Test mutual TLS authentication for TestInfluxDBClient object.""" + cli = InfluxDBClient(ssl=True, cert='/etc/pki/tls/private/dummy.crt') + self.assertEqual(cli._session.cert, '/etc/pki/tls/private/dummy.crt') + + with self.assertRaises(ValueError): + cli = InfluxDBClient(cert='/etc/pki/tls/private/dummy.crt') + def test_switch_database(self): """Test switch database in TestInfluxDBClient object.""" cli = InfluxDBClient('host', 8086, 'username', 'password', 'database') From dc83fc6576b6463dcc77a0c101475a2a71ed655a Mon Sep 17 00:00:00 2001 From: Song Yihan Date: Fri, 12 Jul 2019 06:36:05 +0800 Subject: [PATCH 057/133] Fix newline in tag value cause partial commit (#716) When a tag value contains newline(\n), the request sent to db would be splitted into two parts and the first part would fail to write to db but the second woudl be succeed. The reason is that before sending we do serialization (make_lines) the _escape_tag method in line_protocol.py won't handle it well, we need somehow more specific on newline instead of only handling escape character (\) --- influxdb/line_protocol.py | 2 ++ influxdb/tests/test_line_protocol.py | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/influxdb/line_protocol.py b/influxdb/line_protocol.py index e8816fc0..249511d3 100644 --- a/influxdb/line_protocol.py +++ b/influxdb/line_protocol.py @@ -54,6 +54,8 @@ def _escape_tag(tag): ",", "\\," ).replace( "=", "\\=" + ).replace( + "\n", "\\n" ) diff --git a/influxdb/tests/test_line_protocol.py b/influxdb/tests/test_line_protocol.py index a3d84793..bccd7727 100644 --- a/influxdb/tests/test_line_protocol.py +++ b/influxdb/tests/test_line_protocol.py @@ -115,6 +115,27 @@ def test_make_lines_unicode(self): 'test,unicode_tag=\'Привет!\' unicode_val="Привет!"\n' ) + def test_tag_value_newline(self): + """Test make lines with tag value contains newline.""" + data = { + "tags": { + "t1": "line1\nline2" + }, + "points": [ + { + "measurement": "test", + "fields": { + "val": "hello" + } + } + ] + } + + self.assertEqual( + line_protocol.make_lines(data), + 'test,t1=line1\\nline2 val="hello"\n' + ) + def test_quote_ident(self): """Test quote indentation in TestLineProtocol object.""" self.assertEqual( From 9903974e936766edb8a4bf5434aa174f790ef0ec Mon Sep 17 00:00:00 2001 From: Nathan Epstein Date: Thu, 8 Aug 2019 00:08:59 -0400 Subject: [PATCH 058/133] this tutorial did not write to the database with the protocol set to json. It worked when changed to line (#737) --- examples/tutorial_pandas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/tutorial_pandas.py b/examples/tutorial_pandas.py index 67a5457d..13e72f8c 100644 --- a/examples/tutorial_pandas.py +++ b/examples/tutorial_pandas.py @@ -12,7 +12,7 @@ def main(host='localhost', port=8086): user = 'root' password = 'root' dbname = 'demo' - protocol = 'json' + protocol = 'line' client = DataFrameClient(host, port, user, password, dbname) From b9504e05db83b8cd159da6e02deab2faac8fbc59 Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Wed, 7 Aug 2019 23:15:47 -0500 Subject: [PATCH 059/133] chore(CHANGELOG): update with latest merged PR #797 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f1503b5..59eb55c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Update test suite to add support for Python 3.7 and InfluxDB v1.6.4 and 1.7.4 (#692 thx @clslgrnc) - Update classifiers tuple to list in setup.py (#697 thx @Hanaasagi) - Update documentation for empty `delete_series` confusion +- Update tests/tutorials_pandas.py to reference `line` protocol, bug in `json` (#797 thx @Aeium) ### Removed From 5de0162c17d3a3137ec97e277420485b392f0634 Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Mon, 19 Aug 2019 08:16:12 -0500 Subject: [PATCH 060/133] chore(CHANGELOG): update for v5.2.3 release --- CHANGELOG.md | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 59eb55c7..025a1d71 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,16 +7,30 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] ### Added + +### Changed + +### Removed + +## [v5.2.3] - 2019-08-19 + +### Added +- Add consistency param to InfluxDBClient.write_points (#643 thx @RonRothman) +- Add UDP example (#648 thx @shantanoo-desai) +- Add consistency paramter to `write_points` (#664 tx @RonRothman) +- The query() function now accepts a bind_params argument for parameter binding (#678 thx @clslgrnc) - Add `get_list_continuous_queries`, `drop_continuous_query`, and `create_continuous_query` management methods for continuous queries (#681 thx @lukaszdudek-silvair) -- query() now accepts a bind_params argument for parameter binding (#678 thx @clslgrnc) +- Mutual TLS authentication (#702 thx @LloydW93) ### Changed -- Add consistency param to InfluxDBClient.write_points (#643 thx @RonRothman) - Update test suite to add support for Python 3.7 and InfluxDB v1.6.4 and 1.7.4 (#692 thx @clslgrnc) +- Update supported versions of influxdb + python (#693 thx @clslgrnc) +- Fix for the line protocol issue with leading comma (#694 thx @d3banjan) - Update classifiers tuple to list in setup.py (#697 thx @Hanaasagi) -- Update documentation for empty `delete_series` confusion -- Update tests/tutorials_pandas.py to reference `line` protocol, bug in `json` (#797 thx @Aeium) +- Update documentation for empty `delete_series` confusion (#699 thx @xginn8) +- Fix newline character issue in tag value (#716 thx @syhan) +- Update tests/tutorials_pandas.py to reference `line` protocol, bug in `json` (#737 thx @Aeium) ### Removed From f3b3df50423bfba5cf11652cec049ec285ed7227 Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Mon, 26 Aug 2019 12:25:51 -0500 Subject: [PATCH 061/133] chore(influxdb): update release version number --- influxdb/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/influxdb/__init__.py b/influxdb/__init__.py index 288880b1..b31170bb 100644 --- a/influxdb/__init__.py +++ b/influxdb/__init__.py @@ -18,4 +18,4 @@ ] -__version__ = '5.2.2' +__version__ = '5.2.3' From 6baf7eee61e968c70d4c7f111e7179bc92a80e8d Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Wed, 4 Sep 2019 08:24:19 -0500 Subject: [PATCH 062/133] chore(client_test): adding in an old test from legacy PR#315 (#752) * chore(client_test): adding in an old test from legacy PR#315 * chore(client_test): update from GET to POST * chore(client_test): update to clear out flake8 issues --- influxdb/tests/client_test.py | 43 +++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/influxdb/tests/client_test.py b/influxdb/tests/client_test.py index b741cf7a..571b7ebc 100644 --- a/influxdb/tests/client_test.py +++ b/influxdb/tests/client_test.py @@ -706,6 +706,49 @@ def test_create_retention_policy(self): '"db" duration 1d replication 4 shard duration 0s' ) + def test_create_retention_policy_shard_duration(self): + """Test create retention policy with a custom shard duration.""" + example_response = '{"results":[{}]}' + + with requests_mock.Mocker() as m: + m.register_uri( + requests_mock.POST, + "http://localhost:8086/query", + text=example_response + ) + self.cli.create_retention_policy( + 'somename2', '1d', 4, database='db', + shard_duration='1h' + ) + + self.assertEqual( + m.last_request.qs['q'][0], + 'create retention policy "somename2" on ' + '"db" duration 1d replication 4 shard duration 1h' + ) + + def test_create_retention_policy_shard_duration_default(self): + """Test create retention policy with a default shard duration.""" + example_response = '{"results":[{}]}' + + with requests_mock.Mocker() as m: + m.register_uri( + requests_mock.POST, + "http://localhost:8086/query", + text=example_response + ) + self.cli.create_retention_policy( + 'somename3', '1d', 4, database='db', + shard_duration='1h', default=True + ) + + self.assertEqual( + m.last_request.qs['q'][0], + 'create retention policy "somename3" on ' + '"db" duration 1d replication 4 shard duration 1h ' + 'default' + ) + def test_alter_retention_policy(self): """Test alter retention policy for TestInfluxDBClient object.""" example_response = '{"results":[{}]}' From bfc582bdd7e3d1009c28fb874c13778ed1522e35 Mon Sep 17 00:00:00 2001 From: sblondon Date: Mon, 9 Sep 2019 21:59:50 +0200 Subject: [PATCH 063/133] Fix minor spelling error (#744) `the connections is` -> `the connection is` Or should it be `the connections are` ? --- docs/source/api-documentation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/api-documentation.rst b/docs/source/api-documentation.rst index d00600e6..35fdb291 100644 --- a/docs/source/api-documentation.rst +++ b/docs/source/api-documentation.rst @@ -30,7 +30,7 @@ These clients are initiated in the same way as the client = DataFrameClient(host='127.0.0.1', port=8086, username='root', password='root', database='dbname') -.. note:: Only when using UDP (use_udp=True) the connections is established. +.. note:: Only when using UDP (use_udp=True) the connection is established. .. _InfluxDBClient-api: From 7b5c3d54985cc3503457c1a731a8f9f605ac6190 Mon Sep 17 00:00:00 2001 From: xginn8 Date: Mon, 9 Sep 2019 23:27:53 +0200 Subject: [PATCH 064/133] meta: add .github config for ISSUE_TEMPLATE/CODEOWNERS/PR_TEMPLATE (#754) Signed-off-by: Matthew McGinn --- .github/CODEOWNERS | 1 + .github/ISSUE_TEMPLATE.md | 4 ++++ .github/PULL_REQUEST_TEMPLATE.md | 5 +++++ 3 files changed, 10 insertions(+) create mode 100644 .github/CODEOWNERS create mode 100644 .github/ISSUE_TEMPLATE.md create mode 100644 .github/PULL_REQUEST_TEMPLATE.md diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..0248ade1 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @aviau @sebito91 @xginn8 diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 00000000..7a7927c1 --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,4 @@ +- **InfluxDB version:** e.g. 1.7.7 (output of the `influx version` command) +- **InfluxDB-python version:** e.g. 5.2.2 (output of the `python -c "from __future__ import print_function; import influxdb; print(influxdb.__version__)"` command) +- **Python version:** e.g. 3.7.4 (output of the `python --version` command) +- **Operating system version:** e.g. Windows 10, Ubuntu 18.04, macOS 10.14.5 diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 00000000..84729d17 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,5 @@ +--- +##### Contributor checklist + +- [ ] Builds are passing +- [ ] New tests have been added (for feature additions) From beb236038200fb99c119260b476e92c33487dab9 Mon Sep 17 00:00:00 2001 From: Matthew McGinn Date: Tue, 10 Sep 2019 00:54:47 +0200 Subject: [PATCH 065/133] meta: remove (old) CODEOWNERS (replaced in 7b5c3d5) Signed-off-by: Matthew McGinn --- CODEOWNERS | 1 - 1 file changed, 1 deletion(-) delete mode 100644 CODEOWNERS diff --git a/CODEOWNERS b/CODEOWNERS deleted file mode 100644 index 0acbd7c8..00000000 --- a/CODEOWNERS +++ /dev/null @@ -1 +0,0 @@ -* @aviau @xginn8 @sebito91 From 71426a68aec6661ec975db9e7fd0c5971e2b6817 Mon Sep 17 00:00:00 2001 From: xginn8 Date: Tue, 10 Sep 2019 01:41:08 +0200 Subject: [PATCH 066/133] meta: clean up travis CI config (#755) bump non-code tests to Python3.7 Signed-off-by: Matthew McGinn --- .travis.yml | 31 ++++--------------------------- README.rst | 8 ++++---- 2 files changed, 8 insertions(+), 31 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8c660b67..580e0c8f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,7 @@ python: - "2.7" - "3.5" - "3.6" + - "3.7" - "pypy" - "pypy3" @@ -23,36 +24,12 @@ addons: matrix: include: - python: 3.7 - dist: xenial - sudo: true - env: INFLUXDB_VER=1.2.4 - - python: 3.7 - dist: xenial - sudo: true - env: INFLUXDB_VER=1.3.9 - - python: 3.7 - dist: xenial - sudo: true - env: INFLUXDB_VER=1.4.3 - - python: 3.7 - dist: xenial - sudo: true - env: INFLUXDB_VER=1.5.4 - - python: 3.7 - dist: xenial - sudo: true - env: INFLUXDB_VER=1.6.4 - - python: 3.7 - dist: xenial - sudo: true - env: INFLUXDB_VER=1.7.4 - - python: 3.6 env: TOX_ENV=pep257 - - python: 3.6 + - python: 3.7 env: TOX_ENV=docs - - python: 3.6 + - python: 3.7 env: TOX_ENV=flake8 - - python: 3.6 + - python: 3.7 env: TOX_ENV=coverage install: diff --git a/README.rst b/README.rst index 026171b2..a40ed148 100644 --- a/README.rst +++ b/README.rst @@ -19,13 +19,13 @@ InfluxDB-Python is a client for interacting with InfluxDB_. Development of this library is maintained by: -+-----------+-------------------------------+ ++-----------+-------------------------------+ | Github ID | URL | -+===========+===============================+ ++===========+===============================+ | @aviau | (https://github.com/aviau) | -+-----------+-------------------------------+ ++-----------+-------------------------------+ | @xginn8 | (https://github.com/xginn8) | -+-----------+-------------------------------+ ++-----------+-------------------------------+ | @sebito91 | (https://github.com/sebito91) | +-----------+-------------------------------+ From f749764d066b6a298981ad78b754b168a6612858 Mon Sep 17 00:00:00 2001 From: xginn8 Date: Mon, 16 Sep 2019 20:20:11 +0200 Subject: [PATCH 067/133] tests: add basic type checking with mypy (#756) Closes #736 --- .gitignore | 1 + .travis.yml | 2 ++ influxdb/dataframe_client.py | 2 +- influxdb/tests/server_tests/base.py | 11 +++++++++-- mypy.ini | 8 ++++++++ tox.ini | 7 ++++++- 6 files changed, 27 insertions(+), 4 deletions(-) create mode 100644 mypy.ini diff --git a/.gitignore b/.gitignore index 7720b658..d970c44c 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ build/ mock*/ nose*/ .pybuild/ +.mypy_cache/ debian/files debian/python-influxdb.debhelper.log debian/python-influxdb.postinst.debhelper diff --git a/.travis.yml b/.travis.yml index 580e0c8f..9d45f19b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,6 +31,8 @@ matrix: env: TOX_ENV=flake8 - python: 3.7 env: TOX_ENV=coverage + - python: 3.7 + env: TOX_ENV=mypy install: - pip install tox-travis diff --git a/influxdb/dataframe_client.py b/influxdb/dataframe_client.py index 97258644..babfe0dd 100644 --- a/influxdb/dataframe_client.py +++ b/influxdb/dataframe_client.py @@ -25,4 +25,4 @@ def __init__(self, *a, **kw): raise ImportError("DataFrameClient requires Pandas " "which couldn't be imported: %s" % self.err) else: - from ._dataframe_client import DataFrameClient + from ._dataframe_client import DataFrameClient # type: ignore diff --git a/influxdb/tests/server_tests/base.py b/influxdb/tests/server_tests/base.py index f4bd3ff9..fe722870 100644 --- a/influxdb/tests/server_tests/base.py +++ b/influxdb/tests/server_tests/base.py @@ -51,8 +51,15 @@ class SingleTestCaseWithServerMixin(object): # 'influxdb_template_conf' attribute must be set # on the TestCase class or instance. - setUp = _setup_influxdb_server - tearDown = _teardown_influxdb_server + @classmethod + def setUp(cls): + """Set up an instance of the SingleTestCaseWithServerMixin.""" + _setup_influxdb_server(cls) + + @classmethod + def tearDown(cls): + """Tear down an instance of the SingleTestCaseWithServerMixin.""" + _teardown_influxdb_server(cls) class ManyTestCasesWithServerMixin(object): diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 00000000..308aa62d --- /dev/null +++ b/mypy.ini @@ -0,0 +1,8 @@ +[mypy] +ignore_missing_imports = True +warn_unused_ignores = True +warn_unused_configs = True +warn_redundant_casts = True +warn_no_return = True +no_implicit_optional = True +strict_equality = True diff --git a/tox.ini b/tox.ini index 4a1921e2..ff30ebac 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27, py35, py36, py37, pypy, pypy3, flake8, pep257, coverage, docs +envlist = py27, py35, py36, py37, pypy, pypy3, flake8, pep257, coverage, docs, mypy [testenv] passenv = INFLUXDB_PYTHON_INFLUXD_PATH @@ -44,6 +44,11 @@ deps = -r{toxinidir}/requirements.txt sphinx_rtd_theme commands = sphinx-build -b html docs/source docs/build +[testenv:mypy] +deps = -r{toxinidir}/test-requirements.txt + mypy==0.720 +commands = mypy --config-file mypy.ini -p influxdb + [flake8] ignore = W503,W504,W605,N802,F821,E402 # W503: Line break occurred before a binary operator From 3d61f1f465c0fbfbe404c77a4728c5d7268aaf1e Mon Sep 17 00:00:00 2001 From: Matthew McGinn Date: Mon, 16 Sep 2019 22:20:50 +0200 Subject: [PATCH 068/133] meta: add missing changelog entries for last few commits Signed-off-by: Matthew McGinn --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 025a1d71..8aa8de84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,8 +7,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] ### Added +- Add mypy testing framework (#756) ### Changed +- Clean up stale CI config (#755) +- Add legacy client test (#752 & #318 thx @oldmantaiter & @sebito91) ### Removed From 3a5e6219a2b62af076caa4f37fab9258165ec266 Mon Sep 17 00:00:00 2001 From: Ophir LOJKINE Date: Thu, 5 Dec 2019 17:42:49 +0100 Subject: [PATCH 069/133] Add support for messagepack (#734) * Add support for messagepack * Remove unnecessary blank line Fixes https://github.com/influxdata/influxdb-python/pull/734/files/57daf8ccd5027c796a2fd3934b8e88d3982d300e#r302769403 * Small code reorganization * Small code reorganization Fixes https://github.com/influxdata/influxdb-python/pull/734#discussion_r302770011 --- influxdb/client.py | 50 ++++++++++++++++++++++++++++------- influxdb/tests/client_test.py | 23 ++++++++++++++++ requirements.txt | 1 + 3 files changed, 64 insertions(+), 10 deletions(-) diff --git a/influxdb/client.py b/influxdb/client.py index ad4c6b66..5e39f490 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -6,11 +6,14 @@ from __future__ import print_function from __future__ import unicode_literals -import time -import random - +import datetime import json +import random import socket +import struct +import time + +import msgpack import requests import requests.exceptions from six.moves import xrange @@ -144,7 +147,7 @@ def __init__(self, self._headers = { 'Content-Type': 'application/json', - 'Accept': 'text/plain' + 'Accept': 'application/x-msgpack' } @property @@ -293,13 +296,30 @@ def request(self, url, method='GET', params=None, data=None, time.sleep((2 ** _try) * random.random() / 100.0) if not retry: raise + + type_header = response.headers and response.headers.get("Content-Type") + if type_header == "application/x-msgpack" and response.content: + response._msgpack = msgpack.unpackb( + packed=response.content, + ext_hook=_msgpack_parse_hook, + raw=False) + else: + response._msgpack = None + + def reformat_error(response): + if response._msgpack: + return json.dumps(response._msgpack, separators=(',', ':')) + else: + return response.content + # if there's not an error, there must have been a successful response if 500 <= response.status_code < 600: - raise InfluxDBServerError(response.content) + raise InfluxDBServerError(reformat_error(response)) elif response.status_code == expected_response_code: return response else: - raise InfluxDBClientError(response.content, response.status_code) + err_msg = reformat_error(response) + raise InfluxDBClientError(err_msg, response.status_code) def write(self, data, params=None, expected_response_code=204, protocol='json'): @@ -450,10 +470,11 @@ def query(self, expected_response_code=expected_response_code ) - if chunked: - return self._read_chunked_response(response) - - data = response.json() + data = response._msgpack + if not data: + if chunked: + return self._read_chunked_response(response) + data = response.json() results = [ ResultSet(result, raise_errors=raise_errors) @@ -1119,3 +1140,12 @@ def _parse_netloc(netloc): 'password': info.password or None, 'host': info.hostname or 'localhost', 'port': info.port or 8086} + + +def _msgpack_parse_hook(code, data): + if code == 5: + (epoch_s, epoch_ns) = struct.unpack(">QI", data) + timestamp = datetime.datetime.utcfromtimestamp(epoch_s) + timestamp += datetime.timedelta(microseconds=(epoch_ns / 1000)) + return timestamp.isoformat() + 'Z' + return msgpack.ExtType(code, data) diff --git a/influxdb/tests/client_test.py b/influxdb/tests/client_test.py index 571b7ebc..54116f7e 100644 --- a/influxdb/tests/client_test.py +++ b/influxdb/tests/client_test.py @@ -473,6 +473,29 @@ def test_query(self): [{'value': 0.64, 'time': '2009-11-10T23:00:00Z'}] ) + def test_query_msgpack(self): + """Test query method with a messagepack response.""" + example_response = bytes(bytearray.fromhex( + "81a7726573756c74739182ac73746174656d656e745f696400a673657269" + "65739183a46e616d65a161a7636f6c756d6e7392a474696d65a176a67661" + "6c7565739192c70c05000000005d26178a019096c8cb3ff0000000000000" + )) + + with requests_mock.Mocker() as m: + m.register_uri( + requests_mock.GET, + "http://localhost:8086/query", + request_headers={"Accept": "application/x-msgpack"}, + headers={"Content-Type": "application/x-msgpack"}, + content=example_response + ) + rs = self.cli.query('select * from a') + + self.assertListEqual( + list(rs.get_points()), + [{'v': 1.0, 'time': '2019-07-10T16:51:22.026253Z'}] + ) + def test_select_into_post(self): """Test SELECT.*INTO is POSTed.""" example_response = ( diff --git a/requirements.txt b/requirements.txt index db5f6f85..77d7306f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ python-dateutil>=2.6.0 pytz requests>=2.17.0 six>=1.10.0 +msgpack==0.6.1 From cb15c2ebc4d337cac606638004cb794870b52c2e Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Thu, 5 Dec 2019 10:44:25 -0600 Subject: [PATCH 070/133] chore(CHANGELOG): adding in #754 from lovasoa, thanks! --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8aa8de84..13445e97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added - Add mypy testing framework (#756) +- Add support for messagepack (#734 thx @lovasoa) ### Changed - Clean up stale CI config (#755) From ea1b995ac3b12b7693254167ea8443f4a5263386 Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Tue, 7 Apr 2020 17:30:49 -0500 Subject: [PATCH 071/133] feat(client): re-add support for 'show series' from legacy PR #357. Closes #353 (#806) * feat(client): re-add support for 'show series' from legacy PR #357. Closes #353 * chore(client): fix failing tests * chore(client): update linters to pass --- CHANGELOG.md | 7 +++ influxdb/client.py | 35 +++++++++++ influxdb/tests/client_test.py | 60 +++++++++++++++++++ .../server_tests/client_test_with_server.py | 58 ++++++++++++++++++ 4 files changed, 160 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 13445e97..bdc08569 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,9 +6,16 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] +### Added + +### Changed + +## [v5.2.4] - 2020-04-10 + ### Added - Add mypy testing framework (#756) - Add support for messagepack (#734 thx @lovasoa) +- Add support for 'show series' (#357 thx @gaker) ### Changed - Clean up stale CI config (#755) diff --git a/influxdb/client.py b/influxdb/client.py index 5e39f490..43427a11 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -7,6 +7,7 @@ from __future__ import unicode_literals import datetime +import itertools import json import random import socket @@ -637,6 +638,40 @@ def get_list_database(self): """ return list(self.query("SHOW DATABASES").get_points()) + def get_list_series(self, database=None, measurement=None, tags=None): + """ + Query SHOW SERIES returns the distinct series in your database. + + FROM and WHERE clauses are optional. + + :param measurement: Show all series from a measurement + :type id: string + :param tags: Show all series that match given tags + :type id: dict + :param database: the database from which the series should be + shows, defaults to client's current database + :type database: str + """ + database = database or self._database + query_str = 'SHOW SERIES' + + if measurement: + query_str += ' FROM "{0}"'.format(measurement) + + if tags: + query_str += ' WHERE ' + ' and '.join(["{0}='{1}'".format(k, v) + for k, v in tags.items()]) + + return list( + itertools.chain.from_iterable( + [ + x.values() + for x in (self.query(query_str, database=database) + .get_points()) + ] + ) + ) + def create_database(self, dbname): """Create a new database in InfluxDB. diff --git a/influxdb/tests/client_test.py b/influxdb/tests/client_test.py index 54116f7e..99a7f42b 100644 --- a/influxdb/tests/client_test.py +++ b/influxdb/tests/client_test.py @@ -689,6 +689,66 @@ def test_get_list_measurements(self): [{'name': 'cpu'}, {'name': 'disk'}] ) + def test_get_list_series(self): + """Test get a list of series from the database.""" + data = {'results': [ + {'series': [ + { + 'values': [ + ['cpu_load_short,host=server01,region=us-west'], + ['memory_usage,host=server02,region=us-east']], + 'columns': ['key'] + } + ]} + ]} + + with _mocked_session(self.cli, 'get', 200, json.dumps(data)): + self.assertListEqual( + self.cli.get_list_series(), + ['cpu_load_short,host=server01,region=us-west', + 'memory_usage,host=server02,region=us-east']) + + def test_get_list_series_with_measurement(self): + """Test get a list of series from the database by filter.""" + data = {'results': [ + {'series': [ + { + 'values': [ + ['cpu_load_short,host=server01,region=us-west']], + 'columns': ['key'] + } + ]} + ]} + + with _mocked_session(self.cli, 'get', 200, json.dumps(data)): + self.assertListEqual( + self.cli.get_list_series(measurement='cpu_load_short'), + ['cpu_load_short,host=server01,region=us-west']) + + def test_get_list_series_with_tags(self): + """Test get a list of series from the database by tags.""" + data = {'results': [ + {'series': [ + { + 'values': [ + ['cpu_load_short,host=server01,region=us-west']], + 'columns': ['key'] + } + ]} + ]} + + with _mocked_session(self.cli, 'get', 200, json.dumps(data)): + self.assertListEqual( + self.cli.get_list_series(tags={'region': 'us-west'}), + ['cpu_load_short,host=server01,region=us-west']) + + @raises(Exception) + def test_get_list_series_fails(self): + """Test get a list of series from the database but fail.""" + cli = InfluxDBClient('host', 8086, 'username', 'password') + with _mocked_session(cli, 'get', 401): + cli.get_list_series() + def test_create_retention_policy_default(self): """Test create default ret policy for TestInfluxDBClient object.""" example_response = '{"results":[{}]}' diff --git a/influxdb/tests/server_tests/client_test_with_server.py b/influxdb/tests/server_tests/client_test_with_server.py index fda3f720..94f28b66 100644 --- a/influxdb/tests/server_tests/client_test_with_server.py +++ b/influxdb/tests/server_tests/client_test_with_server.py @@ -817,6 +817,64 @@ def test_query_multiple_series(self): ] self.cli.write_points(pts) + def test_get_list_series(self): + """Test get a list of series from the database.""" + dummy_points = [ + { + "measurement": "cpu_load_short", + "tags": { + "host": "server01", + "region": "us-west" + }, + "time": "2009-11-10T23:00:00.123456Z", + "fields": { + "value": 0.64 + } + } + ] + + dummy_points_2 = [ + { + "measurement": "memory_usage", + "tags": { + "host": "server02", + "region": "us-east" + }, + "time": "2009-11-10T23:00:00.123456Z", + "fields": { + "value": 80 + } + } + ] + + self.cli.write_points(dummy_points) + self.cli.write_points(dummy_points_2) + + self.assertEquals( + self.cli.get_list_series(), + ['cpu_load_short,host=server01,region=us-west', + 'memory_usage,host=server02,region=us-east'] + ) + + self.assertEquals( + self.cli.get_list_series(measurement='memory_usage'), + ['memory_usage,host=server02,region=us-east'] + ) + + self.assertEquals( + self.cli.get_list_series(measurement='memory_usage'), + ['memory_usage,host=server02,region=us-east'] + ) + + self.assertEquals( + self.cli.get_list_series(tags={'host': 'server02'}), + ['memory_usage,host=server02,region=us-east']) + + self.assertEquals( + self.cli.get_list_series( + measurement='cpu_load_short', tags={'host': 'server02'}), + []) + @skip_server_tests class UdpTests(ManyTestCasesWithServerMixin, unittest.TestCase): From ad5e5b616130f83f6f63532d3e7ae8b005687ba7 Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Tue, 7 Apr 2020 18:36:21 -0500 Subject: [PATCH 072/133] feat(client): allow custom requests session in InfluxDBClient (#807) * feat(client): enable client request to provide custom requests session * feat(client): allow custom requests session in InfluxDBClient --- CHANGELOG.md | 1 + influxdb/client.py | 10 +++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bdc08569..59777b5f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Add mypy testing framework (#756) - Add support for messagepack (#734 thx @lovasoa) - Add support for 'show series' (#357 thx @gaker) +- Add support for custom request session in InfluxDBClient (#360 thx @dschien) ### Changed - Clean up stale CI config (#755) diff --git a/influxdb/client.py b/influxdb/client.py index 43427a11..390d8e16 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -70,6 +70,9 @@ class InfluxDBClient(object): as a single file containing the private key and the certificate, or as a tuple of both files’ paths, defaults to None :type cert: str + :param session: allow for the new client request to use an existing + requests Session, defaults to None + :type session: requests.Session :raises ValueError: if cert is provided but ssl is disabled (set to False) """ @@ -90,6 +93,7 @@ def __init__(self, pool_size=10, path='', cert=None, + session=None, ): """Construct a new InfluxDBClient object.""" self.__host = host @@ -104,7 +108,11 @@ def __init__(self, self.__use_udp = use_udp self.__udp_port = udp_port - self._session = requests.Session() + + if not session: + session = requests.Session() + + self._session = session adapter = requests.adapters.HTTPAdapter( pool_connections=int(pool_size), pool_maxsize=int(pool_size) From 4799c588789c2425693c419da308c62158063174 Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Wed, 8 Apr 2020 12:39:13 -0500 Subject: [PATCH 073/133] feat(line_protocol): split out make_line function from core make_lines (#810) * feat(line_protocol): split out make_line function from core make_lines * chore(line_protocol): fix malformed testcase --- CHANGELOG.md | 1 + influxdb/line_protocol.py | 123 ++++++++++++++++----------- influxdb/tests/test_line_protocol.py | 2 +- 3 files changed, 75 insertions(+), 51 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 59777b5f..81809bcd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Changed - Clean up stale CI config (#755) - Add legacy client test (#752 & #318 thx @oldmantaiter & @sebito91) +- Update make_lines section in line_protocol.py to split out core function (#375 thx @aisbaa) ### Removed diff --git a/influxdb/line_protocol.py b/influxdb/line_protocol.py index 249511d3..ec59ef47 100644 --- a/influxdb/line_protocol.py +++ b/influxdb/line_protocol.py @@ -11,7 +11,7 @@ from pytz import UTC from dateutil.parser import parse -from six import iteritems, binary_type, text_type, integer_types, PY2 +from six import binary_type, text_type, integer_types, PY2 EPOCH = UTC.localize(datetime.utcfromtimestamp(0)) @@ -30,15 +30,20 @@ def _convert_timestamp(timestamp, precision=None): ns = (timestamp - EPOCH).total_seconds() * 1e9 if precision is None or precision == 'n': return ns - elif precision == 'u': + + if precision == 'u': return ns / 1e3 - elif precision == 'ms': + + if precision == 'ms': return ns / 1e6 - elif precision == 's': + + if precision == 's': return ns / 1e9 - elif precision == 'm': + + if precision == 'm': return ns / 1e9 / 60 - elif precision == 'h': + + if precision == 'h': return ns / 1e9 / 3600 raise ValueError(timestamp) @@ -95,9 +100,11 @@ def _escape_value(value): if isinstance(value, text_type) and value != '': return quote_ident(value) - elif isinstance(value, integer_types) and not isinstance(value, bool): + + if isinstance(value, integer_types) and not isinstance(value, bool): return str(value) + 'i' - elif _is_float(value): + + if _is_float(value): return repr(value) return str(value) @@ -107,15 +114,60 @@ def _get_unicode(data, force=False): """Try to return a text aka unicode object from the given data.""" if isinstance(data, binary_type): return data.decode('utf-8') - elif data is None: + + if data is None: return '' - elif force: + + if force: if PY2: return unicode(data) - else: - return str(data) - else: - return data + return str(data) + + return data + + +def make_line(measurement, tags=None, fields=None, time=None, precision=None): + """Extract the actual point from a given measurement line.""" + tags = tags or {} + fields = fields or {} + + line = _escape_tag(_get_unicode(measurement)) + + # tags should be sorted client-side to take load off server + tag_list = [] + for tag_key in sorted(tags.keys()): + key = _escape_tag(tag_key) + value = _escape_tag(tags[tag_key]) + + if key != '' and value != '': + tag_list.append( + "{key}={value}".format(key=key, value=value) + ) + + if tag_list: + line += ',' + ','.join(tag_list) + + field_list = [] + for field_key in sorted(fields.keys()): + key = _escape_tag(field_key) + value = _escape_value(fields[field_key]) + + if key != '' and value != '': + field_list.append("{key}={value}".format( + key=key, + value=value + )) + + if field_list: + line += ' ' + ','.join(field_list) + + if time is not None: + timestamp = _get_unicode(str(int( + _convert_timestamp(time, precision) + ))) + line += ' ' + timestamp + + return line def make_lines(data, precision=None): @@ -127,48 +179,19 @@ def make_lines(data, precision=None): lines = [] static_tags = data.get('tags') for point in data['points']: - elements = [] - - # add measurement name - measurement = _escape_tag(_get_unicode( - point.get('measurement', data.get('measurement')))) - key_values = [measurement] - - # add tags if static_tags: tags = dict(static_tags) # make a copy, since we'll modify tags.update(point.get('tags') or {}) else: tags = point.get('tags') or {} - # tags should be sorted client-side to take load off server - for tag_key, tag_value in sorted(iteritems(tags)): - key = _escape_tag(tag_key) - value = _escape_tag_value(tag_value) - - if key != '' and value != '': - key_values.append(key + "=" + value) - - elements.append(','.join(key_values)) - - # add fields - field_values = [] - for field_key, field_value in sorted(iteritems(point['fields'])): - key = _escape_tag(field_key) - value = _escape_value(field_value) - - if key != '' and value != '': - field_values.append(key + "=" + value) - - elements.append(','.join(field_values)) - - # add timestamp - if 'time' in point: - timestamp = _get_unicode(str(int( - _convert_timestamp(point['time'], precision)))) - elements.append(timestamp) - - line = ' '.join(elements) + line = make_line( + point.get('measurement', data.get('measurement')), + tags=tags, + fields=point.get('fields'), + precision=precision, + time=point.get('time') + ) lines.append(line) return '\n'.join(lines) + '\n' diff --git a/influxdb/tests/test_line_protocol.py b/influxdb/tests/test_line_protocol.py index bccd7727..71828f62 100644 --- a/influxdb/tests/test_line_protocol.py +++ b/influxdb/tests/test_line_protocol.py @@ -42,7 +42,7 @@ def test_make_lines(self): self.assertEqual( line_protocol.make_lines(data), - 'test,backslash_tag=C:\\\\ ,integer_tag=2,string_tag=hello ' + 'test,backslash_tag=C:\\\\,integer_tag=2,string_tag=hello ' 'bool_val=True,float_val=1.1,int_val=1i,string_val="hello!"\n' ) From 04205cf5858520fe2362432090232a871f1860ea Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Wed, 8 Apr 2020 16:09:04 -0500 Subject: [PATCH 074/133] chore(line_protocol): fix nanosecond timestamp resolution for points (#811) --- CHANGELOG.md | 1 + influxdb/line_protocol.py | 20 ++++++++++++++------ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 81809bcd..0556d942 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Clean up stale CI config (#755) - Add legacy client test (#752 & #318 thx @oldmantaiter & @sebito91) - Update make_lines section in line_protocol.py to split out core function (#375 thx @aisbaa) +- Fix nanosecond time resolution for points (#407 thx @AndreCAndersen && @clslgrnc) ### Removed diff --git a/influxdb/line_protocol.py b/influxdb/line_protocol.py index ec59ef47..3a5eb4e8 100644 --- a/influxdb/line_protocol.py +++ b/influxdb/line_protocol.py @@ -16,6 +16,14 @@ EPOCH = UTC.localize(datetime.utcfromtimestamp(0)) +def _to_nanos(timestamp): + delta = timestamp - EPOCH + nanos_in_days = delta.days * 86400 * 10 ** 9 + nanos_in_seconds = delta.seconds * 10 ** 9 + nanos_in_micros = delta.microseconds * 10 ** 3 + return nanos_in_days + nanos_in_seconds + nanos_in_micros + + def _convert_timestamp(timestamp, precision=None): if isinstance(timestamp, Integral): return timestamp # assume precision is correct if timestamp is int @@ -27,24 +35,24 @@ def _convert_timestamp(timestamp, precision=None): if not timestamp.tzinfo: timestamp = UTC.localize(timestamp) - ns = (timestamp - EPOCH).total_seconds() * 1e9 + ns = _to_nanos(timestamp) if precision is None or precision == 'n': return ns if precision == 'u': - return ns / 1e3 + return ns / 10**3 if precision == 'ms': - return ns / 1e6 + return ns / 10**6 if precision == 's': - return ns / 1e9 + return ns / 10**9 if precision == 'm': - return ns / 1e9 / 60 + return ns / 10**9 / 60 if precision == 'h': - return ns / 1e9 / 3600 + return ns / 10**9 / 3600 raise ValueError(timestamp) From a368951aca91c89d978fe29ee0ddc8a246711ea0 Mon Sep 17 00:00:00 2001 From: Sergei Smolianinov Date: Wed, 8 Apr 2020 21:21:12 +0000 Subject: [PATCH 075/133] Add CQs management methods to the client (#414) * Add CQs management methods to the client * chore(server_tests): update pep257 and flake8 commentary * chore(client_test): update comments based on pep257 and flake8 Co-authored-by: Sergei Smolianinov Co-authored-by: Sebastian Borza Co-authored-by: xginn8 --- influxdb/tests/client_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/influxdb/tests/client_test.py b/influxdb/tests/client_test.py index 99a7f42b..fe7381ea 100644 --- a/influxdb/tests/client_test.py +++ b/influxdb/tests/client_test.py @@ -1161,7 +1161,7 @@ def test_revoke_privilege_invalid(self): self.cli.revoke_privilege('', 'testdb', 'test') def test_get_list_privileges(self): - """Tst get list of privs for TestInfluxDBClient object.""" + """Test get list of privs for TestInfluxDBClient object.""" data = {'results': [ {'series': [ {'columns': ['database', 'privilege'], From 516274f9bae9edfc9c6f35c6e56281b29b23b12f Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Wed, 8 Apr 2020 16:22:27 -0500 Subject: [PATCH 076/133] chore(CHANGELOG): add smolse to PR merge for #681 and #414 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0556d942..2a887f90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,7 +34,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Add consistency paramter to `write_points` (#664 tx @RonRothman) - The query() function now accepts a bind_params argument for parameter binding (#678 thx @clslgrnc) - Add `get_list_continuous_queries`, `drop_continuous_query`, and `create_continuous_query` management methods for - continuous queries (#681 thx @lukaszdudek-silvair) + continuous queries (#681 thx @lukaszdudek-silvair && @smolse) - Mutual TLS authentication (#702 thx @LloydW93) ### Changed From 7c858bf2b069061005a4525c5a5252fd5f0b27ad Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Wed, 8 Apr 2020 19:22:31 -0500 Subject: [PATCH 077/133] feat(dataframe_client): handle np.nan, np.inf values in DataFrameClient (#812) * feat(dataframe_client): handle np.nan, np.inf values in DataFrameClient * chore(dataframe): handle cases where tagset is empty * chore(dataframe): add tests for Nan lines but with tag values --- CHANGELOG.md | 1 + influxdb/_dataframe_client.py | 40 +++++--- influxdb/tests/dataframe_client_test.py | 122 +++++++++++++++++++++++- 3 files changed, 147 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a887f90..be15d52b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Add support for messagepack (#734 thx @lovasoa) - Add support for 'show series' (#357 thx @gaker) - Add support for custom request session in InfluxDBClient (#360 thx @dschien) +- Add support for handling np.nan and np.inf values in DataFrameClient (#436 thx @nmerket) ### Changed - Clean up stale CI config (#755) diff --git a/influxdb/_dataframe_client.py b/influxdb/_dataframe_client.py index d16e29ca..f411bb37 100644 --- a/influxdb/_dataframe_client.py +++ b/influxdb/_dataframe_client.py @@ -270,14 +270,31 @@ def _convert_dataframe_to_json(dataframe, "h": 1e9 * 3600, }.get(time_precision, 1) + if not tag_columns: + points = [ + {'measurement': measurement, + 'fields': + rec.replace([np.inf, -np.inf], np.nan).dropna().to_dict(), + 'time': np.int64(ts.value / precision_factor)} + for ts, (_, rec) in zip( + dataframe.index, + dataframe[field_columns].iterrows() + ) + ] + + return points + points = [ {'measurement': measurement, 'tags': dict(list(tag.items()) + list(tags.items())), - 'fields': rec, + 'fields': + rec.replace([np.inf, -np.inf], np.nan).dropna().to_dict(), 'time': np.int64(ts.value / precision_factor)} - for ts, tag, rec in zip(dataframe.index, - dataframe[tag_columns].to_dict('record'), - dataframe[field_columns].to_dict('record')) + for ts, tag, (_, rec) in zip( + dataframe.index, + dataframe[tag_columns].to_dict('record'), + dataframe[field_columns].iterrows() + ) ] return points @@ -379,21 +396,18 @@ def _convert_dataframe_to_lines(self, tags = '' # Make an array of formatted field keys and values - field_df = dataframe[field_columns] - # Keep the positions where Null values are found - mask_null = field_df.isnull().values + field_df = dataframe[field_columns].replace([np.inf, -np.inf], np.nan) + nans = pd.isnull(field_df) field_df = self._stringify_dataframe(field_df, numeric_precision, datatype='field') field_df = (field_df.columns.values + '=').tolist() + field_df - field_df[field_df.columns[1:]] = ',' + field_df[ - field_df.columns[1:]] - field_df = field_df.where(~mask_null, '') # drop Null entries - fields = field_df.sum(axis=1) - # take out leading , where first column has a Null value - fields = fields.str.lstrip(",") + field_df[field_df.columns[1:]] = ',' + field_df[field_df.columns[1:]] + field_df[nans] = '' + + fields = field_df.sum(axis=1).map(lambda x: x.lstrip(',')) del field_df # Generate line protocol string diff --git a/influxdb/tests/dataframe_client_test.py b/influxdb/tests/dataframe_client_test.py index 90312ed8..0573d5c3 100644 --- a/influxdb/tests/dataframe_client_test.py +++ b/influxdb/tests/dataframe_client_test.py @@ -13,8 +13,8 @@ import warnings import requests_mock -from influxdb.tests import skip_if_pypy, using_pypy from nose.tools import raises +from influxdb.tests import skip_if_pypy, using_pypy from .client_test import _mocked_session @@ -22,7 +22,7 @@ import pandas as pd from pandas.util.testing import assert_frame_equal from influxdb import DataFrameClient - import numpy + import numpy as np @skip_if_pypy @@ -462,7 +462,7 @@ def test_write_points_from_dataframe_with_numeric_precision(self): ["2", 2, 2.2222222222222]], index=[now, now + timedelta(hours=1)]) - if numpy.lib.NumpyVersion(numpy.__version__) <= '1.13.3': + if np.lib.NumpyVersion(np.__version__) <= '1.13.3': expected_default_precision = ( b'foo,hello=there 0=\"1\",1=1i,2=1.11111111111 0\n' b'foo,hello=there 0=\"2\",1=2i,2=2.22222222222 3600000000000\n' @@ -1032,3 +1032,119 @@ def test_dsn_constructor(self): client = DataFrameClient.from_dsn('influxdb://localhost:8086') self.assertIsInstance(client, DataFrameClient) self.assertEqual('http://localhost:8086', client._baseurl) + + def test_write_points_from_dataframe_with_nan_line(self): + """Test write points from dataframe with Nan lines.""" + now = pd.Timestamp('1970-01-01 00:00+00:00') + dataframe = pd.DataFrame(data=[["1", 1, np.inf], ["2", 2, np.nan]], + index=[now, now + timedelta(hours=1)], + columns=["column_one", "column_two", + "column_three"]) + expected = ( + b"foo column_one=\"1\",column_two=1i 0\n" + b"foo column_one=\"2\",column_two=2i " + b"3600000000000\n" + ) + + with requests_mock.Mocker() as m: + m.register_uri(requests_mock.POST, + "http://localhost:8086/write", + status_code=204) + + cli = DataFrameClient(database='db') + + cli.write_points(dataframe, 'foo', protocol='line') + self.assertEqual(m.last_request.body, expected) + + cli.write_points(dataframe, 'foo', tags=None, protocol='line') + self.assertEqual(m.last_request.body, expected) + + def test_write_points_from_dataframe_with_nan_json(self): + """Test write points from json with NaN lines.""" + now = pd.Timestamp('1970-01-01 00:00+00:00') + dataframe = pd.DataFrame(data=[["1", 1, np.inf], ["2", 2, np.nan]], + index=[now, now + timedelta(hours=1)], + columns=["column_one", "column_two", + "column_three"]) + expected = ( + b"foo column_one=\"1\",column_two=1i 0\n" + b"foo column_one=\"2\",column_two=2i " + b"3600000000000\n" + ) + + with requests_mock.Mocker() as m: + m.register_uri(requests_mock.POST, + "http://localhost:8086/write", + status_code=204) + + cli = DataFrameClient(database='db') + + cli.write_points(dataframe, 'foo', protocol='json') + self.assertEqual(m.last_request.body, expected) + + cli.write_points(dataframe, 'foo', tags=None, protocol='json') + self.assertEqual(m.last_request.body, expected) + + def test_write_points_from_dataframe_with_tags_and_nan_line(self): + """Test write points from dataframe with NaN lines and tags.""" + now = pd.Timestamp('1970-01-01 00:00+00:00') + dataframe = pd.DataFrame(data=[['blue', 1, "1", 1, np.inf], + ['red', 0, "2", 2, np.nan]], + index=[now, now + timedelta(hours=1)], + columns=["tag_one", "tag_two", "column_one", + "column_two", "column_three"]) + expected = ( + b"foo,tag_one=blue,tag_two=1 " + b"column_one=\"1\",column_two=1i " + b"0\n" + b"foo,tag_one=red,tag_two=0 " + b"column_one=\"2\",column_two=2i " + b"3600000000000\n" + ) + + with requests_mock.Mocker() as m: + m.register_uri(requests_mock.POST, + "http://localhost:8086/write", + status_code=204) + + cli = DataFrameClient(database='db') + + cli.write_points(dataframe, 'foo', protocol='line', + tag_columns=['tag_one', 'tag_two']) + self.assertEqual(m.last_request.body, expected) + + cli.write_points(dataframe, 'foo', tags=None, protocol='line', + tag_columns=['tag_one', 'tag_two']) + self.assertEqual(m.last_request.body, expected) + + def test_write_points_from_dataframe_with_tags_and_nan_json(self): + """Test write points from json with NaN lines and tags.""" + now = pd.Timestamp('1970-01-01 00:00+00:00') + dataframe = pd.DataFrame(data=[['blue', 1, "1", 1, np.inf], + ['red', 0, "2", 2, np.nan]], + index=[now, now + timedelta(hours=1)], + columns=["tag_one", "tag_two", "column_one", + "column_two", "column_three"]) + expected = ( + b"foo,tag_one=blue,tag_two=1 " + b"column_one=\"1\",column_two=1i " + b"0\n" + b"foo,tag_one=red,tag_two=0 " + b"column_one=\"2\",column_two=2i " + b"3600000000000\n" + ) + + with requests_mock.Mocker() as m: + m.register_uri(requests_mock.POST, + "http://localhost:8086/write", + status_code=204) + + cli = DataFrameClient(database='db') + + cli.write_points(dataframe, 'foo', protocol='json', + tag_columns=['tag_one', 'tag_two']) + self.assertEqual(m.last_request.body, expected) + + cli.write_points(dataframe, 'foo', tags=None, protocol='json', + tag_columns=['tag_one', 'tag_two']) + self.assertEqual(m.last_request.body, expected) From 9b4a51ebc811bce92f66123ef17f46f10c8c943f Mon Sep 17 00:00:00 2001 From: Christopher Head Date: Thu, 9 Apr 2020 15:33:11 -0700 Subject: [PATCH 078/133] Fix import of distutils.spawn (#805) --- influxdb/tests/server_tests/influxdb_instance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/influxdb/tests/server_tests/influxdb_instance.py b/influxdb/tests/server_tests/influxdb_instance.py index 1dcd7567..2dd823ff 100644 --- a/influxdb/tests/server_tests/influxdb_instance.py +++ b/influxdb/tests/server_tests/influxdb_instance.py @@ -7,7 +7,7 @@ from __future__ import unicode_literals import datetime -import distutils +import distutils.spawn import os import tempfile import shutil From 57c14083215ed2012dc3eb755869544c30f4f809 Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Thu, 9 Apr 2020 17:34:33 -0500 Subject: [PATCH 079/133] chore(CHANGELOG): update to include PR from #805 --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index be15d52b..44f7f4d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Changed -## [v5.2.4] - 2020-04-10 +## [v5.3.0] - 2020-04-10 ### Added - Add mypy testing framework (#756) @@ -24,6 +24,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Add legacy client test (#752 & #318 thx @oldmantaiter & @sebito91) - Update make_lines section in line_protocol.py to split out core function (#375 thx @aisbaa) - Fix nanosecond time resolution for points (#407 thx @AndreCAndersen && @clslgrnc) +- Fix import of distutils.spawn (#805 thx @Hawk777) ### Removed From 896f6237f9217354fd365b7157bb4dd01af633a3 Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Thu, 9 Apr 2020 19:05:34 -0500 Subject: [PATCH 080/133] chore(line_protocol): update repr value of floats to properly handle precision (#813) * chore(line_protocol): update repr value of floats to properly handle precision. Closes #488 * chore(line_protocol): fix repr and handle boolean values * chore(CHANGELOG): update to include reference to PR#488 --- CHANGELOG.md | 1 + influxdb/line_protocol.py | 5 ++++- influxdb/tests/test_line_protocol.py | 23 +++++++++++++++++++++-- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 44f7f4d1..943fb83f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Update make_lines section in line_protocol.py to split out core function (#375 thx @aisbaa) - Fix nanosecond time resolution for points (#407 thx @AndreCAndersen && @clslgrnc) - Fix import of distutils.spawn (#805 thx @Hawk777) +- Update repr of float values including properly handling of boolean (#488 thx @ghost) ### Removed diff --git a/influxdb/line_protocol.py b/influxdb/line_protocol.py index 3a5eb4e8..d6cbf46f 100644 --- a/influxdb/line_protocol.py +++ b/influxdb/line_protocol.py @@ -112,8 +112,11 @@ def _escape_value(value): if isinstance(value, integer_types) and not isinstance(value, bool): return str(value) + 'i' + if isinstance(value, bool): + return str(value) + if _is_float(value): - return repr(value) + return repr(float(value)) return str(value) diff --git a/influxdb/tests/test_line_protocol.py b/influxdb/tests/test_line_protocol.py index 71828f62..c48d5edc 100644 --- a/influxdb/tests/test_line_protocol.py +++ b/influxdb/tests/test_line_protocol.py @@ -6,10 +6,12 @@ from __future__ import print_function from __future__ import unicode_literals -from datetime import datetime import unittest -from pytz import UTC, timezone +from datetime import datetime +from decimal import Decimal + +from pytz import UTC, timezone from influxdb import line_protocol @@ -166,3 +168,20 @@ def test_float_with_long_decimal_fraction(self): line_protocol.make_lines(data), 'test float_val=1.0000000000000009\n' ) + + def test_float_with_long_decimal_fraction_as_type_decimal(self): + """Ensure precision is preserved when casting Decimal into strings.""" + data = { + "points": [ + { + "measurement": "test", + "fields": { + "float_val": Decimal(0.8289445733333332), + } + } + ] + } + self.assertEqual( + line_protocol.make_lines(data), + 'test float_val=0.8289445733333332\n' + ) From 42d172d1db5eab8d881325d24404aa240bf342ac Mon Sep 17 00:00:00 2001 From: Kenzyme Le Date: Thu, 9 Apr 2020 22:15:05 -0400 Subject: [PATCH 081/133] Fix tests for existing 'Adding time_precision optional option to SeriesHelper' PR (#719) * Adding time_precision into Meta of SeriesHelper time_precision option not currently supported in influx-db python * Making Time Precision optional Changed it from required to optional * Fixing Typo in _time_precision Attribute * Fixing coding conventions for Travis CI * Appunni: Test for invalid time_precision on SeriesHelper * Appunni: Intendation problem resolution in master * - Fix flake7 errors : E131 continuation line unaligned for hanging indent - Fix typo - Fix cls._client declaration ordering - Remove duplicate code cls._autocommit ... Co-authored-by: appunni-dishq <31534711+appunni-dishq@users.noreply.github.com> Co-authored-by: xginn8 Co-authored-by: appunni --- influxdb/helper.py | 20 +++++++++++++++++--- influxdb/tests/helper_test.py | 13 ++++++++++++- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/influxdb/helper.py b/influxdb/helper.py index e622526d..fa79c079 100644 --- a/influxdb/helper.py +++ b/influxdb/helper.py @@ -41,6 +41,10 @@ class Meta: # Only applicable if autocommit is True. autocommit = True # If True and no bulk_size, then will set bulk_size to 1. + time_precision = "h"|"m"|s"|"ms"|"u"|"ns" + # Default is ns (nanoseconds) + # Setting time precision while writing point + # You should also make sure time is set in the given precision """ @@ -71,6 +75,13 @@ def __new__(cls, *args, **kwargs): cls.__name__)) cls._autocommit = getattr(_meta, 'autocommit', False) + cls._time_precision = getattr(_meta, 'time_precision', None) + + allowed_time_precisions = ['h', 'm', 's', 'ms', 'u', 'ns', None] + if cls._time_precision not in allowed_time_precisions: + raise AttributeError( + 'In {0}, time_precision is set, but invalid use any of {}.' + .format(cls.__name__, ','.join(allowed_time_precisions))) cls._client = getattr(_meta, 'client', None) if cls._autocommit and not cls._client: @@ -116,11 +127,11 @@ def __init__(self, **kw): keys = set(kw.keys()) # all tags should be passed, and keys - tags should be a subset of keys - if not(tags <= keys): + if not (tags <= keys): raise NameError( 'Expected arguments to contain all tags {0}, instead got {1}.' .format(cls._tags, kw.keys())) - if not(keys - tags <= fields): + if not (keys - tags <= fields): raise NameError('Got arguments not in tags or fields: {0}' .format(keys - tags - fields)) @@ -143,7 +154,10 @@ def commit(cls, client=None): """ if not client: client = cls._client - rtn = client.write_points(cls._json_body_()) + rtn = client.write_points( + cls._json_body_(), + time_precision=cls._time_precision) + # will be None if not set and will default to ns cls._reset_() return rtn diff --git a/influxdb/tests/helper_test.py b/influxdb/tests/helper_test.py index 6f24e85d..6aa8f15a 100644 --- a/influxdb/tests/helper_test.py +++ b/influxdb/tests/helper_test.py @@ -310,8 +310,19 @@ class Meta: series_name = 'events.stats.{server_name}' + class InvalidTimePrecision(SeriesHelper): + """Define instance of SeriesHelper for invalid time precision.""" + + class Meta: + """Define metadata for InvalidTimePrecision.""" + + series_name = 'events.stats.{server_name}' + time_precision = "ks" + fields = ['time', 'server_name'] + autocommit = True + for cls in [MissingMeta, MissingClient, MissingFields, - MissingSeriesName]: + MissingSeriesName, InvalidTimePrecision]: self.assertRaises( AttributeError, cls, **{'time': 159, 'server_name': 'us.east-1'}) From 8a8b8ff3c7d517959146c9daeb09c982a9c184b1 Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Thu, 9 Apr 2020 21:27:23 -0500 Subject: [PATCH 082/133] chore(CHANGELOG): update for PR#502 and #719 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 943fb83f..b94e438e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Add support for 'show series' (#357 thx @gaker) - Add support for custom request session in InfluxDBClient (#360 thx @dschien) - Add support for handling np.nan and np.inf values in DataFrameClient (#436 thx @nmerket) +- Add support for optional `time_precision` in the SeriesHelper (#502 && #719 thx @appunni-dishq && @klDen) ### Changed - Clean up stale CI config (#755) From 72c372f2412d445ea205e9b3cc696e242a90e35e Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Fri, 10 Apr 2020 08:59:46 -0500 Subject: [PATCH 083/133] chore(dataframe_client): update to handle empty tags in dataframe client (#814) --- CHANGELOG.md | 1 + influxdb/_dataframe_client.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b94e438e..e7f40dc7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Fix nanosecond time resolution for points (#407 thx @AndreCAndersen && @clslgrnc) - Fix import of distutils.spawn (#805 thx @Hawk777) - Update repr of float values including properly handling of boolean (#488 thx @ghost) +- Update dataframe_client to fix faulty empty tags (#770 thx @michelfripiat) ### Removed diff --git a/influxdb/_dataframe_client.py b/influxdb/_dataframe_client.py index f411bb37..a977754e 100644 --- a/influxdb/_dataframe_client.py +++ b/influxdb/_dataframe_client.py @@ -388,7 +388,8 @@ def _convert_dataframe_to_lines(self, del tag_df elif global_tags: tag_string = ''.join( - [",{}={}".format(k, _escape_tag(v)) if v else '' + [",{}={}".format(k, _escape_tag(v)) + if v not in [None, ''] else "" for k, v in sorted(global_tags.items())] ) tags = pd.Series(tag_string, index=dataframe.index) From c35e49ef6dd919c87dc20a01003922818bb30096 Mon Sep 17 00:00:00 2001 From: jgspiro <45625897+jgspiro@users.noreply.github.com> Date: Fri, 10 Apr 2020 16:07:18 +0200 Subject: [PATCH 084/133] Bugfix dropna in DataFrameClient. Add unit test. (#778) --- influxdb/_dataframe_client.py | 3 +- influxdb/tests/dataframe_client_test.py | 92 +++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 1 deletion(-) diff --git a/influxdb/_dataframe_client.py b/influxdb/_dataframe_client.py index a977754e..d7e67baa 100644 --- a/influxdb/_dataframe_client.py +++ b/influxdb/_dataframe_client.py @@ -203,7 +203,8 @@ def query(self, def _to_dataframe(self, rs, dropna=True): result = defaultdict(list) if isinstance(rs, list): - return map(self._to_dataframe, rs) + return map(self._to_dataframe, rs, + [dropna for _ in range(len(rs))]) for key, data in rs.items(): name, tags = key diff --git a/influxdb/tests/dataframe_client_test.py b/influxdb/tests/dataframe_client_test.py index 0573d5c3..4e172ea7 100644 --- a/influxdb/tests/dataframe_client_test.py +++ b/influxdb/tests/dataframe_client_test.py @@ -968,6 +968,98 @@ def test_multiquery_into_dataframe(self): for k in e: assert_frame_equal(e[k], r[k]) + def test_multiquery_into_dataframe_dropna(self): + """Test multiquery into df for TestDataFrameClient object.""" + data = { + "results": [ + { + "series": [ + { + "name": "cpu_load_short", + "columns": ["time", "value", "value2", "value3"], + "values": [ + ["2015-01-29T21:55:43.702900257Z", + 0.55, 0.254, numpy.NaN], + ["2015-01-29T21:55:43.702900257Z", + 23422, 122878, numpy.NaN], + ["2015-06-11T20:46:02Z", + 0.64, 0.5434, numpy.NaN] + ] + } + ] + }, { + "series": [ + { + "name": "cpu_load_short", + "columns": ["time", "count"], + "values": [ + ["1970-01-01T00:00:00Z", 3] + ] + } + ] + } + ] + } + + pd1 = pd.DataFrame( + [[0.55, 0.254, numpy.NaN], + [23422.0, 122878, numpy.NaN], + [0.64, 0.5434, numpy.NaN]], + columns=['value', 'value2', 'value3'], + index=pd.to_datetime([ + "2015-01-29 21:55:43.702900257+0000", + "2015-01-29 21:55:43.702900257+0000", + "2015-06-11 20:46:02+0000"])) + + if pd1.index.tzinfo is None: + pd1.index = pd1.index.tz_localize('UTC') + + pd1_dropna = pd.DataFrame( + [[0.55, 0.254], [23422.0, 122878], [0.64, 0.5434]], + columns=['value', 'value2'], + index=pd.to_datetime([ + "2015-01-29 21:55:43.702900257+0000", + "2015-01-29 21:55:43.702900257+0000", + "2015-06-11 20:46:02+0000"])) + + if pd1_dropna.index.tzinfo is None: + pd1_dropna.index = pd1_dropna.index.tz_localize('UTC') + + pd2 = pd.DataFrame( + [[3]], columns=['count'], + index=pd.to_datetime(["1970-01-01 00:00:00+00:00"])) + + if pd2.index.tzinfo is None: + pd2.index = pd2.index.tz_localize('UTC') + + expected_dropna_true = [ + {'cpu_load_short': pd1_dropna}, + {'cpu_load_short': pd2}] + expected_dropna_false = [ + {'cpu_load_short': pd1}, + {'cpu_load_short': pd2}] + + cli = DataFrameClient('host', 8086, 'username', 'password', 'db') + iql = "SELECT value FROM cpu_load_short WHERE region=$region;" \ + "SELECT count(value) FROM cpu_load_short WHERE region=$region" + bind_params = {'region': 'us-west'} + + for dropna in [True, False]: + with _mocked_session(cli, 'GET', 200, data): + result = cli.query(iql, bind_params=bind_params, dropna=dropna) + expected = \ + expected_dropna_true if dropna else expected_dropna_false + for r, e in zip(result, expected): + for k in e: + assert_frame_equal(e[k], r[k]) + + # test default value (dropna = True) + with _mocked_session(cli, 'GET', 200, data): + result = cli.query(iql, bind_params=bind_params) + for r, e in zip(result, expected_dropna_true): + for k in e: + assert_frame_equal(e[k], r[k]) + def test_query_with_empty_result(self): """Test query with empty results in TestDataFrameClient object.""" cli = DataFrameClient('host', 8086, 'username', 'password', 'db') From d85ecee3aa132980e11005a3ba8a248ccac6965b Mon Sep 17 00:00:00 2001 From: testforvln <1694611+testforvln@users.noreply.github.com> Date: Fri, 10 Apr 2020 22:21:24 +0800 Subject: [PATCH 085/133] fix bug in _convert_dataframe_to_json function that need not transform index if data type of index is already DatatimeIndex (#623) --- influxdb/_dataframe_client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/influxdb/_dataframe_client.py b/influxdb/_dataframe_client.py index d7e67baa..600bc1ec 100644 --- a/influxdb/_dataframe_client.py +++ b/influxdb/_dataframe_client.py @@ -252,7 +252,8 @@ def _convert_dataframe_to_json(dataframe, field_columns = list( set(dataframe.columns).difference(set(tag_columns))) - dataframe.index = pd.to_datetime(dataframe.index) + if not isinstance(dataframe.index, pd.DatetimeIndex): + dataframe.index = pd.to_datetime(dataframe.index) if dataframe.index.tzinfo is None: dataframe.index = dataframe.index.tz_localize('UTC') From ed975b4b5ab754d09c804bd69a26c6bddc160fa1 Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Fri, 10 Apr 2020 09:22:36 -0500 Subject: [PATCH 086/133] chore(CHANGELOG): update with more recent PR merges --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e7f40dc7..11f47e38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,7 +27,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Fix nanosecond time resolution for points (#407 thx @AndreCAndersen && @clslgrnc) - Fix import of distutils.spawn (#805 thx @Hawk777) - Update repr of float values including properly handling of boolean (#488 thx @ghost) -- Update dataframe_client to fix faulty empty tags (#770 thx @michelfripiat) +- Update DataFrameClient to fix faulty empty tags (#770 thx @michelfripiat) +- Update DataFrameClient to properly return `dropna` values (#778 thx @jgspiro) +- Update DataFrameClient to test for pd.DataTimeIndex before blind conversion (#623 thx @testforvin) ### Removed From 5929fef5df91f49df7981688df832958d9fa7e89 Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Fri, 10 Apr 2020 09:41:52 -0500 Subject: [PATCH 087/133] chore(dataframe_client_test): fix incorrect package naming convention --- influxdb/tests/dataframe_client_test.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/influxdb/tests/dataframe_client_test.py b/influxdb/tests/dataframe_client_test.py index 4e172ea7..a80498f3 100644 --- a/influxdb/tests/dataframe_client_test.py +++ b/influxdb/tests/dataframe_client_test.py @@ -979,11 +979,11 @@ def test_multiquery_into_dataframe_dropna(self): "columns": ["time", "value", "value2", "value3"], "values": [ ["2015-01-29T21:55:43.702900257Z", - 0.55, 0.254, numpy.NaN], + 0.55, 0.254, np.NaN], ["2015-01-29T21:55:43.702900257Z", - 23422, 122878, numpy.NaN], + 23422, 122878, np.NaN], ["2015-06-11T20:46:02Z", - 0.64, 0.5434, numpy.NaN] + 0.64, 0.5434, np.NaN] ] } ] @@ -1002,9 +1002,9 @@ def test_multiquery_into_dataframe_dropna(self): } pd1 = pd.DataFrame( - [[0.55, 0.254, numpy.NaN], - [23422.0, 122878, numpy.NaN], - [0.64, 0.5434, numpy.NaN]], + [[0.55, 0.254, np.NaN], + [23422.0, 122878, np.NaN], + [0.64, 0.5434, np.NaN]], columns=['value', 'value2', 'value3'], index=pd.to_datetime([ "2015-01-29 21:55:43.702900257+0000", From aad8349db4638696fbbc3d253ca353b5f7a31759 Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Fri, 10 Apr 2020 10:50:15 -0500 Subject: [PATCH 088/133] chore(client): ensure UDP port is actually an int. Closes #651. (#815) * chore(client): ensure UDP port is actually an int. Closes #651. * chore(CHANGELOG): update to include PR#651 --- CHANGELOG.md | 1 + influxdb/client.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 11f47e38..31e36680 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Update DataFrameClient to fix faulty empty tags (#770 thx @michelfripiat) - Update DataFrameClient to properly return `dropna` values (#778 thx @jgspiro) - Update DataFrameClient to test for pd.DataTimeIndex before blind conversion (#623 thx @testforvin) +- Update client to type-set UDP port to int (#651 thx @yifeikong) ### Removed diff --git a/influxdb/client.py b/influxdb/client.py index 390d8e16..0f0350c8 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -107,7 +107,7 @@ def __init__(self, self._verify_ssl = verify_ssl self.__use_udp = use_udp - self.__udp_port = udp_port + self.__udp_port = int(udp_port) if not session: session = requests.Session() From d2bf36c5be1cafb48f11e095f2a269901085dcf7 Mon Sep 17 00:00:00 2001 From: Cesar Sanz Date: Fri, 10 Apr 2020 18:28:53 +0200 Subject: [PATCH 089/133] Specify Retention Policy in SeriesHelper (#723) * Allow specify a retention policy in SeriesHelper * Complete the annotation example with the retention policy * Fix formatting * Fix formatting again * Add helper write with retention policy * Add helper write without retention policy * Remove blank line after the docstring. Co-authored-by: Sebastian Borza --- influxdb/helper.py | 8 +++++- influxdb/tests/helper_test.py | 51 +++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/influxdb/helper.py b/influxdb/helper.py index fa79c079..f49f40ad 100644 --- a/influxdb/helper.py +++ b/influxdb/helper.py @@ -41,6 +41,8 @@ class Meta: # Only applicable if autocommit is True. autocommit = True # If True and no bulk_size, then will set bulk_size to 1. + retention_policy = 'your_retention_policy' + # Specify the retention policy for the data points time_precision = "h"|"m"|s"|"ms"|"u"|"ns" # Default is ns (nanoseconds) # Setting time precision while writing point @@ -83,6 +85,8 @@ def __new__(cls, *args, **kwargs): 'In {0}, time_precision is set, but invalid use any of {}.' .format(cls.__name__, ','.join(allowed_time_precisions))) + cls._retention_policy = getattr(_meta, 'retention_policy', None) + cls._client = getattr(_meta, 'client', None) if cls._autocommit and not cls._client: raise AttributeError( @@ -154,9 +158,11 @@ def commit(cls, client=None): """ if not client: client = cls._client + rtn = client.write_points( cls._json_body_(), - time_precision=cls._time_precision) + time_precision=cls._time_precision, + retention_policy=cls._retention_policy) # will be None if not set and will default to ns cls._reset_() return rtn diff --git a/influxdb/tests/helper_test.py b/influxdb/tests/helper_test.py index 6aa8f15a..16924936 100644 --- a/influxdb/tests/helper_test.py +++ b/influxdb/tests/helper_test.py @@ -376,3 +376,54 @@ class Meta: .format(WarnBulkSizeNoEffect)) self.assertIn('has no affect', str(w[-1].message), 'Warning message did not contain "has not affect".') + + def testSeriesWithRetentionPolicy(self): + """Test that the data is saved with the specified retention policy.""" + my_policy = 'my_policy' + + class RetentionPolicySeriesHelper(SeriesHelper): + + class Meta: + client = InfluxDBClient() + series_name = 'events.stats.{server_name}' + fields = ['some_stat', 'time'] + tags = ['server_name', 'other_tag'] + bulk_size = 2 + autocommit = True + retention_policy = my_policy + + fake_write_points = mock.MagicMock() + RetentionPolicySeriesHelper( + server_name='us.east-1', some_stat=159, other_tag='gg') + RetentionPolicySeriesHelper._client.write_points = fake_write_points + RetentionPolicySeriesHelper( + server_name='us.east-1', some_stat=158, other_tag='aa') + + kall = fake_write_points.call_args + args, kwargs = kall + self.assertTrue('retention_policy' in kwargs) + self.assertEqual(kwargs['retention_policy'], my_policy) + + def testSeriesWithoutRetentionPolicy(self): + """Test that the data is saved without any retention policy.""" + class NoRetentionPolicySeriesHelper(SeriesHelper): + + class Meta: + client = InfluxDBClient() + series_name = 'events.stats.{server_name}' + fields = ['some_stat', 'time'] + tags = ['server_name', 'other_tag'] + bulk_size = 2 + autocommit = True + + fake_write_points = mock.MagicMock() + NoRetentionPolicySeriesHelper( + server_name='us.east-1', some_stat=159, other_tag='gg') + NoRetentionPolicySeriesHelper._client.write_points = fake_write_points + NoRetentionPolicySeriesHelper( + server_name='us.east-1', some_stat=158, other_tag='aa') + + kall = fake_write_points.call_args + args, kwargs = kall + self.assertTrue('retention_policy' in kwargs) + self.assertEqual(kwargs['retention_policy'], None) From 72c18b899db03432bca37d08c3f9efeb0875d30d Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Fri, 10 Apr 2020 11:29:58 -0500 Subject: [PATCH 090/133] chore(CHANGELOG): update for PR#723 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 31e36680..f2fad368 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Add support for custom request session in InfluxDBClient (#360 thx @dschien) - Add support for handling np.nan and np.inf values in DataFrameClient (#436 thx @nmerket) - Add support for optional `time_precision` in the SeriesHelper (#502 && #719 thx @appunni-dishq && @klDen) +- Add ability to specify retention policy in SeriesHelper (#723 thx @csanz91) ### Changed - Clean up stale CI config (#755) From c858a46ae5fd74cfe9220a8356dd5825ee2a66dd Mon Sep 17 00:00:00 2001 From: Kevin Claytor Date: Fri, 10 Apr 2020 12:49:26 -0400 Subject: [PATCH 091/133] gzip compression for data (post and responses) (#732) * gzip compression working in my influx stack. Needs proper tests. * Also gzip data from server, slightly more straightforward data handling. * Adding in test cases. * Switching back to zlib with gzip headers. * flake8 compatibility * Move parameter into correct position. per review * Switching back to gzip for the headers. * Fixing python 2.7 compatability with gzip. * flake8 compatibility. * flake8 testing Co-authored-by: Kevin Claytor Co-authored-by: Sebastian Borza --- influxdb/client.py | 25 ++++++- influxdb/tests/client_test.py | 67 +++++++++++++++++++ influxdb/tests/server_tests/base.py | 47 +++++++++++++ .../server_tests/client_test_with_server.py | 24 +++++++ 4 files changed, 162 insertions(+), 1 deletion(-) diff --git a/influxdb/client.py b/influxdb/client.py index 0f0350c8..3262c242 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -7,7 +7,9 @@ from __future__ import unicode_literals import datetime +import gzip import itertools +import io import json import random import socket @@ -70,10 +72,11 @@ class InfluxDBClient(object): as a single file containing the private key and the certificate, or as a tuple of both files’ paths, defaults to None :type cert: str + :param gzip: use gzip content encoding to compress requests + :type gzip: bool :param session: allow for the new client request to use an existing requests Session, defaults to None :type session: requests.Session - :raises ValueError: if cert is provided but ssl is disabled (set to False) """ @@ -93,6 +96,7 @@ def __init__(self, pool_size=10, path='', cert=None, + gzip=False, session=None, ): """Construct a new InfluxDBClient object.""" @@ -159,6 +163,8 @@ def __init__(self, 'Accept': 'application/x-msgpack' } + self._gzip = gzip + @property def _baseurl(self): return self.__baseurl @@ -278,6 +284,23 @@ def request(self, url, method='GET', params=None, data=None, if isinstance(data, (dict, list)): data = json.dumps(data) + if self._gzip: + # Receive and send compressed data + headers.update({ + 'Accept-Encoding': 'gzip', + 'Content-Encoding': 'gzip', + }) + if data is not None: + # For Py 2.7 compatability use Gzipfile + compressed = io.BytesIO() + with gzip.GzipFile( + compresslevel=9, + fileobj=compressed, + mode='w' + ) as f: + f.write(data) + data = compressed.getvalue() + # Try to send the request more than once by default (see #103) retry = True _try = 0 diff --git a/influxdb/tests/client_test.py b/influxdb/tests/client_test.py index fe7381ea..f8f3cb00 100644 --- a/influxdb/tests/client_test.py +++ b/influxdb/tests/client_test.py @@ -24,6 +24,8 @@ import unittest import warnings +import io +import gzip import json import mock import requests @@ -214,6 +216,71 @@ def test_write_points(self): m.last_request.body.decode('utf-8'), ) + def test_write_gzip(self): + """Test write in TestInfluxDBClient object.""" + with requests_mock.Mocker() as m: + m.register_uri( + requests_mock.POST, + "http://localhost:8086/write", + status_code=204 + ) + + cli = InfluxDBClient(database='db', gzip=True) + cli.write( + {"database": "mydb", + "retentionPolicy": "mypolicy", + "points": [{"measurement": "cpu_load_short", + "tags": {"host": "server01", + "region": "us-west"}, + "time": "2009-11-10T23:00:00Z", + "fields": {"value": 0.64}}]} + ) + + compressed = io.BytesIO() + with gzip.GzipFile( + compresslevel=9, + fileobj=compressed, + mode='w' + ) as f: + f.write( + b"cpu_load_short,host=server01,region=us-west " + b"value=0.64 1257894000000000000\n" + ) + + self.assertEqual( + m.last_request.body, + compressed.getvalue(), + ) + + def test_write_points_gzip(self): + """Test write points for TestInfluxDBClient object.""" + with requests_mock.Mocker() as m: + m.register_uri( + requests_mock.POST, + "http://localhost:8086/write", + status_code=204 + ) + + cli = InfluxDBClient(database='db', gzip=True) + cli.write_points( + self.dummy_points, + ) + + compressed = io.BytesIO() + with gzip.GzipFile( + compresslevel=9, + fileobj=compressed, + mode='w' + ) as f: + f.write( + b'cpu_load_short,host=server01,region=us-west ' + b'value=0.64 1257894000123456000\n' + ) + self.assertEqual( + m.last_request.body, + compressed.getvalue(), + ) + def test_write_points_toplevel_attributes(self): """Test write points attrs for TestInfluxDBClient object.""" with requests_mock.Mocker() as m: diff --git a/influxdb/tests/server_tests/base.py b/influxdb/tests/server_tests/base.py index fe722870..45a9ec80 100644 --- a/influxdb/tests/server_tests/base.py +++ b/influxdb/tests/server_tests/base.py @@ -36,6 +36,15 @@ def _setup_influxdb_server(inst): database='db') +def _setup_gzip_client(inst): + inst.cli = InfluxDBClient('localhost', + inst.influxd_inst.http_port, + 'root', + '', + database='db', + gzip=True) + + def _teardown_influxdb_server(inst): remove_tree = sys.exc_info() == (None, None, None) inst.influxd_inst.close(remove_tree=remove_tree) @@ -89,3 +98,41 @@ def tearDownClass(cls): def tearDown(self): """Deconstruct an instance of ManyTestCasesWithServerMixin.""" self.cli.drop_database('db') + + +class SingleTestCaseWithServerGzipMixin(object): + """Define the single testcase with server with gzip client mixin. + + Same as the SingleTestCaseWithServerGzipMixin but the InfluxDBClient has + gzip=True + """ + + @classmethod + def setUp(cls): + """Set up an instance of the SingleTestCaseWithServerGzipMixin.""" + _setup_influxdb_server(cls) + _setup_gzip_client(cls) + + @classmethod + def tearDown(cls): + """Tear down an instance of the SingleTestCaseWithServerMixin.""" + _teardown_influxdb_server(cls) + + +class ManyTestCasesWithServerGzipMixin(object): + """Define the many testcase with server with gzip client mixin. + + Same as the ManyTestCasesWithServerMixin but the InfluxDBClient has + gzip=True. + """ + + @classmethod + def setUpClass(cls): + """Set up an instance of the ManyTestCasesWithServerGzipMixin.""" + _setup_influxdb_server(cls) + _setup_gzip_client(cls) + + @classmethod + def tearDown(cls): + """Tear down an instance of the SingleTestCaseWithServerMixin.""" + _teardown_influxdb_server(cls) diff --git a/influxdb/tests/server_tests/client_test_with_server.py b/influxdb/tests/server_tests/client_test_with_server.py index 94f28b66..020014c3 100644 --- a/influxdb/tests/server_tests/client_test_with_server.py +++ b/influxdb/tests/server_tests/client_test_with_server.py @@ -26,6 +26,8 @@ from influxdb.tests import skip_if_pypy, using_pypy, skip_server_tests from influxdb.tests.server_tests.base import ManyTestCasesWithServerMixin from influxdb.tests.server_tests.base import SingleTestCaseWithServerMixin +from influxdb.tests.server_tests.base import ManyTestCasesWithServerGzipMixin +from influxdb.tests.server_tests.base import SingleTestCaseWithServerGzipMixin # By default, raise exceptions on warnings warnings.simplefilter('error', FutureWarning) @@ -913,3 +915,25 @@ def test_write_points_udp(self): ], list(rsp['cpu_load_short']) ) + + +# Run the tests again, but with gzip enabled this time +@skip_server_tests +class GzipSimpleTests(SimpleTests, SingleTestCaseWithServerGzipMixin): + """Repeat the simple tests with InfluxDBClient where gzip=True.""" + + pass + + +@skip_server_tests +class GzipCommonTests(CommonTests, ManyTestCasesWithServerGzipMixin): + """Repeat the common tests with InfluxDBClient where gzip=True.""" + + pass + + +@skip_server_tests +class GzipUdpTests(UdpTests, ManyTestCasesWithServerGzipMixin): + """Repeat the UDP tests with InfluxDBClient where gzip=True.""" + + pass From d59011945435d747d8446cd2453b1f0e0850754f Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Fri, 10 Apr 2020 11:50:38 -0500 Subject: [PATCH 092/133] chore(CHANGELOG): update to add PR#732 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f2fad368..3a706e7b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Add support for handling np.nan and np.inf values in DataFrameClient (#436 thx @nmerket) - Add support for optional `time_precision` in the SeriesHelper (#502 && #719 thx @appunni-dishq && @klDen) - Add ability to specify retention policy in SeriesHelper (#723 thx @csanz91) +- Add gzip compression for post and response data (#732 thx @KEClaytor) ### Changed - Clean up stale CI config (#755) From de44289e0167d690895e8e70eff37edeef36240f Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Fri, 10 Apr 2020 18:58:31 +0200 Subject: [PATCH 093/133] Make batched writing support all iterables (#746) * Make batched writing support all iterables * Also test batching generator against real server * Fix PEP257 error * Import itertools functions directly --- influxdb/client.py | 15 ++++++++-- influxdb/tests/client_test.py | 30 +++++++++++++++++++ .../server_tests/client_test_with_server.py | 27 +++++++++++++++++ 3 files changed, 69 insertions(+), 3 deletions(-) diff --git a/influxdb/client.py b/influxdb/client.py index 3262c242..46424bc2 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -15,11 +15,11 @@ import socket import struct import time +from itertools import chain, islice import msgpack import requests import requests.exceptions -from six.moves import xrange from six.moves.urllib.parse import urlparse from influxdb.line_protocol import make_lines, quote_ident, quote_literal @@ -597,8 +597,17 @@ def ping(self): @staticmethod def _batches(iterable, size): - for i in xrange(0, len(iterable), size): - yield iterable[i:i + size] + # Iterate over an iterable producing iterables of batches. Based on: + # http://code.activestate.com/recipes/303279-getting-items-in-batches/ + iterator = iter(iterable) + while True: + try: # Try get the first element in the iterator... + head = (next(iterator),) + except StopIteration: + return # ...so that we can stop if there isn't one + # Otherwise, lazily slice the rest of the batch + rest = islice(iterator, size - 1) + yield chain(head, rest) def _write_points(self, points, diff --git a/influxdb/tests/client_test.py b/influxdb/tests/client_test.py index f8f3cb00..a8f8e864 100644 --- a/influxdb/tests/client_test.py +++ b/influxdb/tests/client_test.py @@ -332,6 +332,36 @@ def test_write_points_batch(self): self.assertEqual(expected_last_body, m.last_request.body.decode('utf-8')) + def test_write_points_batch_generator(self): + """Test write points batch from a generator for TestInfluxDBClient.""" + dummy_points = [ + {"measurement": "cpu_usage", "tags": {"unit": "percent"}, + "time": "2009-11-10T23:00:00Z", "fields": {"value": 12.34}}, + {"measurement": "network", "tags": {"direction": "in"}, + "time": "2009-11-10T23:00:00Z", "fields": {"value": 123.00}}, + {"measurement": "network", "tags": {"direction": "out"}, + "time": "2009-11-10T23:00:00Z", "fields": {"value": 12.00}} + ] + dummy_points_generator = (point for point in dummy_points) + expected_last_body = ( + "network,direction=out,host=server01,region=us-west " + "value=12.0 1257894000000000000\n" + ) + + with requests_mock.Mocker() as m: + m.register_uri(requests_mock.POST, + "http://localhost:8086/write", + status_code=204) + cli = InfluxDBClient(database='db') + cli.write_points(points=dummy_points_generator, + database='db', + tags={"host": "server01", + "region": "us-west"}, + batch_size=2) + self.assertEqual(m.call_count, 2) + self.assertEqual(expected_last_body, + m.last_request.body.decode('utf-8')) + def test_write_points_udp(self): """Test write points UDP for TestInfluxDBClient object.""" s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) diff --git a/influxdb/tests/server_tests/client_test_with_server.py b/influxdb/tests/server_tests/client_test_with_server.py index 020014c3..a0263243 100644 --- a/influxdb/tests/server_tests/client_test_with_server.py +++ b/influxdb/tests/server_tests/client_test_with_server.py @@ -452,6 +452,33 @@ def test_write_points_batch(self): self.assertIn(12, net_out['series'][0]['values'][0]) self.assertIn(12.34, cpu['series'][0]['values'][0]) + def test_write_points_batch_generator(self): + """Test writing points in a batch from a generator.""" + dummy_points = [ + {"measurement": "cpu_usage", "tags": {"unit": "percent"}, + "time": "2009-11-10T23:00:00Z", "fields": {"value": 12.34}}, + {"measurement": "network", "tags": {"direction": "in"}, + "time": "2009-11-10T23:00:00Z", "fields": {"value": 123.00}}, + {"measurement": "network", "tags": {"direction": "out"}, + "time": "2009-11-10T23:00:00Z", "fields": {"value": 12.00}} + ] + dummy_points_generator = (point for point in dummy_points) + self.cli.write_points(points=dummy_points_generator, + tags={"host": "server01", + "region": "us-west"}, + batch_size=2) + time.sleep(5) + net_in = self.cli.query("SELECT value FROM network " + "WHERE direction=$dir", + bind_params={'dir': 'in'} + ).raw + net_out = self.cli.query("SELECT value FROM network " + "WHERE direction='out'").raw + cpu = self.cli.query("SELECT value FROM cpu_usage").raw + self.assertIn(123, net_in['series'][0]['values'][0]) + self.assertIn(12, net_out['series'][0]['values'][0]) + self.assertIn(12.34, cpu['series'][0]['values'][0]) + def test_query(self): """Test querying data back from server.""" self.assertIs(True, self.cli.write_points(dummy_point)) From d6192a759c106da1317fc60d0630022bf7e829cf Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Fri, 10 Apr 2020 11:59:25 -0500 Subject: [PATCH 094/133] chore(CHANGELOG): update to include PR#746 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a706e7b..5a49c84a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Update DataFrameClient to properly return `dropna` values (#778 thx @jgspiro) - Update DataFrameClient to test for pd.DataTimeIndex before blind conversion (#623 thx @testforvin) - Update client to type-set UDP port to int (#651 thx @yifeikong) +- Update batched writing support for all iterables (#746 thx @JayH5) ### Removed From c903d73efcf49b4e340490072d777d8f34ac8e1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20=22hr=22=20Berder=20=28=E7=99=BD=E5=B3=B0=29?= Date: Sat, 11 Apr 2020 01:10:40 +0800 Subject: [PATCH 095/133] Fix chunked query to return chunk resultsets (#753) When querying large data sets, it's vital to get a chunked responses to manage memory usage. Wrapping the query response in a generator and streaming the request provides the desired result. It also fixes `InfluxDBClient.query()` behavior for chunked queries that is currently not working according to [specs](https://github.com/influxdata/influxdb-python/blob/master/influxdb/client.py#L429) Closes #585. Closes #531. Closes #538. --- influxdb/client.py | 10 +++++--- influxdb/tests/client_test.py | 44 ++++++++++++++--------------------- 2 files changed, 25 insertions(+), 29 deletions(-) diff --git a/influxdb/client.py b/influxdb/client.py index 46424bc2..b28ed1b5 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -249,7 +249,7 @@ def switch_user(self, username, password): self._username = username self._password = password - def request(self, url, method='GET', params=None, data=None, + def request(self, url, method='GET', params=None, data=None, stream=False, expected_response_code=200, headers=None): """Make a HTTP request to the InfluxDB API. @@ -261,6 +261,8 @@ def request(self, url, method='GET', params=None, data=None, :type params: dict :param data: the data of the request, defaults to None :type data: str + :param stream: True if a query uses chunked responses + :type stream: bool :param expected_response_code: the expected response code of the request, defaults to 200 :type expected_response_code: int @@ -312,6 +314,7 @@ def request(self, url, method='GET', params=None, data=None, auth=(self._username, self._password), params=params, data=data, + stream=stream, headers=headers, proxies=self._proxies, verify=self._verify_ssl, @@ -398,17 +401,17 @@ def write(self, data, params=None, expected_response_code=204, @staticmethod def _read_chunked_response(response, raise_errors=True): - result_set = {} for line in response.iter_lines(): if isinstance(line, bytes): line = line.decode('utf-8') data = json.loads(line) + result_set = {} for result in data.get('results', []): for _key in result: if isinstance(result[_key], list): result_set.setdefault( _key, []).extend(result[_key]) - return ResultSet(result_set, raise_errors=raise_errors) + yield ResultSet(result_set, raise_errors=raise_errors) def query(self, query, @@ -499,6 +502,7 @@ def query(self, method=method, params=params, data=None, + stream=chunked, expected_response_code=expected_response_code ) diff --git a/influxdb/tests/client_test.py b/influxdb/tests/client_test.py index a8f8e864..fd3c06bb 100644 --- a/influxdb/tests/client_test.py +++ b/influxdb/tests/client_test.py @@ -1400,16 +1400,11 @@ def test_invalid_port_fails(self): def test_chunked_response(self): """Test chunked reponse for TestInfluxDBClient object.""" example_response = \ - u'{"results":[{"statement_id":0,"series":' \ - '[{"name":"cpu","columns":["fieldKey","fieldType"],"values":' \ - '[["value","integer"]]}],"partial":true}]}\n{"results":' \ - '[{"statement_id":0,"series":[{"name":"iops","columns":' \ - '["fieldKey","fieldType"],"values":[["value","integer"]]}],' \ - '"partial":true}]}\n{"results":[{"statement_id":0,"series":' \ - '[{"name":"load","columns":["fieldKey","fieldType"],"values":' \ - '[["value","integer"]]}],"partial":true}]}\n{"results":' \ - '[{"statement_id":0,"series":[{"name":"memory","columns":' \ - '["fieldKey","fieldType"],"values":[["value","integer"]]}]}]}\n' + u'{"results":[{"statement_id":0,"series":[{"columns":["key"],' \ + '"values":[["cpu"],["memory"],["iops"],["network"]],"partial":' \ + 'true}],"partial":true}]}\n{"results":[{"statement_id":0,' \ + '"series":[{"columns":["key"],"values":[["qps"],["uptime"],' \ + '["df"],["mount"]]}]}]}\n' with requests_mock.Mocker() as m: m.register_uri( @@ -1417,23 +1412,20 @@ def test_chunked_response(self): "http://localhost:8086/query", text=example_response ) - response = self.cli.query('show series limit 4 offset 0', + response = self.cli.query('show series', chunked=True, chunk_size=4) - self.assertTrue(len(response) == 4) - self.assertEqual(response.__repr__(), ResultSet( - {'series': [{'values': [['value', 'integer']], - 'name': 'cpu', - 'columns': ['fieldKey', 'fieldType']}, - {'values': [['value', 'integer']], - 'name': 'iops', - 'columns': ['fieldKey', 'fieldType']}, - {'values': [['value', 'integer']], - 'name': 'load', - 'columns': ['fieldKey', 'fieldType']}, - {'values': [['value', 'integer']], - 'name': 'memory', - 'columns': ['fieldKey', 'fieldType']}]} - ).__repr__()) + res = list(response) + self.assertTrue(len(res) == 2) + self.assertEqual(res[0].__repr__(), ResultSet( + {'series': [{ + 'columns': ['key'], + 'values': [['cpu'], ['memory'], ['iops'], ['network']] + }]}).__repr__()) + self.assertEqual(res[1].__repr__(), ResultSet( + {'series': [{ + 'columns': ['key'], + 'values': [['qps'], ['uptime'], ['df'], ['mount']] + }]}).__repr__()) class FakeClient(InfluxDBClient): From e884631148dda96f586ab9da358deb7119bb57d2 Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Fri, 10 Apr 2020 12:12:03 -0500 Subject: [PATCH 096/133] chore(CHANGELOG): update to include PR#753 and #538 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a49c84a..00d2a4f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Add support for optional `time_precision` in the SeriesHelper (#502 && #719 thx @appunni-dishq && @klDen) - Add ability to specify retention policy in SeriesHelper (#723 thx @csanz91) - Add gzip compression for post and response data (#732 thx @KEClaytor) +- Add support for chunked responses in ResultSet (#753 and #538 thx @hrbonz && @psy0rz) ### Changed - Clean up stale CI config (#755) From 7d82f9371176699f8d6f9c4d0b0133ed5bd20275 Mon Sep 17 00:00:00 2001 From: Greg Schrock Date: Fri, 10 Apr 2020 13:29:54 -0400 Subject: [PATCH 097/133] Fix make_lines excludes fields with empty strings (#655) (#766) * Fix make_lines excludes fields with empty strings (#655) Converting to unicode required something to be done with None values. They were converted to empty strings which were subsequently ignored. This makes it impossible to write an explicitly empty string, which should be possible. This change distinguishes between None and empty strings. * Fix linting failure due to long comment line Co-authored-by: Greg Schrock --- influxdb/line_protocol.py | 6 ++++-- influxdb/tests/test_line_protocol.py | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/influxdb/line_protocol.py b/influxdb/line_protocol.py index d6cbf46f..25dd2ad7 100644 --- a/influxdb/line_protocol.py +++ b/influxdb/line_protocol.py @@ -104,9 +104,11 @@ def _is_float(value): def _escape_value(value): - value = _get_unicode(value) + if value is None: + return '' - if isinstance(value, text_type) and value != '': + value = _get_unicode(value) + if isinstance(value, text_type): return quote_ident(value) if isinstance(value, integer_types) and not isinstance(value, bool): diff --git a/influxdb/tests/test_line_protocol.py b/influxdb/tests/test_line_protocol.py index c48d5edc..5b344990 100644 --- a/influxdb/tests/test_line_protocol.py +++ b/influxdb/tests/test_line_protocol.py @@ -117,6 +117,24 @@ def test_make_lines_unicode(self): 'test,unicode_tag=\'Привет!\' unicode_val="Привет!"\n' ) + def test_make_lines_empty_field_string(self): + """Test make lines with an empty string field.""" + data = { + "points": [ + { + "measurement": "test", + "fields": { + "string": "", + } + } + ] + } + + self.assertEqual( + line_protocol.make_lines(data), + 'test string=""\n' + ) + def test_tag_value_newline(self): """Test make lines with tag value contains newline.""" data = { From 6290994590c4164535d59879e50482b44bb4d7ff Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Fri, 10 Apr 2020 12:30:50 -0500 Subject: [PATCH 098/133] chore(CHANGELOG): update for PR #766 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 00d2a4f7..4ee39303 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Add ability to specify retention policy in SeriesHelper (#723 thx @csanz91) - Add gzip compression for post and response data (#732 thx @KEClaytor) - Add support for chunked responses in ResultSet (#753 and #538 thx @hrbonz && @psy0rz) +- Add support for empty string fields (#766 thx @gregschrock) ### Changed - Clean up stale CI config (#755) From 6f73ea12d7d9ca0184ba3c487264e58146ce2bac Mon Sep 17 00:00:00 2001 From: Keunhyun Oh Date: Sat, 11 Apr 2020 02:33:14 +0900 Subject: [PATCH 099/133] fix: Calling commit and _json_body make raising an exception when any datapoints are not added. (#772) * If any SerialHelper is not generated, calling the commit function makes raising an exception because _datapoints is not allocated. It is hard to find the reason of this error. it is because reviewing influxdb-python's code is needed. I think that it is important that is producing predictable results. Results when calling first time is needed to equal to calling resetting datapoints in json_body. So, I've fixed that if not initialized when calling _json_body() function, _datapoints is reset to avoid raising error. In Unittest, the setup function is added. When calling setup function firstly, __initialized__ is False and _datapoints is not assigned. But, because of this commit, it is OK. Contacts: Keunhyun Oh * fix build fail Contacts: Keunhyun Oh * fix build fail Contacts: Keunhyun Oh * fix build fail Contacts: Keunhyun Oh * Update helper_test.py * Update helper_test.py * Update helper_test.py --- influxdb/helper.py | 2 ++ influxdb/influxdb08/helper.py | 2 ++ influxdb/tests/helper_test.py | 8 ++++++++ 3 files changed, 12 insertions(+) diff --git a/influxdb/helper.py b/influxdb/helper.py index f49f40ad..74209354 100644 --- a/influxdb/helper.py +++ b/influxdb/helper.py @@ -174,6 +174,8 @@ def _json_body_(cls): :return: JSON body of these datapoints. """ json = [] + if not cls.__initialized__: + cls._reset_() for series_name, data in six.iteritems(cls._datapoints): for point in data: json_point = { diff --git a/influxdb/influxdb08/helper.py b/influxdb/influxdb08/helper.py index f3dec33c..5f2d4614 100644 --- a/influxdb/influxdb08/helper.py +++ b/influxdb/influxdb08/helper.py @@ -139,6 +139,8 @@ def _json_body_(cls): :return: JSON body of the datapoints. """ json = [] + if not cls.__initialized__: + cls._reset_() for series_name, data in six.iteritems(cls._datapoints): json.append({'name': series_name, 'columns': cls._fields, diff --git a/influxdb/tests/helper_test.py b/influxdb/tests/helper_test.py index 16924936..6737f921 100644 --- a/influxdb/tests/helper_test.py +++ b/influxdb/tests/helper_test.py @@ -47,6 +47,14 @@ class Meta: TestSeriesHelper.MySeriesHelper = MySeriesHelper + def setUp(self): + """Check that MySeriesHelper has empty datapoints.""" + super(TestSeriesHelper, self).setUp() + self.assertEqual( + TestSeriesHelper.MySeriesHelper._json_body_(), + [], + 'Resetting helper in teardown did not empty datapoints.') + def tearDown(self): """Deconstruct the TestSeriesHelper object.""" super(TestSeriesHelper, self).tearDown() From 351b98a880d7a62188e00a240df4edaa2d75f499 Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Fri, 10 Apr 2020 12:34:05 -0500 Subject: [PATCH 100/133] chore(CHANGELOG): add PR#772 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ee39303..322c952b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Update DataFrameClient to test for pd.DataTimeIndex before blind conversion (#623 thx @testforvin) - Update client to type-set UDP port to int (#651 thx @yifeikong) - Update batched writing support for all iterables (#746 thx @JayH5) +- Update SeriesHelper to enable class instantiation when not initialized (#772 thx @ocworld) ### Removed From f7f30b5ed3d7bc9ded164288af8b1a7ca1a3a6b0 Mon Sep 17 00:00:00 2001 From: Shan Desai Date: Fri, 10 Apr 2020 19:59:04 +0200 Subject: [PATCH 101/133] Refactor `tutorial_udp` example for timestamps (#808) - `time` key should be within each datapoint - This PR addresses #788 regarding the structure of the data when inserting via UDP. - The original documentation contributed by me took the structure of the `tutorial.py` as base. However, upon testing, the timestamp in the example are not written (2020 is written as opposed to 2009). - Tested for `influxdb-python` v5.2.3 and InfluxDB v1.6.1 --- examples/tutorial_udp.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/examples/tutorial_udp.py b/examples/tutorial_udp.py index 517ae858..93b923d7 100644 --- a/examples/tutorial_udp.py +++ b/examples/tutorial_udp.py @@ -29,18 +29,19 @@ def main(uport): "host": "server01", "region": "us-west" }, - "time": "2009-11-10T23:00:00Z", "points": [{ "measurement": "cpu_load_short", "fields": { "value": 0.64 - } + }, + "time": "2009-11-10T23:00:00Z", }, - { - "measurement": "cpu_load_short", - "fields": { - "value": 0.67 - } + { + "measurement": "cpu_load_short", + "fields": { + "value": 0.67 + }, + "time": "2009-11-10T23:05:00Z" }] } From f8705f9f474e260b0455fc44d1b1f74c64d22162 Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Fri, 10 Apr 2020 14:27:14 -0500 Subject: [PATCH 102/133] feat(client): add support for context managers (#816) * feat(client): add support for context managers * chore(CHANGELOG): rebase against master --- CHANGELOG.md | 2 ++ influxdb/client.py | 12 ++++++++++++ 2 files changed, 14 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 322c952b..411b46a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Add gzip compression for post and response data (#732 thx @KEClaytor) - Add support for chunked responses in ResultSet (#753 and #538 thx @hrbonz && @psy0rz) - Add support for empty string fields (#766 thx @gregschrock) +- Add support for context managers to InfluxDBClient (#721 thx @JustusAdam) ### Changed - Clean up stale CI config (#755) @@ -37,6 +38,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Update client to type-set UDP port to int (#651 thx @yifeikong) - Update batched writing support for all iterables (#746 thx @JayH5) - Update SeriesHelper to enable class instantiation when not initialized (#772 thx @ocworld) +- Update UDP test case to add proper timestamp to datapoints (#808 thx @shantanoo-desai) ### Removed diff --git a/influxdb/client.py b/influxdb/client.py index b28ed1b5..a0f571f5 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -35,6 +35,9 @@ class InfluxDBClient(object): connect to InfluxDB. Requests can be made to InfluxDB directly through the client. + The client supports the use as a `context manager + `_. + :param host: hostname to connect to InfluxDB, defaults to 'localhost' :type host: str :param port: port to connect to InfluxDB, defaults to 8086 @@ -78,6 +81,7 @@ class InfluxDBClient(object): requests Session, defaults to None :type session: requests.Session :raises ValueError: if cert is provided but ssl is disabled (set to False) + """ def __init__(self, @@ -165,6 +169,14 @@ def __init__(self, self._gzip = gzip + def __enter__(self): + """Enter function as used by context manager.""" + pass + + def __exit__(self, _exc_type, _exc_value, _traceback): + """Exit function as used by context manager.""" + self.close() + @property def _baseurl(self): return self.__baseurl From cf83d1d429ecebb093f6eaab773bd41c251530ca Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Fri, 10 Apr 2020 14:38:30 -0500 Subject: [PATCH 103/133] chore(CHANGELOG): tagging release to 5.3.0 --- LICENSE | 2 +- influxdb/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LICENSE b/LICENSE index 38ee2491..a49a5410 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2013 InfluxDB +Copyright (c) 2020 InfluxDB Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff --git a/influxdb/__init__.py b/influxdb/__init__.py index b31170bb..56f2f619 100644 --- a/influxdb/__init__.py +++ b/influxdb/__init__.py @@ -18,4 +18,4 @@ ] -__version__ = '5.2.3' +__version__ = '5.3.0' From e7ef0454524bf0617097a2f303d784242e5ddfe8 Mon Sep 17 00:00:00 2001 From: Krzysztof Baranski Date: Sun, 12 Apr 2020 01:45:32 +0200 Subject: [PATCH 104/133] do not sleep after last retry before raising exception (#790) * do not sleep after last retry before raising exception * documentation: clarification of retry parameter retry=0 - retry forever retry=1 - try once, on error don't do any retry retry=2 - 2 tries, one original and one retry on error retry=3 - 3 tries, one original and maximum two retries on errors * retries - move raise before sleep * retries - documentation * fix line length --- influxdb/client.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/influxdb/client.py b/influxdb/client.py index a0f571f5..80994190 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -59,8 +59,12 @@ class InfluxDBClient(object): :param timeout: number of seconds Requests will wait for your client to establish a connection, defaults to None :type timeout: int - :param retries: number of retries your client will try before aborting, - defaults to 3. 0 indicates try until success + :param retries: number of attempts your client will make before aborting, + defaults to 3 + 0 - try until success + 1 - attempt only once (without retry) + 2 - maximum two attempts (including one retry) + 3 - maximum three attempts (default option) :type retries: int :param use_udp: use UDP to connect to InfluxDB, defaults to False :type use_udp: bool @@ -339,10 +343,10 @@ def request(self, url, method='GET', params=None, data=None, stream=False, _try += 1 if self._retries != 0: retry = _try < self._retries - if method == "POST": - time.sleep((2 ** _try) * random.random() / 100.0) if not retry: raise + if method == "POST": + time.sleep((2 ** _try) * random.random() / 100.0) type_header = response.headers and response.headers.get("Content-Type") if type_header == "application/x-msgpack" and response.content: From b4390cc8680abd0931b14b8f2ee1e83998a53eb2 Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Sat, 11 Apr 2020 18:48:26 -0500 Subject: [PATCH 105/133] chore(CHANGELOG): add PR#790 --- CHANGELOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 411b46a4..c2f3edcc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,11 +4,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [v5.3.1] - Unreleased ### Added ### Changed +- Amend retry to avoid sleep after last retry before raising exception (#790 thx @krzysbaranski) + +### Removed ## [v5.3.0] - 2020-04-10 From cc41e290f690c4eb67f75c98fa9f027bdb6eb16b Mon Sep 17 00:00:00 2001 From: Matthew Thode Date: Sat, 11 Apr 2020 18:49:33 -0500 Subject: [PATCH 106/133] remove msgpack pin (#818) The hard lock prevents this from being co-installed with many other packages. For instance, it's preventing it from being included in openstack (which is on 0.6.2 and working on 1.0.0 now). Signed-off-by: Matthew Thode --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 77d7306f..548b17c8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,4 @@ python-dateutil>=2.6.0 pytz requests>=2.17.0 six>=1.10.0 -msgpack==0.6.1 +msgpack From cdc2c665808f2cf572973029f81b9a0175b43277 Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Sat, 11 Apr 2020 18:53:49 -0500 Subject: [PATCH 107/133] chore(CHANGELOG): add PR#818 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c2f3edcc..d380b73e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Changed - Amend retry to avoid sleep after last retry before raising exception (#790 thx @krzysbaranski) +- Remove msgpack pinning for requirements (#818 thx @prometheanfire) ### Removed From 5bcfadf7a525232078b875b2fb94d831287d6c8e Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Sun, 12 Apr 2020 22:14:38 -0500 Subject: [PATCH 108/133] chore(dataframe_client): update param definition. Closes #525. (#819) --- influxdb/_dataframe_client.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/influxdb/_dataframe_client.py b/influxdb/_dataframe_client.py index 600bc1ec..ec58cebb 100644 --- a/influxdb/_dataframe_client.py +++ b/influxdb/_dataframe_client.py @@ -59,6 +59,8 @@ def write_points(self, :param dataframe: data points in a DataFrame :param measurement: name of measurement :param tags: dictionary of tags, with string key-values + :param tag_columns: [Optional, default None] List of data tag names + :param field_columns: [Options, default None] List of data field names :param time_precision: [Optional, default None] Either 's', 'ms', 'u' or 'n'. :param batch_size: [Optional] Value to write the points in batches From 7fb5e946062dd36a84801e4a03012a3c032a70db Mon Sep 17 00:00:00 2001 From: Adam Suban-Loewen Date: Mon, 13 Apr 2020 22:05:49 -0400 Subject: [PATCH 109/133] Added headers parameter to InfluxDBClient (#710) --- influxdb/client.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/influxdb/client.py b/influxdb/client.py index 80994190..404e14be 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -84,8 +84,10 @@ class InfluxDBClient(object): :param session: allow for the new client request to use an existing requests Session, defaults to None :type session: requests.Session + :param headers: headers to add to Requests, will add 'Content-Type' + and 'Accept' unless these are already present, defaults to {} + :type headers: dict :raises ValueError: if cert is provided but ssl is disabled (set to False) - """ def __init__(self, @@ -106,6 +108,7 @@ def __init__(self, cert=None, gzip=False, session=None, + headers=None, ): """Construct a new InfluxDBClient object.""" self.__host = host @@ -166,10 +169,11 @@ def __init__(self, self._port, self._path) - self._headers = { - 'Content-Type': 'application/json', - 'Accept': 'application/x-msgpack' - } + if headers is None: + headers = {} + headers.setdefault('Content-Type', 'application/json') + headers.setdefault('Accept', 'application/x-msgpack') + self._headers = headers self._gzip = gzip @@ -390,7 +394,7 @@ def write(self, data, params=None, expected_response_code=204, :returns: True, if the write operation is successful :rtype: bool """ - headers = self._headers + headers = self._headers.copy() headers['Content-Type'] = 'application/octet-stream' if params: From 95e0efb0821d44bf06aebe0b2c4700e4d3b084c8 Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Mon, 13 Apr 2020 21:09:22 -0500 Subject: [PATCH 110/133] chore(CHANGELOG): add PR #710 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d380b73e..92bbe42e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [v5.3.1] - Unreleased ### Added +- Add support for custom headers in the InfluxDBClient (#710 thx @nathanielatom) ### Changed - Amend retry to avoid sleep after last retry before raising exception (#790 thx @krzysbaranski) From de75e7351ce3fc084c145d16df38f96d18603bf5 Mon Sep 17 00:00:00 2001 From: Pavlina Rolincova Date: Thu, 28 May 2020 10:41:50 +0200 Subject: [PATCH 111/133] Add support for custom indexes for query in the DataFrameClient (#785) --- CHANGELOG.md | 1 + influxdb/_dataframe_client.py | 22 +++++++++++------ influxdb/tests/dataframe_client_test.py | 33 +++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 92bbe42e..2b374faa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added - Add support for custom headers in the InfluxDBClient (#710 thx @nathanielatom) +- Add support for custom indexes for query in the DataFrameClient (#785) ### Changed - Amend retry to avoid sleep after last retry before raising exception (#790 thx @krzysbaranski) diff --git a/influxdb/_dataframe_client.py b/influxdb/_dataframe_client.py index ec58cebb..58063500 100644 --- a/influxdb/_dataframe_client.py +++ b/influxdb/_dataframe_client.py @@ -8,6 +8,7 @@ import math from collections import defaultdict +from typing import List import pandas as pd import numpy as np @@ -152,7 +153,8 @@ def query(self, chunked=False, chunk_size=0, method="GET", - dropna=True): + dropna=True, + data_frame_index: List[str] = None): """ Query data into a DataFrame. @@ -181,6 +183,7 @@ def query(self, containing all results within that chunk :param chunk_size: Size of each chunk to tell InfluxDB to use. :param dropna: drop columns where all values are missing + :param data_frame_index: the list of columns that are used as DataFrame index :returns: the queried data :rtype: :class:`~.ResultSet` """ @@ -196,13 +199,13 @@ def query(self, results = super(DataFrameClient, self).query(query, **query_args) if query.strip().upper().startswith("SELECT"): if len(results) > 0: - return self._to_dataframe(results, dropna) + return self._to_dataframe(results, dropna, data_frame_index=data_frame_index) else: return {} else: return results - def _to_dataframe(self, rs, dropna=True): + def _to_dataframe(self, rs, dropna=True, data_frame_index: List[str] = None): result = defaultdict(list) if isinstance(rs, list): return map(self._to_dataframe, rs, @@ -216,10 +219,15 @@ def _to_dataframe(self, rs, dropna=True): key = (name, tuple(sorted(tags.items()))) df = pd.DataFrame(data) df.time = pd.to_datetime(df.time) - df.set_index('time', inplace=True) - if df.index.tzinfo is None: - df.index = df.index.tz_localize('UTC') - df.index.name = None + + if data_frame_index: + df.set_index(data_frame_index, inplace=True) + else: + df.set_index('time', inplace=True) + if df.index.tzinfo is None: + df.index = df.index.tz_localize('UTC') + df.index.name = 'time' + result[key].append(df) for key, data in result.items(): df = pd.concat(data).sort_index() diff --git a/influxdb/tests/dataframe_client_test.py b/influxdb/tests/dataframe_client_test.py index a80498f3..cf82b49c 100644 --- a/influxdb/tests/dataframe_client_test.py +++ b/influxdb/tests/dataframe_client_test.py @@ -1240,3 +1240,36 @@ def test_write_points_from_dataframe_with_tags_and_nan_json(self): cli.write_points(dataframe, 'foo', tags=None, protocol='json', tag_columns=['tag_one', 'tag_two']) self.assertEqual(m.last_request.body, expected) + + def test_query_custom_index(self): + data = { + "results": [ + { + "series": [ + { + "name": "cpu_load_short", + "columns": ["time", "value", "host"], + "values": [ + [1, 0.55, "local"], + [2, 23422, "local"], + [3, 0.64, "local"] + ] + } + ] + } + ] + } + + cli = DataFrameClient('host', 8086, 'username', 'password', 'db') + iql = "SELECT value FROM cpu_load_short WHERE region=$region;" \ + "SELECT count(value) FROM cpu_load_short WHERE region=$region" + bind_params = {'region': 'us-west'} + with _mocked_session(cli, 'GET', 200, data): + result = cli.query(iql, bind_params=bind_params, data_frame_index=["time", "host"]) + + _data_frame = result['cpu_load_short'] + print(_data_frame) + + self.assertListEqual(["time", "host"], list(_data_frame.index.names)) + + From d3fd851c8e99350524fad710c753a0d978a5f978 Mon Sep 17 00:00:00 2001 From: Pavlina Rolincova Date: Mon, 1 Jun 2020 08:26:39 +0200 Subject: [PATCH 112/133] Add support for custom indexes for query in the DataFrameClient (#785) --- influxdb/_dataframe_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/influxdb/_dataframe_client.py b/influxdb/_dataframe_client.py index 58063500..0b9f282d 100644 --- a/influxdb/_dataframe_client.py +++ b/influxdb/_dataframe_client.py @@ -226,7 +226,7 @@ def _to_dataframe(self, rs, dropna=True, data_frame_index: List[str] = None): df.set_index('time', inplace=True) if df.index.tzinfo is None: df.index = df.index.tz_localize('UTC') - df.index.name = 'time' + df.index.name = None result[key].append(df) for key, data in result.items(): From 9a110a1abda2677340bc7fbd00890d27b8d3d5ab Mon Sep 17 00:00:00 2001 From: Pavlina Rolincova Date: Mon, 1 Jun 2020 09:57:12 +0200 Subject: [PATCH 113/133] Add support for custom indexes for query in the DataFrameClient (#785) --- influxdb/_dataframe_client.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/influxdb/_dataframe_client.py b/influxdb/_dataframe_client.py index 0b9f282d..afd75b39 100644 --- a/influxdb/_dataframe_client.py +++ b/influxdb/_dataframe_client.py @@ -8,7 +8,6 @@ import math from collections import defaultdict -from typing import List import pandas as pd import numpy as np @@ -154,7 +153,7 @@ def query(self, chunk_size=0, method="GET", dropna=True, - data_frame_index: List[str] = None): + data_frame_index=None): """ Query data into a DataFrame. @@ -205,7 +204,7 @@ def query(self, else: return results - def _to_dataframe(self, rs, dropna=True, data_frame_index: List[str] = None): + def _to_dataframe(self, rs, dropna=True, data_frame_index=None): result = defaultdict(list) if isinstance(rs, list): return map(self._to_dataframe, rs, From 055d71fb83aeb83603bd9b5423d6b05d1f01c59e Mon Sep 17 00:00:00 2001 From: Pavlina Rolincova Date: Mon, 1 Jun 2020 10:37:56 +0200 Subject: [PATCH 114/133] Add support for custom indexes for query in the DataFrameClient (#785) --- influxdb/tests/dataframe_client_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/influxdb/tests/dataframe_client_test.py b/influxdb/tests/dataframe_client_test.py index cf82b49c..2dd98398 100644 --- a/influxdb/tests/dataframe_client_test.py +++ b/influxdb/tests/dataframe_client_test.py @@ -1242,6 +1242,7 @@ def test_write_points_from_dataframe_with_tags_and_nan_json(self): self.assertEqual(m.last_request.body, expected) def test_query_custom_index(self): + """Test query with custom indexes.""" data = { "results": [ { From ddd82612f57da90476cd0d10539d7edf1ca25d49 Mon Sep 17 00:00:00 2001 From: Pavlina Rolincova Date: Mon, 1 Jun 2020 10:40:39 +0200 Subject: [PATCH 115/133] Add support for custom indexes for query in the DataFrameClient (#785) --- influxdb/tests/dataframe_client_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/influxdb/tests/dataframe_client_test.py b/influxdb/tests/dataframe_client_test.py index 2dd98398..f6db3c22 100644 --- a/influxdb/tests/dataframe_client_test.py +++ b/influxdb/tests/dataframe_client_test.py @@ -1266,11 +1266,11 @@ def test_query_custom_index(self): "SELECT count(value) FROM cpu_load_short WHERE region=$region" bind_params = {'region': 'us-west'} with _mocked_session(cli, 'GET', 200, data): - result = cli.query(iql, bind_params=bind_params, data_frame_index=["time", "host"]) + result = cli.query(iql, bind_params=bind_params, + data_frame_index=["time", "host"]) _data_frame = result['cpu_load_short'] print(_data_frame) - self.assertListEqual(["time", "host"], list(_data_frame.index.names)) - - + self.assertListEqual(["time", "host"], + list(_data_frame.index.names)) From 64aeddd82358f68b1e5b662d80bfaee6485d0de7 Mon Sep 17 00:00:00 2001 From: Pavlina Rolincova Date: Mon, 1 Jun 2020 11:53:15 +0200 Subject: [PATCH 116/133] Add support for custom indexes for query in the DataFrameClient (#785) --- influxdb/_dataframe_client.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/influxdb/_dataframe_client.py b/influxdb/_dataframe_client.py index afd75b39..e7ae9c17 100644 --- a/influxdb/_dataframe_client.py +++ b/influxdb/_dataframe_client.py @@ -182,7 +182,8 @@ def query(self, containing all results within that chunk :param chunk_size: Size of each chunk to tell InfluxDB to use. :param dropna: drop columns where all values are missing - :param data_frame_index: the list of columns that are used as DataFrame index + :param data_frame_index: the list of columns that + are used as DataFrame index :returns: the queried data :rtype: :class:`~.ResultSet` """ @@ -198,7 +199,8 @@ def query(self, results = super(DataFrameClient, self).query(query, **query_args) if query.strip().upper().startswith("SELECT"): if len(results) > 0: - return self._to_dataframe(results, dropna, data_frame_index=data_frame_index) + return self._to_dataframe(results, dropna, + data_frame_index=data_frame_index) else: return {} else: From 6c45f30d6a7142c17cca0ca7736eb1b693f212e8 Mon Sep 17 00:00:00 2001 From: Pavlina Rolincova Date: Mon, 1 Jun 2020 13:53:30 +0200 Subject: [PATCH 117/133] Add support for custom indexes for query in the DataFrameClient (#785) --- influxdb/influxdb08/client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/influxdb/influxdb08/client.py b/influxdb/influxdb08/client.py index 965a91db..40c58145 100644 --- a/influxdb/influxdb08/client.py +++ b/influxdb/influxdb08/client.py @@ -292,10 +292,10 @@ def write_points(self, data, time_precision='s', *args, **kwargs): :type batch_size: int """ - def list_chunks(l, n): + def list_chunks(data_list, n): """Yield successive n-sized chunks from l.""" - for i in xrange(0, len(l), n): - yield l[i:i + n] + for i in xrange(0, len(data_list), n): + yield data_list[i:i + n] batch_size = kwargs.get('batch_size') if batch_size and batch_size > 0: From 49165bedaab5fad158f742fdf16020b44c92decf Mon Sep 17 00:00:00 2001 From: Pavlina Rolincova Date: Tue, 2 Jun 2020 14:42:29 +0200 Subject: [PATCH 118/133] Add support for custom indexes for query in the DataFrameClient (#785) --- influxdb/client.py | 4 ++-- influxdb/helper.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/influxdb/client.py b/influxdb/client.py index 404e14be..df9ef966 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -860,7 +860,7 @@ def alter_retention_policy(self, name, database=None, query_string = ( "ALTER RETENTION POLICY {0} ON {1}" ).format(quote_ident(name), - quote_ident(database or self._database), shard_duration) + quote_ident(database or self._database)) if duration: query_string += " DURATION {0}".format(duration) if shard_duration: @@ -958,7 +958,7 @@ def drop_user(self, username): :param username: the username to drop :type username: str """ - text = "DROP USER {0}".format(quote_ident(username), method="POST") + text = "DROP USER {0}".format(quote_ident(username)) self.query(text, method="POST") def set_user_password(self, username, password): diff --git a/influxdb/helper.py b/influxdb/helper.py index 74209354..fbf6b65d 100644 --- a/influxdb/helper.py +++ b/influxdb/helper.py @@ -82,7 +82,7 @@ def __new__(cls, *args, **kwargs): allowed_time_precisions = ['h', 'm', 's', 'ms', 'u', 'ns', None] if cls._time_precision not in allowed_time_precisions: raise AttributeError( - 'In {0}, time_precision is set, but invalid use any of {}.' + 'In {0}, time_precision is set, but invalid use any of {1}.' .format(cls.__name__, ','.join(allowed_time_precisions))) cls._retention_policy = getattr(_meta, 'retention_policy', None) From cb3156c1fa9181f142f09e93afccf10355b4e5a1 Mon Sep 17 00:00:00 2001 From: Pavlina Rolincova Date: Tue, 2 Jun 2020 15:11:33 +0200 Subject: [PATCH 119/133] Add support for custom indexes for query in the DataFrameClient (#785) --- influxdb/helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/influxdb/helper.py b/influxdb/helper.py index fbf6b65d..138cf6e8 100644 --- a/influxdb/helper.py +++ b/influxdb/helper.py @@ -82,7 +82,7 @@ def __new__(cls, *args, **kwargs): allowed_time_precisions = ['h', 'm', 's', 'ms', 'u', 'ns', None] if cls._time_precision not in allowed_time_precisions: raise AttributeError( - 'In {0}, time_precision is set, but invalid use any of {1}.' + 'In {}, time_precision is set, but invalid use any of {}.' .format(cls.__name__, ','.join(allowed_time_precisions))) cls._retention_policy = getattr(_meta, 'retention_policy', None) From de5878ae83cbae20ee0a3ec3eba4d0403868c939 Mon Sep 17 00:00:00 2001 From: Yevgen Antymyrov Date: Mon, 15 Jun 2020 16:40:44 +0200 Subject: [PATCH 120/133] Fix #828 "Context manager for InfluxDBClient not working correctly?" by returning a clientt instance from __enter__ --- influxdb/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/influxdb/client.py b/influxdb/client.py index df9ef966..5eea9d1f 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -179,7 +179,7 @@ def __init__(self, def __enter__(self): """Enter function as used by context manager.""" - pass + return self def __exit__(self, _exc_type, _exc_value, _traceback): """Exit function as used by context manager.""" From c49c79adfd8700982d48c01c81e6e2dea539825d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Bedn=C3=A1=C5=99?= Date: Thu, 8 Oct 2020 14:07:58 +0200 Subject: [PATCH 121/133] feat: add support for custom authorization token --- docs/source/examples.rst | 6 +++ examples/tutorial_authorization.py | 32 ++++++++++++++ influxdb/client.py | 3 +- influxdb/tests/client_test.py | 71 ++++++++++++++++++++++++++++++ tox.ini | 2 +- 5 files changed, 112 insertions(+), 2 deletions(-) create mode 100644 examples/tutorial_authorization.py diff --git a/docs/source/examples.rst b/docs/source/examples.rst index fdda62a9..b4ada447 100644 --- a/docs/source/examples.rst +++ b/docs/source/examples.rst @@ -31,3 +31,9 @@ Tutorials - UDP .. literalinclude:: ../../examples/tutorial_udp.py :language: python + +Tutorials - Authorization by Token +=============== + +.. literalinclude:: ../../examples/tutorial_authorization.py + :language: python diff --git a/examples/tutorial_authorization.py b/examples/tutorial_authorization.py new file mode 100644 index 00000000..9d9a800f --- /dev/null +++ b/examples/tutorial_authorization.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +"""Tutorial how to authorize InfluxDB client by custom Authorization token.""" + +import argparse +from influxdb import InfluxDBClient + + +def main(token='my-token'): + """Instantiate a connection to the InfluxDB.""" + client = InfluxDBClient(username=None, password=None, + headers={"Authorization": token}) + + print("Use authorization token: " + token) + + version = client.ping() + print("Successfully connected to InfluxDB: " + version) + pass + + +def parse_args(): + """Parse the args from main.""" + parser = argparse.ArgumentParser( + description='example code to play with InfluxDB') + parser.add_argument('--token', type=str, required=False, + default='my-token', + help='Authorization token for the proxy that is ahead the InfluxDB.') + return parser.parse_args() + + +if __name__ == '__main__': + args = parse_args() + main(token=args.token) diff --git a/influxdb/client.py b/influxdb/client.py index df9ef966..51a64ac3 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -328,10 +328,11 @@ def request(self, url, method='GET', params=None, data=None, stream=False, _try = 0 while retry: try: + auth = (self._username, self._password) response = self._session.request( method=method, url=url, - auth=(self._username, self._password), + auth=auth if None not in auth else None, params=params, data=data, stream=stream, diff --git a/influxdb/tests/client_test.py b/influxdb/tests/client_test.py index fd3c06bb..e511ca9b 100644 --- a/influxdb/tests/client_test.py +++ b/influxdb/tests/client_test.py @@ -1427,6 +1427,77 @@ def test_chunked_response(self): 'values': [['qps'], ['uptime'], ['df'], ['mount']] }]}).__repr__()) + def test_auth_default(self): + """Test auth with default settings.""" + with requests_mock.Mocker() as m: + m.register_uri( + requests_mock.GET, + "http://localhost:8086/ping", + status_code=204, + headers={'X-Influxdb-Version': '1.2.3'} + ) + + cli = InfluxDBClient() + cli.ping() + + self.assertEqual(m.last_request.headers["Authorization"], + "Basic cm9vdDpyb290") + + def test_auth_username_password(self): + """Test auth with custom username and password.""" + with requests_mock.Mocker() as m: + m.register_uri( + requests_mock.GET, + "http://localhost:8086/ping", + status_code=204, + headers={'X-Influxdb-Version': '1.2.3'} + ) + + cli = InfluxDBClient(username='my-username', + password='my-password') + cli.ping() + + self.assertEqual(m.last_request.headers["Authorization"], + "Basic bXktdXNlcm5hbWU6bXktcGFzc3dvcmQ=") + + def test_auth_username_password_none(self): + """Test auth with not defined username or password.""" + with requests_mock.Mocker() as m: + m.register_uri( + requests_mock.GET, + "http://localhost:8086/ping", + status_code=204, + headers={'X-Influxdb-Version': '1.2.3'} + ) + + cli = InfluxDBClient(username=None, password=None) + cli.ping() + self.assertFalse('Authorization' in m.last_request.headers) + + cli = InfluxDBClient(username=None) + cli.ping() + self.assertFalse('Authorization' in m.last_request.headers) + + cli = InfluxDBClient(password=None) + cli.ping() + self.assertFalse('Authorization' in m.last_request.headers) + + def test_auth_token(self): + """Test auth with custom authorization header.""" + with requests_mock.Mocker() as m: + m.register_uri( + requests_mock.GET, + "http://localhost:8086/ping", + status_code=204, + headers={'X-Influxdb-Version': '1.2.3'} + ) + + cli = InfluxDBClient(username=None, password=None, + headers={"Authorization": "my-token"}) + cli.ping() + self.assertEqual(m.last_request.headers["Authorization"], + "my-token") + class FakeClient(InfluxDBClient): """Set up a fake client instance of InfluxDBClient.""" diff --git a/tox.ini b/tox.ini index ff30ebac..1e59b415 100644 --- a/tox.ini +++ b/tox.ini @@ -31,7 +31,7 @@ commands = pydocstyle --count -ve examples influxdb [testenv:coverage] deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt - pandas + pandas==0.24.2 coverage numpy commands = nosetests -v --with-coverage --cover-html --cover-package=influxdb From 7b0367309b118e8be97bd85af4433b25129a4618 Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Wed, 11 Nov 2020 15:03:24 -0600 Subject: [PATCH 122/133] chore(CHANGELOG): update to include v5.3.1 updates --- CHANGELOG.md | 1 + influxdb/__init__.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b374faa..f3e86086 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Changed - Amend retry to avoid sleep after last retry before raising exception (#790 thx @krzysbaranski) - Remove msgpack pinning for requirements (#818 thx @prometheanfire) +- Update support for HTTP headers in the InfluxDBClient (#851 thx @bednar) ### Removed diff --git a/influxdb/__init__.py b/influxdb/__init__.py index 56f2f619..59916c26 100644 --- a/influxdb/__init__.py +++ b/influxdb/__init__.py @@ -18,4 +18,4 @@ ] -__version__ = '5.3.0' +__version__ = '5.3.1' From c3903dda515d4f7efcb8c55250fd8b75c8446034 Mon Sep 17 00:00:00 2001 From: Tim Gates Date: Fri, 1 Jan 2021 06:10:01 +1100 Subject: [PATCH 123/133] docs: fix simple typo, reponse -> response (#873) There is a small typo in influxdb/tests/client_test.py. Should read `response` rather than `reponse`. --- influxdb/tests/client_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/influxdb/tests/client_test.py b/influxdb/tests/client_test.py index e511ca9b..1f9d704a 100644 --- a/influxdb/tests/client_test.py +++ b/influxdb/tests/client_test.py @@ -1398,7 +1398,7 @@ def test_invalid_port_fails(self): InfluxDBClient('host', '80/redir', 'username', 'password') def test_chunked_response(self): - """Test chunked reponse for TestInfluxDBClient object.""" + """Test chunked response for TestInfluxDBClient object.""" example_response = \ u'{"results":[{"statement_id":0,"series":[{"columns":["key"],' \ '"values":[["cpu"],["memory"],["iops"],["network"]],"partial":' \ From cf51d026469252d06d57f65489561fc5b1e337c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Bedn=C3=A1=C5=99?= Date: Fri, 19 Mar 2021 16:04:03 +0100 Subject: [PATCH 124/133] docs: add link to v2 client (#881) --- README.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.rst b/README.rst index a40ed148..01e054e8 100644 --- a/README.rst +++ b/README.rst @@ -17,6 +17,8 @@ InfluxDB-Python InfluxDB-Python is a client for interacting with InfluxDB_. +**Note: This library is for use with InfluxDB 1.x. For connecting to InfluxDB 2.x instances, please use the the** `influxdb-client-python `_ **client.** + Development of this library is maintained by: +-----------+-------------------------------+ From dec2f0587ca0592cb94072d58b5f545ebfe68ba2 Mon Sep 17 00:00:00 2001 From: Robert Hajek Date: Wed, 28 Apr 2021 15:55:50 +0200 Subject: [PATCH 125/133] feat: Add custom socket_options --- influxdb/client.py | 24 +++++++++++++++++++++--- influxdb/tests/client_test.py | 25 +++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/influxdb/client.py b/influxdb/client.py index c413181b..2d0b0ddb 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -20,6 +20,7 @@ import msgpack import requests import requests.exceptions +from requests.adapters import HTTPAdapter from six.moves.urllib.parse import urlparse from influxdb.line_protocol import make_lines, quote_ident, quote_literal @@ -87,6 +88,10 @@ class InfluxDBClient(object): :param headers: headers to add to Requests, will add 'Content-Type' and 'Accept' unless these are already present, defaults to {} :type headers: dict + :param socket_options: use custom tcp socket options, If not specified, then defaults are loaded from + ``HTTPConnection.default_socket_options`` + :type socket_options: list + :raises ValueError: if cert is provided but ssl is disabled (set to False) """ @@ -109,6 +114,7 @@ def __init__(self, gzip=False, session=None, headers=None, + socket_options=None, ): """Construct a new InfluxDBClient object.""" self.__host = host @@ -128,9 +134,10 @@ def __init__(self, session = requests.Session() self._session = session - adapter = requests.adapters.HTTPAdapter( + adapter = SocketOptionsAdapter( pool_connections=int(pool_size), - pool_maxsize=int(pool_size) + pool_maxsize=int(pool_size), + socket_options=socket_options ) if use_udp: @@ -626,7 +633,7 @@ def _batches(iterable, size): # http://code.activestate.com/recipes/303279-getting-items-in-batches/ iterator = iter(iterable) while True: - try: # Try get the first element in the iterator... + try: # Try get the first element in the iterator... head = (next(iterator),) except StopIteration: return # ...so that we can stop if there isn't one @@ -1249,3 +1256,14 @@ def _msgpack_parse_hook(code, data): timestamp += datetime.timedelta(microseconds=(epoch_ns / 1000)) return timestamp.isoformat() + 'Z' return msgpack.ExtType(code, data) + + +class SocketOptionsAdapter(HTTPAdapter): + def __init__(self, *args, **kwargs): + self.socket_options = kwargs.pop("socket_options", None) + super(SocketOptionsAdapter, self).__init__(*args, **kwargs) + + def init_poolmanager(self, *args, **kwargs): + if self.socket_options is not None: + kwargs["socket_options"] = self.socket_options + super(SocketOptionsAdapter, self).init_poolmanager(*args, **kwargs) diff --git a/influxdb/tests/client_test.py b/influxdb/tests/client_test.py index 1f9d704a..6b65249c 100644 --- a/influxdb/tests/client_test.py +++ b/influxdb/tests/client_test.py @@ -33,6 +33,7 @@ import requests_mock from nose.tools import raises +from urllib3.connection import HTTPConnection from influxdb import InfluxDBClient from influxdb.resultset import ResultSet @@ -1498,6 +1499,30 @@ def test_auth_token(self): self.assertEqual(m.last_request.headers["Authorization"], "my-token") + def test_custom_socket_options(self): + test_socket_options = HTTPConnection.default_socket_options + [(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1), + (socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, 60), + (socket.IPPROTO_TCP, socket.TCP_KEEPCNT, 15)] + + cli = InfluxDBClient(username=None, password=None, socket_options=test_socket_options) + + self.assertEquals(cli._session.adapters.get("http://").socket_options, test_socket_options) + self.assertEquals(cli._session.adapters.get("http://").poolmanager.connection_pool_kw.get("socket_options"), + test_socket_options) + + connection_pool = cli._session.adapters.get("http://").poolmanager.connection_from_url( + url="http://localhost:8086") + new_connection = connection_pool._new_conn() + self.assertEquals(new_connection.socket_options, test_socket_options) + + def test_none_socket_options(self): + cli = InfluxDBClient(username=None, password=None) + self.assertEquals(cli._session.adapters.get("http://").socket_options, None) + connection_pool = cli._session.adapters.get("http://").poolmanager.connection_from_url( + url="http://localhost:8086") + new_connection = connection_pool._new_conn() + self.assertEquals(new_connection.socket_options, HTTPConnection.default_socket_options) + class FakeClient(InfluxDBClient): """Set up a fake client instance of InfluxDBClient.""" From 6ba88c064e7fd99df7251409c1b8aad2c88335e2 Mon Sep 17 00:00:00 2001 From: Robert Hajek Date: Thu, 29 Apr 2021 13:30:47 +0200 Subject: [PATCH 126/133] feat: Add custom socket_options --- influxdb/client.py | 13 ++++++++----- influxdb/tests/client_test.py | 30 ++++++++++++++++++++---------- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/influxdb/client.py b/influxdb/client.py index 2d0b0ddb..548c5772 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -88,7 +88,8 @@ class InfluxDBClient(object): :param headers: headers to add to Requests, will add 'Content-Type' and 'Accept' unless these are already present, defaults to {} :type headers: dict - :param socket_options: use custom tcp socket options, If not specified, then defaults are loaded from + :param socket_options: use custom tcp socket options, + If not specified, then defaults are loaded from ``HTTPConnection.default_socket_options`` :type socket_options: list @@ -134,7 +135,7 @@ def __init__(self, session = requests.Session() self._session = session - adapter = SocketOptionsAdapter( + adapter = _SocketOptionsAdapter( pool_connections=int(pool_size), pool_maxsize=int(pool_size), socket_options=socket_options @@ -1258,12 +1259,14 @@ def _msgpack_parse_hook(code, data): return msgpack.ExtType(code, data) -class SocketOptionsAdapter(HTTPAdapter): +class _SocketOptionsAdapter(HTTPAdapter): + """_SocketOptionsAdapter injects socket_options into HTTP Adapter.""" + def __init__(self, *args, **kwargs): self.socket_options = kwargs.pop("socket_options", None) - super(SocketOptionsAdapter, self).__init__(*args, **kwargs) + super(_SocketOptionsAdapter, self).__init__(*args, **kwargs) def init_poolmanager(self, *args, **kwargs): if self.socket_options is not None: kwargs["socket_options"] = self.socket_options - super(SocketOptionsAdapter, self).init_poolmanager(*args, **kwargs) + super(_SocketOptionsAdapter, self).init_poolmanager(*args, **kwargs) diff --git a/influxdb/tests/client_test.py b/influxdb/tests/client_test.py index 6b65249c..115fbc48 100644 --- a/influxdb/tests/client_test.py +++ b/influxdb/tests/client_test.py @@ -1500,28 +1500,38 @@ def test_auth_token(self): "my-token") def test_custom_socket_options(self): - test_socket_options = HTTPConnection.default_socket_options + [(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1), - (socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, 60), - (socket.IPPROTO_TCP, socket.TCP_KEEPCNT, 15)] + """Test custom socket options.""" + test_socket_options = HTTPConnection.default_socket_options + \ + [(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1), + (socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, 60), + (socket.IPPROTO_TCP, socket.TCP_KEEPCNT, 15)] - cli = InfluxDBClient(username=None, password=None, socket_options=test_socket_options) + cli = InfluxDBClient(username=None, password=None, + socket_options=test_socket_options) - self.assertEquals(cli._session.adapters.get("http://").socket_options, test_socket_options) - self.assertEquals(cli._session.adapters.get("http://").poolmanager.connection_pool_kw.get("socket_options"), + self.assertEquals(cli._session.adapters.get("http://").socket_options, + test_socket_options) + self.assertEquals(cli._session.adapters.get("http://").poolmanager. + connection_pool_kw.get("socket_options"), test_socket_options) - connection_pool = cli._session.adapters.get("http://").poolmanager.connection_from_url( + connection_pool = cli._session.adapters.get("http://").poolmanager \ + .connection_from_url( url="http://localhost:8086") new_connection = connection_pool._new_conn() self.assertEquals(new_connection.socket_options, test_socket_options) def test_none_socket_options(self): + """Test default socket options.""" cli = InfluxDBClient(username=None, password=None) - self.assertEquals(cli._session.adapters.get("http://").socket_options, None) - connection_pool = cli._session.adapters.get("http://").poolmanager.connection_from_url( + self.assertEquals(cli._session.adapters.get("http://").socket_options, + None) + connection_pool = cli._session.adapters.get("http://").poolmanager \ + .connection_from_url( url="http://localhost:8086") new_connection = connection_pool._new_conn() - self.assertEquals(new_connection.socket_options, HTTPConnection.default_socket_options) + self.assertEquals(new_connection.socket_options, + HTTPConnection.default_socket_options) class FakeClient(InfluxDBClient): From 7cb565698c88bfbf9f4804650231bd28d09e2e6d Mon Sep 17 00:00:00 2001 From: Grant7z Date: Thu, 9 Sep 2021 08:50:11 +0800 Subject: [PATCH 127/133] Prevent overwriting Authorization with basic auth (#901) --- influxdb/client.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/influxdb/client.py b/influxdb/client.py index 548c5772..adab4edc 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -336,7 +336,10 @@ def request(self, url, method='GET', params=None, data=None, stream=False, _try = 0 while retry: try: - auth = (self._username, self._password) + if "Authorization" in headers: + auth = (None, None) + else: + auth = (self._username, self._password) response = self._session.request( method=method, url=url, From 3d1f1ce32524e40f4db33b9d8f11faf9b4925bf2 Mon Sep 17 00:00:00 2001 From: Jamie Strandboge Date: Tue, 15 Nov 2022 15:25:21 -0600 Subject: [PATCH 128/133] README.md: archive repo --- README.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.rst b/README.rst index 01e054e8..58bfcd1a 100644 --- a/README.rst +++ b/README.rst @@ -1,3 +1,13 @@ +The v1 client libraries for InfluxDB were typically developed and maintained by +community members. They have all now been succeeded by v2 client libraries. +They are being archived it in favor of the v2 client library. See +https://github.com/influxdata/influxdb-python/issues/918. + +If there are still users of this v1 client library, and they or somebody else +are willing to keep them updated with security fixes at a minimum please reach +out on the [Community Forums](https://community.influxdata.com/) or +[InfluxData Slack](https://influxdata.com/slack). + InfluxDB-Python =============== From 1df9a816a93c561779b54bb82472c83ce990282b Mon Sep 17 00:00:00 2001 From: Jamie Strandboge Date: Tue, 18 Jul 2023 16:29:05 -0500 Subject: [PATCH 129/133] chore: update README.rst based on feedback from Product --- README.rst | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index 58bfcd1a..d214d41a 100644 --- a/README.rst +++ b/README.rst @@ -1,12 +1,11 @@ The v1 client libraries for InfluxDB were typically developed and maintained by -community members. They have all now been succeeded by v2 client libraries. -They are being archived it in favor of the v2 client library. See -https://github.com/influxdata/influxdb-python/issues/918. +community members. For InfluxDB 3.0 users, this library is succeeded by the +lightweight `v3 client library `_. If there are still users of this v1 client library, and they or somebody else are willing to keep them updated with security fixes at a minimum please reach -out on the [Community Forums](https://community.influxdata.com/) or -[InfluxData Slack](https://influxdata.com/slack). +out on the `Community Forums `_ or +`InfluxData Slack `_. InfluxDB-Python =============== From b614474e82c621dc77f52d36a9d54b8be2af45c6 Mon Sep 17 00:00:00 2001 From: Josh Powers Date: Tue, 16 Apr 2024 08:29:25 -0600 Subject: [PATCH 130/133] fix: Correctly serialize nanosecond dataframe timestamps Co-authored-by: @bednar --- influxdb/_dataframe_client.py | 4 +- influxdb/tests/dataframe_client_test.py | 74 ++++++++++++++++++++++++- 2 files changed, 75 insertions(+), 3 deletions(-) diff --git a/influxdb/_dataframe_client.py b/influxdb/_dataframe_client.py index e7ae9c17..907db2cb 100644 --- a/influxdb/_dataframe_client.py +++ b/influxdb/_dataframe_client.py @@ -372,10 +372,10 @@ def _convert_dataframe_to_lines(self, # Make array of timestamp ints if isinstance(dataframe.index, pd.PeriodIndex): - time = ((dataframe.index.to_timestamp().values.astype(np.int64) / + time = ((dataframe.index.to_timestamp().values.astype(np.int64) // precision_factor).astype(np.int64).astype(str)) else: - time = ((pd.to_datetime(dataframe.index).values.astype(np.int64) / + time = ((pd.to_datetime(dataframe.index).values.astype(np.int64) // precision_factor).astype(np.int64).astype(str)) # If tag columns exist, make an array of formatted tag keys and values diff --git a/influxdb/tests/dataframe_client_test.py b/influxdb/tests/dataframe_client_test.py index f6db3c22..87b8e0d8 100644 --- a/influxdb/tests/dataframe_client_test.py +++ b/influxdb/tests/dataframe_client_test.py @@ -877,7 +877,7 @@ def test_query_into_dataframe(self): {"measurement": "network", "tags": {"direction": ""}, "columns": ["time", "value"], - "values":[["2009-11-10T23:00:00Z", 23422]] + "values": [["2009-11-10T23:00:00Z", 23422]] }, {"measurement": "network", "tags": {"direction": "in"}, @@ -1274,3 +1274,75 @@ def test_query_custom_index(self): self.assertListEqual(["time", "host"], list(_data_frame.index.names)) + + def test_dataframe_nanosecond_precision(self): + """Test nanosecond precision.""" + for_df_dict = { + "nanFloats": [1.1, float('nan'), 3.3, 4.4], + "onlyFloats": [1.1, 2.2, 3.3, 4.4], + "strings": ['one_one', 'two_two', 'three_three', 'four_four'] + } + df = pd.DataFrame.from_dict(for_df_dict) + df['time'] = ['2019-10-04 06:27:19.850557111+00:00', + '2019-10-04 06:27:19.850557184+00:00', + '2019-10-04 06:27:42.251396864+00:00', + '2019-10-04 06:27:42.251396974+00:00'] + df['time'] = pd.to_datetime(df['time'], unit='ns') + df = df.set_index('time') + + expected = ( + b'foo nanFloats=1.1,onlyFloats=1.1,strings="one_one" 1570170439850557111\n' # noqa E501 line too long + b'foo onlyFloats=2.2,strings="two_two" 1570170439850557184\n' # noqa E501 line too long + b'foo nanFloats=3.3,onlyFloats=3.3,strings="three_three" 1570170462251396864\n' # noqa E501 line too long + b'foo nanFloats=4.4,onlyFloats=4.4,strings="four_four" 1570170462251396974\n' # noqa E501 line too long + ) + + with requests_mock.Mocker() as m: + m.register_uri( + requests_mock.POST, + "http://localhost:8086/write", + status_code=204 + ) + + cli = DataFrameClient(database='db') + cli.write_points(df, 'foo', time_precision='n') + + self.assertEqual(m.last_request.body, expected) + + def test_dataframe_nanosecond_precision_one_microsecond(self): + """Test nanosecond precision within one microsecond.""" + # 1 microsecond = 1000 nanoseconds + start = np.datetime64('2019-10-04T06:27:19.850557000') + end = np.datetime64('2019-10-04T06:27:19.850558000') + + # generate timestamps with nanosecond precision + timestamps = np.arange( + start, + end + np.timedelta64(1, 'ns'), + np.timedelta64(1, 'ns') + ) + # generate values + values = np.arange(0.0, len(timestamps)) + + df = pd.DataFrame({'value': values}, index=timestamps) + with requests_mock.Mocker() as m: + m.register_uri( + requests_mock.POST, + "http://localhost:8086/write", + status_code=204 + ) + + cli = DataFrameClient(database='db') + cli.write_points(df, 'foo', time_precision='n') + + lines = m.last_request.body.decode('utf-8').split('\n') + self.assertEqual(len(lines), 1002) + + for index, line in enumerate(lines): + if index == 1001: + self.assertEqual(line, '') + continue + self.assertEqual( + line, + f"foo value={index}.0 157017043985055{7000 + index:04}" + ) From 5ad04f696c71514967c9d7419bff457dbfbe8400 Mon Sep 17 00:00:00 2001 From: Joshua Powers Date: Tue, 16 Apr 2024 08:35:24 -0600 Subject: [PATCH 131/133] Update README.rst Add link to v2 library --- README.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.rst b/README.rst index d214d41a..b78e626d 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,8 @@ The v1 client libraries for InfluxDB were typically developed and maintained by community members. For InfluxDB 3.0 users, this library is succeeded by the lightweight `v3 client library `_. +For InfluxDB 2.0 users, look at the `v2 client library +`_. If there are still users of this v1 client library, and they or somebody else are willing to keep them updated with security fixes at a minimum please reach From 37ff905fbefe33bc321e619ea970d015ccd8b434 Mon Sep 17 00:00:00 2001 From: Josh Powers Date: Wed, 17 Apr 2024 07:14:43 -0600 Subject: [PATCH 132/133] chore(CHANGELOG): Release v5.3.2 --- CHANGELOG.md | 7 ++++++- influxdb/__init__.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f3e86086..bfd27d38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). -## [v5.3.1] - Unreleased +## [v5.3.2] - 2024-04-17 + +### Changed +- Correctly serialize nanosecond dataframe timestamps (#926) + +## [v5.3.1] - 2022-11-14 ### Added - Add support for custom headers in the InfluxDBClient (#710 thx @nathanielatom) diff --git a/influxdb/__init__.py b/influxdb/__init__.py index 59916c26..e66f80ea 100644 --- a/influxdb/__init__.py +++ b/influxdb/__init__.py @@ -18,4 +18,4 @@ ] -__version__ = '5.3.1' +__version__ = '5.3.2' From bbe80ed32cf57b252be131d3edcda5ca610fc223 Mon Sep 17 00:00:00 2001 From: Jason Stirnaman Date: Tue, 29 Oct 2024 10:54:14 -0500 Subject: [PATCH 133/133] Update docs (#929) * chore(docs): Update version advice and tox build requirements: - Updates the version advice and removes the succession statement to be consistent with the OSS 1.11 release. - Adds explicit relations to v2 and v3. - Removes redundancies. - Fixes formatting errors reported by the linter. - Updates dependencies for docs build. * fix(docs): mention 1.x compat endpoints in 2.x --- README.rst | 78 ++++++++++++++++++++++------------------ docs/source/conf.py | 3 +- docs/source/examples.rst | 2 +- influxdb/client.py | 7 ++-- pyproject.toml | 3 ++ requirements.txt | 4 +-- setup.py | 5 +++ tox.ini | 10 +++--- 8 files changed, 65 insertions(+), 47 deletions(-) create mode 100644 pyproject.toml diff --git a/README.rst b/README.rst index b78e626d..048db045 100644 --- a/README.rst +++ b/README.rst @@ -1,14 +1,3 @@ -The v1 client libraries for InfluxDB were typically developed and maintained by -community members. For InfluxDB 3.0 users, this library is succeeded by the -lightweight `v3 client library `_. -For InfluxDB 2.0 users, look at the `v2 client library -`_. - -If there are still users of this v1 client library, and they or somebody else -are willing to keep them updated with security fixes at a minimum please reach -out on the `Community Forums `_ or -`InfluxData Slack `_. - InfluxDB-Python =============== @@ -26,38 +15,45 @@ InfluxDB-Python :target: https://pypi.python.org/pypi/influxdb :alt: PyPI Status -InfluxDB-Python is a client for interacting with InfluxDB_. -**Note: This library is for use with InfluxDB 1.x. For connecting to InfluxDB 2.x instances, please use the the** `influxdb-client-python `_ **client.** +.. important:: -Development of this library is maintained by: + **This project is no longer in development** + + This v1 client library is for interacting with `InfluxDB 1.x `_ and 1.x-compatible endpoints in `InfluxDB 2.x `_. + Use it to: + + - Write data in line protocol. + - Query data with `InfluxQL `_. -+-----------+-------------------------------+ -| Github ID | URL | -+===========+===============================+ -| @aviau | (https://github.com/aviau) | -+-----------+-------------------------------+ -| @xginn8 | (https://github.com/xginn8) | -+-----------+-------------------------------+ -| @sebito91 | (https://github.com/sebito91) | -+-----------+-------------------------------+ + If you use `InfluxDB 2.x (TSM storage engine) `_ and `Flux `_, see the `v2 client library `_. + + If you use `InfluxDB 3.0 `_, see the `v3 client library `_. + + For new projects, consider using InfluxDB 3.0 and v3 client libraries. + +Description +=========== + +InfluxDB-python, the InfluxDB Python Client (1.x), is a client library for interacting with `InfluxDB 1.x `_ instances. .. _readme-about: -InfluxDB is an open-source distributed time series database, find more about InfluxDB_ at https://docs.influxdata.com/influxdb/latest +`InfluxDB`_ is the time series platform designed to handle high write and query loads. .. _installation: -InfluxDB pre v1.1.0 users -------------------------- -This module is tested with InfluxDB versions: v1.2.4, v1.3.9, v1.4.3, v1.5.4, v1.6.4, and 1.7.4. +For InfluxDB pre-v1.1.0 users +----------------------------- -Those users still on InfluxDB v0.8.x users may still use the legacy client by importing ``from influxdb.influxdb08 import InfluxDBClient``. +This module is tested with InfluxDB versions v1.2.4, v1.3.9, v1.4.3, v1.5.4, v1.6.4, and 1.7.4. -Installation ------------- +Users on InfluxDB v0.8.x may still use the legacy client by importing ``from influxdb.influxdb08 import InfluxDBClient``. + +For InfluxDB v1.1+ users +------------------------ Install, upgrade and uninstall influxdb-python with these commands:: @@ -165,21 +161,33 @@ We are also lurking on the following: Development ----------- +The v1 client libraries for InfluxDB 1.x were typically developed and maintained by InfluxDB community members. If you are an InfluxDB v1 user interested in maintaining this client library (at a minimum, keeping it updated with security patches) please contact the InfluxDB team at on the `Community Forums `_ or +`InfluxData Slack `_. + All development is done on Github_. Use Issues_ to report problems or submit contributions. .. _Github: https://github.com/influxdb/influxdb-python/ .. _Issues: https://github.com/influxdb/influxdb-python/issues -Please note that we WILL get to your questions/issues/concerns as quickly as possible. We maintain many -software repositories and sometimes things may get pushed to the backburner. Please don't take offense, -we will do our best to reply as soon as possible! +Please note that we will answer you question as quickly as possible. +Maintainers: + ++-----------+-------------------------------+ +| Github ID | URL | ++===========+===============================+ +| @aviau | (https://github.com/aviau) | ++-----------+-------------------------------+ +| @xginn8 | (https://github.com/xginn8) | ++-----------+-------------------------------+ +| @sebito91 | (https://github.com/sebito91) | ++-----------+-------------------------------+ Source code ----------- -The source code is currently available on Github: https://github.com/influxdata/influxdb-python +The source code for the InfluxDB Python Client (1.x) is currently available on Github: https://github.com/influxdata/influxdb-python TODO @@ -188,6 +196,6 @@ TODO The TODO/Roadmap can be found in Github bug tracker: https://github.com/influxdata/influxdb-python/issues -.. _InfluxDB: https://influxdata.com/time-series-platform/influxdb/ +.. _InfluxDB: https://influxdata.com/ .. _Sphinx: http://sphinx.pocoo.org/ .. _Tox: https://tox.readthedocs.org diff --git a/docs/source/conf.py b/docs/source/conf.py index 231c776c..efc22f88 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -117,7 +117,8 @@ # Add any paths that contain custom themes here, relative to this directory. #html_theme_path = [] -html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] +# Calling get_html_theme_path is deprecated. +# html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". diff --git a/docs/source/examples.rst b/docs/source/examples.rst index b4ada447..841ad8b1 100644 --- a/docs/source/examples.rst +++ b/docs/source/examples.rst @@ -33,7 +33,7 @@ Tutorials - UDP :language: python Tutorials - Authorization by Token -=============== +================================== .. literalinclude:: ../../examples/tutorial_authorization.py :language: python diff --git a/influxdb/client.py b/influxdb/client.py index adab4edc..c535a3f1 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -395,7 +395,7 @@ def write(self, data, params=None, expected_response_code=204, :param data: the data to be written :type data: (if protocol is 'json') dict (if protocol is 'line') sequence of line protocol strings - or single string + or single string :param params: additional parameters for the request, defaults to None :type params: dict :param expected_response_code: the expected response code of the write @@ -571,8 +571,9 @@ def write_points(self, :param points: the list of points to be written in the database :type points: list of dictionaries, each dictionary represents a point :type points: (if protocol is 'json') list of dicts, where each dict - represents a point. - (if protocol is 'line') sequence of line protocol strings. + represents a point. + (if protocol is 'line') sequence of line protocol strings. + :param time_precision: Either 's', 'm', 'ms' or 'u', defaults to None :type time_precision: str :param database: the database to write the points to. Defaults to diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..1b68d94e --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools>=42", "wheel"] +build-backend = "setuptools.build_meta" \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 548b17c8..a3df3154 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ python-dateutil>=2.6.0 -pytz +pytz>=2016.10 requests>=2.17.0 six>=1.10.0 -msgpack +msgpack>=0.5.0 diff --git a/setup.py b/setup.py index d44875f6..8ac7d1a7 100755 --- a/setup.py +++ b/setup.py @@ -23,6 +23,11 @@ with open('requirements.txt', 'r') as f: requires = [x.strip() for x in f if x.strip()] +# Debugging: Print the requires values +print("install_requires values:") +for req in requires: + print(f"- {req}") + with open('test-requirements.txt', 'r') as f: test_requires = [x.strip() for x in f if x.strip()] diff --git a/tox.ini b/tox.ini index 1e59b415..a1005abb 100644 --- a/tox.ini +++ b/tox.ini @@ -12,8 +12,8 @@ deps = -r{toxinidir}/requirements.txt py35: numpy==1.14.6 py36: pandas==0.23.4 py36: numpy==1.15.4 - py37: pandas==0.24.2 - py37: numpy==1.16.2 + py37: pandas>=0.24.2 + py37: numpy>=1.16.2 # Only install pandas with non-pypy interpreters # Testing all combinations would be too expensive commands = nosetests -v --with-doctest {posargs} @@ -38,9 +38,9 @@ commands = nosetests -v --with-coverage --cover-html --cover-package=influxdb [testenv:docs] deps = -r{toxinidir}/requirements.txt - pandas==0.24.2 - numpy==1.16.2 - Sphinx==1.8.5 + pandas>=0.24.2 + numpy>=1.16.2 + Sphinx>=1.8.5 sphinx_rtd_theme commands = sphinx-build -b html docs/source docs/build