From 04205cf5858520fe2362432090232a871f1860ea Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Wed, 8 Apr 2020 16:09:04 -0500 Subject: [PATCH 01/60] chore(line_protocol): fix nanosecond timestamp resolution for points (#811) --- CHANGELOG.md | 1 + influxdb/line_protocol.py | 20 ++++++++++++++------ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 81809bcd..0556d942 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Clean up stale CI config (#755) - Add legacy client test (#752 & #318 thx @oldmantaiter & @sebito91) - Update make_lines section in line_protocol.py to split out core function (#375 thx @aisbaa) +- Fix nanosecond time resolution for points (#407 thx @AndreCAndersen && @clslgrnc) ### Removed diff --git a/influxdb/line_protocol.py b/influxdb/line_protocol.py index ec59ef47..3a5eb4e8 100644 --- a/influxdb/line_protocol.py +++ b/influxdb/line_protocol.py @@ -16,6 +16,14 @@ EPOCH = UTC.localize(datetime.utcfromtimestamp(0)) +def _to_nanos(timestamp): + delta = timestamp - EPOCH + nanos_in_days = delta.days * 86400 * 10 ** 9 + nanos_in_seconds = delta.seconds * 10 ** 9 + nanos_in_micros = delta.microseconds * 10 ** 3 + return nanos_in_days + nanos_in_seconds + nanos_in_micros + + def _convert_timestamp(timestamp, precision=None): if isinstance(timestamp, Integral): return timestamp # assume precision is correct if timestamp is int @@ -27,24 +35,24 @@ def _convert_timestamp(timestamp, precision=None): if not timestamp.tzinfo: timestamp = UTC.localize(timestamp) - ns = (timestamp - EPOCH).total_seconds() * 1e9 + ns = _to_nanos(timestamp) if precision is None or precision == 'n': return ns if precision == 'u': - return ns / 1e3 + return ns / 10**3 if precision == 'ms': - return ns / 1e6 + return ns / 10**6 if precision == 's': - return ns / 1e9 + return ns / 10**9 if precision == 'm': - return ns / 1e9 / 60 + return ns / 10**9 / 60 if precision == 'h': - return ns / 1e9 / 3600 + return ns / 10**9 / 3600 raise ValueError(timestamp) From a368951aca91c89d978fe29ee0ddc8a246711ea0 Mon Sep 17 00:00:00 2001 From: Sergei Smolianinov Date: Wed, 8 Apr 2020 21:21:12 +0000 Subject: [PATCH 02/60] Add CQs management methods to the client (#414) * Add CQs management methods to the client * chore(server_tests): update pep257 and flake8 commentary * chore(client_test): update comments based on pep257 and flake8 Co-authored-by: Sergei Smolianinov Co-authored-by: Sebastian Borza Co-authored-by: xginn8 --- influxdb/tests/client_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/influxdb/tests/client_test.py b/influxdb/tests/client_test.py index 99a7f42b..fe7381ea 100644 --- a/influxdb/tests/client_test.py +++ b/influxdb/tests/client_test.py @@ -1161,7 +1161,7 @@ def test_revoke_privilege_invalid(self): self.cli.revoke_privilege('', 'testdb', 'test') def test_get_list_privileges(self): - """Tst get list of privs for TestInfluxDBClient object.""" + """Test get list of privs for TestInfluxDBClient object.""" data = {'results': [ {'series': [ {'columns': ['database', 'privilege'], From 516274f9bae9edfc9c6f35c6e56281b29b23b12f Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Wed, 8 Apr 2020 16:22:27 -0500 Subject: [PATCH 03/60] chore(CHANGELOG): add smolse to PR merge for #681 and #414 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0556d942..2a887f90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,7 +34,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Add consistency paramter to `write_points` (#664 tx @RonRothman) - The query() function now accepts a bind_params argument for parameter binding (#678 thx @clslgrnc) - Add `get_list_continuous_queries`, `drop_continuous_query`, and `create_continuous_query` management methods for - continuous queries (#681 thx @lukaszdudek-silvair) + continuous queries (#681 thx @lukaszdudek-silvair && @smolse) - Mutual TLS authentication (#702 thx @LloydW93) ### Changed From 7c858bf2b069061005a4525c5a5252fd5f0b27ad Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Wed, 8 Apr 2020 19:22:31 -0500 Subject: [PATCH 04/60] feat(dataframe_client): handle np.nan, np.inf values in DataFrameClient (#812) * feat(dataframe_client): handle np.nan, np.inf values in DataFrameClient * chore(dataframe): handle cases where tagset is empty * chore(dataframe): add tests for Nan lines but with tag values --- CHANGELOG.md | 1 + influxdb/_dataframe_client.py | 40 +++++--- influxdb/tests/dataframe_client_test.py | 122 +++++++++++++++++++++++- 3 files changed, 147 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a887f90..be15d52b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Add support for messagepack (#734 thx @lovasoa) - Add support for 'show series' (#357 thx @gaker) - Add support for custom request session in InfluxDBClient (#360 thx @dschien) +- Add support for handling np.nan and np.inf values in DataFrameClient (#436 thx @nmerket) ### Changed - Clean up stale CI config (#755) diff --git a/influxdb/_dataframe_client.py b/influxdb/_dataframe_client.py index d16e29ca..f411bb37 100644 --- a/influxdb/_dataframe_client.py +++ b/influxdb/_dataframe_client.py @@ -270,14 +270,31 @@ def _convert_dataframe_to_json(dataframe, "h": 1e9 * 3600, }.get(time_precision, 1) + if not tag_columns: + points = [ + {'measurement': measurement, + 'fields': + rec.replace([np.inf, -np.inf], np.nan).dropna().to_dict(), + 'time': np.int64(ts.value / precision_factor)} + for ts, (_, rec) in zip( + dataframe.index, + dataframe[field_columns].iterrows() + ) + ] + + return points + points = [ {'measurement': measurement, 'tags': dict(list(tag.items()) + list(tags.items())), - 'fields': rec, + 'fields': + rec.replace([np.inf, -np.inf], np.nan).dropna().to_dict(), 'time': np.int64(ts.value / precision_factor)} - for ts, tag, rec in zip(dataframe.index, - dataframe[tag_columns].to_dict('record'), - dataframe[field_columns].to_dict('record')) + for ts, tag, (_, rec) in zip( + dataframe.index, + dataframe[tag_columns].to_dict('record'), + dataframe[field_columns].iterrows() + ) ] return points @@ -379,21 +396,18 @@ def _convert_dataframe_to_lines(self, tags = '' # Make an array of formatted field keys and values - field_df = dataframe[field_columns] - # Keep the positions where Null values are found - mask_null = field_df.isnull().values + field_df = dataframe[field_columns].replace([np.inf, -np.inf], np.nan) + nans = pd.isnull(field_df) field_df = self._stringify_dataframe(field_df, numeric_precision, datatype='field') field_df = (field_df.columns.values + '=').tolist() + field_df - field_df[field_df.columns[1:]] = ',' + field_df[ - field_df.columns[1:]] - field_df = field_df.where(~mask_null, '') # drop Null entries - fields = field_df.sum(axis=1) - # take out leading , where first column has a Null value - fields = fields.str.lstrip(",") + field_df[field_df.columns[1:]] = ',' + field_df[field_df.columns[1:]] + field_df[nans] = '' + + fields = field_df.sum(axis=1).map(lambda x: x.lstrip(',')) del field_df # Generate line protocol string diff --git a/influxdb/tests/dataframe_client_test.py b/influxdb/tests/dataframe_client_test.py index 90312ed8..0573d5c3 100644 --- a/influxdb/tests/dataframe_client_test.py +++ b/influxdb/tests/dataframe_client_test.py @@ -13,8 +13,8 @@ import warnings import requests_mock -from influxdb.tests import skip_if_pypy, using_pypy from nose.tools import raises +from influxdb.tests import skip_if_pypy, using_pypy from .client_test import _mocked_session @@ -22,7 +22,7 @@ import pandas as pd from pandas.util.testing import assert_frame_equal from influxdb import DataFrameClient - import numpy + import numpy as np @skip_if_pypy @@ -462,7 +462,7 @@ def test_write_points_from_dataframe_with_numeric_precision(self): ["2", 2, 2.2222222222222]], index=[now, now + timedelta(hours=1)]) - if numpy.lib.NumpyVersion(numpy.__version__) <= '1.13.3': + if np.lib.NumpyVersion(np.__version__) <= '1.13.3': expected_default_precision = ( b'foo,hello=there 0=\"1\",1=1i,2=1.11111111111 0\n' b'foo,hello=there 0=\"2\",1=2i,2=2.22222222222 3600000000000\n' @@ -1032,3 +1032,119 @@ def test_dsn_constructor(self): client = DataFrameClient.from_dsn('influxdb://localhost:8086') self.assertIsInstance(client, DataFrameClient) self.assertEqual('http://localhost:8086', client._baseurl) + + def test_write_points_from_dataframe_with_nan_line(self): + """Test write points from dataframe with Nan lines.""" + now = pd.Timestamp('1970-01-01 00:00+00:00') + dataframe = pd.DataFrame(data=[["1", 1, np.inf], ["2", 2, np.nan]], + index=[now, now + timedelta(hours=1)], + columns=["column_one", "column_two", + "column_three"]) + expected = ( + b"foo column_one=\"1\",column_two=1i 0\n" + b"foo column_one=\"2\",column_two=2i " + b"3600000000000\n" + ) + + with requests_mock.Mocker() as m: + m.register_uri(requests_mock.POST, + "http://localhost:8086/write", + status_code=204) + + cli = DataFrameClient(database='db') + + cli.write_points(dataframe, 'foo', protocol='line') + self.assertEqual(m.last_request.body, expected) + + cli.write_points(dataframe, 'foo', tags=None, protocol='line') + self.assertEqual(m.last_request.body, expected) + + def test_write_points_from_dataframe_with_nan_json(self): + """Test write points from json with NaN lines.""" + now = pd.Timestamp('1970-01-01 00:00+00:00') + dataframe = pd.DataFrame(data=[["1", 1, np.inf], ["2", 2, np.nan]], + index=[now, now + timedelta(hours=1)], + columns=["column_one", "column_two", + "column_three"]) + expected = ( + b"foo column_one=\"1\",column_two=1i 0\n" + b"foo column_one=\"2\",column_two=2i " + b"3600000000000\n" + ) + + with requests_mock.Mocker() as m: + m.register_uri(requests_mock.POST, + "http://localhost:8086/write", + status_code=204) + + cli = DataFrameClient(database='db') + + cli.write_points(dataframe, 'foo', protocol='json') + self.assertEqual(m.last_request.body, expected) + + cli.write_points(dataframe, 'foo', tags=None, protocol='json') + self.assertEqual(m.last_request.body, expected) + + def test_write_points_from_dataframe_with_tags_and_nan_line(self): + """Test write points from dataframe with NaN lines and tags.""" + now = pd.Timestamp('1970-01-01 00:00+00:00') + dataframe = pd.DataFrame(data=[['blue', 1, "1", 1, np.inf], + ['red', 0, "2", 2, np.nan]], + index=[now, now + timedelta(hours=1)], + columns=["tag_one", "tag_two", "column_one", + "column_two", "column_three"]) + expected = ( + b"foo,tag_one=blue,tag_two=1 " + b"column_one=\"1\",column_two=1i " + b"0\n" + b"foo,tag_one=red,tag_two=0 " + b"column_one=\"2\",column_two=2i " + b"3600000000000\n" + ) + + with requests_mock.Mocker() as m: + m.register_uri(requests_mock.POST, + "http://localhost:8086/write", + status_code=204) + + cli = DataFrameClient(database='db') + + cli.write_points(dataframe, 'foo', protocol='line', + tag_columns=['tag_one', 'tag_two']) + self.assertEqual(m.last_request.body, expected) + + cli.write_points(dataframe, 'foo', tags=None, protocol='line', + tag_columns=['tag_one', 'tag_two']) + self.assertEqual(m.last_request.body, expected) + + def test_write_points_from_dataframe_with_tags_and_nan_json(self): + """Test write points from json with NaN lines and tags.""" + now = pd.Timestamp('1970-01-01 00:00+00:00') + dataframe = pd.DataFrame(data=[['blue', 1, "1", 1, np.inf], + ['red', 0, "2", 2, np.nan]], + index=[now, now + timedelta(hours=1)], + columns=["tag_one", "tag_two", "column_one", + "column_two", "column_three"]) + expected = ( + b"foo,tag_one=blue,tag_two=1 " + b"column_one=\"1\",column_two=1i " + b"0\n" + b"foo,tag_one=red,tag_two=0 " + b"column_one=\"2\",column_two=2i " + b"3600000000000\n" + ) + + with requests_mock.Mocker() as m: + m.register_uri(requests_mock.POST, + "http://localhost:8086/write", + status_code=204) + + cli = DataFrameClient(database='db') + + cli.write_points(dataframe, 'foo', protocol='json', + tag_columns=['tag_one', 'tag_two']) + self.assertEqual(m.last_request.body, expected) + + cli.write_points(dataframe, 'foo', tags=None, protocol='json', + tag_columns=['tag_one', 'tag_two']) + self.assertEqual(m.last_request.body, expected) From 9b4a51ebc811bce92f66123ef17f46f10c8c943f Mon Sep 17 00:00:00 2001 From: Christopher Head Date: Thu, 9 Apr 2020 15:33:11 -0700 Subject: [PATCH 05/60] Fix import of distutils.spawn (#805) --- influxdb/tests/server_tests/influxdb_instance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/influxdb/tests/server_tests/influxdb_instance.py b/influxdb/tests/server_tests/influxdb_instance.py index 1dcd7567..2dd823ff 100644 --- a/influxdb/tests/server_tests/influxdb_instance.py +++ b/influxdb/tests/server_tests/influxdb_instance.py @@ -7,7 +7,7 @@ from __future__ import unicode_literals import datetime -import distutils +import distutils.spawn import os import tempfile import shutil From 57c14083215ed2012dc3eb755869544c30f4f809 Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Thu, 9 Apr 2020 17:34:33 -0500 Subject: [PATCH 06/60] chore(CHANGELOG): update to include PR from #805 --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index be15d52b..44f7f4d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Changed -## [v5.2.4] - 2020-04-10 +## [v5.3.0] - 2020-04-10 ### Added - Add mypy testing framework (#756) @@ -24,6 +24,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Add legacy client test (#752 & #318 thx @oldmantaiter & @sebito91) - Update make_lines section in line_protocol.py to split out core function (#375 thx @aisbaa) - Fix nanosecond time resolution for points (#407 thx @AndreCAndersen && @clslgrnc) +- Fix import of distutils.spawn (#805 thx @Hawk777) ### Removed From 896f6237f9217354fd365b7157bb4dd01af633a3 Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Thu, 9 Apr 2020 19:05:34 -0500 Subject: [PATCH 07/60] chore(line_protocol): update repr value of floats to properly handle precision (#813) * chore(line_protocol): update repr value of floats to properly handle precision. Closes #488 * chore(line_protocol): fix repr and handle boolean values * chore(CHANGELOG): update to include reference to PR#488 --- CHANGELOG.md | 1 + influxdb/line_protocol.py | 5 ++++- influxdb/tests/test_line_protocol.py | 23 +++++++++++++++++++++-- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 44f7f4d1..943fb83f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Update make_lines section in line_protocol.py to split out core function (#375 thx @aisbaa) - Fix nanosecond time resolution for points (#407 thx @AndreCAndersen && @clslgrnc) - Fix import of distutils.spawn (#805 thx @Hawk777) +- Update repr of float values including properly handling of boolean (#488 thx @ghost) ### Removed diff --git a/influxdb/line_protocol.py b/influxdb/line_protocol.py index 3a5eb4e8..d6cbf46f 100644 --- a/influxdb/line_protocol.py +++ b/influxdb/line_protocol.py @@ -112,8 +112,11 @@ def _escape_value(value): if isinstance(value, integer_types) and not isinstance(value, bool): return str(value) + 'i' + if isinstance(value, bool): + return str(value) + if _is_float(value): - return repr(value) + return repr(float(value)) return str(value) diff --git a/influxdb/tests/test_line_protocol.py b/influxdb/tests/test_line_protocol.py index 71828f62..c48d5edc 100644 --- a/influxdb/tests/test_line_protocol.py +++ b/influxdb/tests/test_line_protocol.py @@ -6,10 +6,12 @@ from __future__ import print_function from __future__ import unicode_literals -from datetime import datetime import unittest -from pytz import UTC, timezone +from datetime import datetime +from decimal import Decimal + +from pytz import UTC, timezone from influxdb import line_protocol @@ -166,3 +168,20 @@ def test_float_with_long_decimal_fraction(self): line_protocol.make_lines(data), 'test float_val=1.0000000000000009\n' ) + + def test_float_with_long_decimal_fraction_as_type_decimal(self): + """Ensure precision is preserved when casting Decimal into strings.""" + data = { + "points": [ + { + "measurement": "test", + "fields": { + "float_val": Decimal(0.8289445733333332), + } + } + ] + } + self.assertEqual( + line_protocol.make_lines(data), + 'test float_val=0.8289445733333332\n' + ) From 42d172d1db5eab8d881325d24404aa240bf342ac Mon Sep 17 00:00:00 2001 From: Kenzyme Le Date: Thu, 9 Apr 2020 22:15:05 -0400 Subject: [PATCH 08/60] Fix tests for existing 'Adding time_precision optional option to SeriesHelper' PR (#719) * Adding time_precision into Meta of SeriesHelper time_precision option not currently supported in influx-db python * Making Time Precision optional Changed it from required to optional * Fixing Typo in _time_precision Attribute * Fixing coding conventions for Travis CI * Appunni: Test for invalid time_precision on SeriesHelper * Appunni: Intendation problem resolution in master * - Fix flake7 errors : E131 continuation line unaligned for hanging indent - Fix typo - Fix cls._client declaration ordering - Remove duplicate code cls._autocommit ... Co-authored-by: appunni-dishq <31534711+appunni-dishq@users.noreply.github.com> Co-authored-by: xginn8 Co-authored-by: appunni --- influxdb/helper.py | 20 +++++++++++++++++--- influxdb/tests/helper_test.py | 13 ++++++++++++- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/influxdb/helper.py b/influxdb/helper.py index e622526d..fa79c079 100644 --- a/influxdb/helper.py +++ b/influxdb/helper.py @@ -41,6 +41,10 @@ class Meta: # Only applicable if autocommit is True. autocommit = True # If True and no bulk_size, then will set bulk_size to 1. + time_precision = "h"|"m"|s"|"ms"|"u"|"ns" + # Default is ns (nanoseconds) + # Setting time precision while writing point + # You should also make sure time is set in the given precision """ @@ -71,6 +75,13 @@ def __new__(cls, *args, **kwargs): cls.__name__)) cls._autocommit = getattr(_meta, 'autocommit', False) + cls._time_precision = getattr(_meta, 'time_precision', None) + + allowed_time_precisions = ['h', 'm', 's', 'ms', 'u', 'ns', None] + if cls._time_precision not in allowed_time_precisions: + raise AttributeError( + 'In {0}, time_precision is set, but invalid use any of {}.' + .format(cls.__name__, ','.join(allowed_time_precisions))) cls._client = getattr(_meta, 'client', None) if cls._autocommit and not cls._client: @@ -116,11 +127,11 @@ def __init__(self, **kw): keys = set(kw.keys()) # all tags should be passed, and keys - tags should be a subset of keys - if not(tags <= keys): + if not (tags <= keys): raise NameError( 'Expected arguments to contain all tags {0}, instead got {1}.' .format(cls._tags, kw.keys())) - if not(keys - tags <= fields): + if not (keys - tags <= fields): raise NameError('Got arguments not in tags or fields: {0}' .format(keys - tags - fields)) @@ -143,7 +154,10 @@ def commit(cls, client=None): """ if not client: client = cls._client - rtn = client.write_points(cls._json_body_()) + rtn = client.write_points( + cls._json_body_(), + time_precision=cls._time_precision) + # will be None if not set and will default to ns cls._reset_() return rtn diff --git a/influxdb/tests/helper_test.py b/influxdb/tests/helper_test.py index 6f24e85d..6aa8f15a 100644 --- a/influxdb/tests/helper_test.py +++ b/influxdb/tests/helper_test.py @@ -310,8 +310,19 @@ class Meta: series_name = 'events.stats.{server_name}' + class InvalidTimePrecision(SeriesHelper): + """Define instance of SeriesHelper for invalid time precision.""" + + class Meta: + """Define metadata for InvalidTimePrecision.""" + + series_name = 'events.stats.{server_name}' + time_precision = "ks" + fields = ['time', 'server_name'] + autocommit = True + for cls in [MissingMeta, MissingClient, MissingFields, - MissingSeriesName]: + MissingSeriesName, InvalidTimePrecision]: self.assertRaises( AttributeError, cls, **{'time': 159, 'server_name': 'us.east-1'}) From 8a8b8ff3c7d517959146c9daeb09c982a9c184b1 Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Thu, 9 Apr 2020 21:27:23 -0500 Subject: [PATCH 09/60] chore(CHANGELOG): update for PR#502 and #719 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 943fb83f..b94e438e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Add support for 'show series' (#357 thx @gaker) - Add support for custom request session in InfluxDBClient (#360 thx @dschien) - Add support for handling np.nan and np.inf values in DataFrameClient (#436 thx @nmerket) +- Add support for optional `time_precision` in the SeriesHelper (#502 && #719 thx @appunni-dishq && @klDen) ### Changed - Clean up stale CI config (#755) From 72c372f2412d445ea205e9b3cc696e242a90e35e Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Fri, 10 Apr 2020 08:59:46 -0500 Subject: [PATCH 10/60] chore(dataframe_client): update to handle empty tags in dataframe client (#814) --- CHANGELOG.md | 1 + influxdb/_dataframe_client.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b94e438e..e7f40dc7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Fix nanosecond time resolution for points (#407 thx @AndreCAndersen && @clslgrnc) - Fix import of distutils.spawn (#805 thx @Hawk777) - Update repr of float values including properly handling of boolean (#488 thx @ghost) +- Update dataframe_client to fix faulty empty tags (#770 thx @michelfripiat) ### Removed diff --git a/influxdb/_dataframe_client.py b/influxdb/_dataframe_client.py index f411bb37..a977754e 100644 --- a/influxdb/_dataframe_client.py +++ b/influxdb/_dataframe_client.py @@ -388,7 +388,8 @@ def _convert_dataframe_to_lines(self, del tag_df elif global_tags: tag_string = ''.join( - [",{}={}".format(k, _escape_tag(v)) if v else '' + [",{}={}".format(k, _escape_tag(v)) + if v not in [None, ''] else "" for k, v in sorted(global_tags.items())] ) tags = pd.Series(tag_string, index=dataframe.index) From c35e49ef6dd919c87dc20a01003922818bb30096 Mon Sep 17 00:00:00 2001 From: jgspiro <45625897+jgspiro@users.noreply.github.com> Date: Fri, 10 Apr 2020 16:07:18 +0200 Subject: [PATCH 11/60] Bugfix dropna in DataFrameClient. Add unit test. (#778) --- influxdb/_dataframe_client.py | 3 +- influxdb/tests/dataframe_client_test.py | 92 +++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 1 deletion(-) diff --git a/influxdb/_dataframe_client.py b/influxdb/_dataframe_client.py index a977754e..d7e67baa 100644 --- a/influxdb/_dataframe_client.py +++ b/influxdb/_dataframe_client.py @@ -203,7 +203,8 @@ def query(self, def _to_dataframe(self, rs, dropna=True): result = defaultdict(list) if isinstance(rs, list): - return map(self._to_dataframe, rs) + return map(self._to_dataframe, rs, + [dropna for _ in range(len(rs))]) for key, data in rs.items(): name, tags = key diff --git a/influxdb/tests/dataframe_client_test.py b/influxdb/tests/dataframe_client_test.py index 0573d5c3..4e172ea7 100644 --- a/influxdb/tests/dataframe_client_test.py +++ b/influxdb/tests/dataframe_client_test.py @@ -968,6 +968,98 @@ def test_multiquery_into_dataframe(self): for k in e: assert_frame_equal(e[k], r[k]) + def test_multiquery_into_dataframe_dropna(self): + """Test multiquery into df for TestDataFrameClient object.""" + data = { + "results": [ + { + "series": [ + { + "name": "cpu_load_short", + "columns": ["time", "value", "value2", "value3"], + "values": [ + ["2015-01-29T21:55:43.702900257Z", + 0.55, 0.254, numpy.NaN], + ["2015-01-29T21:55:43.702900257Z", + 23422, 122878, numpy.NaN], + ["2015-06-11T20:46:02Z", + 0.64, 0.5434, numpy.NaN] + ] + } + ] + }, { + "series": [ + { + "name": "cpu_load_short", + "columns": ["time", "count"], + "values": [ + ["1970-01-01T00:00:00Z", 3] + ] + } + ] + } + ] + } + + pd1 = pd.DataFrame( + [[0.55, 0.254, numpy.NaN], + [23422.0, 122878, numpy.NaN], + [0.64, 0.5434, numpy.NaN]], + columns=['value', 'value2', 'value3'], + index=pd.to_datetime([ + "2015-01-29 21:55:43.702900257+0000", + "2015-01-29 21:55:43.702900257+0000", + "2015-06-11 20:46:02+0000"])) + + if pd1.index.tzinfo is None: + pd1.index = pd1.index.tz_localize('UTC') + + pd1_dropna = pd.DataFrame( + [[0.55, 0.254], [23422.0, 122878], [0.64, 0.5434]], + columns=['value', 'value2'], + index=pd.to_datetime([ + "2015-01-29 21:55:43.702900257+0000", + "2015-01-29 21:55:43.702900257+0000", + "2015-06-11 20:46:02+0000"])) + + if pd1_dropna.index.tzinfo is None: + pd1_dropna.index = pd1_dropna.index.tz_localize('UTC') + + pd2 = pd.DataFrame( + [[3]], columns=['count'], + index=pd.to_datetime(["1970-01-01 00:00:00+00:00"])) + + if pd2.index.tzinfo is None: + pd2.index = pd2.index.tz_localize('UTC') + + expected_dropna_true = [ + {'cpu_load_short': pd1_dropna}, + {'cpu_load_short': pd2}] + expected_dropna_false = [ + {'cpu_load_short': pd1}, + {'cpu_load_short': pd2}] + + cli = DataFrameClient('host', 8086, 'username', 'password', 'db') + iql = "SELECT value FROM cpu_load_short WHERE region=$region;" \ + "SELECT count(value) FROM cpu_load_short WHERE region=$region" + bind_params = {'region': 'us-west'} + + for dropna in [True, False]: + with _mocked_session(cli, 'GET', 200, data): + result = cli.query(iql, bind_params=bind_params, dropna=dropna) + expected = \ + expected_dropna_true if dropna else expected_dropna_false + for r, e in zip(result, expected): + for k in e: + assert_frame_equal(e[k], r[k]) + + # test default value (dropna = True) + with _mocked_session(cli, 'GET', 200, data): + result = cli.query(iql, bind_params=bind_params) + for r, e in zip(result, expected_dropna_true): + for k in e: + assert_frame_equal(e[k], r[k]) + def test_query_with_empty_result(self): """Test query with empty results in TestDataFrameClient object.""" cli = DataFrameClient('host', 8086, 'username', 'password', 'db') From d85ecee3aa132980e11005a3ba8a248ccac6965b Mon Sep 17 00:00:00 2001 From: testforvln <1694611+testforvln@users.noreply.github.com> Date: Fri, 10 Apr 2020 22:21:24 +0800 Subject: [PATCH 12/60] fix bug in _convert_dataframe_to_json function that need not transform index if data type of index is already DatatimeIndex (#623) --- influxdb/_dataframe_client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/influxdb/_dataframe_client.py b/influxdb/_dataframe_client.py index d7e67baa..600bc1ec 100644 --- a/influxdb/_dataframe_client.py +++ b/influxdb/_dataframe_client.py @@ -252,7 +252,8 @@ def _convert_dataframe_to_json(dataframe, field_columns = list( set(dataframe.columns).difference(set(tag_columns))) - dataframe.index = pd.to_datetime(dataframe.index) + if not isinstance(dataframe.index, pd.DatetimeIndex): + dataframe.index = pd.to_datetime(dataframe.index) if dataframe.index.tzinfo is None: dataframe.index = dataframe.index.tz_localize('UTC') From ed975b4b5ab754d09c804bd69a26c6bddc160fa1 Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Fri, 10 Apr 2020 09:22:36 -0500 Subject: [PATCH 13/60] chore(CHANGELOG): update with more recent PR merges --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e7f40dc7..11f47e38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,7 +27,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Fix nanosecond time resolution for points (#407 thx @AndreCAndersen && @clslgrnc) - Fix import of distutils.spawn (#805 thx @Hawk777) - Update repr of float values including properly handling of boolean (#488 thx @ghost) -- Update dataframe_client to fix faulty empty tags (#770 thx @michelfripiat) +- Update DataFrameClient to fix faulty empty tags (#770 thx @michelfripiat) +- Update DataFrameClient to properly return `dropna` values (#778 thx @jgspiro) +- Update DataFrameClient to test for pd.DataTimeIndex before blind conversion (#623 thx @testforvin) ### Removed From 5929fef5df91f49df7981688df832958d9fa7e89 Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Fri, 10 Apr 2020 09:41:52 -0500 Subject: [PATCH 14/60] chore(dataframe_client_test): fix incorrect package naming convention --- influxdb/tests/dataframe_client_test.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/influxdb/tests/dataframe_client_test.py b/influxdb/tests/dataframe_client_test.py index 4e172ea7..a80498f3 100644 --- a/influxdb/tests/dataframe_client_test.py +++ b/influxdb/tests/dataframe_client_test.py @@ -979,11 +979,11 @@ def test_multiquery_into_dataframe_dropna(self): "columns": ["time", "value", "value2", "value3"], "values": [ ["2015-01-29T21:55:43.702900257Z", - 0.55, 0.254, numpy.NaN], + 0.55, 0.254, np.NaN], ["2015-01-29T21:55:43.702900257Z", - 23422, 122878, numpy.NaN], + 23422, 122878, np.NaN], ["2015-06-11T20:46:02Z", - 0.64, 0.5434, numpy.NaN] + 0.64, 0.5434, np.NaN] ] } ] @@ -1002,9 +1002,9 @@ def test_multiquery_into_dataframe_dropna(self): } pd1 = pd.DataFrame( - [[0.55, 0.254, numpy.NaN], - [23422.0, 122878, numpy.NaN], - [0.64, 0.5434, numpy.NaN]], + [[0.55, 0.254, np.NaN], + [23422.0, 122878, np.NaN], + [0.64, 0.5434, np.NaN]], columns=['value', 'value2', 'value3'], index=pd.to_datetime([ "2015-01-29 21:55:43.702900257+0000", From aad8349db4638696fbbc3d253ca353b5f7a31759 Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Fri, 10 Apr 2020 10:50:15 -0500 Subject: [PATCH 15/60] chore(client): ensure UDP port is actually an int. Closes #651. (#815) * chore(client): ensure UDP port is actually an int. Closes #651. * chore(CHANGELOG): update to include PR#651 --- CHANGELOG.md | 1 + influxdb/client.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 11f47e38..31e36680 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Update DataFrameClient to fix faulty empty tags (#770 thx @michelfripiat) - Update DataFrameClient to properly return `dropna` values (#778 thx @jgspiro) - Update DataFrameClient to test for pd.DataTimeIndex before blind conversion (#623 thx @testforvin) +- Update client to type-set UDP port to int (#651 thx @yifeikong) ### Removed diff --git a/influxdb/client.py b/influxdb/client.py index 390d8e16..0f0350c8 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -107,7 +107,7 @@ def __init__(self, self._verify_ssl = verify_ssl self.__use_udp = use_udp - self.__udp_port = udp_port + self.__udp_port = int(udp_port) if not session: session = requests.Session() From d2bf36c5be1cafb48f11e095f2a269901085dcf7 Mon Sep 17 00:00:00 2001 From: Cesar Sanz Date: Fri, 10 Apr 2020 18:28:53 +0200 Subject: [PATCH 16/60] Specify Retention Policy in SeriesHelper (#723) * Allow specify a retention policy in SeriesHelper * Complete the annotation example with the retention policy * Fix formatting * Fix formatting again * Add helper write with retention policy * Add helper write without retention policy * Remove blank line after the docstring. Co-authored-by: Sebastian Borza --- influxdb/helper.py | 8 +++++- influxdb/tests/helper_test.py | 51 +++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/influxdb/helper.py b/influxdb/helper.py index fa79c079..f49f40ad 100644 --- a/influxdb/helper.py +++ b/influxdb/helper.py @@ -41,6 +41,8 @@ class Meta: # Only applicable if autocommit is True. autocommit = True # If True and no bulk_size, then will set bulk_size to 1. + retention_policy = 'your_retention_policy' + # Specify the retention policy for the data points time_precision = "h"|"m"|s"|"ms"|"u"|"ns" # Default is ns (nanoseconds) # Setting time precision while writing point @@ -83,6 +85,8 @@ def __new__(cls, *args, **kwargs): 'In {0}, time_precision is set, but invalid use any of {}.' .format(cls.__name__, ','.join(allowed_time_precisions))) + cls._retention_policy = getattr(_meta, 'retention_policy', None) + cls._client = getattr(_meta, 'client', None) if cls._autocommit and not cls._client: raise AttributeError( @@ -154,9 +158,11 @@ def commit(cls, client=None): """ if not client: client = cls._client + rtn = client.write_points( cls._json_body_(), - time_precision=cls._time_precision) + time_precision=cls._time_precision, + retention_policy=cls._retention_policy) # will be None if not set and will default to ns cls._reset_() return rtn diff --git a/influxdb/tests/helper_test.py b/influxdb/tests/helper_test.py index 6aa8f15a..16924936 100644 --- a/influxdb/tests/helper_test.py +++ b/influxdb/tests/helper_test.py @@ -376,3 +376,54 @@ class Meta: .format(WarnBulkSizeNoEffect)) self.assertIn('has no affect', str(w[-1].message), 'Warning message did not contain "has not affect".') + + def testSeriesWithRetentionPolicy(self): + """Test that the data is saved with the specified retention policy.""" + my_policy = 'my_policy' + + class RetentionPolicySeriesHelper(SeriesHelper): + + class Meta: + client = InfluxDBClient() + series_name = 'events.stats.{server_name}' + fields = ['some_stat', 'time'] + tags = ['server_name', 'other_tag'] + bulk_size = 2 + autocommit = True + retention_policy = my_policy + + fake_write_points = mock.MagicMock() + RetentionPolicySeriesHelper( + server_name='us.east-1', some_stat=159, other_tag='gg') + RetentionPolicySeriesHelper._client.write_points = fake_write_points + RetentionPolicySeriesHelper( + server_name='us.east-1', some_stat=158, other_tag='aa') + + kall = fake_write_points.call_args + args, kwargs = kall + self.assertTrue('retention_policy' in kwargs) + self.assertEqual(kwargs['retention_policy'], my_policy) + + def testSeriesWithoutRetentionPolicy(self): + """Test that the data is saved without any retention policy.""" + class NoRetentionPolicySeriesHelper(SeriesHelper): + + class Meta: + client = InfluxDBClient() + series_name = 'events.stats.{server_name}' + fields = ['some_stat', 'time'] + tags = ['server_name', 'other_tag'] + bulk_size = 2 + autocommit = True + + fake_write_points = mock.MagicMock() + NoRetentionPolicySeriesHelper( + server_name='us.east-1', some_stat=159, other_tag='gg') + NoRetentionPolicySeriesHelper._client.write_points = fake_write_points + NoRetentionPolicySeriesHelper( + server_name='us.east-1', some_stat=158, other_tag='aa') + + kall = fake_write_points.call_args + args, kwargs = kall + self.assertTrue('retention_policy' in kwargs) + self.assertEqual(kwargs['retention_policy'], None) From 72c18b899db03432bca37d08c3f9efeb0875d30d Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Fri, 10 Apr 2020 11:29:58 -0500 Subject: [PATCH 17/60] chore(CHANGELOG): update for PR#723 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 31e36680..f2fad368 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Add support for custom request session in InfluxDBClient (#360 thx @dschien) - Add support for handling np.nan and np.inf values in DataFrameClient (#436 thx @nmerket) - Add support for optional `time_precision` in the SeriesHelper (#502 && #719 thx @appunni-dishq && @klDen) +- Add ability to specify retention policy in SeriesHelper (#723 thx @csanz91) ### Changed - Clean up stale CI config (#755) From c858a46ae5fd74cfe9220a8356dd5825ee2a66dd Mon Sep 17 00:00:00 2001 From: Kevin Claytor Date: Fri, 10 Apr 2020 12:49:26 -0400 Subject: [PATCH 18/60] gzip compression for data (post and responses) (#732) * gzip compression working in my influx stack. Needs proper tests. * Also gzip data from server, slightly more straightforward data handling. * Adding in test cases. * Switching back to zlib with gzip headers. * flake8 compatibility * Move parameter into correct position. per review * Switching back to gzip for the headers. * Fixing python 2.7 compatability with gzip. * flake8 compatibility. * flake8 testing Co-authored-by: Kevin Claytor Co-authored-by: Sebastian Borza --- influxdb/client.py | 25 ++++++- influxdb/tests/client_test.py | 67 +++++++++++++++++++ influxdb/tests/server_tests/base.py | 47 +++++++++++++ .../server_tests/client_test_with_server.py | 24 +++++++ 4 files changed, 162 insertions(+), 1 deletion(-) diff --git a/influxdb/client.py b/influxdb/client.py index 0f0350c8..3262c242 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -7,7 +7,9 @@ from __future__ import unicode_literals import datetime +import gzip import itertools +import io import json import random import socket @@ -70,10 +72,11 @@ class InfluxDBClient(object): as a single file containing the private key and the certificate, or as a tuple of both files’ paths, defaults to None :type cert: str + :param gzip: use gzip content encoding to compress requests + :type gzip: bool :param session: allow for the new client request to use an existing requests Session, defaults to None :type session: requests.Session - :raises ValueError: if cert is provided but ssl is disabled (set to False) """ @@ -93,6 +96,7 @@ def __init__(self, pool_size=10, path='', cert=None, + gzip=False, session=None, ): """Construct a new InfluxDBClient object.""" @@ -159,6 +163,8 @@ def __init__(self, 'Accept': 'application/x-msgpack' } + self._gzip = gzip + @property def _baseurl(self): return self.__baseurl @@ -278,6 +284,23 @@ def request(self, url, method='GET', params=None, data=None, if isinstance(data, (dict, list)): data = json.dumps(data) + if self._gzip: + # Receive and send compressed data + headers.update({ + 'Accept-Encoding': 'gzip', + 'Content-Encoding': 'gzip', + }) + if data is not None: + # For Py 2.7 compatability use Gzipfile + compressed = io.BytesIO() + with gzip.GzipFile( + compresslevel=9, + fileobj=compressed, + mode='w' + ) as f: + f.write(data) + data = compressed.getvalue() + # Try to send the request more than once by default (see #103) retry = True _try = 0 diff --git a/influxdb/tests/client_test.py b/influxdb/tests/client_test.py index fe7381ea..f8f3cb00 100644 --- a/influxdb/tests/client_test.py +++ b/influxdb/tests/client_test.py @@ -24,6 +24,8 @@ import unittest import warnings +import io +import gzip import json import mock import requests @@ -214,6 +216,71 @@ def test_write_points(self): m.last_request.body.decode('utf-8'), ) + def test_write_gzip(self): + """Test write in TestInfluxDBClient object.""" + with requests_mock.Mocker() as m: + m.register_uri( + requests_mock.POST, + "http://localhost:8086/write", + status_code=204 + ) + + cli = InfluxDBClient(database='db', gzip=True) + cli.write( + {"database": "mydb", + "retentionPolicy": "mypolicy", + "points": [{"measurement": "cpu_load_short", + "tags": {"host": "server01", + "region": "us-west"}, + "time": "2009-11-10T23:00:00Z", + "fields": {"value": 0.64}}]} + ) + + compressed = io.BytesIO() + with gzip.GzipFile( + compresslevel=9, + fileobj=compressed, + mode='w' + ) as f: + f.write( + b"cpu_load_short,host=server01,region=us-west " + b"value=0.64 1257894000000000000\n" + ) + + self.assertEqual( + m.last_request.body, + compressed.getvalue(), + ) + + def test_write_points_gzip(self): + """Test write points for TestInfluxDBClient object.""" + with requests_mock.Mocker() as m: + m.register_uri( + requests_mock.POST, + "http://localhost:8086/write", + status_code=204 + ) + + cli = InfluxDBClient(database='db', gzip=True) + cli.write_points( + self.dummy_points, + ) + + compressed = io.BytesIO() + with gzip.GzipFile( + compresslevel=9, + fileobj=compressed, + mode='w' + ) as f: + f.write( + b'cpu_load_short,host=server01,region=us-west ' + b'value=0.64 1257894000123456000\n' + ) + self.assertEqual( + m.last_request.body, + compressed.getvalue(), + ) + def test_write_points_toplevel_attributes(self): """Test write points attrs for TestInfluxDBClient object.""" with requests_mock.Mocker() as m: diff --git a/influxdb/tests/server_tests/base.py b/influxdb/tests/server_tests/base.py index fe722870..45a9ec80 100644 --- a/influxdb/tests/server_tests/base.py +++ b/influxdb/tests/server_tests/base.py @@ -36,6 +36,15 @@ def _setup_influxdb_server(inst): database='db') +def _setup_gzip_client(inst): + inst.cli = InfluxDBClient('localhost', + inst.influxd_inst.http_port, + 'root', + '', + database='db', + gzip=True) + + def _teardown_influxdb_server(inst): remove_tree = sys.exc_info() == (None, None, None) inst.influxd_inst.close(remove_tree=remove_tree) @@ -89,3 +98,41 @@ def tearDownClass(cls): def tearDown(self): """Deconstruct an instance of ManyTestCasesWithServerMixin.""" self.cli.drop_database('db') + + +class SingleTestCaseWithServerGzipMixin(object): + """Define the single testcase with server with gzip client mixin. + + Same as the SingleTestCaseWithServerGzipMixin but the InfluxDBClient has + gzip=True + """ + + @classmethod + def setUp(cls): + """Set up an instance of the SingleTestCaseWithServerGzipMixin.""" + _setup_influxdb_server(cls) + _setup_gzip_client(cls) + + @classmethod + def tearDown(cls): + """Tear down an instance of the SingleTestCaseWithServerMixin.""" + _teardown_influxdb_server(cls) + + +class ManyTestCasesWithServerGzipMixin(object): + """Define the many testcase with server with gzip client mixin. + + Same as the ManyTestCasesWithServerMixin but the InfluxDBClient has + gzip=True. + """ + + @classmethod + def setUpClass(cls): + """Set up an instance of the ManyTestCasesWithServerGzipMixin.""" + _setup_influxdb_server(cls) + _setup_gzip_client(cls) + + @classmethod + def tearDown(cls): + """Tear down an instance of the SingleTestCaseWithServerMixin.""" + _teardown_influxdb_server(cls) diff --git a/influxdb/tests/server_tests/client_test_with_server.py b/influxdb/tests/server_tests/client_test_with_server.py index 94f28b66..020014c3 100644 --- a/influxdb/tests/server_tests/client_test_with_server.py +++ b/influxdb/tests/server_tests/client_test_with_server.py @@ -26,6 +26,8 @@ from influxdb.tests import skip_if_pypy, using_pypy, skip_server_tests from influxdb.tests.server_tests.base import ManyTestCasesWithServerMixin from influxdb.tests.server_tests.base import SingleTestCaseWithServerMixin +from influxdb.tests.server_tests.base import ManyTestCasesWithServerGzipMixin +from influxdb.tests.server_tests.base import SingleTestCaseWithServerGzipMixin # By default, raise exceptions on warnings warnings.simplefilter('error', FutureWarning) @@ -913,3 +915,25 @@ def test_write_points_udp(self): ], list(rsp['cpu_load_short']) ) + + +# Run the tests again, but with gzip enabled this time +@skip_server_tests +class GzipSimpleTests(SimpleTests, SingleTestCaseWithServerGzipMixin): + """Repeat the simple tests with InfluxDBClient where gzip=True.""" + + pass + + +@skip_server_tests +class GzipCommonTests(CommonTests, ManyTestCasesWithServerGzipMixin): + """Repeat the common tests with InfluxDBClient where gzip=True.""" + + pass + + +@skip_server_tests +class GzipUdpTests(UdpTests, ManyTestCasesWithServerGzipMixin): + """Repeat the UDP tests with InfluxDBClient where gzip=True.""" + + pass From d59011945435d747d8446cd2453b1f0e0850754f Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Fri, 10 Apr 2020 11:50:38 -0500 Subject: [PATCH 19/60] chore(CHANGELOG): update to add PR#732 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f2fad368..3a706e7b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Add support for handling np.nan and np.inf values in DataFrameClient (#436 thx @nmerket) - Add support for optional `time_precision` in the SeriesHelper (#502 && #719 thx @appunni-dishq && @klDen) - Add ability to specify retention policy in SeriesHelper (#723 thx @csanz91) +- Add gzip compression for post and response data (#732 thx @KEClaytor) ### Changed - Clean up stale CI config (#755) From de44289e0167d690895e8e70eff37edeef36240f Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Fri, 10 Apr 2020 18:58:31 +0200 Subject: [PATCH 20/60] Make batched writing support all iterables (#746) * Make batched writing support all iterables * Also test batching generator against real server * Fix PEP257 error * Import itertools functions directly --- influxdb/client.py | 15 ++++++++-- influxdb/tests/client_test.py | 30 +++++++++++++++++++ .../server_tests/client_test_with_server.py | 27 +++++++++++++++++ 3 files changed, 69 insertions(+), 3 deletions(-) diff --git a/influxdb/client.py b/influxdb/client.py index 3262c242..46424bc2 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -15,11 +15,11 @@ import socket import struct import time +from itertools import chain, islice import msgpack import requests import requests.exceptions -from six.moves import xrange from six.moves.urllib.parse import urlparse from influxdb.line_protocol import make_lines, quote_ident, quote_literal @@ -597,8 +597,17 @@ def ping(self): @staticmethod def _batches(iterable, size): - for i in xrange(0, len(iterable), size): - yield iterable[i:i + size] + # Iterate over an iterable producing iterables of batches. Based on: + # http://code.activestate.com/recipes/303279-getting-items-in-batches/ + iterator = iter(iterable) + while True: + try: # Try get the first element in the iterator... + head = (next(iterator),) + except StopIteration: + return # ...so that we can stop if there isn't one + # Otherwise, lazily slice the rest of the batch + rest = islice(iterator, size - 1) + yield chain(head, rest) def _write_points(self, points, diff --git a/influxdb/tests/client_test.py b/influxdb/tests/client_test.py index f8f3cb00..a8f8e864 100644 --- a/influxdb/tests/client_test.py +++ b/influxdb/tests/client_test.py @@ -332,6 +332,36 @@ def test_write_points_batch(self): self.assertEqual(expected_last_body, m.last_request.body.decode('utf-8')) + def test_write_points_batch_generator(self): + """Test write points batch from a generator for TestInfluxDBClient.""" + dummy_points = [ + {"measurement": "cpu_usage", "tags": {"unit": "percent"}, + "time": "2009-11-10T23:00:00Z", "fields": {"value": 12.34}}, + {"measurement": "network", "tags": {"direction": "in"}, + "time": "2009-11-10T23:00:00Z", "fields": {"value": 123.00}}, + {"measurement": "network", "tags": {"direction": "out"}, + "time": "2009-11-10T23:00:00Z", "fields": {"value": 12.00}} + ] + dummy_points_generator = (point for point in dummy_points) + expected_last_body = ( + "network,direction=out,host=server01,region=us-west " + "value=12.0 1257894000000000000\n" + ) + + with requests_mock.Mocker() as m: + m.register_uri(requests_mock.POST, + "http://localhost:8086/write", + status_code=204) + cli = InfluxDBClient(database='db') + cli.write_points(points=dummy_points_generator, + database='db', + tags={"host": "server01", + "region": "us-west"}, + batch_size=2) + self.assertEqual(m.call_count, 2) + self.assertEqual(expected_last_body, + m.last_request.body.decode('utf-8')) + def test_write_points_udp(self): """Test write points UDP for TestInfluxDBClient object.""" s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) diff --git a/influxdb/tests/server_tests/client_test_with_server.py b/influxdb/tests/server_tests/client_test_with_server.py index 020014c3..a0263243 100644 --- a/influxdb/tests/server_tests/client_test_with_server.py +++ b/influxdb/tests/server_tests/client_test_with_server.py @@ -452,6 +452,33 @@ def test_write_points_batch(self): self.assertIn(12, net_out['series'][0]['values'][0]) self.assertIn(12.34, cpu['series'][0]['values'][0]) + def test_write_points_batch_generator(self): + """Test writing points in a batch from a generator.""" + dummy_points = [ + {"measurement": "cpu_usage", "tags": {"unit": "percent"}, + "time": "2009-11-10T23:00:00Z", "fields": {"value": 12.34}}, + {"measurement": "network", "tags": {"direction": "in"}, + "time": "2009-11-10T23:00:00Z", "fields": {"value": 123.00}}, + {"measurement": "network", "tags": {"direction": "out"}, + "time": "2009-11-10T23:00:00Z", "fields": {"value": 12.00}} + ] + dummy_points_generator = (point for point in dummy_points) + self.cli.write_points(points=dummy_points_generator, + tags={"host": "server01", + "region": "us-west"}, + batch_size=2) + time.sleep(5) + net_in = self.cli.query("SELECT value FROM network " + "WHERE direction=$dir", + bind_params={'dir': 'in'} + ).raw + net_out = self.cli.query("SELECT value FROM network " + "WHERE direction='out'").raw + cpu = self.cli.query("SELECT value FROM cpu_usage").raw + self.assertIn(123, net_in['series'][0]['values'][0]) + self.assertIn(12, net_out['series'][0]['values'][0]) + self.assertIn(12.34, cpu['series'][0]['values'][0]) + def test_query(self): """Test querying data back from server.""" self.assertIs(True, self.cli.write_points(dummy_point)) From d6192a759c106da1317fc60d0630022bf7e829cf Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Fri, 10 Apr 2020 11:59:25 -0500 Subject: [PATCH 21/60] chore(CHANGELOG): update to include PR#746 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a706e7b..5a49c84a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Update DataFrameClient to properly return `dropna` values (#778 thx @jgspiro) - Update DataFrameClient to test for pd.DataTimeIndex before blind conversion (#623 thx @testforvin) - Update client to type-set UDP port to int (#651 thx @yifeikong) +- Update batched writing support for all iterables (#746 thx @JayH5) ### Removed From c903d73efcf49b4e340490072d777d8f34ac8e1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20=22hr=22=20Berder=20=28=E7=99=BD=E5=B3=B0=29?= Date: Sat, 11 Apr 2020 01:10:40 +0800 Subject: [PATCH 22/60] Fix chunked query to return chunk resultsets (#753) When querying large data sets, it's vital to get a chunked responses to manage memory usage. Wrapping the query response in a generator and streaming the request provides the desired result. It also fixes `InfluxDBClient.query()` behavior for chunked queries that is currently not working according to [specs](https://github.com/influxdata/influxdb-python/blob/master/influxdb/client.py#L429) Closes #585. Closes #531. Closes #538. --- influxdb/client.py | 10 +++++--- influxdb/tests/client_test.py | 44 ++++++++++++++--------------------- 2 files changed, 25 insertions(+), 29 deletions(-) diff --git a/influxdb/client.py b/influxdb/client.py index 46424bc2..b28ed1b5 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -249,7 +249,7 @@ def switch_user(self, username, password): self._username = username self._password = password - def request(self, url, method='GET', params=None, data=None, + def request(self, url, method='GET', params=None, data=None, stream=False, expected_response_code=200, headers=None): """Make a HTTP request to the InfluxDB API. @@ -261,6 +261,8 @@ def request(self, url, method='GET', params=None, data=None, :type params: dict :param data: the data of the request, defaults to None :type data: str + :param stream: True if a query uses chunked responses + :type stream: bool :param expected_response_code: the expected response code of the request, defaults to 200 :type expected_response_code: int @@ -312,6 +314,7 @@ def request(self, url, method='GET', params=None, data=None, auth=(self._username, self._password), params=params, data=data, + stream=stream, headers=headers, proxies=self._proxies, verify=self._verify_ssl, @@ -398,17 +401,17 @@ def write(self, data, params=None, expected_response_code=204, @staticmethod def _read_chunked_response(response, raise_errors=True): - result_set = {} for line in response.iter_lines(): if isinstance(line, bytes): line = line.decode('utf-8') data = json.loads(line) + result_set = {} for result in data.get('results', []): for _key in result: if isinstance(result[_key], list): result_set.setdefault( _key, []).extend(result[_key]) - return ResultSet(result_set, raise_errors=raise_errors) + yield ResultSet(result_set, raise_errors=raise_errors) def query(self, query, @@ -499,6 +502,7 @@ def query(self, method=method, params=params, data=None, + stream=chunked, expected_response_code=expected_response_code ) diff --git a/influxdb/tests/client_test.py b/influxdb/tests/client_test.py index a8f8e864..fd3c06bb 100644 --- a/influxdb/tests/client_test.py +++ b/influxdb/tests/client_test.py @@ -1400,16 +1400,11 @@ def test_invalid_port_fails(self): def test_chunked_response(self): """Test chunked reponse for TestInfluxDBClient object.""" example_response = \ - u'{"results":[{"statement_id":0,"series":' \ - '[{"name":"cpu","columns":["fieldKey","fieldType"],"values":' \ - '[["value","integer"]]}],"partial":true}]}\n{"results":' \ - '[{"statement_id":0,"series":[{"name":"iops","columns":' \ - '["fieldKey","fieldType"],"values":[["value","integer"]]}],' \ - '"partial":true}]}\n{"results":[{"statement_id":0,"series":' \ - '[{"name":"load","columns":["fieldKey","fieldType"],"values":' \ - '[["value","integer"]]}],"partial":true}]}\n{"results":' \ - '[{"statement_id":0,"series":[{"name":"memory","columns":' \ - '["fieldKey","fieldType"],"values":[["value","integer"]]}]}]}\n' + u'{"results":[{"statement_id":0,"series":[{"columns":["key"],' \ + '"values":[["cpu"],["memory"],["iops"],["network"]],"partial":' \ + 'true}],"partial":true}]}\n{"results":[{"statement_id":0,' \ + '"series":[{"columns":["key"],"values":[["qps"],["uptime"],' \ + '["df"],["mount"]]}]}]}\n' with requests_mock.Mocker() as m: m.register_uri( @@ -1417,23 +1412,20 @@ def test_chunked_response(self): "http://localhost:8086/query", text=example_response ) - response = self.cli.query('show series limit 4 offset 0', + response = self.cli.query('show series', chunked=True, chunk_size=4) - self.assertTrue(len(response) == 4) - self.assertEqual(response.__repr__(), ResultSet( - {'series': [{'values': [['value', 'integer']], - 'name': 'cpu', - 'columns': ['fieldKey', 'fieldType']}, - {'values': [['value', 'integer']], - 'name': 'iops', - 'columns': ['fieldKey', 'fieldType']}, - {'values': [['value', 'integer']], - 'name': 'load', - 'columns': ['fieldKey', 'fieldType']}, - {'values': [['value', 'integer']], - 'name': 'memory', - 'columns': ['fieldKey', 'fieldType']}]} - ).__repr__()) + res = list(response) + self.assertTrue(len(res) == 2) + self.assertEqual(res[0].__repr__(), ResultSet( + {'series': [{ + 'columns': ['key'], + 'values': [['cpu'], ['memory'], ['iops'], ['network']] + }]}).__repr__()) + self.assertEqual(res[1].__repr__(), ResultSet( + {'series': [{ + 'columns': ['key'], + 'values': [['qps'], ['uptime'], ['df'], ['mount']] + }]}).__repr__()) class FakeClient(InfluxDBClient): From e884631148dda96f586ab9da358deb7119bb57d2 Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Fri, 10 Apr 2020 12:12:03 -0500 Subject: [PATCH 23/60] chore(CHANGELOG): update to include PR#753 and #538 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a49c84a..00d2a4f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Add support for optional `time_precision` in the SeriesHelper (#502 && #719 thx @appunni-dishq && @klDen) - Add ability to specify retention policy in SeriesHelper (#723 thx @csanz91) - Add gzip compression for post and response data (#732 thx @KEClaytor) +- Add support for chunked responses in ResultSet (#753 and #538 thx @hrbonz && @psy0rz) ### Changed - Clean up stale CI config (#755) From 7d82f9371176699f8d6f9c4d0b0133ed5bd20275 Mon Sep 17 00:00:00 2001 From: Greg Schrock Date: Fri, 10 Apr 2020 13:29:54 -0400 Subject: [PATCH 24/60] Fix make_lines excludes fields with empty strings (#655) (#766) * Fix make_lines excludes fields with empty strings (#655) Converting to unicode required something to be done with None values. They were converted to empty strings which were subsequently ignored. This makes it impossible to write an explicitly empty string, which should be possible. This change distinguishes between None and empty strings. * Fix linting failure due to long comment line Co-authored-by: Greg Schrock --- influxdb/line_protocol.py | 6 ++++-- influxdb/tests/test_line_protocol.py | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/influxdb/line_protocol.py b/influxdb/line_protocol.py index d6cbf46f..25dd2ad7 100644 --- a/influxdb/line_protocol.py +++ b/influxdb/line_protocol.py @@ -104,9 +104,11 @@ def _is_float(value): def _escape_value(value): - value = _get_unicode(value) + if value is None: + return '' - if isinstance(value, text_type) and value != '': + value = _get_unicode(value) + if isinstance(value, text_type): return quote_ident(value) if isinstance(value, integer_types) and not isinstance(value, bool): diff --git a/influxdb/tests/test_line_protocol.py b/influxdb/tests/test_line_protocol.py index c48d5edc..5b344990 100644 --- a/influxdb/tests/test_line_protocol.py +++ b/influxdb/tests/test_line_protocol.py @@ -117,6 +117,24 @@ def test_make_lines_unicode(self): 'test,unicode_tag=\'Привет!\' unicode_val="Привет!"\n' ) + def test_make_lines_empty_field_string(self): + """Test make lines with an empty string field.""" + data = { + "points": [ + { + "measurement": "test", + "fields": { + "string": "", + } + } + ] + } + + self.assertEqual( + line_protocol.make_lines(data), + 'test string=""\n' + ) + def test_tag_value_newline(self): """Test make lines with tag value contains newline.""" data = { From 6290994590c4164535d59879e50482b44bb4d7ff Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Fri, 10 Apr 2020 12:30:50 -0500 Subject: [PATCH 25/60] chore(CHANGELOG): update for PR #766 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 00d2a4f7..4ee39303 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Add ability to specify retention policy in SeriesHelper (#723 thx @csanz91) - Add gzip compression for post and response data (#732 thx @KEClaytor) - Add support for chunked responses in ResultSet (#753 and #538 thx @hrbonz && @psy0rz) +- Add support for empty string fields (#766 thx @gregschrock) ### Changed - Clean up stale CI config (#755) From 6f73ea12d7d9ca0184ba3c487264e58146ce2bac Mon Sep 17 00:00:00 2001 From: Keunhyun Oh Date: Sat, 11 Apr 2020 02:33:14 +0900 Subject: [PATCH 26/60] fix: Calling commit and _json_body make raising an exception when any datapoints are not added. (#772) * If any SerialHelper is not generated, calling the commit function makes raising an exception because _datapoints is not allocated. It is hard to find the reason of this error. it is because reviewing influxdb-python's code is needed. I think that it is important that is producing predictable results. Results when calling first time is needed to equal to calling resetting datapoints in json_body. So, I've fixed that if not initialized when calling _json_body() function, _datapoints is reset to avoid raising error. In Unittest, the setup function is added. When calling setup function firstly, __initialized__ is False and _datapoints is not assigned. But, because of this commit, it is OK. Contacts: Keunhyun Oh * fix build fail Contacts: Keunhyun Oh * fix build fail Contacts: Keunhyun Oh * fix build fail Contacts: Keunhyun Oh * Update helper_test.py * Update helper_test.py * Update helper_test.py --- influxdb/helper.py | 2 ++ influxdb/influxdb08/helper.py | 2 ++ influxdb/tests/helper_test.py | 8 ++++++++ 3 files changed, 12 insertions(+) diff --git a/influxdb/helper.py b/influxdb/helper.py index f49f40ad..74209354 100644 --- a/influxdb/helper.py +++ b/influxdb/helper.py @@ -174,6 +174,8 @@ def _json_body_(cls): :return: JSON body of these datapoints. """ json = [] + if not cls.__initialized__: + cls._reset_() for series_name, data in six.iteritems(cls._datapoints): for point in data: json_point = { diff --git a/influxdb/influxdb08/helper.py b/influxdb/influxdb08/helper.py index f3dec33c..5f2d4614 100644 --- a/influxdb/influxdb08/helper.py +++ b/influxdb/influxdb08/helper.py @@ -139,6 +139,8 @@ def _json_body_(cls): :return: JSON body of the datapoints. """ json = [] + if not cls.__initialized__: + cls._reset_() for series_name, data in six.iteritems(cls._datapoints): json.append({'name': series_name, 'columns': cls._fields, diff --git a/influxdb/tests/helper_test.py b/influxdb/tests/helper_test.py index 16924936..6737f921 100644 --- a/influxdb/tests/helper_test.py +++ b/influxdb/tests/helper_test.py @@ -47,6 +47,14 @@ class Meta: TestSeriesHelper.MySeriesHelper = MySeriesHelper + def setUp(self): + """Check that MySeriesHelper has empty datapoints.""" + super(TestSeriesHelper, self).setUp() + self.assertEqual( + TestSeriesHelper.MySeriesHelper._json_body_(), + [], + 'Resetting helper in teardown did not empty datapoints.') + def tearDown(self): """Deconstruct the TestSeriesHelper object.""" super(TestSeriesHelper, self).tearDown() From 351b98a880d7a62188e00a240df4edaa2d75f499 Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Fri, 10 Apr 2020 12:34:05 -0500 Subject: [PATCH 27/60] chore(CHANGELOG): add PR#772 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ee39303..322c952b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Update DataFrameClient to test for pd.DataTimeIndex before blind conversion (#623 thx @testforvin) - Update client to type-set UDP port to int (#651 thx @yifeikong) - Update batched writing support for all iterables (#746 thx @JayH5) +- Update SeriesHelper to enable class instantiation when not initialized (#772 thx @ocworld) ### Removed From f7f30b5ed3d7bc9ded164288af8b1a7ca1a3a6b0 Mon Sep 17 00:00:00 2001 From: Shan Desai Date: Fri, 10 Apr 2020 19:59:04 +0200 Subject: [PATCH 28/60] Refactor `tutorial_udp` example for timestamps (#808) - `time` key should be within each datapoint - This PR addresses #788 regarding the structure of the data when inserting via UDP. - The original documentation contributed by me took the structure of the `tutorial.py` as base. However, upon testing, the timestamp in the example are not written (2020 is written as opposed to 2009). - Tested for `influxdb-python` v5.2.3 and InfluxDB v1.6.1 --- examples/tutorial_udp.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/examples/tutorial_udp.py b/examples/tutorial_udp.py index 517ae858..93b923d7 100644 --- a/examples/tutorial_udp.py +++ b/examples/tutorial_udp.py @@ -29,18 +29,19 @@ def main(uport): "host": "server01", "region": "us-west" }, - "time": "2009-11-10T23:00:00Z", "points": [{ "measurement": "cpu_load_short", "fields": { "value": 0.64 - } + }, + "time": "2009-11-10T23:00:00Z", }, - { - "measurement": "cpu_load_short", - "fields": { - "value": 0.67 - } + { + "measurement": "cpu_load_short", + "fields": { + "value": 0.67 + }, + "time": "2009-11-10T23:05:00Z" }] } From f8705f9f474e260b0455fc44d1b1f74c64d22162 Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Fri, 10 Apr 2020 14:27:14 -0500 Subject: [PATCH 29/60] feat(client): add support for context managers (#816) * feat(client): add support for context managers * chore(CHANGELOG): rebase against master --- CHANGELOG.md | 2 ++ influxdb/client.py | 12 ++++++++++++ 2 files changed, 14 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 322c952b..411b46a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Add gzip compression for post and response data (#732 thx @KEClaytor) - Add support for chunked responses in ResultSet (#753 and #538 thx @hrbonz && @psy0rz) - Add support for empty string fields (#766 thx @gregschrock) +- Add support for context managers to InfluxDBClient (#721 thx @JustusAdam) ### Changed - Clean up stale CI config (#755) @@ -37,6 +38,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Update client to type-set UDP port to int (#651 thx @yifeikong) - Update batched writing support for all iterables (#746 thx @JayH5) - Update SeriesHelper to enable class instantiation when not initialized (#772 thx @ocworld) +- Update UDP test case to add proper timestamp to datapoints (#808 thx @shantanoo-desai) ### Removed diff --git a/influxdb/client.py b/influxdb/client.py index b28ed1b5..a0f571f5 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -35,6 +35,9 @@ class InfluxDBClient(object): connect to InfluxDB. Requests can be made to InfluxDB directly through the client. + The client supports the use as a `context manager + `_. + :param host: hostname to connect to InfluxDB, defaults to 'localhost' :type host: str :param port: port to connect to InfluxDB, defaults to 8086 @@ -78,6 +81,7 @@ class InfluxDBClient(object): requests Session, defaults to None :type session: requests.Session :raises ValueError: if cert is provided but ssl is disabled (set to False) + """ def __init__(self, @@ -165,6 +169,14 @@ def __init__(self, self._gzip = gzip + def __enter__(self): + """Enter function as used by context manager.""" + pass + + def __exit__(self, _exc_type, _exc_value, _traceback): + """Exit function as used by context manager.""" + self.close() + @property def _baseurl(self): return self.__baseurl From cf83d1d429ecebb093f6eaab773bd41c251530ca Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Fri, 10 Apr 2020 14:38:30 -0500 Subject: [PATCH 30/60] chore(CHANGELOG): tagging release to 5.3.0 --- LICENSE | 2 +- influxdb/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LICENSE b/LICENSE index 38ee2491..a49a5410 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2013 InfluxDB +Copyright (c) 2020 InfluxDB Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff --git a/influxdb/__init__.py b/influxdb/__init__.py index b31170bb..56f2f619 100644 --- a/influxdb/__init__.py +++ b/influxdb/__init__.py @@ -18,4 +18,4 @@ ] -__version__ = '5.2.3' +__version__ = '5.3.0' From e7ef0454524bf0617097a2f303d784242e5ddfe8 Mon Sep 17 00:00:00 2001 From: Krzysztof Baranski Date: Sun, 12 Apr 2020 01:45:32 +0200 Subject: [PATCH 31/60] do not sleep after last retry before raising exception (#790) * do not sleep after last retry before raising exception * documentation: clarification of retry parameter retry=0 - retry forever retry=1 - try once, on error don't do any retry retry=2 - 2 tries, one original and one retry on error retry=3 - 3 tries, one original and maximum two retries on errors * retries - move raise before sleep * retries - documentation * fix line length --- influxdb/client.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/influxdb/client.py b/influxdb/client.py index a0f571f5..80994190 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -59,8 +59,12 @@ class InfluxDBClient(object): :param timeout: number of seconds Requests will wait for your client to establish a connection, defaults to None :type timeout: int - :param retries: number of retries your client will try before aborting, - defaults to 3. 0 indicates try until success + :param retries: number of attempts your client will make before aborting, + defaults to 3 + 0 - try until success + 1 - attempt only once (without retry) + 2 - maximum two attempts (including one retry) + 3 - maximum three attempts (default option) :type retries: int :param use_udp: use UDP to connect to InfluxDB, defaults to False :type use_udp: bool @@ -339,10 +343,10 @@ def request(self, url, method='GET', params=None, data=None, stream=False, _try += 1 if self._retries != 0: retry = _try < self._retries - if method == "POST": - time.sleep((2 ** _try) * random.random() / 100.0) if not retry: raise + if method == "POST": + time.sleep((2 ** _try) * random.random() / 100.0) type_header = response.headers and response.headers.get("Content-Type") if type_header == "application/x-msgpack" and response.content: From b4390cc8680abd0931b14b8f2ee1e83998a53eb2 Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Sat, 11 Apr 2020 18:48:26 -0500 Subject: [PATCH 32/60] chore(CHANGELOG): add PR#790 --- CHANGELOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 411b46a4..c2f3edcc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,11 +4,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [v5.3.1] - Unreleased ### Added ### Changed +- Amend retry to avoid sleep after last retry before raising exception (#790 thx @krzysbaranski) + +### Removed ## [v5.3.0] - 2020-04-10 From cc41e290f690c4eb67f75c98fa9f027bdb6eb16b Mon Sep 17 00:00:00 2001 From: Matthew Thode Date: Sat, 11 Apr 2020 18:49:33 -0500 Subject: [PATCH 33/60] remove msgpack pin (#818) The hard lock prevents this from being co-installed with many other packages. For instance, it's preventing it from being included in openstack (which is on 0.6.2 and working on 1.0.0 now). Signed-off-by: Matthew Thode --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 77d7306f..548b17c8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,4 @@ python-dateutil>=2.6.0 pytz requests>=2.17.0 six>=1.10.0 -msgpack==0.6.1 +msgpack From cdc2c665808f2cf572973029f81b9a0175b43277 Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Sat, 11 Apr 2020 18:53:49 -0500 Subject: [PATCH 34/60] chore(CHANGELOG): add PR#818 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c2f3edcc..d380b73e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Changed - Amend retry to avoid sleep after last retry before raising exception (#790 thx @krzysbaranski) +- Remove msgpack pinning for requirements (#818 thx @prometheanfire) ### Removed From 5bcfadf7a525232078b875b2fb94d831287d6c8e Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Sun, 12 Apr 2020 22:14:38 -0500 Subject: [PATCH 35/60] chore(dataframe_client): update param definition. Closes #525. (#819) --- influxdb/_dataframe_client.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/influxdb/_dataframe_client.py b/influxdb/_dataframe_client.py index 600bc1ec..ec58cebb 100644 --- a/influxdb/_dataframe_client.py +++ b/influxdb/_dataframe_client.py @@ -59,6 +59,8 @@ def write_points(self, :param dataframe: data points in a DataFrame :param measurement: name of measurement :param tags: dictionary of tags, with string key-values + :param tag_columns: [Optional, default None] List of data tag names + :param field_columns: [Options, default None] List of data field names :param time_precision: [Optional, default None] Either 's', 'ms', 'u' or 'n'. :param batch_size: [Optional] Value to write the points in batches From 7fb5e946062dd36a84801e4a03012a3c032a70db Mon Sep 17 00:00:00 2001 From: Adam Suban-Loewen Date: Mon, 13 Apr 2020 22:05:49 -0400 Subject: [PATCH 36/60] Added headers parameter to InfluxDBClient (#710) --- influxdb/client.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/influxdb/client.py b/influxdb/client.py index 80994190..404e14be 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -84,8 +84,10 @@ class InfluxDBClient(object): :param session: allow for the new client request to use an existing requests Session, defaults to None :type session: requests.Session + :param headers: headers to add to Requests, will add 'Content-Type' + and 'Accept' unless these are already present, defaults to {} + :type headers: dict :raises ValueError: if cert is provided but ssl is disabled (set to False) - """ def __init__(self, @@ -106,6 +108,7 @@ def __init__(self, cert=None, gzip=False, session=None, + headers=None, ): """Construct a new InfluxDBClient object.""" self.__host = host @@ -166,10 +169,11 @@ def __init__(self, self._port, self._path) - self._headers = { - 'Content-Type': 'application/json', - 'Accept': 'application/x-msgpack' - } + if headers is None: + headers = {} + headers.setdefault('Content-Type', 'application/json') + headers.setdefault('Accept', 'application/x-msgpack') + self._headers = headers self._gzip = gzip @@ -390,7 +394,7 @@ def write(self, data, params=None, expected_response_code=204, :returns: True, if the write operation is successful :rtype: bool """ - headers = self._headers + headers = self._headers.copy() headers['Content-Type'] = 'application/octet-stream' if params: From 95e0efb0821d44bf06aebe0b2c4700e4d3b084c8 Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Mon, 13 Apr 2020 21:09:22 -0500 Subject: [PATCH 37/60] chore(CHANGELOG): add PR #710 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d380b73e..92bbe42e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [v5.3.1] - Unreleased ### Added +- Add support for custom headers in the InfluxDBClient (#710 thx @nathanielatom) ### Changed - Amend retry to avoid sleep after last retry before raising exception (#790 thx @krzysbaranski) From de75e7351ce3fc084c145d16df38f96d18603bf5 Mon Sep 17 00:00:00 2001 From: Pavlina Rolincova Date: Thu, 28 May 2020 10:41:50 +0200 Subject: [PATCH 38/60] Add support for custom indexes for query in the DataFrameClient (#785) --- CHANGELOG.md | 1 + influxdb/_dataframe_client.py | 22 +++++++++++------ influxdb/tests/dataframe_client_test.py | 33 +++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 92bbe42e..2b374faa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added - Add support for custom headers in the InfluxDBClient (#710 thx @nathanielatom) +- Add support for custom indexes for query in the DataFrameClient (#785) ### Changed - Amend retry to avoid sleep after last retry before raising exception (#790 thx @krzysbaranski) diff --git a/influxdb/_dataframe_client.py b/influxdb/_dataframe_client.py index ec58cebb..58063500 100644 --- a/influxdb/_dataframe_client.py +++ b/influxdb/_dataframe_client.py @@ -8,6 +8,7 @@ import math from collections import defaultdict +from typing import List import pandas as pd import numpy as np @@ -152,7 +153,8 @@ def query(self, chunked=False, chunk_size=0, method="GET", - dropna=True): + dropna=True, + data_frame_index: List[str] = None): """ Query data into a DataFrame. @@ -181,6 +183,7 @@ def query(self, containing all results within that chunk :param chunk_size: Size of each chunk to tell InfluxDB to use. :param dropna: drop columns where all values are missing + :param data_frame_index: the list of columns that are used as DataFrame index :returns: the queried data :rtype: :class:`~.ResultSet` """ @@ -196,13 +199,13 @@ def query(self, results = super(DataFrameClient, self).query(query, **query_args) if query.strip().upper().startswith("SELECT"): if len(results) > 0: - return self._to_dataframe(results, dropna) + return self._to_dataframe(results, dropna, data_frame_index=data_frame_index) else: return {} else: return results - def _to_dataframe(self, rs, dropna=True): + def _to_dataframe(self, rs, dropna=True, data_frame_index: List[str] = None): result = defaultdict(list) if isinstance(rs, list): return map(self._to_dataframe, rs, @@ -216,10 +219,15 @@ def _to_dataframe(self, rs, dropna=True): key = (name, tuple(sorted(tags.items()))) df = pd.DataFrame(data) df.time = pd.to_datetime(df.time) - df.set_index('time', inplace=True) - if df.index.tzinfo is None: - df.index = df.index.tz_localize('UTC') - df.index.name = None + + if data_frame_index: + df.set_index(data_frame_index, inplace=True) + else: + df.set_index('time', inplace=True) + if df.index.tzinfo is None: + df.index = df.index.tz_localize('UTC') + df.index.name = 'time' + result[key].append(df) for key, data in result.items(): df = pd.concat(data).sort_index() diff --git a/influxdb/tests/dataframe_client_test.py b/influxdb/tests/dataframe_client_test.py index a80498f3..cf82b49c 100644 --- a/influxdb/tests/dataframe_client_test.py +++ b/influxdb/tests/dataframe_client_test.py @@ -1240,3 +1240,36 @@ def test_write_points_from_dataframe_with_tags_and_nan_json(self): cli.write_points(dataframe, 'foo', tags=None, protocol='json', tag_columns=['tag_one', 'tag_two']) self.assertEqual(m.last_request.body, expected) + + def test_query_custom_index(self): + data = { + "results": [ + { + "series": [ + { + "name": "cpu_load_short", + "columns": ["time", "value", "host"], + "values": [ + [1, 0.55, "local"], + [2, 23422, "local"], + [3, 0.64, "local"] + ] + } + ] + } + ] + } + + cli = DataFrameClient('host', 8086, 'username', 'password', 'db') + iql = "SELECT value FROM cpu_load_short WHERE region=$region;" \ + "SELECT count(value) FROM cpu_load_short WHERE region=$region" + bind_params = {'region': 'us-west'} + with _mocked_session(cli, 'GET', 200, data): + result = cli.query(iql, bind_params=bind_params, data_frame_index=["time", "host"]) + + _data_frame = result['cpu_load_short'] + print(_data_frame) + + self.assertListEqual(["time", "host"], list(_data_frame.index.names)) + + From d3fd851c8e99350524fad710c753a0d978a5f978 Mon Sep 17 00:00:00 2001 From: Pavlina Rolincova Date: Mon, 1 Jun 2020 08:26:39 +0200 Subject: [PATCH 39/60] Add support for custom indexes for query in the DataFrameClient (#785) --- influxdb/_dataframe_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/influxdb/_dataframe_client.py b/influxdb/_dataframe_client.py index 58063500..0b9f282d 100644 --- a/influxdb/_dataframe_client.py +++ b/influxdb/_dataframe_client.py @@ -226,7 +226,7 @@ def _to_dataframe(self, rs, dropna=True, data_frame_index: List[str] = None): df.set_index('time', inplace=True) if df.index.tzinfo is None: df.index = df.index.tz_localize('UTC') - df.index.name = 'time' + df.index.name = None result[key].append(df) for key, data in result.items(): From 9a110a1abda2677340bc7fbd00890d27b8d3d5ab Mon Sep 17 00:00:00 2001 From: Pavlina Rolincova Date: Mon, 1 Jun 2020 09:57:12 +0200 Subject: [PATCH 40/60] Add support for custom indexes for query in the DataFrameClient (#785) --- influxdb/_dataframe_client.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/influxdb/_dataframe_client.py b/influxdb/_dataframe_client.py index 0b9f282d..afd75b39 100644 --- a/influxdb/_dataframe_client.py +++ b/influxdb/_dataframe_client.py @@ -8,7 +8,6 @@ import math from collections import defaultdict -from typing import List import pandas as pd import numpy as np @@ -154,7 +153,7 @@ def query(self, chunk_size=0, method="GET", dropna=True, - data_frame_index: List[str] = None): + data_frame_index=None): """ Query data into a DataFrame. @@ -205,7 +204,7 @@ def query(self, else: return results - def _to_dataframe(self, rs, dropna=True, data_frame_index: List[str] = None): + def _to_dataframe(self, rs, dropna=True, data_frame_index=None): result = defaultdict(list) if isinstance(rs, list): return map(self._to_dataframe, rs, From 055d71fb83aeb83603bd9b5423d6b05d1f01c59e Mon Sep 17 00:00:00 2001 From: Pavlina Rolincova Date: Mon, 1 Jun 2020 10:37:56 +0200 Subject: [PATCH 41/60] Add support for custom indexes for query in the DataFrameClient (#785) --- influxdb/tests/dataframe_client_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/influxdb/tests/dataframe_client_test.py b/influxdb/tests/dataframe_client_test.py index cf82b49c..2dd98398 100644 --- a/influxdb/tests/dataframe_client_test.py +++ b/influxdb/tests/dataframe_client_test.py @@ -1242,6 +1242,7 @@ def test_write_points_from_dataframe_with_tags_and_nan_json(self): self.assertEqual(m.last_request.body, expected) def test_query_custom_index(self): + """Test query with custom indexes.""" data = { "results": [ { From ddd82612f57da90476cd0d10539d7edf1ca25d49 Mon Sep 17 00:00:00 2001 From: Pavlina Rolincova Date: Mon, 1 Jun 2020 10:40:39 +0200 Subject: [PATCH 42/60] Add support for custom indexes for query in the DataFrameClient (#785) --- influxdb/tests/dataframe_client_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/influxdb/tests/dataframe_client_test.py b/influxdb/tests/dataframe_client_test.py index 2dd98398..f6db3c22 100644 --- a/influxdb/tests/dataframe_client_test.py +++ b/influxdb/tests/dataframe_client_test.py @@ -1266,11 +1266,11 @@ def test_query_custom_index(self): "SELECT count(value) FROM cpu_load_short WHERE region=$region" bind_params = {'region': 'us-west'} with _mocked_session(cli, 'GET', 200, data): - result = cli.query(iql, bind_params=bind_params, data_frame_index=["time", "host"]) + result = cli.query(iql, bind_params=bind_params, + data_frame_index=["time", "host"]) _data_frame = result['cpu_load_short'] print(_data_frame) - self.assertListEqual(["time", "host"], list(_data_frame.index.names)) - - + self.assertListEqual(["time", "host"], + list(_data_frame.index.names)) From 64aeddd82358f68b1e5b662d80bfaee6485d0de7 Mon Sep 17 00:00:00 2001 From: Pavlina Rolincova Date: Mon, 1 Jun 2020 11:53:15 +0200 Subject: [PATCH 43/60] Add support for custom indexes for query in the DataFrameClient (#785) --- influxdb/_dataframe_client.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/influxdb/_dataframe_client.py b/influxdb/_dataframe_client.py index afd75b39..e7ae9c17 100644 --- a/influxdb/_dataframe_client.py +++ b/influxdb/_dataframe_client.py @@ -182,7 +182,8 @@ def query(self, containing all results within that chunk :param chunk_size: Size of each chunk to tell InfluxDB to use. :param dropna: drop columns where all values are missing - :param data_frame_index: the list of columns that are used as DataFrame index + :param data_frame_index: the list of columns that + are used as DataFrame index :returns: the queried data :rtype: :class:`~.ResultSet` """ @@ -198,7 +199,8 @@ def query(self, results = super(DataFrameClient, self).query(query, **query_args) if query.strip().upper().startswith("SELECT"): if len(results) > 0: - return self._to_dataframe(results, dropna, data_frame_index=data_frame_index) + return self._to_dataframe(results, dropna, + data_frame_index=data_frame_index) else: return {} else: From 6c45f30d6a7142c17cca0ca7736eb1b693f212e8 Mon Sep 17 00:00:00 2001 From: Pavlina Rolincova Date: Mon, 1 Jun 2020 13:53:30 +0200 Subject: [PATCH 44/60] Add support for custom indexes for query in the DataFrameClient (#785) --- influxdb/influxdb08/client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/influxdb/influxdb08/client.py b/influxdb/influxdb08/client.py index 965a91db..40c58145 100644 --- a/influxdb/influxdb08/client.py +++ b/influxdb/influxdb08/client.py @@ -292,10 +292,10 @@ def write_points(self, data, time_precision='s', *args, **kwargs): :type batch_size: int """ - def list_chunks(l, n): + def list_chunks(data_list, n): """Yield successive n-sized chunks from l.""" - for i in xrange(0, len(l), n): - yield l[i:i + n] + for i in xrange(0, len(data_list), n): + yield data_list[i:i + n] batch_size = kwargs.get('batch_size') if batch_size and batch_size > 0: From 49165bedaab5fad158f742fdf16020b44c92decf Mon Sep 17 00:00:00 2001 From: Pavlina Rolincova Date: Tue, 2 Jun 2020 14:42:29 +0200 Subject: [PATCH 45/60] Add support for custom indexes for query in the DataFrameClient (#785) --- influxdb/client.py | 4 ++-- influxdb/helper.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/influxdb/client.py b/influxdb/client.py index 404e14be..df9ef966 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -860,7 +860,7 @@ def alter_retention_policy(self, name, database=None, query_string = ( "ALTER RETENTION POLICY {0} ON {1}" ).format(quote_ident(name), - quote_ident(database or self._database), shard_duration) + quote_ident(database or self._database)) if duration: query_string += " DURATION {0}".format(duration) if shard_duration: @@ -958,7 +958,7 @@ def drop_user(self, username): :param username: the username to drop :type username: str """ - text = "DROP USER {0}".format(quote_ident(username), method="POST") + text = "DROP USER {0}".format(quote_ident(username)) self.query(text, method="POST") def set_user_password(self, username, password): diff --git a/influxdb/helper.py b/influxdb/helper.py index 74209354..fbf6b65d 100644 --- a/influxdb/helper.py +++ b/influxdb/helper.py @@ -82,7 +82,7 @@ def __new__(cls, *args, **kwargs): allowed_time_precisions = ['h', 'm', 's', 'ms', 'u', 'ns', None] if cls._time_precision not in allowed_time_precisions: raise AttributeError( - 'In {0}, time_precision is set, but invalid use any of {}.' + 'In {0}, time_precision is set, but invalid use any of {1}.' .format(cls.__name__, ','.join(allowed_time_precisions))) cls._retention_policy = getattr(_meta, 'retention_policy', None) From cb3156c1fa9181f142f09e93afccf10355b4e5a1 Mon Sep 17 00:00:00 2001 From: Pavlina Rolincova Date: Tue, 2 Jun 2020 15:11:33 +0200 Subject: [PATCH 46/60] Add support for custom indexes for query in the DataFrameClient (#785) --- influxdb/helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/influxdb/helper.py b/influxdb/helper.py index fbf6b65d..138cf6e8 100644 --- a/influxdb/helper.py +++ b/influxdb/helper.py @@ -82,7 +82,7 @@ def __new__(cls, *args, **kwargs): allowed_time_precisions = ['h', 'm', 's', 'ms', 'u', 'ns', None] if cls._time_precision not in allowed_time_precisions: raise AttributeError( - 'In {0}, time_precision is set, but invalid use any of {1}.' + 'In {}, time_precision is set, but invalid use any of {}.' .format(cls.__name__, ','.join(allowed_time_precisions))) cls._retention_policy = getattr(_meta, 'retention_policy', None) From de5878ae83cbae20ee0a3ec3eba4d0403868c939 Mon Sep 17 00:00:00 2001 From: Yevgen Antymyrov Date: Mon, 15 Jun 2020 16:40:44 +0200 Subject: [PATCH 47/60] Fix #828 "Context manager for InfluxDBClient not working correctly?" by returning a clientt instance from __enter__ --- influxdb/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/influxdb/client.py b/influxdb/client.py index df9ef966..5eea9d1f 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -179,7 +179,7 @@ def __init__(self, def __enter__(self): """Enter function as used by context manager.""" - pass + return self def __exit__(self, _exc_type, _exc_value, _traceback): """Exit function as used by context manager.""" From c49c79adfd8700982d48c01c81e6e2dea539825d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Bedn=C3=A1=C5=99?= Date: Thu, 8 Oct 2020 14:07:58 +0200 Subject: [PATCH 48/60] feat: add support for custom authorization token --- docs/source/examples.rst | 6 +++ examples/tutorial_authorization.py | 32 ++++++++++++++ influxdb/client.py | 3 +- influxdb/tests/client_test.py | 71 ++++++++++++++++++++++++++++++ tox.ini | 2 +- 5 files changed, 112 insertions(+), 2 deletions(-) create mode 100644 examples/tutorial_authorization.py diff --git a/docs/source/examples.rst b/docs/source/examples.rst index fdda62a9..b4ada447 100644 --- a/docs/source/examples.rst +++ b/docs/source/examples.rst @@ -31,3 +31,9 @@ Tutorials - UDP .. literalinclude:: ../../examples/tutorial_udp.py :language: python + +Tutorials - Authorization by Token +=============== + +.. literalinclude:: ../../examples/tutorial_authorization.py + :language: python diff --git a/examples/tutorial_authorization.py b/examples/tutorial_authorization.py new file mode 100644 index 00000000..9d9a800f --- /dev/null +++ b/examples/tutorial_authorization.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +"""Tutorial how to authorize InfluxDB client by custom Authorization token.""" + +import argparse +from influxdb import InfluxDBClient + + +def main(token='my-token'): + """Instantiate a connection to the InfluxDB.""" + client = InfluxDBClient(username=None, password=None, + headers={"Authorization": token}) + + print("Use authorization token: " + token) + + version = client.ping() + print("Successfully connected to InfluxDB: " + version) + pass + + +def parse_args(): + """Parse the args from main.""" + parser = argparse.ArgumentParser( + description='example code to play with InfluxDB') + parser.add_argument('--token', type=str, required=False, + default='my-token', + help='Authorization token for the proxy that is ahead the InfluxDB.') + return parser.parse_args() + + +if __name__ == '__main__': + args = parse_args() + main(token=args.token) diff --git a/influxdb/client.py b/influxdb/client.py index df9ef966..51a64ac3 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -328,10 +328,11 @@ def request(self, url, method='GET', params=None, data=None, stream=False, _try = 0 while retry: try: + auth = (self._username, self._password) response = self._session.request( method=method, url=url, - auth=(self._username, self._password), + auth=auth if None not in auth else None, params=params, data=data, stream=stream, diff --git a/influxdb/tests/client_test.py b/influxdb/tests/client_test.py index fd3c06bb..e511ca9b 100644 --- a/influxdb/tests/client_test.py +++ b/influxdb/tests/client_test.py @@ -1427,6 +1427,77 @@ def test_chunked_response(self): 'values': [['qps'], ['uptime'], ['df'], ['mount']] }]}).__repr__()) + def test_auth_default(self): + """Test auth with default settings.""" + with requests_mock.Mocker() as m: + m.register_uri( + requests_mock.GET, + "http://localhost:8086/ping", + status_code=204, + headers={'X-Influxdb-Version': '1.2.3'} + ) + + cli = InfluxDBClient() + cli.ping() + + self.assertEqual(m.last_request.headers["Authorization"], + "Basic cm9vdDpyb290") + + def test_auth_username_password(self): + """Test auth with custom username and password.""" + with requests_mock.Mocker() as m: + m.register_uri( + requests_mock.GET, + "http://localhost:8086/ping", + status_code=204, + headers={'X-Influxdb-Version': '1.2.3'} + ) + + cli = InfluxDBClient(username='my-username', + password='my-password') + cli.ping() + + self.assertEqual(m.last_request.headers["Authorization"], + "Basic bXktdXNlcm5hbWU6bXktcGFzc3dvcmQ=") + + def test_auth_username_password_none(self): + """Test auth with not defined username or password.""" + with requests_mock.Mocker() as m: + m.register_uri( + requests_mock.GET, + "http://localhost:8086/ping", + status_code=204, + headers={'X-Influxdb-Version': '1.2.3'} + ) + + cli = InfluxDBClient(username=None, password=None) + cli.ping() + self.assertFalse('Authorization' in m.last_request.headers) + + cli = InfluxDBClient(username=None) + cli.ping() + self.assertFalse('Authorization' in m.last_request.headers) + + cli = InfluxDBClient(password=None) + cli.ping() + self.assertFalse('Authorization' in m.last_request.headers) + + def test_auth_token(self): + """Test auth with custom authorization header.""" + with requests_mock.Mocker() as m: + m.register_uri( + requests_mock.GET, + "http://localhost:8086/ping", + status_code=204, + headers={'X-Influxdb-Version': '1.2.3'} + ) + + cli = InfluxDBClient(username=None, password=None, + headers={"Authorization": "my-token"}) + cli.ping() + self.assertEqual(m.last_request.headers["Authorization"], + "my-token") + class FakeClient(InfluxDBClient): """Set up a fake client instance of InfluxDBClient.""" diff --git a/tox.ini b/tox.ini index ff30ebac..1e59b415 100644 --- a/tox.ini +++ b/tox.ini @@ -31,7 +31,7 @@ commands = pydocstyle --count -ve examples influxdb [testenv:coverage] deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt - pandas + pandas==0.24.2 coverage numpy commands = nosetests -v --with-coverage --cover-html --cover-package=influxdb From 7b0367309b118e8be97bd85af4433b25129a4618 Mon Sep 17 00:00:00 2001 From: Sebastian Borza Date: Wed, 11 Nov 2020 15:03:24 -0600 Subject: [PATCH 49/60] chore(CHANGELOG): update to include v5.3.1 updates --- CHANGELOG.md | 1 + influxdb/__init__.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b374faa..f3e86086 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Changed - Amend retry to avoid sleep after last retry before raising exception (#790 thx @krzysbaranski) - Remove msgpack pinning for requirements (#818 thx @prometheanfire) +- Update support for HTTP headers in the InfluxDBClient (#851 thx @bednar) ### Removed diff --git a/influxdb/__init__.py b/influxdb/__init__.py index 56f2f619..59916c26 100644 --- a/influxdb/__init__.py +++ b/influxdb/__init__.py @@ -18,4 +18,4 @@ ] -__version__ = '5.3.0' +__version__ = '5.3.1' From c3903dda515d4f7efcb8c55250fd8b75c8446034 Mon Sep 17 00:00:00 2001 From: Tim Gates Date: Fri, 1 Jan 2021 06:10:01 +1100 Subject: [PATCH 50/60] docs: fix simple typo, reponse -> response (#873) There is a small typo in influxdb/tests/client_test.py. Should read `response` rather than `reponse`. --- influxdb/tests/client_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/influxdb/tests/client_test.py b/influxdb/tests/client_test.py index e511ca9b..1f9d704a 100644 --- a/influxdb/tests/client_test.py +++ b/influxdb/tests/client_test.py @@ -1398,7 +1398,7 @@ def test_invalid_port_fails(self): InfluxDBClient('host', '80/redir', 'username', 'password') def test_chunked_response(self): - """Test chunked reponse for TestInfluxDBClient object.""" + """Test chunked response for TestInfluxDBClient object.""" example_response = \ u'{"results":[{"statement_id":0,"series":[{"columns":["key"],' \ '"values":[["cpu"],["memory"],["iops"],["network"]],"partial":' \ From cf51d026469252d06d57f65489561fc5b1e337c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Bedn=C3=A1=C5=99?= Date: Fri, 19 Mar 2021 16:04:03 +0100 Subject: [PATCH 51/60] docs: add link to v2 client (#881) --- README.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.rst b/README.rst index a40ed148..01e054e8 100644 --- a/README.rst +++ b/README.rst @@ -17,6 +17,8 @@ InfluxDB-Python InfluxDB-Python is a client for interacting with InfluxDB_. +**Note: This library is for use with InfluxDB 1.x. For connecting to InfluxDB 2.x instances, please use the the** `influxdb-client-python `_ **client.** + Development of this library is maintained by: +-----------+-------------------------------+ From dec2f0587ca0592cb94072d58b5f545ebfe68ba2 Mon Sep 17 00:00:00 2001 From: Robert Hajek Date: Wed, 28 Apr 2021 15:55:50 +0200 Subject: [PATCH 52/60] feat: Add custom socket_options --- influxdb/client.py | 24 +++++++++++++++++++++--- influxdb/tests/client_test.py | 25 +++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/influxdb/client.py b/influxdb/client.py index c413181b..2d0b0ddb 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -20,6 +20,7 @@ import msgpack import requests import requests.exceptions +from requests.adapters import HTTPAdapter from six.moves.urllib.parse import urlparse from influxdb.line_protocol import make_lines, quote_ident, quote_literal @@ -87,6 +88,10 @@ class InfluxDBClient(object): :param headers: headers to add to Requests, will add 'Content-Type' and 'Accept' unless these are already present, defaults to {} :type headers: dict + :param socket_options: use custom tcp socket options, If not specified, then defaults are loaded from + ``HTTPConnection.default_socket_options`` + :type socket_options: list + :raises ValueError: if cert is provided but ssl is disabled (set to False) """ @@ -109,6 +114,7 @@ def __init__(self, gzip=False, session=None, headers=None, + socket_options=None, ): """Construct a new InfluxDBClient object.""" self.__host = host @@ -128,9 +134,10 @@ def __init__(self, session = requests.Session() self._session = session - adapter = requests.adapters.HTTPAdapter( + adapter = SocketOptionsAdapter( pool_connections=int(pool_size), - pool_maxsize=int(pool_size) + pool_maxsize=int(pool_size), + socket_options=socket_options ) if use_udp: @@ -626,7 +633,7 @@ def _batches(iterable, size): # http://code.activestate.com/recipes/303279-getting-items-in-batches/ iterator = iter(iterable) while True: - try: # Try get the first element in the iterator... + try: # Try get the first element in the iterator... head = (next(iterator),) except StopIteration: return # ...so that we can stop if there isn't one @@ -1249,3 +1256,14 @@ def _msgpack_parse_hook(code, data): timestamp += datetime.timedelta(microseconds=(epoch_ns / 1000)) return timestamp.isoformat() + 'Z' return msgpack.ExtType(code, data) + + +class SocketOptionsAdapter(HTTPAdapter): + def __init__(self, *args, **kwargs): + self.socket_options = kwargs.pop("socket_options", None) + super(SocketOptionsAdapter, self).__init__(*args, **kwargs) + + def init_poolmanager(self, *args, **kwargs): + if self.socket_options is not None: + kwargs["socket_options"] = self.socket_options + super(SocketOptionsAdapter, self).init_poolmanager(*args, **kwargs) diff --git a/influxdb/tests/client_test.py b/influxdb/tests/client_test.py index 1f9d704a..6b65249c 100644 --- a/influxdb/tests/client_test.py +++ b/influxdb/tests/client_test.py @@ -33,6 +33,7 @@ import requests_mock from nose.tools import raises +from urllib3.connection import HTTPConnection from influxdb import InfluxDBClient from influxdb.resultset import ResultSet @@ -1498,6 +1499,30 @@ def test_auth_token(self): self.assertEqual(m.last_request.headers["Authorization"], "my-token") + def test_custom_socket_options(self): + test_socket_options = HTTPConnection.default_socket_options + [(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1), + (socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, 60), + (socket.IPPROTO_TCP, socket.TCP_KEEPCNT, 15)] + + cli = InfluxDBClient(username=None, password=None, socket_options=test_socket_options) + + self.assertEquals(cli._session.adapters.get("http://").socket_options, test_socket_options) + self.assertEquals(cli._session.adapters.get("http://").poolmanager.connection_pool_kw.get("socket_options"), + test_socket_options) + + connection_pool = cli._session.adapters.get("http://").poolmanager.connection_from_url( + url="http://localhost:8086") + new_connection = connection_pool._new_conn() + self.assertEquals(new_connection.socket_options, test_socket_options) + + def test_none_socket_options(self): + cli = InfluxDBClient(username=None, password=None) + self.assertEquals(cli._session.adapters.get("http://").socket_options, None) + connection_pool = cli._session.adapters.get("http://").poolmanager.connection_from_url( + url="http://localhost:8086") + new_connection = connection_pool._new_conn() + self.assertEquals(new_connection.socket_options, HTTPConnection.default_socket_options) + class FakeClient(InfluxDBClient): """Set up a fake client instance of InfluxDBClient.""" From 6ba88c064e7fd99df7251409c1b8aad2c88335e2 Mon Sep 17 00:00:00 2001 From: Robert Hajek Date: Thu, 29 Apr 2021 13:30:47 +0200 Subject: [PATCH 53/60] feat: Add custom socket_options --- influxdb/client.py | 13 ++++++++----- influxdb/tests/client_test.py | 30 ++++++++++++++++++++---------- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/influxdb/client.py b/influxdb/client.py index 2d0b0ddb..548c5772 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -88,7 +88,8 @@ class InfluxDBClient(object): :param headers: headers to add to Requests, will add 'Content-Type' and 'Accept' unless these are already present, defaults to {} :type headers: dict - :param socket_options: use custom tcp socket options, If not specified, then defaults are loaded from + :param socket_options: use custom tcp socket options, + If not specified, then defaults are loaded from ``HTTPConnection.default_socket_options`` :type socket_options: list @@ -134,7 +135,7 @@ def __init__(self, session = requests.Session() self._session = session - adapter = SocketOptionsAdapter( + adapter = _SocketOptionsAdapter( pool_connections=int(pool_size), pool_maxsize=int(pool_size), socket_options=socket_options @@ -1258,12 +1259,14 @@ def _msgpack_parse_hook(code, data): return msgpack.ExtType(code, data) -class SocketOptionsAdapter(HTTPAdapter): +class _SocketOptionsAdapter(HTTPAdapter): + """_SocketOptionsAdapter injects socket_options into HTTP Adapter.""" + def __init__(self, *args, **kwargs): self.socket_options = kwargs.pop("socket_options", None) - super(SocketOptionsAdapter, self).__init__(*args, **kwargs) + super(_SocketOptionsAdapter, self).__init__(*args, **kwargs) def init_poolmanager(self, *args, **kwargs): if self.socket_options is not None: kwargs["socket_options"] = self.socket_options - super(SocketOptionsAdapter, self).init_poolmanager(*args, **kwargs) + super(_SocketOptionsAdapter, self).init_poolmanager(*args, **kwargs) diff --git a/influxdb/tests/client_test.py b/influxdb/tests/client_test.py index 6b65249c..115fbc48 100644 --- a/influxdb/tests/client_test.py +++ b/influxdb/tests/client_test.py @@ -1500,28 +1500,38 @@ def test_auth_token(self): "my-token") def test_custom_socket_options(self): - test_socket_options = HTTPConnection.default_socket_options + [(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1), - (socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, 60), - (socket.IPPROTO_TCP, socket.TCP_KEEPCNT, 15)] + """Test custom socket options.""" + test_socket_options = HTTPConnection.default_socket_options + \ + [(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1), + (socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, 60), + (socket.IPPROTO_TCP, socket.TCP_KEEPCNT, 15)] - cli = InfluxDBClient(username=None, password=None, socket_options=test_socket_options) + cli = InfluxDBClient(username=None, password=None, + socket_options=test_socket_options) - self.assertEquals(cli._session.adapters.get("http://").socket_options, test_socket_options) - self.assertEquals(cli._session.adapters.get("http://").poolmanager.connection_pool_kw.get("socket_options"), + self.assertEquals(cli._session.adapters.get("http://").socket_options, + test_socket_options) + self.assertEquals(cli._session.adapters.get("http://").poolmanager. + connection_pool_kw.get("socket_options"), test_socket_options) - connection_pool = cli._session.adapters.get("http://").poolmanager.connection_from_url( + connection_pool = cli._session.adapters.get("http://").poolmanager \ + .connection_from_url( url="http://localhost:8086") new_connection = connection_pool._new_conn() self.assertEquals(new_connection.socket_options, test_socket_options) def test_none_socket_options(self): + """Test default socket options.""" cli = InfluxDBClient(username=None, password=None) - self.assertEquals(cli._session.adapters.get("http://").socket_options, None) - connection_pool = cli._session.adapters.get("http://").poolmanager.connection_from_url( + self.assertEquals(cli._session.adapters.get("http://").socket_options, + None) + connection_pool = cli._session.adapters.get("http://").poolmanager \ + .connection_from_url( url="http://localhost:8086") new_connection = connection_pool._new_conn() - self.assertEquals(new_connection.socket_options, HTTPConnection.default_socket_options) + self.assertEquals(new_connection.socket_options, + HTTPConnection.default_socket_options) class FakeClient(InfluxDBClient): From 7cb565698c88bfbf9f4804650231bd28d09e2e6d Mon Sep 17 00:00:00 2001 From: Grant7z Date: Thu, 9 Sep 2021 08:50:11 +0800 Subject: [PATCH 54/60] Prevent overwriting Authorization with basic auth (#901) --- influxdb/client.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/influxdb/client.py b/influxdb/client.py index 548c5772..adab4edc 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -336,7 +336,10 @@ def request(self, url, method='GET', params=None, data=None, stream=False, _try = 0 while retry: try: - auth = (self._username, self._password) + if "Authorization" in headers: + auth = (None, None) + else: + auth = (self._username, self._password) response = self._session.request( method=method, url=url, From 3d1f1ce32524e40f4db33b9d8f11faf9b4925bf2 Mon Sep 17 00:00:00 2001 From: Jamie Strandboge Date: Tue, 15 Nov 2022 15:25:21 -0600 Subject: [PATCH 55/60] README.md: archive repo --- README.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.rst b/README.rst index 01e054e8..58bfcd1a 100644 --- a/README.rst +++ b/README.rst @@ -1,3 +1,13 @@ +The v1 client libraries for InfluxDB were typically developed and maintained by +community members. They have all now been succeeded by v2 client libraries. +They are being archived it in favor of the v2 client library. See +https://github.com/influxdata/influxdb-python/issues/918. + +If there are still users of this v1 client library, and they or somebody else +are willing to keep them updated with security fixes at a minimum please reach +out on the [Community Forums](https://community.influxdata.com/) or +[InfluxData Slack](https://influxdata.com/slack). + InfluxDB-Python =============== From 1df9a816a93c561779b54bb82472c83ce990282b Mon Sep 17 00:00:00 2001 From: Jamie Strandboge Date: Tue, 18 Jul 2023 16:29:05 -0500 Subject: [PATCH 56/60] chore: update README.rst based on feedback from Product --- README.rst | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index 58bfcd1a..d214d41a 100644 --- a/README.rst +++ b/README.rst @@ -1,12 +1,11 @@ The v1 client libraries for InfluxDB were typically developed and maintained by -community members. They have all now been succeeded by v2 client libraries. -They are being archived it in favor of the v2 client library. See -https://github.com/influxdata/influxdb-python/issues/918. +community members. For InfluxDB 3.0 users, this library is succeeded by the +lightweight `v3 client library `_. If there are still users of this v1 client library, and they or somebody else are willing to keep them updated with security fixes at a minimum please reach -out on the [Community Forums](https://community.influxdata.com/) or -[InfluxData Slack](https://influxdata.com/slack). +out on the `Community Forums `_ or +`InfluxData Slack `_. InfluxDB-Python =============== From b614474e82c621dc77f52d36a9d54b8be2af45c6 Mon Sep 17 00:00:00 2001 From: Josh Powers Date: Tue, 16 Apr 2024 08:29:25 -0600 Subject: [PATCH 57/60] fix: Correctly serialize nanosecond dataframe timestamps Co-authored-by: @bednar --- influxdb/_dataframe_client.py | 4 +- influxdb/tests/dataframe_client_test.py | 74 ++++++++++++++++++++++++- 2 files changed, 75 insertions(+), 3 deletions(-) diff --git a/influxdb/_dataframe_client.py b/influxdb/_dataframe_client.py index e7ae9c17..907db2cb 100644 --- a/influxdb/_dataframe_client.py +++ b/influxdb/_dataframe_client.py @@ -372,10 +372,10 @@ def _convert_dataframe_to_lines(self, # Make array of timestamp ints if isinstance(dataframe.index, pd.PeriodIndex): - time = ((dataframe.index.to_timestamp().values.astype(np.int64) / + time = ((dataframe.index.to_timestamp().values.astype(np.int64) // precision_factor).astype(np.int64).astype(str)) else: - time = ((pd.to_datetime(dataframe.index).values.astype(np.int64) / + time = ((pd.to_datetime(dataframe.index).values.astype(np.int64) // precision_factor).astype(np.int64).astype(str)) # If tag columns exist, make an array of formatted tag keys and values diff --git a/influxdb/tests/dataframe_client_test.py b/influxdb/tests/dataframe_client_test.py index f6db3c22..87b8e0d8 100644 --- a/influxdb/tests/dataframe_client_test.py +++ b/influxdb/tests/dataframe_client_test.py @@ -877,7 +877,7 @@ def test_query_into_dataframe(self): {"measurement": "network", "tags": {"direction": ""}, "columns": ["time", "value"], - "values":[["2009-11-10T23:00:00Z", 23422]] + "values": [["2009-11-10T23:00:00Z", 23422]] }, {"measurement": "network", "tags": {"direction": "in"}, @@ -1274,3 +1274,75 @@ def test_query_custom_index(self): self.assertListEqual(["time", "host"], list(_data_frame.index.names)) + + def test_dataframe_nanosecond_precision(self): + """Test nanosecond precision.""" + for_df_dict = { + "nanFloats": [1.1, float('nan'), 3.3, 4.4], + "onlyFloats": [1.1, 2.2, 3.3, 4.4], + "strings": ['one_one', 'two_two', 'three_three', 'four_four'] + } + df = pd.DataFrame.from_dict(for_df_dict) + df['time'] = ['2019-10-04 06:27:19.850557111+00:00', + '2019-10-04 06:27:19.850557184+00:00', + '2019-10-04 06:27:42.251396864+00:00', + '2019-10-04 06:27:42.251396974+00:00'] + df['time'] = pd.to_datetime(df['time'], unit='ns') + df = df.set_index('time') + + expected = ( + b'foo nanFloats=1.1,onlyFloats=1.1,strings="one_one" 1570170439850557111\n' # noqa E501 line too long + b'foo onlyFloats=2.2,strings="two_two" 1570170439850557184\n' # noqa E501 line too long + b'foo nanFloats=3.3,onlyFloats=3.3,strings="three_three" 1570170462251396864\n' # noqa E501 line too long + b'foo nanFloats=4.4,onlyFloats=4.4,strings="four_four" 1570170462251396974\n' # noqa E501 line too long + ) + + with requests_mock.Mocker() as m: + m.register_uri( + requests_mock.POST, + "http://localhost:8086/write", + status_code=204 + ) + + cli = DataFrameClient(database='db') + cli.write_points(df, 'foo', time_precision='n') + + self.assertEqual(m.last_request.body, expected) + + def test_dataframe_nanosecond_precision_one_microsecond(self): + """Test nanosecond precision within one microsecond.""" + # 1 microsecond = 1000 nanoseconds + start = np.datetime64('2019-10-04T06:27:19.850557000') + end = np.datetime64('2019-10-04T06:27:19.850558000') + + # generate timestamps with nanosecond precision + timestamps = np.arange( + start, + end + np.timedelta64(1, 'ns'), + np.timedelta64(1, 'ns') + ) + # generate values + values = np.arange(0.0, len(timestamps)) + + df = pd.DataFrame({'value': values}, index=timestamps) + with requests_mock.Mocker() as m: + m.register_uri( + requests_mock.POST, + "http://localhost:8086/write", + status_code=204 + ) + + cli = DataFrameClient(database='db') + cli.write_points(df, 'foo', time_precision='n') + + lines = m.last_request.body.decode('utf-8').split('\n') + self.assertEqual(len(lines), 1002) + + for index, line in enumerate(lines): + if index == 1001: + self.assertEqual(line, '') + continue + self.assertEqual( + line, + f"foo value={index}.0 157017043985055{7000 + index:04}" + ) From 5ad04f696c71514967c9d7419bff457dbfbe8400 Mon Sep 17 00:00:00 2001 From: Joshua Powers Date: Tue, 16 Apr 2024 08:35:24 -0600 Subject: [PATCH 58/60] Update README.rst Add link to v2 library --- README.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.rst b/README.rst index d214d41a..b78e626d 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,8 @@ The v1 client libraries for InfluxDB were typically developed and maintained by community members. For InfluxDB 3.0 users, this library is succeeded by the lightweight `v3 client library `_. +For InfluxDB 2.0 users, look at the `v2 client library +`_. If there are still users of this v1 client library, and they or somebody else are willing to keep them updated with security fixes at a minimum please reach From 37ff905fbefe33bc321e619ea970d015ccd8b434 Mon Sep 17 00:00:00 2001 From: Josh Powers Date: Wed, 17 Apr 2024 07:14:43 -0600 Subject: [PATCH 59/60] chore(CHANGELOG): Release v5.3.2 --- CHANGELOG.md | 7 ++++++- influxdb/__init__.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f3e86086..bfd27d38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). -## [v5.3.1] - Unreleased +## [v5.3.2] - 2024-04-17 + +### Changed +- Correctly serialize nanosecond dataframe timestamps (#926) + +## [v5.3.1] - 2022-11-14 ### Added - Add support for custom headers in the InfluxDBClient (#710 thx @nathanielatom) diff --git a/influxdb/__init__.py b/influxdb/__init__.py index 59916c26..e66f80ea 100644 --- a/influxdb/__init__.py +++ b/influxdb/__init__.py @@ -18,4 +18,4 @@ ] -__version__ = '5.3.1' +__version__ = '5.3.2' From bbe80ed32cf57b252be131d3edcda5ca610fc223 Mon Sep 17 00:00:00 2001 From: Jason Stirnaman Date: Tue, 29 Oct 2024 10:54:14 -0500 Subject: [PATCH 60/60] Update docs (#929) * chore(docs): Update version advice and tox build requirements: - Updates the version advice and removes the succession statement to be consistent with the OSS 1.11 release. - Adds explicit relations to v2 and v3. - Removes redundancies. - Fixes formatting errors reported by the linter. - Updates dependencies for docs build. * fix(docs): mention 1.x compat endpoints in 2.x --- README.rst | 78 ++++++++++++++++++++++------------------ docs/source/conf.py | 3 +- docs/source/examples.rst | 2 +- influxdb/client.py | 7 ++-- pyproject.toml | 3 ++ requirements.txt | 4 +-- setup.py | 5 +++ tox.ini | 10 +++--- 8 files changed, 65 insertions(+), 47 deletions(-) create mode 100644 pyproject.toml diff --git a/README.rst b/README.rst index b78e626d..048db045 100644 --- a/README.rst +++ b/README.rst @@ -1,14 +1,3 @@ -The v1 client libraries for InfluxDB were typically developed and maintained by -community members. For InfluxDB 3.0 users, this library is succeeded by the -lightweight `v3 client library `_. -For InfluxDB 2.0 users, look at the `v2 client library -`_. - -If there are still users of this v1 client library, and they or somebody else -are willing to keep them updated with security fixes at a minimum please reach -out on the `Community Forums `_ or -`InfluxData Slack `_. - InfluxDB-Python =============== @@ -26,38 +15,45 @@ InfluxDB-Python :target: https://pypi.python.org/pypi/influxdb :alt: PyPI Status -InfluxDB-Python is a client for interacting with InfluxDB_. -**Note: This library is for use with InfluxDB 1.x. For connecting to InfluxDB 2.x instances, please use the the** `influxdb-client-python `_ **client.** +.. important:: -Development of this library is maintained by: + **This project is no longer in development** + + This v1 client library is for interacting with `InfluxDB 1.x `_ and 1.x-compatible endpoints in `InfluxDB 2.x `_. + Use it to: + + - Write data in line protocol. + - Query data with `InfluxQL `_. -+-----------+-------------------------------+ -| Github ID | URL | -+===========+===============================+ -| @aviau | (https://github.com/aviau) | -+-----------+-------------------------------+ -| @xginn8 | (https://github.com/xginn8) | -+-----------+-------------------------------+ -| @sebito91 | (https://github.com/sebito91) | -+-----------+-------------------------------+ + If you use `InfluxDB 2.x (TSM storage engine) `_ and `Flux `_, see the `v2 client library `_. + + If you use `InfluxDB 3.0 `_, see the `v3 client library `_. + + For new projects, consider using InfluxDB 3.0 and v3 client libraries. + +Description +=========== + +InfluxDB-python, the InfluxDB Python Client (1.x), is a client library for interacting with `InfluxDB 1.x `_ instances. .. _readme-about: -InfluxDB is an open-source distributed time series database, find more about InfluxDB_ at https://docs.influxdata.com/influxdb/latest +`InfluxDB`_ is the time series platform designed to handle high write and query loads. .. _installation: -InfluxDB pre v1.1.0 users -------------------------- -This module is tested with InfluxDB versions: v1.2.4, v1.3.9, v1.4.3, v1.5.4, v1.6.4, and 1.7.4. +For InfluxDB pre-v1.1.0 users +----------------------------- -Those users still on InfluxDB v0.8.x users may still use the legacy client by importing ``from influxdb.influxdb08 import InfluxDBClient``. +This module is tested with InfluxDB versions v1.2.4, v1.3.9, v1.4.3, v1.5.4, v1.6.4, and 1.7.4. -Installation ------------- +Users on InfluxDB v0.8.x may still use the legacy client by importing ``from influxdb.influxdb08 import InfluxDBClient``. + +For InfluxDB v1.1+ users +------------------------ Install, upgrade and uninstall influxdb-python with these commands:: @@ -165,21 +161,33 @@ We are also lurking on the following: Development ----------- +The v1 client libraries for InfluxDB 1.x were typically developed and maintained by InfluxDB community members. If you are an InfluxDB v1 user interested in maintaining this client library (at a minimum, keeping it updated with security patches) please contact the InfluxDB team at on the `Community Forums `_ or +`InfluxData Slack `_. + All development is done on Github_. Use Issues_ to report problems or submit contributions. .. _Github: https://github.com/influxdb/influxdb-python/ .. _Issues: https://github.com/influxdb/influxdb-python/issues -Please note that we WILL get to your questions/issues/concerns as quickly as possible. We maintain many -software repositories and sometimes things may get pushed to the backburner. Please don't take offense, -we will do our best to reply as soon as possible! +Please note that we will answer you question as quickly as possible. +Maintainers: + ++-----------+-------------------------------+ +| Github ID | URL | ++===========+===============================+ +| @aviau | (https://github.com/aviau) | ++-----------+-------------------------------+ +| @xginn8 | (https://github.com/xginn8) | ++-----------+-------------------------------+ +| @sebito91 | (https://github.com/sebito91) | ++-----------+-------------------------------+ Source code ----------- -The source code is currently available on Github: https://github.com/influxdata/influxdb-python +The source code for the InfluxDB Python Client (1.x) is currently available on Github: https://github.com/influxdata/influxdb-python TODO @@ -188,6 +196,6 @@ TODO The TODO/Roadmap can be found in Github bug tracker: https://github.com/influxdata/influxdb-python/issues -.. _InfluxDB: https://influxdata.com/time-series-platform/influxdb/ +.. _InfluxDB: https://influxdata.com/ .. _Sphinx: http://sphinx.pocoo.org/ .. _Tox: https://tox.readthedocs.org diff --git a/docs/source/conf.py b/docs/source/conf.py index 231c776c..efc22f88 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -117,7 +117,8 @@ # Add any paths that contain custom themes here, relative to this directory. #html_theme_path = [] -html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] +# Calling get_html_theme_path is deprecated. +# html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". diff --git a/docs/source/examples.rst b/docs/source/examples.rst index b4ada447..841ad8b1 100644 --- a/docs/source/examples.rst +++ b/docs/source/examples.rst @@ -33,7 +33,7 @@ Tutorials - UDP :language: python Tutorials - Authorization by Token -=============== +================================== .. literalinclude:: ../../examples/tutorial_authorization.py :language: python diff --git a/influxdb/client.py b/influxdb/client.py index adab4edc..c535a3f1 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -395,7 +395,7 @@ def write(self, data, params=None, expected_response_code=204, :param data: the data to be written :type data: (if protocol is 'json') dict (if protocol is 'line') sequence of line protocol strings - or single string + or single string :param params: additional parameters for the request, defaults to None :type params: dict :param expected_response_code: the expected response code of the write @@ -571,8 +571,9 @@ def write_points(self, :param points: the list of points to be written in the database :type points: list of dictionaries, each dictionary represents a point :type points: (if protocol is 'json') list of dicts, where each dict - represents a point. - (if protocol is 'line') sequence of line protocol strings. + represents a point. + (if protocol is 'line') sequence of line protocol strings. + :param time_precision: Either 's', 'm', 'ms' or 'u', defaults to None :type time_precision: str :param database: the database to write the points to. Defaults to diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..1b68d94e --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools>=42", "wheel"] +build-backend = "setuptools.build_meta" \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 548b17c8..a3df3154 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ python-dateutil>=2.6.0 -pytz +pytz>=2016.10 requests>=2.17.0 six>=1.10.0 -msgpack +msgpack>=0.5.0 diff --git a/setup.py b/setup.py index d44875f6..8ac7d1a7 100755 --- a/setup.py +++ b/setup.py @@ -23,6 +23,11 @@ with open('requirements.txt', 'r') as f: requires = [x.strip() for x in f if x.strip()] +# Debugging: Print the requires values +print("install_requires values:") +for req in requires: + print(f"- {req}") + with open('test-requirements.txt', 'r') as f: test_requires = [x.strip() for x in f if x.strip()] diff --git a/tox.ini b/tox.ini index 1e59b415..a1005abb 100644 --- a/tox.ini +++ b/tox.ini @@ -12,8 +12,8 @@ deps = -r{toxinidir}/requirements.txt py35: numpy==1.14.6 py36: pandas==0.23.4 py36: numpy==1.15.4 - py37: pandas==0.24.2 - py37: numpy==1.16.2 + py37: pandas>=0.24.2 + py37: numpy>=1.16.2 # Only install pandas with non-pypy interpreters # Testing all combinations would be too expensive commands = nosetests -v --with-doctest {posargs} @@ -38,9 +38,9 @@ commands = nosetests -v --with-coverage --cover-html --cover-package=influxdb [testenv:docs] deps = -r{toxinidir}/requirements.txt - pandas==0.24.2 - numpy==1.16.2 - Sphinx==1.8.5 + pandas>=0.24.2 + numpy>=1.16.2 + Sphinx>=1.8.5 sphinx_rtd_theme commands = sphinx-build -b html docs/source docs/build