From b614474e82c621dc77f52d36a9d54b8be2af45c6 Mon Sep 17 00:00:00 2001 From: Josh Powers Date: Tue, 16 Apr 2024 08:29:25 -0600 Subject: [PATCH 1/4] fix: Correctly serialize nanosecond dataframe timestamps Co-authored-by: @bednar --- influxdb/_dataframe_client.py | 4 +- influxdb/tests/dataframe_client_test.py | 74 ++++++++++++++++++++++++- 2 files changed, 75 insertions(+), 3 deletions(-) diff --git a/influxdb/_dataframe_client.py b/influxdb/_dataframe_client.py index e7ae9c17..907db2cb 100644 --- a/influxdb/_dataframe_client.py +++ b/influxdb/_dataframe_client.py @@ -372,10 +372,10 @@ def _convert_dataframe_to_lines(self, # Make array of timestamp ints if isinstance(dataframe.index, pd.PeriodIndex): - time = ((dataframe.index.to_timestamp().values.astype(np.int64) / + time = ((dataframe.index.to_timestamp().values.astype(np.int64) // precision_factor).astype(np.int64).astype(str)) else: - time = ((pd.to_datetime(dataframe.index).values.astype(np.int64) / + time = ((pd.to_datetime(dataframe.index).values.astype(np.int64) // precision_factor).astype(np.int64).astype(str)) # If tag columns exist, make an array of formatted tag keys and values diff --git a/influxdb/tests/dataframe_client_test.py b/influxdb/tests/dataframe_client_test.py index f6db3c22..87b8e0d8 100644 --- a/influxdb/tests/dataframe_client_test.py +++ b/influxdb/tests/dataframe_client_test.py @@ -877,7 +877,7 @@ def test_query_into_dataframe(self): {"measurement": "network", "tags": {"direction": ""}, "columns": ["time", "value"], - "values":[["2009-11-10T23:00:00Z", 23422]] + "values": [["2009-11-10T23:00:00Z", 23422]] }, {"measurement": "network", "tags": {"direction": "in"}, @@ -1274,3 +1274,75 @@ def test_query_custom_index(self): self.assertListEqual(["time", "host"], list(_data_frame.index.names)) + + def test_dataframe_nanosecond_precision(self): + """Test nanosecond precision.""" + for_df_dict = { + "nanFloats": [1.1, float('nan'), 3.3, 4.4], + "onlyFloats": [1.1, 2.2, 3.3, 4.4], + "strings": ['one_one', 'two_two', 'three_three', 'four_four'] + } + df = pd.DataFrame.from_dict(for_df_dict) + df['time'] = ['2019-10-04 06:27:19.850557111+00:00', + '2019-10-04 06:27:19.850557184+00:00', + '2019-10-04 06:27:42.251396864+00:00', + '2019-10-04 06:27:42.251396974+00:00'] + df['time'] = pd.to_datetime(df['time'], unit='ns') + df = df.set_index('time') + + expected = ( + b'foo nanFloats=1.1,onlyFloats=1.1,strings="one_one" 1570170439850557111\n' # noqa E501 line too long + b'foo onlyFloats=2.2,strings="two_two" 1570170439850557184\n' # noqa E501 line too long + b'foo nanFloats=3.3,onlyFloats=3.3,strings="three_three" 1570170462251396864\n' # noqa E501 line too long + b'foo nanFloats=4.4,onlyFloats=4.4,strings="four_four" 1570170462251396974\n' # noqa E501 line too long + ) + + with requests_mock.Mocker() as m: + m.register_uri( + requests_mock.POST, + "http://localhost:8086/write", + status_code=204 + ) + + cli = DataFrameClient(database='db') + cli.write_points(df, 'foo', time_precision='n') + + self.assertEqual(m.last_request.body, expected) + + def test_dataframe_nanosecond_precision_one_microsecond(self): + """Test nanosecond precision within one microsecond.""" + # 1 microsecond = 1000 nanoseconds + start = np.datetime64('2019-10-04T06:27:19.850557000') + end = np.datetime64('2019-10-04T06:27:19.850558000') + + # generate timestamps with nanosecond precision + timestamps = np.arange( + start, + end + np.timedelta64(1, 'ns'), + np.timedelta64(1, 'ns') + ) + # generate values + values = np.arange(0.0, len(timestamps)) + + df = pd.DataFrame({'value': values}, index=timestamps) + with requests_mock.Mocker() as m: + m.register_uri( + requests_mock.POST, + "http://localhost:8086/write", + status_code=204 + ) + + cli = DataFrameClient(database='db') + cli.write_points(df, 'foo', time_precision='n') + + lines = m.last_request.body.decode('utf-8').split('\n') + self.assertEqual(len(lines), 1002) + + for index, line in enumerate(lines): + if index == 1001: + self.assertEqual(line, '') + continue + self.assertEqual( + line, + f"foo value={index}.0 157017043985055{7000 + index:04}" + ) From 5ad04f696c71514967c9d7419bff457dbfbe8400 Mon Sep 17 00:00:00 2001 From: Joshua Powers Date: Tue, 16 Apr 2024 08:35:24 -0600 Subject: [PATCH 2/4] Update README.rst Add link to v2 library --- README.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.rst b/README.rst index d214d41a..b78e626d 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,8 @@ The v1 client libraries for InfluxDB were typically developed and maintained by community members. For InfluxDB 3.0 users, this library is succeeded by the lightweight `v3 client library `_. +For InfluxDB 2.0 users, look at the `v2 client library +`_. If there are still users of this v1 client library, and they or somebody else are willing to keep them updated with security fixes at a minimum please reach From 37ff905fbefe33bc321e619ea970d015ccd8b434 Mon Sep 17 00:00:00 2001 From: Josh Powers Date: Wed, 17 Apr 2024 07:14:43 -0600 Subject: [PATCH 3/4] chore(CHANGELOG): Release v5.3.2 --- CHANGELOG.md | 7 ++++++- influxdb/__init__.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f3e86086..bfd27d38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). -## [v5.3.1] - Unreleased +## [v5.3.2] - 2024-04-17 + +### Changed +- Correctly serialize nanosecond dataframe timestamps (#926) + +## [v5.3.1] - 2022-11-14 ### Added - Add support for custom headers in the InfluxDBClient (#710 thx @nathanielatom) diff --git a/influxdb/__init__.py b/influxdb/__init__.py index 59916c26..e66f80ea 100644 --- a/influxdb/__init__.py +++ b/influxdb/__init__.py @@ -18,4 +18,4 @@ ] -__version__ = '5.3.1' +__version__ = '5.3.2' From bbe80ed32cf57b252be131d3edcda5ca610fc223 Mon Sep 17 00:00:00 2001 From: Jason Stirnaman Date: Tue, 29 Oct 2024 10:54:14 -0500 Subject: [PATCH 4/4] Update docs (#929) * chore(docs): Update version advice and tox build requirements: - Updates the version advice and removes the succession statement to be consistent with the OSS 1.11 release. - Adds explicit relations to v2 and v3. - Removes redundancies. - Fixes formatting errors reported by the linter. - Updates dependencies for docs build. * fix(docs): mention 1.x compat endpoints in 2.x --- README.rst | 78 ++++++++++++++++++++++------------------ docs/source/conf.py | 3 +- docs/source/examples.rst | 2 +- influxdb/client.py | 7 ++-- pyproject.toml | 3 ++ requirements.txt | 4 +-- setup.py | 5 +++ tox.ini | 10 +++--- 8 files changed, 65 insertions(+), 47 deletions(-) create mode 100644 pyproject.toml diff --git a/README.rst b/README.rst index b78e626d..048db045 100644 --- a/README.rst +++ b/README.rst @@ -1,14 +1,3 @@ -The v1 client libraries for InfluxDB were typically developed and maintained by -community members. For InfluxDB 3.0 users, this library is succeeded by the -lightweight `v3 client library `_. -For InfluxDB 2.0 users, look at the `v2 client library -`_. - -If there are still users of this v1 client library, and they or somebody else -are willing to keep them updated with security fixes at a minimum please reach -out on the `Community Forums `_ or -`InfluxData Slack `_. - InfluxDB-Python =============== @@ -26,38 +15,45 @@ InfluxDB-Python :target: https://pypi.python.org/pypi/influxdb :alt: PyPI Status -InfluxDB-Python is a client for interacting with InfluxDB_. -**Note: This library is for use with InfluxDB 1.x. For connecting to InfluxDB 2.x instances, please use the the** `influxdb-client-python `_ **client.** +.. important:: -Development of this library is maintained by: + **This project is no longer in development** + + This v1 client library is for interacting with `InfluxDB 1.x `_ and 1.x-compatible endpoints in `InfluxDB 2.x `_. + Use it to: + + - Write data in line protocol. + - Query data with `InfluxQL `_. -+-----------+-------------------------------+ -| Github ID | URL | -+===========+===============================+ -| @aviau | (https://github.com/aviau) | -+-----------+-------------------------------+ -| @xginn8 | (https://github.com/xginn8) | -+-----------+-------------------------------+ -| @sebito91 | (https://github.com/sebito91) | -+-----------+-------------------------------+ + If you use `InfluxDB 2.x (TSM storage engine) `_ and `Flux `_, see the `v2 client library `_. + + If you use `InfluxDB 3.0 `_, see the `v3 client library `_. + + For new projects, consider using InfluxDB 3.0 and v3 client libraries. + +Description +=========== + +InfluxDB-python, the InfluxDB Python Client (1.x), is a client library for interacting with `InfluxDB 1.x `_ instances. .. _readme-about: -InfluxDB is an open-source distributed time series database, find more about InfluxDB_ at https://docs.influxdata.com/influxdb/latest +`InfluxDB`_ is the time series platform designed to handle high write and query loads. .. _installation: -InfluxDB pre v1.1.0 users -------------------------- -This module is tested with InfluxDB versions: v1.2.4, v1.3.9, v1.4.3, v1.5.4, v1.6.4, and 1.7.4. +For InfluxDB pre-v1.1.0 users +----------------------------- -Those users still on InfluxDB v0.8.x users may still use the legacy client by importing ``from influxdb.influxdb08 import InfluxDBClient``. +This module is tested with InfluxDB versions v1.2.4, v1.3.9, v1.4.3, v1.5.4, v1.6.4, and 1.7.4. -Installation ------------- +Users on InfluxDB v0.8.x may still use the legacy client by importing ``from influxdb.influxdb08 import InfluxDBClient``. + +For InfluxDB v1.1+ users +------------------------ Install, upgrade and uninstall influxdb-python with these commands:: @@ -165,21 +161,33 @@ We are also lurking on the following: Development ----------- +The v1 client libraries for InfluxDB 1.x were typically developed and maintained by InfluxDB community members. If you are an InfluxDB v1 user interested in maintaining this client library (at a minimum, keeping it updated with security patches) please contact the InfluxDB team at on the `Community Forums `_ or +`InfluxData Slack `_. + All development is done on Github_. Use Issues_ to report problems or submit contributions. .. _Github: https://github.com/influxdb/influxdb-python/ .. _Issues: https://github.com/influxdb/influxdb-python/issues -Please note that we WILL get to your questions/issues/concerns as quickly as possible. We maintain many -software repositories and sometimes things may get pushed to the backburner. Please don't take offense, -we will do our best to reply as soon as possible! +Please note that we will answer you question as quickly as possible. +Maintainers: + ++-----------+-------------------------------+ +| Github ID | URL | ++===========+===============================+ +| @aviau | (https://github.com/aviau) | ++-----------+-------------------------------+ +| @xginn8 | (https://github.com/xginn8) | ++-----------+-------------------------------+ +| @sebito91 | (https://github.com/sebito91) | ++-----------+-------------------------------+ Source code ----------- -The source code is currently available on Github: https://github.com/influxdata/influxdb-python +The source code for the InfluxDB Python Client (1.x) is currently available on Github: https://github.com/influxdata/influxdb-python TODO @@ -188,6 +196,6 @@ TODO The TODO/Roadmap can be found in Github bug tracker: https://github.com/influxdata/influxdb-python/issues -.. _InfluxDB: https://influxdata.com/time-series-platform/influxdb/ +.. _InfluxDB: https://influxdata.com/ .. _Sphinx: http://sphinx.pocoo.org/ .. _Tox: https://tox.readthedocs.org diff --git a/docs/source/conf.py b/docs/source/conf.py index 231c776c..efc22f88 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -117,7 +117,8 @@ # Add any paths that contain custom themes here, relative to this directory. #html_theme_path = [] -html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] +# Calling get_html_theme_path is deprecated. +# html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". diff --git a/docs/source/examples.rst b/docs/source/examples.rst index b4ada447..841ad8b1 100644 --- a/docs/source/examples.rst +++ b/docs/source/examples.rst @@ -33,7 +33,7 @@ Tutorials - UDP :language: python Tutorials - Authorization by Token -=============== +================================== .. literalinclude:: ../../examples/tutorial_authorization.py :language: python diff --git a/influxdb/client.py b/influxdb/client.py index adab4edc..c535a3f1 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -395,7 +395,7 @@ def write(self, data, params=None, expected_response_code=204, :param data: the data to be written :type data: (if protocol is 'json') dict (if protocol is 'line') sequence of line protocol strings - or single string + or single string :param params: additional parameters for the request, defaults to None :type params: dict :param expected_response_code: the expected response code of the write @@ -571,8 +571,9 @@ def write_points(self, :param points: the list of points to be written in the database :type points: list of dictionaries, each dictionary represents a point :type points: (if protocol is 'json') list of dicts, where each dict - represents a point. - (if protocol is 'line') sequence of line protocol strings. + represents a point. + (if protocol is 'line') sequence of line protocol strings. + :param time_precision: Either 's', 'm', 'ms' or 'u', defaults to None :type time_precision: str :param database: the database to write the points to. Defaults to diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..1b68d94e --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools>=42", "wheel"] +build-backend = "setuptools.build_meta" \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 548b17c8..a3df3154 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ python-dateutil>=2.6.0 -pytz +pytz>=2016.10 requests>=2.17.0 six>=1.10.0 -msgpack +msgpack>=0.5.0 diff --git a/setup.py b/setup.py index d44875f6..8ac7d1a7 100755 --- a/setup.py +++ b/setup.py @@ -23,6 +23,11 @@ with open('requirements.txt', 'r') as f: requires = [x.strip() for x in f if x.strip()] +# Debugging: Print the requires values +print("install_requires values:") +for req in requires: + print(f"- {req}") + with open('test-requirements.txt', 'r') as f: test_requires = [x.strip() for x in f if x.strip()] diff --git a/tox.ini b/tox.ini index 1e59b415..a1005abb 100644 --- a/tox.ini +++ b/tox.ini @@ -12,8 +12,8 @@ deps = -r{toxinidir}/requirements.txt py35: numpy==1.14.6 py36: pandas==0.23.4 py36: numpy==1.15.4 - py37: pandas==0.24.2 - py37: numpy==1.16.2 + py37: pandas>=0.24.2 + py37: numpy>=1.16.2 # Only install pandas with non-pypy interpreters # Testing all combinations would be too expensive commands = nosetests -v --with-doctest {posargs} @@ -38,9 +38,9 @@ commands = nosetests -v --with-coverage --cover-html --cover-package=influxdb [testenv:docs] deps = -r{toxinidir}/requirements.txt - pandas==0.24.2 - numpy==1.16.2 - Sphinx==1.8.5 + pandas>=0.24.2 + numpy>=1.16.2 + Sphinx>=1.8.5 sphinx_rtd_theme commands = sphinx-build -b html docs/source docs/build