From eb5afd19c2014eddc33247bffef4a360c486277c Mon Sep 17 00:00:00 2001 From: Jakub Bednar Date: Tue, 5 Dec 2023 08:51:38 +0100 Subject: [PATCH 01/51] chore(release): prepare for next development iteration --- CHANGELOG.md | 2 ++ conda/meta.yaml | 6 +++--- influxdb_client/version.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3947b346..a0e4ebc1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +## 1.40.0 [unreleased] + ## 1.39.0 [2023-12-05] ### Features diff --git a/conda/meta.yaml b/conda/meta.yaml index 2ce4b993..a2291164 100644 --- a/conda/meta.yaml +++ b/conda/meta.yaml @@ -1,5 +1,5 @@ {% set name = "influxdb_client" %} -{% set version = "1.38.0" %} +{% set version = "1.39.0" %} package: @@ -7,8 +7,8 @@ package: version: {{ version }} source: - url: https://files.pythonhosted.org/packages/8b/b5/03786e2e47f65c9d799e579dcef3cf325b92adda5c726beb3769b0bfdcbd/influxdb_client-1.38.0.tar.gz - sha256: 88ee8c1beb6b3b1359f4117d51704d5da5ac70e598b9fe786823e36ac86175a8 + url: https://files.pythonhosted.org/packages/f1/0e/d4da1d18316eab78b7041e60dbf4fe6062ae7e32dd55ed22bda316b1d217/influxdb_client-1.39.0.tar.gz + sha256: 6a534913523bd262f1928e4ff80046bf95e313c1694ce13e45fd17eea90fe691 build: number: 0 diff --git a/influxdb_client/version.py b/influxdb_client/version.py index e302b74d..e33b8426 100644 --- a/influxdb_client/version.py +++ b/influxdb_client/version.py @@ -1,3 +1,3 @@ """Version of the Client that is used in User-Agent header.""" -VERSION = '1.39.0' +VERSION = '1.40.0dev0' From 8286f454885ff7938c27d703e65e1287798f165e Mon Sep 17 00:00:00 2001 From: "H.John Choi" Date: Thu, 4 Jan 2024 18:24:08 +0900 Subject: [PATCH 02/51] fix: prevent creating unnecessary threads repeatedly (#562) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: prevent creating unnecessary threads repeatedly Use ThreadPoolScheduler for WriteApi batch subject instead of TimeoutScheduler. Fixes #561 * docs: Update CHANGELOG.md * docs: update CHANGELOG.md --------- Co-authored-by: Jakub Bednář --- CHANGELOG.md | 3 +++ influxdb_client/client/write_api.py | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a0e4ebc1..23f7dc23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ ## 1.40.0 [unreleased] +### Bug Fixes +1. [#562](https://github.com/influxdata/influxdb-client-python/pull/562): Use `ThreadPoolScheduler` for `WriteApi`'s batch subject instead of `TimeoutScheduler` to prevent creating unnecessary threads repeatedly + ## 1.39.0 [2023-12-05] ### Features diff --git a/influxdb_client/client/write_api.py b/influxdb_client/client/write_api.py index 61242446..050a7a5c 100644 --- a/influxdb_client/client/write_api.py +++ b/influxdb_client/client/write_api.py @@ -258,7 +258,8 @@ def __init__(self, self._disposable = self._subject.pipe( # Split incoming data to windows by batch_size or flush_interval ops.window_with_time_or_count(count=write_options.batch_size, - timespan=timedelta(milliseconds=write_options.flush_interval)), + timespan=timedelta(milliseconds=write_options.flush_interval), + scheduler=ThreadPoolScheduler(1)), # Map window into groups defined by 'organization', 'bucket' and 'precision' ops.flat_map(lambda window: window.pipe( # Group window by 'organization', 'bucket' and 'precision' From 024c1b1b6f48bfec5a2dc7d274baa4ae456a4e7c Mon Sep 17 00:00:00 2001 From: konstantin Date: Thu, 4 Jan 2024 09:30:43 +0000 Subject: [PATCH 03/51] feat: Add `__eq__` implementation to class Point (#625) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: Add `__eq__` implementation to class Point * docs: Add Changelog Entry * docs: Update CHANGELOG.md --------- Co-authored-by: Jakub Bednář --- CHANGELOG.md | 3 + influxdb_client/client/write/point.py | 12 +++ tests/test_point.py | 104 ++++++++++++++++++++++++++ 3 files changed, 119 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 23f7dc23..2f2cb4df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ ## 1.40.0 [unreleased] +### Features +1. [#625](https://github.com/influxdata/influxdb-client-python/pull/625): Make class `Point` equatable + ### Bug Fixes 1. [#562](https://github.com/influxdata/influxdb-client-python/pull/562): Use `ThreadPoolScheduler` for `WriteApi`'s batch subject instead of `TimeoutScheduler` to prevent creating unnecessary threads repeatedly diff --git a/influxdb_client/client/write/point.py b/influxdb_client/client/write/point.py index 60ce7c40..31d44d5c 100644 --- a/influxdb_client/client/write/point.py +++ b/influxdb_client/client/write/point.py @@ -251,6 +251,18 @@ def __str__(self): """Create string representation of this Point.""" return self.to_line_protocol() + def __eq__(self, other): + """Return true iff other is equal to self.""" + if not isinstance(other, Point): + return False + # assume points are equal iff their instance fields are equal + return (self._tags == other._tags and + self._fields == other._fields and + self._name == other._name and + self._time == other._time and + self._write_precision == other._write_precision and + self._field_types == other._field_types) + def _append_tags(tags): _return = [] diff --git a/tests/test_point.py b/tests/test_point.py index 992ac354..e799ae9c 100644 --- a/tests/test_point.py +++ b/tests/test_point.py @@ -557,6 +557,110 @@ def test_name_start_with_hash(self): self.assertEqual('#hash_start,location=europe level=2.2', point.to_line_protocol()) self.assertEqual(1, len(warnings)) + def test_equality_from_dict(self): + point_dict = { + "measurement": "h2o_feet", + "tags": {"location": "coyote_creek"}, + "fields": { + "water_level": 1.0, + "some_counter": 108913123234 + }, + "field_types": {"some_counter": "float"}, + "time": 1 + } + point_a = Point.from_dict(point_dict) + point_b = Point.from_dict(point_dict) + self.assertEqual(point_a, point_b) + + def test_equality(self): + # https://github.com/influxdata/influxdb-client-python/issues/623#issue-2048573579 + point_a = ( + Point("asd") + .tag("foo", "bar") + .field("value", 123.45) + .time(datetime(2023, 12, 19, 13, 27, 42, 215000, tzinfo=timezone.utc)) + ) + + point_b = ( + Point("asd") + .tag("foo", "bar") + .field("value", 123.45) + .time(datetime(2023, 12, 19, 13, 27, 42, 215000, tzinfo=timezone.utc)) + ) + self.assertEqual(point_a, point_b) + + def test_not_equal_if_tags_differ(self): + point_a = ( + Point("asd") + .tag("foo", "bar") + .field("value", 123.45) + .time(datetime(2023, 12, 19, 13, 27, 42, 215000, tzinfo=timezone.utc)) + ) + + point_b = ( + Point("asd") + .tag("foo", "baz") # not "bar" + .field("value", 123.45) + .time(datetime(2023, 12, 19, 13, 27, 42, 215000, tzinfo=timezone.utc)) + ) + self.assertNotEqual(point_a, point_b) + + def test_not_equal_if_fields_differ(self): + point_a = ( + Point("asd") + .tag("foo", "bar") + .field("value", 123.45) + .time(datetime(2023, 12, 19, 13, 27, 42, 215000, tzinfo=timezone.utc)) + ) + + point_b = ( + Point("asd") + .tag("foo", "bar") + .field("value", 678.90) # not 123.45 + .time(datetime(2023, 12, 19, 13, 27, 42, 215000, tzinfo=timezone.utc)) + ) + self.assertNotEqual(point_a, point_b) + + def test_not_equal_if_measurements_differ(self): + point_a = ( + Point("asd") + .tag("foo", "bar") + .field("value", 123.45) + .time(datetime(2023, 12, 19, 13, 27, 42, 215000, tzinfo=timezone.utc)) + ) + + point_b = ( + Point("fgh") # not "asd" + .tag("foo", "bar") + .field("value", 123.45) + .time(datetime(2023, 12, 19, 13, 27, 42, 215000, tzinfo=timezone.utc)) + ) + self.assertNotEqual(point_a, point_b) + + def test_not_equal_if_times_differ(self): + point_a = ( + Point("asd") + .tag("foo", "bar") + .field("value", 123.45) + .time(datetime(2023, 12, 19, 13, 27, 42, 215000, tzinfo=timezone.utc)) + ) + + point_b = ( + Point("asd") + .tag("foo", "bar") + .field("value", 123.45) + .time(datetime(2024, 12, 19, 13, 27, 42, 215000, tzinfo=timezone.utc)) + ) + self.assertNotEqual(point_a, point_b) + def test_not_equal_if_other_is_no_point(self): + point_a = ( + Point("asd") + .tag("foo", "bar") + .field("value", 123.45) + .time(datetime(2023, 12, 19, 13, 27, 42, 215000, tzinfo=timezone.utc)) + ) + not_a_point = "not a point but a string" + self.assertNotEqual(point_a, not_a_point) if __name__ == '__main__': unittest.main() From d7181fab9612dd0558822fc40e5d9ccf5a15d56b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 11 Jan 2024 16:18:24 -0600 Subject: [PATCH 04/51] chore(deps-dev): bump jinja2 from 3.1.2 to 3.1.3 (#627) Bumps [jinja2](https://github.com/pallets/jinja) from 3.1.2 to 3.1.3. - [Release notes](https://github.com/pallets/jinja/releases) - [Changelog](https://github.com/pallets/jinja/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/jinja/compare/3.1.2...3.1.3) --- updated-dependencies: - dependency-name: jinja2 dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5c63fa2d..546290de 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ 'aioresponses>=0.7.3', 'sphinx==1.8.5', 'sphinx_rtd_theme', - 'jinja2==3.1.2' + 'jinja2==3.1.3' ] extra_requires = [ From 4f1ab87ea15c6fec86dbc56443ab056d213ae6b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Bedn=C3=A1=C5=99?= Date: Mon, 29 Jan 2024 08:54:23 +0100 Subject: [PATCH 05/51] fix: logging http requests without query parameters (#631) * fix: logging http requests without query parameters * docs: update CHANGELOG.md --- CHANGELOG.md | 1 + influxdb_client/_sync/rest.py | 2 +- tests/test_InfluxDBClient.py | 12 ++++++++++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f2cb4df..6f11f45d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Bug Fixes 1. [#562](https://github.com/influxdata/influxdb-client-python/pull/562): Use `ThreadPoolScheduler` for `WriteApi`'s batch subject instead of `TimeoutScheduler` to prevent creating unnecessary threads repeatedly +1. [#631](https://github.com/influxdata/influxdb-client-python/pull/631): Logging HTTP requests without query parameters ## 1.39.0 [2023-12-05] diff --git a/influxdb_client/_sync/rest.py b/influxdb_client/_sync/rest.py index 2d80de13..eadbf061 100644 --- a/influxdb_client/_sync/rest.py +++ b/influxdb_client/_sync/rest.py @@ -170,7 +170,7 @@ def request(self, method, url, query_params=None, headers=None, headers['Content-Type'] = 'application/json' if self.configuration.debug: - _BaseRESTClient.log_request(method, f"{url}?{urlencode(query_params)}") + _BaseRESTClient.log_request(method, f"{url}{'' if query_params is None else '?' + urlencode(query_params)}") _BaseRESTClient.log_headers(headers, '>>>') _BaseRESTClient.log_body(body, '>>>') diff --git a/tests/test_InfluxDBClient.py b/tests/test_InfluxDBClient.py index ca37291b..c2f9b0a5 100644 --- a/tests/test_InfluxDBClient.py +++ b/tests/test_InfluxDBClient.py @@ -415,6 +415,18 @@ def test_custom_debug_logging_handler(self): logger = logging.getLogger('influxdb_client.client.http') self.assertEqual(2, len(logger.handlers)) + def test_debug_request_without_query_parameters(self): + httpretty.register_uri(httpretty.GET, uri="http://localhost/ping", status=200, body="") + self.influxdb_client = InfluxDBClient("http://localhost", "my-token", debug=True) + + log_stream = StringIO() + logger = logging.getLogger("influxdb_client.client.http") + logger.addHandler(logging.StreamHandler(log_stream)) + + self.influxdb_client.api_client.call_api('/ping', 'GET') + + self.assertIn("'GET http://localhost/ping'", log_stream.getvalue()) + class ServerWithSelfSingedSSL(http.server.SimpleHTTPRequestHandler): def _set_headers(self, response: bytes): From 17c78763678c71c5ab2e26c182e38ba6214a561c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Bedn=C3=A1=C5=99?= Date: Mon, 29 Jan 2024 08:57:59 +0100 Subject: [PATCH 06/51] fix: render README.rst at GitHub (#635) * fix: render README.rst at GitHub Removed unsupported text role :class:. For more info see https://docutils.sourceforge.io/docs/ref/rst/roles.html * docs: update CHANGELOG.md --- CHANGELOG.md | 3 +++ README.rst | 22 +++++++++++----------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f11f45d..f90b3ec4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ 1. [#562](https://github.com/influxdata/influxdb-client-python/pull/562): Use `ThreadPoolScheduler` for `WriteApi`'s batch subject instead of `TimeoutScheduler` to prevent creating unnecessary threads repeatedly 1. [#631](https://github.com/influxdata/influxdb-client-python/pull/631): Logging HTTP requests without query parameters +### Documentation +1. [#635](https://github.com/influxdata/influxdb-client-python/pull/635): Fix render `README.rst` at GitHub + ## 1.39.0 [2023-12-05] ### Features diff --git a/README.rst b/README.rst index a613a41a..e22f8eab 100644 --- a/README.rst +++ b/README.rst @@ -626,7 +626,7 @@ Queries The result retrieved by `QueryApi `_ could be formatted as a: 1. Flux data structure: `FluxTable `_, `FluxColumn `_ and `FluxRecord `_ -2. :class:`~influxdb_client.client.flux_table.CSVIterator` which will iterate over CSV lines +2. :code:`influxdb_client.client.flux_table.CSVIterator` which will iterate over CSV lines 3. Raw unprocessed results as a ``str`` iterator 4. `Pandas DataFrame `_ @@ -1403,12 +1403,12 @@ or use the ``[async]`` extra: Async APIs """""""""" -All async APIs are available via :class:`~influxdb_client.client.influxdb_client_async.InfluxDBClientAsync`. +All async APIs are available via :code:`influxdb_client.client.influxdb_client_async.InfluxDBClientAsync`. The ``async`` version of the client supports following asynchronous APIs: -* :class:`~influxdb_client.client.write_api_async.WriteApiAsync` -* :class:`~influxdb_client.client.query_api_async.QueryApiAsync` -* :class:`~influxdb_client.client.delete_api_async.DeleteApiAsync` +* :code:`influxdb_client.client.write_api_async.WriteApiAsync` +* :code:`influxdb_client.client.query_api_async.QueryApiAsync` +* :code:`influxdb_client.client.delete_api_async.DeleteApiAsync` * Management services into ``influxdb_client.service`` supports async operation and also check to readiness of the InfluxDB via ``/ping`` endpoint: @@ -1432,7 +1432,7 @@ and also check to readiness of the InfluxDB via ``/ping`` endpoint: Async Write API """"""""""""""" -The :class:`~influxdb_client.client.write_api_async.WriteApiAsync` supports ingesting data as: +The :code:`influxdb_client.client.write_api_async.WriteApiAsync` supports ingesting data as: * ``string`` or ``bytes`` that is formatted as a InfluxDB's line protocol * `Data Point `__ structure @@ -1470,13 +1470,13 @@ The :class:`~influxdb_client.client.write_api_async.WriteApiAsync` supports inge Async Query API """"""""""""""" -The :class:`~influxdb_client.client.query_api_async.QueryApiAsync` supports retrieve data as: +The :code:`influxdb_client.client.query_api_async.QueryApiAsync` supports retrieve data as: -* List of :class:`~influxdb_client.client.flux_table.FluxTable` -* Stream of :class:`~influxdb_client.client.flux_table.FluxRecord` via :class:`~typing.AsyncGenerator` +* List of :code:`influxdb_client.client.flux_table.FluxTable` +* Stream of :code:`influxdb_client.client.flux_table.FluxRecord` via :code:`typing.AsyncGenerator` * `Pandas DataFrame `_ -* Stream of `Pandas DataFrame `_ via :class:`~typing.AsyncGenerator` -* Raw :class:`~str` output +* Stream of `Pandas DataFrame `_ via :code:`typing.AsyncGenerator` +* Raw :code:`str` output .. code-block:: python From 903d9b71e5af4cd1edc230e8f23876ae694f057d Mon Sep 17 00:00:00 2001 From: Jakub Bednar Date: Tue, 30 Jan 2024 08:45:08 +0100 Subject: [PATCH 07/51] chore(release): release version 1.40.0 [skip CI] --- CHANGELOG.md | 2 +- influxdb_client/version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f90b3ec4..f9a68af0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## 1.40.0 [unreleased] +## 1.40.0 [2024-01-30] ### Features 1. [#625](https://github.com/influxdata/influxdb-client-python/pull/625): Make class `Point` equatable diff --git a/influxdb_client/version.py b/influxdb_client/version.py index e33b8426..f8c5bb31 100644 --- a/influxdb_client/version.py +++ b/influxdb_client/version.py @@ -1,3 +1,3 @@ """Version of the Client that is used in User-Agent header.""" -VERSION = '1.40.0dev0' +VERSION = '1.40.0' From 27777d1ba5861d6e16a3da4d7f65f6acdefe7e08 Mon Sep 17 00:00:00 2001 From: Jakub Bednar Date: Tue, 30 Jan 2024 08:50:38 +0100 Subject: [PATCH 08/51] chore(release): prepare for next development iteration --- CHANGELOG.md | 2 ++ conda/meta.yaml | 6 +++--- influxdb_client/version.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f9a68af0..7dbc48a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +## 1.41.0 [unreleased] + ## 1.40.0 [2024-01-30] ### Features diff --git a/conda/meta.yaml b/conda/meta.yaml index a2291164..da95cce4 100644 --- a/conda/meta.yaml +++ b/conda/meta.yaml @@ -1,5 +1,5 @@ {% set name = "influxdb_client" %} -{% set version = "1.39.0" %} +{% set version = "1.40.0" %} package: @@ -7,8 +7,8 @@ package: version: {{ version }} source: - url: https://files.pythonhosted.org/packages/f1/0e/d4da1d18316eab78b7041e60dbf4fe6062ae7e32dd55ed22bda316b1d217/influxdb_client-1.39.0.tar.gz - sha256: 6a534913523bd262f1928e4ff80046bf95e313c1694ce13e45fd17eea90fe691 + url: https://files.pythonhosted.org/packages/63/11/07ed82352a28e4e8b623a487337befec77d5bd18293dcc940d769e633f82/influxdb_client-1.40.0.tar.gz + sha256: 027f970af1518479d8806f1cdf5ba20280f943e1b621c2acdbf9ca8dc9bdf1cb build: number: 0 diff --git a/influxdb_client/version.py b/influxdb_client/version.py index f8c5bb31..7a63f210 100644 --- a/influxdb_client/version.py +++ b/influxdb_client/version.py @@ -1,3 +1,3 @@ """Version of the Client that is used in User-Agent header.""" -VERSION = '1.40.0' +VERSION = '1.41.0dev0' From 7a5f655c076a300e6230205fa9bc6df0e1ae6b50 Mon Sep 17 00:00:00 2001 From: alespour <42931850+alespour@users.noreply.github.com> Date: Wed, 31 Jan 2024 14:46:23 +0100 Subject: [PATCH 09/51] fix: handle null values in data (#636) * fix: handle null values in Flux data * test: add tests for null value handling and extension dtypes * test: fix failures with empty warnings cases * test: comment out dtype some extra assertion until solved * test: skip extension dtypes test on pythn 3.7 * fix: single place of dtypes conversion * fix: bump pandas dependency version * docs: update CHANGELOG * chore(build): trigger CI/CD pipeline * fix: add use_extension_dtypes also to async query API methods --- CHANGELOG.md | 3 + influxdb_client/client/_base.py | 18 ++- influxdb_client/client/flux_csv_parser.py | 18 ++- influxdb_client/client/query_api.py | 22 +++- influxdb_client/client/query_api_async.py | 20 +++- setup.py | 2 +- tests/test_FluxCSVParser.py | 131 +++++++++++++++++++++- tests/test_InfluxDBClientAsync.py | 5 +- tests/test_QueryApiDataFrame.py | 82 +++++++++++++- 9 files changed, 272 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7dbc48a8..b1bf2187 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ ## 1.41.0 [unreleased] +### Bug Fixes +1. [#636](https://github.com/influxdata/influxdb-client-python/pull/636): Handle missing data in data frames + ## 1.40.0 [2024-01-30] ### Features diff --git a/influxdb_client/client/_base.py b/influxdb_client/client/_base.py index 08e8ec54..8dcf75e9 100644 --- a/influxdb_client/client/_base.py +++ b/influxdb_client/client/_base.py @@ -277,23 +277,27 @@ async def _to_flux_record_stream_async(self, response, query_options=None, respo return (await _parser.__aenter__()).generator_async() def _to_data_frame_stream(self, data_frame_index, response, query_options=None, - response_metadata_mode: FluxResponseMetadataMode = FluxResponseMetadataMode.full): + response_metadata_mode: FluxResponseMetadataMode = FluxResponseMetadataMode.full, + use_extension_dtypes=False): """ Parse HTTP response to DataFrame stream. :param response: HTTP response from an HTTP client. Expected type: `urllib3.response.HTTPResponse`. """ - _parser = self._to_data_frame_stream_parser(data_frame_index, query_options, response, response_metadata_mode) + _parser = self._to_data_frame_stream_parser(data_frame_index, query_options, response, response_metadata_mode, + use_extension_dtypes) return _parser.generator() async def _to_data_frame_stream_async(self, data_frame_index, response, query_options=None, response_metadata_mode: - FluxResponseMetadataMode = FluxResponseMetadataMode.full): + FluxResponseMetadataMode = FluxResponseMetadataMode.full, + use_extension_dtypes=False): """ Parse HTTP response to DataFrame stream. :param response: HTTP response from an HTTP client. Expected type: `aiohttp.client_reqrep.ClientResponse`. """ - _parser = self._to_data_frame_stream_parser(data_frame_index, query_options, response, response_metadata_mode) + _parser = self._to_data_frame_stream_parser(data_frame_index, query_options, response, response_metadata_mode, + use_extension_dtypes) return (await _parser.__aenter__()).generator_async() def _to_tables_parser(self, response, query_options, response_metadata_mode): @@ -304,10 +308,12 @@ def _to_flux_record_stream_parser(self, query_options, response, response_metada return FluxCsvParser(response=response, serialization_mode=FluxSerializationMode.stream, query_options=query_options, response_metadata_mode=response_metadata_mode) - def _to_data_frame_stream_parser(self, data_frame_index, query_options, response, response_metadata_mode): + def _to_data_frame_stream_parser(self, data_frame_index, query_options, response, response_metadata_mode, + use_extension_dtypes): return FluxCsvParser(response=response, serialization_mode=FluxSerializationMode.dataFrame, data_frame_index=data_frame_index, query_options=query_options, - response_metadata_mode=response_metadata_mode) + response_metadata_mode=response_metadata_mode, + use_extension_dtypes=use_extension_dtypes) def _to_data_frames(self, _generator): """Parse stream of DataFrames into expected type.""" diff --git a/influxdb_client/client/flux_csv_parser.py b/influxdb_client/client/flux_csv_parser.py index 32379622..7a73e3f8 100644 --- a/influxdb_client/client/flux_csv_parser.py +++ b/influxdb_client/client/flux_csv_parser.py @@ -64,7 +64,8 @@ class FluxCsvParser(object): def __init__(self, response, serialization_mode: FluxSerializationMode, data_frame_index: List[str] = None, query_options=None, - response_metadata_mode: FluxResponseMetadataMode = FluxResponseMetadataMode.full) -> None: + response_metadata_mode: FluxResponseMetadataMode = FluxResponseMetadataMode.full, + use_extension_dtypes=False) -> None: """ Initialize defaults. @@ -75,6 +76,7 @@ def __init__(self, response, serialization_mode: FluxSerializationMode, self.tables = TableList() self._serialization_mode = serialization_mode self._response_metadata_mode = response_metadata_mode + self._use_extension_dtypes = use_extension_dtypes self._data_frame_index = data_frame_index self._data_frame_values = [] self._profilers = query_options.profilers if query_options is not None else None @@ -211,7 +213,7 @@ def _parse_flux_response_row(self, metadata, csv): pass else: - # to int converions todo + # to int conversions todo current_id = int(csv[2]) if metadata.table_id == -1: metadata.table_id = current_id @@ -253,7 +255,11 @@ def _prepare_data_frame(self): _temp_df = _temp_df.set_index(self._data_frame_index) # Append data - return pd.concat([self._data_frame.astype(_temp_df.dtypes), _temp_df]) + df = pd.concat([self._data_frame.astype(_temp_df.dtypes), _temp_df]) + + if self._use_extension_dtypes: + return df.convert_dtypes() + return df def parse_record(self, table_index, table, csv): """Parse one record.""" @@ -273,8 +279,10 @@ def _to_value(self, str_val, column): default_value = column.default_value if default_value == '' or default_value is None: if self._serialization_mode is FluxSerializationMode.dataFrame: - from ..extras import np - return self._to_value(np.nan, column) + if self._use_extension_dtypes: + from ..extras import pd + return pd.NA + return None return None return self._to_value(default_value, column) diff --git a/influxdb_client/client/query_api.py b/influxdb_client/client/query_api.py index f1df2041..8611021d 100644 --- a/influxdb_client/client/query_api.py +++ b/influxdb_client/client/query_api.py @@ -222,7 +222,8 @@ def query_stream(self, query: str, org=None, params: dict = None) -> Generator[' async_req=False, _preload_content=False, _return_http_data_only=False) return self._to_flux_record_stream(response, query_options=self._get_query_options()) - def query_data_frame(self, query: str, org=None, data_frame_index: List[str] = None, params: dict = None): + def query_data_frame(self, query: str, org=None, data_frame_index: List[str] = None, params: dict = None, + use_extension_dtypes: bool = False): """ Execute synchronous Flux query and return Pandas DataFrame. @@ -234,6 +235,11 @@ def query_data_frame(self, query: str, org=None, data_frame_index: List[str] = N If not specified the default value from ``InfluxDBClient.org`` is used. :param data_frame_index: the list of columns that are used as DataFrame index :param params: bind parameters + :param use_extension_dtypes: set to ``True`` to use panda's extension data types. + Useful for queries with ``pivot`` function. + When data has missing values, column data type may change (to ``object`` or ``float64``). + Nullable extension types (``Int64``, ``Float64``, ``boolean``) support ``panda.NA`` value. + For more info, see https://pandas.pydata.org/docs/user_guide/missing_data.html. :return: :class:`~DataFrame` or :class:`~List[DataFrame]` .. warning:: For the optimal processing of the query results use the ``pivot() function`` which align results as a table. @@ -250,10 +256,12 @@ def query_data_frame(self, query: str, org=None, data_frame_index: List[str] = N - https://docs.influxdata.com/flux/latest/stdlib/universe/pivot/ - https://docs.influxdata.com/flux/latest/stdlib/influxdata/influxdb/schema/fieldsascols/ """ # noqa: E501 - _generator = self.query_data_frame_stream(query, org=org, data_frame_index=data_frame_index, params=params) + _generator = self.query_data_frame_stream(query, org=org, data_frame_index=data_frame_index, params=params, + use_extension_dtypes=use_extension_dtypes) return self._to_data_frames(_generator) - def query_data_frame_stream(self, query: str, org=None, data_frame_index: List[str] = None, params: dict = None): + def query_data_frame_stream(self, query: str, org=None, data_frame_index: List[str] = None, params: dict = None, + use_extension_dtypes: bool = False): """ Execute synchronous Flux query and return stream of Pandas DataFrame as a :class:`~Generator[DataFrame]`. @@ -265,6 +273,11 @@ def query_data_frame_stream(self, query: str, org=None, data_frame_index: List[s If not specified the default value from ``InfluxDBClient.org`` is used. :param data_frame_index: the list of columns that are used as DataFrame index :param params: bind parameters + :param use_extension_dtypes: set to ``True`` to use panda's extension data types. + Useful for queries with ``pivot`` function. + When data has missing values, column data type may change (to ``object`` or ``float64``). + Nullable extension types (``Int64``, ``Float64``, ``boolean``) support ``panda.NA`` value. + For more info, see https://pandas.pydata.org/docs/user_guide/missing_data.html. :return: :class:`~Generator[DataFrame]` .. warning:: For the optimal processing of the query results use the ``pivot() function`` which align results as a table. @@ -289,7 +302,8 @@ def query_data_frame_stream(self, query: str, org=None, data_frame_index: List[s return self._to_data_frame_stream(data_frame_index=data_frame_index, response=response, - query_options=self._get_query_options()) + query_options=self._get_query_options(), + use_extension_dtypes=use_extension_dtypes) def __del__(self): """Close QueryAPI.""" diff --git a/influxdb_client/client/query_api_async.py b/influxdb_client/client/query_api_async.py index 995adff4..b3b42cb4 100644 --- a/influxdb_client/client/query_api_async.py +++ b/influxdb_client/client/query_api_async.py @@ -120,7 +120,8 @@ async def query_stream(self, query: str, org=None, params: dict = None) -> Async return await self._to_flux_record_stream_async(response, query_options=self._get_query_options()) - async def query_data_frame(self, query: str, org=None, data_frame_index: List[str] = None, params: dict = None): + async def query_data_frame(self, query: str, org=None, data_frame_index: List[str] = None, params: dict = None, + use_extension_dtypes: bool = False): """ Execute asynchronous Flux query and return :class:`~pandas.core.frame.DataFrame`. @@ -132,6 +133,11 @@ async def query_data_frame(self, query: str, org=None, data_frame_index: List[st If not specified the default value from ``InfluxDBClientAsync.org`` is used. :param data_frame_index: the list of columns that are used as DataFrame index :param params: bind parameters + :param use_extension_dtypes: set to ``True`` to use panda's extension data types. + Useful for queries with ``pivot`` function. + When data has missing values, column data type may change (to ``object`` or ``float64``). + Nullable extension types (``Int64``, ``Float64``, ``boolean``) support ``panda.NA`` value. + For more info, see https://pandas.pydata.org/docs/user_guide/missing_data.html. :return: :class:`~DataFrame` or :class:`~List[DataFrame]` .. warning:: For the optimal processing of the query results use the ``pivot() function`` which align results as a table. @@ -149,7 +155,7 @@ async def query_data_frame(self, query: str, org=None, data_frame_index: List[st - https://docs.influxdata.com/flux/latest/stdlib/influxdata/influxdb/schema/fieldsascols/ """ # noqa: E501 _generator = await self.query_data_frame_stream(query, org=org, data_frame_index=data_frame_index, - params=params) + params=params, use_extension_dtypes=use_extension_dtypes) dataframes = [] async for dataframe in _generator: @@ -158,7 +164,7 @@ async def query_data_frame(self, query: str, org=None, data_frame_index: List[st return self._to_data_frames(dataframes) async def query_data_frame_stream(self, query: str, org=None, data_frame_index: List[str] = None, - params: dict = None): + params: dict = None, use_extension_dtypes: bool = False): """ Execute asynchronous Flux query and return stream of :class:`~pandas.core.frame.DataFrame` as an AsyncGenerator[:class:`~pandas.core.frame.DataFrame`]. @@ -170,6 +176,11 @@ async def query_data_frame_stream(self, query: str, org=None, data_frame_index: If not specified the default value from ``InfluxDBClientAsync.org`` is used. :param data_frame_index: the list of columns that are used as DataFrame index :param params: bind parameters + :param use_extension_dtypes: set to ``True`` to use panda's extension data types. + Useful for queries with ``pivot`` function. + When data has missing values, column data type may change (to ``object`` or ``float64``). + Nullable extension types (``Int64``, ``Float64``, ``boolean``) support ``panda.NA`` value. + For more info, see https://pandas.pydata.org/docs/user_guide/missing_data.html. :return: :class:`AsyncGenerator[:class:`DataFrame`]` .. warning:: For the optimal processing of the query results use the ``pivot() function`` which align results as a table. @@ -192,7 +203,8 @@ async def query_data_frame_stream(self, query: str, org=None, data_frame_index: dataframe_query=True)) return await self._to_data_frame_stream_async(data_frame_index=data_frame_index, response=response, - query_options=self._get_query_options()) + query_options=self._get_query_options(), + use_extension_dtypes=use_extension_dtypes) async def query_raw(self, query: str, org=None, dialect=_BaseQueryApi.default_dialect, params: dict = None): """ diff --git a/setup.py b/setup.py index 546290de..a2ed2886 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ ] extra_requires = [ - 'pandas>=0.25.3', + 'pandas>=1.0.0', 'numpy' ] diff --git a/tests/test_FluxCSVParser.py b/tests/test_FluxCSVParser.py index b6831e94..ae9adcec 100644 --- a/tests/test_FluxCSVParser.py +++ b/tests/test_FluxCSVParser.py @@ -1,7 +1,9 @@ import json import math import unittest +import pandas as pd from io import BytesIO +from packaging import version import pytest from urllib3 import HTTPResponse @@ -263,6 +265,31 @@ def test_pandas_column_datatype(self): self.assertEqual('bool', df.dtypes['value4'].name) self.assertEqual('float64', df.dtypes['value5'].name) + def test_pandas_column_datatype_extension_types(self): + data = "#datatype,string,long,dateTime:RFC3339,dateTime:RFC3339,string,string,string,string,long,unsignedLong,string,boolean,double\n" \ + "#group,false,false,true,true,true,true,true,true,false,false,false,false,false\n" \ + "#default,_result,,,,,,,,,,,,\n" \ + ",result,table,_start,_stop,_field,_measurement,host,region,value1,value2,value3,value4,value5\n" \ + ",,0,1977-09-21T00:12:43.145224192Z,2018-07-16T11:21:02.547596934Z,free,mem,A,west,121,11,test,true,6.56\n" + parser = self._parse(data=data, serialization_mode=FluxSerializationMode.dataFrame, + response_metadata_mode=FluxResponseMetadataMode.full, + use_extension_dtypes=True) + df = list(parser.generator())[0] + self.assertEqual(13, df.dtypes.__len__()) + self.assertEqual('string', df.dtypes['result'].name) + self.assertEqual('Int64', df.dtypes['table'].name) + self.assertIn('datetime64[ns,', df.dtypes['_start'].name) + self.assertIn('datetime64[ns,', df.dtypes['_stop'].name) + self.assertEqual('string', df.dtypes['_field'].name) + self.assertEqual('string', df.dtypes['_measurement'].name) + self.assertEqual('string', df.dtypes['host'].name) + self.assertEqual('string', df.dtypes['region'].name) + self.assertEqual('Int64', df.dtypes['value1'].name) + self.assertEqual('Int64', df.dtypes['value2'].name) + self.assertEqual('string', df.dtypes['value3'].name) + self.assertEqual('boolean', df.dtypes['value4'].name) + self.assertEqual('Float64', df.dtypes['value5'].name) + def test_pandas_null_bool_types(self): data = "#datatype,string,long,dateTime:RFC3339,dateTime:RFC3339,string,string,string,string,boolean\n" \ "#group,false,false,true,true,true,true,true,true,false\n" \ @@ -274,7 +301,104 @@ def test_pandas_null_bool_types(self): parser = self._parse(data=data, serialization_mode=FluxSerializationMode.dataFrame, response_metadata_mode=FluxResponseMetadataMode.full) df = list(parser.generator())[0] - self.assertEqual('bool', df.dtypes['value'].name) + self.assertEqual('object', df.dtypes['value'].name) + + def test_pandas_null_bool_types_extension_types(self): + data = "#datatype,string,long,dateTime:RFC3339,dateTime:RFC3339,string,string,string,string,boolean\n" \ + "#group,false,false,true,true,true,true,true,true,false\n" \ + "#default,_result,,,,,,,,\n" \ + ",result,table,_start,_stop,_field,_measurement,host,region,value\n" \ + ",,0,1977-09-21T00:12:43.145224192Z,2018-07-16T11:21:02.547596934Z,free,mem,A,west,true\n" \ + ",,0,1977-09-21T00:12:43.145224192Z,2018-07-16T11:21:02.547596934Z,free,mem,A,west,\n" + + parser = self._parse(data=data, serialization_mode=FluxSerializationMode.dataFrame, + response_metadata_mode=FluxResponseMetadataMode.full, + use_extension_dtypes=True) + df = list(parser.generator())[0] + self.assertEqual('boolean', df.dtypes['value'].name) + + def test_pandas_null_long_types(self): + data = "#datatype,string,long,dateTime:RFC3339,dateTime:RFC3339,string,string,string,string,long\n" \ + "#group,false,false,true,true,true,true,true,true,false\n" \ + "#default,_result,,,,,,,,\n" \ + ",result,table,_start,_stop,_field,_measurement,host,region,value\n" \ + ",,0,1977-09-21T00:12:43.145224192Z,2018-07-16T11:21:02.547596934Z,free,mem,A,west,1\n" \ + ",,0,1977-09-21T00:12:43.145224192Z,2018-07-16T11:21:02.547596934Z,free,mem,A,west,\n" + + parser = self._parse(data=data, serialization_mode=FluxSerializationMode.dataFrame, + response_metadata_mode=FluxResponseMetadataMode.full) + df = list(parser.generator())[0] + self.assertEqual('float64', df.dtypes['value'].name) # pd.NA is converted to float('nan') + + @pytest.mark.skipif(version.parse(pd.__version__).release < (2, 0), reason="numeric nullables require pandas>=2.0 to work correctly") + def test_pandas_null_long_types_extension_types(self): + data = "#datatype,string,long,dateTime:RFC3339,dateTime:RFC3339,string,string,string,string,long\n" \ + "#group,false,false,true,true,true,true,true,true,false\n" \ + "#default,_result,,,,,,,,\n" \ + ",result,table,_start,_stop,_field,_measurement,host,region,value\n" \ + ",,0,1977-09-21T00:12:43.145224192Z,2018-07-16T11:21:02.547596934Z,free,mem,A,west,1\n" \ + ",,0,1977-09-21T00:12:43.145224192Z,2018-07-16T11:21:02.547596934Z,free,mem,A,west,\n" + + parser = self._parse(data=data, serialization_mode=FluxSerializationMode.dataFrame, + response_metadata_mode=FluxResponseMetadataMode.full, + use_extension_dtypes=True) + df = list(parser.generator())[0] + self.assertEqual('Int64', df.dtypes['value'].name) + + def test_pandas_null_double_types(self): + data = "#datatype,string,long,dateTime:RFC3339,dateTime:RFC3339,string,string,string,string,double\n" \ + "#group,false,false,true,true,true,true,true,true,false\n" \ + "#default,_result,,,,,,,,\n" \ + ",result,table,_start,_stop,_field,_measurement,host,region,value\n" \ + ",,0,1977-09-21T00:12:43.145224192Z,2018-07-16T11:21:02.547596934Z,free,mem,A,west,1\n" \ + ",,0,1977-09-21T00:12:43.145224192Z,2018-07-16T11:21:02.547596934Z,free,mem,A,west,\n" + + parser = self._parse(data=data, serialization_mode=FluxSerializationMode.dataFrame, + response_metadata_mode=FluxResponseMetadataMode.full) + df = list(parser.generator())[0] + self.assertEqual('float64', df.dtypes['value'].name) + + @pytest.mark.skipif(version.parse(pd.__version__).release < (2, 0), reason="numeric nullables require pandas>=2.0 to work correctly") + def test_pandas_null_double_types_extension_types(self): + data = "#datatype,string,long,dateTime:RFC3339,dateTime:RFC3339,string,string,string,string,double\n" \ + "#group,false,false,true,true,true,true,true,true,false\n" \ + "#default,_result,,,,,,,,\n" \ + ",result,table,_start,_stop,_field,_measurement,host,region,value\n" \ + ",,0,1977-09-21T00:12:43.145224192Z,2018-07-16T11:21:02.547596934Z,free,mem,A,west,1\n" \ + ",,0,1977-09-21T00:12:43.145224192Z,2018-07-16T11:21:02.547596934Z,free,mem,A,west,\n" + + parser = self._parse(data=data, serialization_mode=FluxSerializationMode.dataFrame, + response_metadata_mode=FluxResponseMetadataMode.full, + use_extension_dtypes=True) + df = list(parser.generator())[0] + self.assertEqual('Float64', df.dtypes['value'].name) + + def test_pandas_null_string_types(self): + data = "#datatype,string,long,dateTime:RFC3339,dateTime:RFC3339,string,string,string,string,string\n" \ + "#group,false,false,true,true,true,true,true,true,false\n" \ + "#default,_result,,,,,,,,\n" \ + ",result,table,_start,_stop,_field,_measurement,host,region,value\n" \ + ",,0,1977-09-21T00:12:43.145224192Z,2018-07-16T11:21:02.547596934Z,free,mem,A,west,hi\n" \ + ",,0,1977-09-21T00:12:43.145224192Z,2018-07-16T11:21:02.547596934Z,free,mem,A,west,\n" + + parser = self._parse(data=data, serialization_mode=FluxSerializationMode.dataFrame, + response_metadata_mode=FluxResponseMetadataMode.full) + df = list(parser.generator())[0] + self.assertEqual('object', df.dtypes['value'].name) + + def test_pandas_null_string_types_extension_types(self): + data = "#datatype,string,long,dateTime:RFC3339,dateTime:RFC3339,string,string,string,string,string\n" \ + "#group,false,false,true,true,true,true,true,true,false\n" \ + "#default,_result,,,,,,,,\n" \ + ",result,table,_start,_stop,_field,_measurement,host,region,value\n" \ + ",,0,1977-09-21T00:12:43.145224192Z,2018-07-16T11:21:02.547596934Z,free,mem,A,west,hi\n" \ + ",,0,1977-09-21T00:12:43.145224192Z,2018-07-16T11:21:02.547596934Z,free,mem,A,west,\n" + + parser = self._parse(data=data, serialization_mode=FluxSerializationMode.dataFrame, + response_metadata_mode=FluxResponseMetadataMode.full, + use_extension_dtypes=True) + df = list(parser.generator())[0] + self.assertEqual('string', df.dtypes['value'].name) def test_parse_without_datatype(self): data = ",result,table,_start,_stop,_field,_measurement,host,region,_value2,value1,value_str\n" \ @@ -399,7 +523,8 @@ def _parse_to_tables(data: str, serialization_mode=FluxSerializationMode.tables, return tables @staticmethod - def _parse(data, serialization_mode, response_metadata_mode): + def _parse(data, serialization_mode, response_metadata_mode, use_extension_dtypes=False): fp = BytesIO(str.encode(data)) return FluxCsvParser(response=HTTPResponse(fp, preload_content=False), - serialization_mode=serialization_mode, response_metadata_mode=response_metadata_mode) + serialization_mode=serialization_mode, response_metadata_mode=response_metadata_mode, + use_extension_dtypes=use_extension_dtypes) diff --git a/tests/test_InfluxDBClientAsync.py b/tests/test_InfluxDBClientAsync.py index af0b0ecd..123967a7 100644 --- a/tests/test_InfluxDBClientAsync.py +++ b/tests/test_InfluxDBClientAsync.py @@ -6,6 +6,7 @@ from io import StringIO import pytest +import warnings from aioresponses import aioresponses from influxdb_client import Point, WritePrecision, BucketsService, OrganizationsService, Organizations @@ -176,10 +177,10 @@ async def test_query_data_frame_without_warning(self): ''' query_api = self.client.query_api() - with pytest.warns(None) as warnings: + with warnings.catch_warnings(record=True) as warns: dataframe = await query_api.query_data_frame(query) self.assertIsNotNone(dataframe) - self.assertEqual(0, len(warnings)) + self.assertEqual(0, len(warns)) @async_test async def test_write_response_type(self): diff --git a/tests/test_QueryApiDataFrame.py b/tests/test_QueryApiDataFrame.py index ed163cdd..670074bc 100644 --- a/tests/test_QueryApiDataFrame.py +++ b/tests/test_QueryApiDataFrame.py @@ -3,6 +3,9 @@ import httpretty import pytest import reactivex as rx +import pandas +import warnings + from pandas import DataFrame from pandas._libs.tslibs.timestamps import Timestamp from reactivex import operators as ops @@ -272,7 +275,7 @@ def test_query_without_warning(self): self.client = InfluxDBClient("http://localhost", "my-token", org="my-org", enable_gzip=False) - with pytest.warns(None) as warnings: + with warnings.catch_warnings(record=True) as warns: self.client.query_api().query_data_frame( 'import "influxdata/influxdb/schema"' '' @@ -281,16 +284,87 @@ def test_query_without_warning(self): '|> filter(fn: (r) => r._measurement == "mem") ' '|> schema.fieldsAsCols() ' "my-org") - self.assertEqual(0, len(warnings)) + self.assertEqual(0, len(warns)) - with pytest.warns(None) as warnings: + with warnings.catch_warnings(record=True) as warns: self.client.query_api().query_data_frame( 'from(bucket: "my-bucket")' '|> range(start: -5s, stop: now()) ' '|> filter(fn: (r) => r._measurement == "mem") ' '|> pivot(rowKey: ["_time"], columnKey: ["_field"], valueColumn: "_value")' "my-org") - self.assertEqual(0, len(warnings)) + self.assertEqual(0, len(warns)) + + def test_pivoted_data(self): + query_response = \ + '#group,false,false,true,true,false,true,false,false,false,false\n' \ + '#datatype,string,long,dateTime:RFC3339,dateTime:RFC3339,dateTime:RFC3339,string,double,long,string,boolean\n' \ + '#default,_result,,,,,,,,,\n' \ + ',result,table,_start,_stop,_time,_measurement,test_double,test_long,test_string,test_boolean\n' \ + ',,0,2023-12-15T13:19:45Z,2023-12-15T13:20:00Z,2023-12-15T13:19:55Z,test,4,,,\n' \ + ',,0,2023-12-15T13:19:45Z,2023-12-15T13:20:00Z,2023-12-15T13:19:56Z,test,,1,,\n' \ + ',,0,2023-12-15T13:19:45Z,2023-12-15T13:20:00Z,2023-12-15T13:19:57Z,test,,,hi,\n' \ + ',,0,2023-12-15T13:19:45Z,2023-12-15T13:20:00Z,2023-12-15T13:19:58Z,test,,,,true\n' \ + '\n\n' + + httpretty.register_uri(httpretty.POST, uri="http://localhost/api/v2/query", status=200, body=query_response) + + self.client = InfluxDBClient("http://localhost", "my-token", org="my-org", enable_gzip=False) + + _dataFrame = self.client.query_api().query_data_frame( + 'from(bucket: "my-bucket") ' + '|> range(start: 2023-12-15T13:19:45Z, stop: 2023-12-15T13:20:00Z)' + '|> filter(fn: (r) => r["_measurement"] == "test")' + '|> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value")' + "my-org", use_extension_dtypes=True) + + self.assertEqual(DataFrame, type(_dataFrame)) + self.assertListEqual( + ["result", "table", "_start", "_stop", "_time", "_measurement", + "test_double", "test_long", "test_string", "test_boolean"], + list(_dataFrame.columns)) + self.assertListEqual([0, 1, 2, 3], list(_dataFrame.index)) + # self.assertEqual('Int64', _dataFrame.dtypes['test_long'].name) + # self.assertEqual('Float64', _dataFrame.dtypes['test_double'].name) + self.assertEqual('string', _dataFrame.dtypes['test_string'].name) + self.assertEqual('boolean', _dataFrame.dtypes['test_boolean'].name) + self.assertEqual(4, len(_dataFrame)) + self.assertEqual("_result", _dataFrame['result'][0]) + self.assertEqual("_result", _dataFrame['result'][1]) + self.assertEqual("_result", _dataFrame['result'][2]) + self.assertEqual("_result", _dataFrame['result'][3]) + self.assertEqual(0, _dataFrame['table'][0], None) + self.assertEqual(0, _dataFrame['table'][1], None) + self.assertEqual(0, _dataFrame['table'][2], None) + self.assertEqual(0, _dataFrame['table'][3], None) + self.assertEqual(Timestamp('2023-12-15 13:19:45+0000'), _dataFrame['_start'][0]) + self.assertEqual(Timestamp('2023-12-15 13:19:45+0000'), _dataFrame['_start'][1]) + self.assertEqual(Timestamp('2023-12-15 13:19:45+0000'), _dataFrame['_start'][2]) + self.assertEqual(Timestamp('2023-12-15 13:19:45+0000'), _dataFrame['_start'][3]) + self.assertEqual(Timestamp('2023-12-15 13:20:00+0000'), _dataFrame['_stop'][0]) + self.assertEqual(Timestamp('2023-12-15 13:20:00+0000'), _dataFrame['_stop'][1]) + self.assertEqual(Timestamp('2023-12-15 13:20:00+0000'), _dataFrame['_stop'][2]) + self.assertEqual(Timestamp('2023-12-15 13:20:00+0000'), _dataFrame['_stop'][3]) + self.assertEqual(Timestamp('2023-12-15 13:19:55+0000'), _dataFrame['_time'][0]) + self.assertEqual(Timestamp('2023-12-15 13:19:56+0000'), _dataFrame['_time'][1]) + self.assertEqual(Timestamp('2023-12-15 13:19:57+0000'), _dataFrame['_time'][2]) + self.assertEqual(Timestamp('2023-12-15 13:19:58+0000'), _dataFrame['_time'][3]) + self.assertEqual(4, _dataFrame['test_double'][0]) + self.assertTrue(pandas.isna(_dataFrame['test_double'][1])) + self.assertTrue(pandas.isna(_dataFrame['test_double'][2])) + self.assertTrue(pandas.isna(_dataFrame['test_double'][3])) + self.assertTrue(pandas.isna(_dataFrame['test_long'][0])) + self.assertEqual(1, _dataFrame['test_long'][1]) + self.assertTrue(pandas.isna(_dataFrame['test_long'][2])) + self.assertTrue(pandas.isna(_dataFrame['test_long'][3])) + self.assertTrue(pandas.isna(_dataFrame['test_string'][0])) + self.assertTrue(pandas.isna(_dataFrame['test_string'][1])) + self.assertEqual('hi', _dataFrame['test_string'][2]) + self.assertTrue(pandas.isna(_dataFrame['test_string'][3])) + self.assertTrue(pandas.isna(_dataFrame['test_boolean'][0])) + self.assertTrue(pandas.isna(_dataFrame['test_boolean'][1])) + self.assertTrue(pandas.isna(_dataFrame['test_boolean'][2])) + self.assertEqual(True, _dataFrame['test_boolean'][3]) class QueryDataFrameIntegrationApi(BaseTest): From 2d91db53147692a5e1810084856882f39ee5a5be Mon Sep 17 00:00:00 2001 From: MattBrth <36452374+MattBrth@users.noreply.github.com> Date: Thu, 1 Feb 2024 13:31:50 +0100 Subject: [PATCH 10/51] refactor(dataframe): avoid chained assignment in replace operation (#638) --- CHANGELOG.md | 1 + influxdb_client/client/write/dataframe_serializer.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b1bf2187..856beb89 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ### Bug Fixes 1. [#636](https://github.com/influxdata/influxdb-client-python/pull/636): Handle missing data in data frames +2. [#638](https://github.com/influxdata/influxdb-client-python/pull/638): Refactor DataFrame operations to avoid chained assignment and resolve FutureWarning in pandas, ensuring compatibility with pandas 3.0. ## 1.40.0 [2024-01-30] diff --git a/influxdb_client/client/write/dataframe_serializer.py b/influxdb_client/client/write/dataframe_serializer.py index 6c028716..98262526 100644 --- a/influxdb_client/client/write/dataframe_serializer.py +++ b/influxdb_client/client/write/dataframe_serializer.py @@ -234,7 +234,7 @@ def __init__(self, data_frame, point_settings, precision=DEFAULT_WRITE_PRECISION for k, v in dict(data_frame.dtypes).items(): if k in data_frame_tag_columns: - data_frame[k].replace('', np.nan, inplace=True) + data_frame.replace({k: ''}, np.nan, inplace=True) self.data_frame = data_frame self.f = f From 293ed7fa45f7901c599c7110e2721bdf3b888ae2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Bedn=C3=A1=C5=99?= Date: Fri, 2 Feb 2024 15:41:28 +0100 Subject: [PATCH 11/51] docs: use markdown instead of rst (#639) * docs: use markdown instead of rst * docs: use markdown instead of rst * fix: generating HTML for pydoc * fix: generating HTML for pydoc * docs: update CHANGELOG.md --- .circleci/config.yml | 1 + CHANGELOG.md | 3 + README.md | 1541 ++++++++++++++++++++++++++++++++++++++ README.rst | 1657 ----------------------------------------- docs/conf.py | 4 +- docs/development.rst | 8 +- docs/index.rst | 7 +- docs/requirements.txt | 3 +- docs/usage.rst | 84 ++- setup.py | 14 +- 10 files changed, 1610 insertions(+), 1712 deletions(-) create mode 100644 README.md delete mode 100644 README.rst diff --git a/.circleci/config.yml b/.circleci/config.yml index f4429ab6..e471dfbb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -153,6 +153,7 @@ jobs: pip install sphinx==1.8.5 --user pip install sphinx_rtd_theme --user pip install jinja2==3.0.3 --user + pip install myst_parser>=0.19.2--user cd docs python -m sphinx -T -E -b html -d _build/doctrees -D language=en . _build/html check-aws-lambda-layer: diff --git a/CHANGELOG.md b/CHANGELOG.md index 856beb89..d5b10565 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ 1. [#636](https://github.com/influxdata/influxdb-client-python/pull/636): Handle missing data in data frames 2. [#638](https://github.com/influxdata/influxdb-client-python/pull/638): Refactor DataFrame operations to avoid chained assignment and resolve FutureWarning in pandas, ensuring compatibility with pandas 3.0. +### Documentation +1. [#639](https://github.com/influxdata/influxdb-client-python/pull/639): Use Markdown for `README` + ## 1.40.0 [2024-01-30] ### Features diff --git a/README.md b/README.md new file mode 100644 index 00000000..ce78bd00 --- /dev/null +++ b/README.md @@ -0,0 +1,1541 @@ +# influxdb-client-python + + + +[![CircleCI](https://circleci.com/gh/influxdata/influxdb-client-python.svg?style=svg)](https://circleci.com/gh/influxdata/influxdb-client-python) +[![codecov](https://codecov.io/gh/influxdata/influxdb-client-python/branch/master/graph/badge.svg)](https://codecov.io/gh/influxdata/influxdb-client-python) +[![CI status](https://img.shields.io/circleci/project/github/influxdata/influxdb-client-python/master.svg)](https://circleci.com/gh/influxdata/influxdb-client-python) +[![PyPI package](https://img.shields.io/pypi/v/influxdb-client.svg)](https://pypi.org/project/influxdb-client/) +[![Anaconda.org package](https://anaconda.org/influxdata/influxdb_client/badges/version.svg)](https://anaconda.org/influxdata/influxdb_client) +[![Supported Python versions](https://img.shields.io/pypi/pyversions/influxdb-client.svg)](https://pypi.python.org/pypi/influxdb-client) +[![Documentation status](https://readthedocs.org/projects/influxdb-client/badge/?version=stable)](https://influxdb-client.readthedocs.io/en/stable/) +[![Slack Status](https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social)](https://www.influxdata.com/slack) + +This repository contains the Python client library for use with InfluxDB 2.x and Flux. InfluxDB 3.x users should instead use the lightweight [v3 client library](https://github.com/InfluxCommunity/influxdb3-python). +InfluxDB 1.x users should use the [v1 client library](https://github.com/influxdata/influxdb-python). + +For ease of migration and a consistent query and write experience, v2 users should consider using InfluxQL and the [v1 client library](https://github.com/influxdata/influxdb-python). + +The API of the **influxdb-client-python** is not the backwards-compatible with the old one - **influxdb-python**. + +## Documentation + +This section contains links to the client library documentation. + +- [Product documentation](https://docs.influxdata.com/influxdb/v2.0/tools/client-libraries/), [Getting Started](#getting-started) +- [Examples](https://github.com/influxdata/influxdb-client-python/tree/master/examples) +- [API Reference](https://influxdb-client.readthedocs.io/en/stable/api.html) +- [Changelog](https://github.com/influxdata/influxdb-client-python/blob/master/CHANGELOG.md) + +## InfluxDB 2.0 client features + +- Querying data + - using the Flux language + - into csv, raw data, [flux_table](https://github.com/influxdata/influxdb-client-python/blob/master/influxdb_client/client/flux_table.py#L33) structure, [Pandas DataFrame](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.html) + - [How to query](#queries) +- Writing data using + - [Line Protocol](https://docs.influxdata.com/influxdb/latest/reference/syntax/line-protocol) + - [Data Point](https://github.com/influxdata/influxdb-client-python/blob/master/influxdb_client/client/write/point.py#L47) + - [RxPY](https://rxpy.readthedocs.io/en/latest/) Observable + - [Pandas DataFrame](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.html) + - [How to write](#writes) +- [InfluxDB 2.0 API](https://github.com/influxdata/influxdb/blob/master/http/swagger.yml) client for management + - the client is generated from the [swagger](https://github.com/influxdata/influxdb/blob/master/http/swagger.yml) by using the [openapi-generator](https://github.com/OpenAPITools/openapi-generator) + - organizations & users management + - buckets management + - tasks management + - authorizations + - health check + - ... +- [InfluxDB 1.8 API compatibility](#influxdb-18-api-compatibility) +- Examples + - [Connect to InfluxDB Cloud](#connect-to-influxdb-cloud) + - [How to efficiently import large dataset](#how-to-efficiently-import-large-dataset) + - [Efficiency write data from IOT sensor](#efficiency-write-data-from-iot-sensor) + - [How to use Jupyter + Pandas + InfluxDB 2](#how-to-use-jupyter--pandas--influxdb-2) +- [Advanced Usage](#advanced-usage) + - [Gzip support](#gzip-support) + - [Proxy configuration](#proxy-configuration) + - [Nanosecond precision](#nanosecond-precision) + - [Delete data](#delete-data) + - [Handling Errors](#handling-errors) + - [Logging](#logging) + +## Installation + +InfluxDB python library uses [RxPY](https://github.com/ReactiveX/RxPY) - The Reactive Extensions for Python (RxPY). + +**Python 3.7** or later is required. + +:warning: +> It is recommended to use `ciso8601` with client for parsing dates. `ciso8601` is much faster than built-in Python datetime. Since it's written as a `C` module the best way is build it from sources: + +**Windows**: + +You have to install [Visual C++ Build Tools 2015](http://go.microsoft.com/fwlink/?LinkId=691126&fixForIE=.exe) to build `ciso8601` by `pip`. + +**conda**: + +Install from sources: `conda install -c conda-forge/label/cf202003 ciso8601`. + +### pip install + +The python package is hosted on [PyPI](https://pypi.org/project/influxdb-client/), you can install latest version directly: + +``` sh +pip install 'influxdb-client[ciso]' +``` + +Then import the package: + +``` python +import influxdb_client +``` + +If your application uses async/await in Python you can install with the `async` extra: + +``` sh +$ pip install influxdb-client[async] +``` + +For more info see [How to use Asyncio](#how-to-use-asyncio). + +### Setuptools + +Install via [Setuptools](http://pypi.python.org/pypi/setuptools). + +``` sh +python setup.py install --user +``` + +(or `sudo python setup.py install` to install the package for all users) + +## Getting Started + +Please follow the [Installation](#installation) and then run the following: + + + +``` python +from influxdb_client import InfluxDBClient, Point +from influxdb_client.client.write_api import SYNCHRONOUS + +bucket = "my-bucket" + +client = InfluxDBClient(url="http://localhost:8086", token="my-token", org="my-org") + +write_api = client.write_api(write_options=SYNCHRONOUS) +query_api = client.query_api() + +p = Point("my_measurement").tag("location", "Prague").field("temperature", 25.3) + +write_api.write(bucket=bucket, record=p) + +## using Table structure +tables = query_api.query('from(bucket:"my-bucket") |> range(start: -10m)') + +for table in tables: + print(table) + for row in table.records: + print (row.values) + + +## using csv library +csv_result = query_api.query_csv('from(bucket:"my-bucket") |> range(start: -10m)') +val_count = 0 +for row in csv_result: + for cell in row: + val_count += 1 +``` + + + +## Client configuration + +### Via File + +A client can be configured via `*.ini` file in segment `influx2`. + +The following options are supported: + +- `url` - the url to connect to InfluxDB +- `org` - default destination organization for writes and queries +- `token` - the token to use for the authorization +- `timeout` - socket timeout in ms (default value is 10000) +- `verify_ssl` - set this to false to skip verifying SSL certificate when calling API from https server +- `ssl_ca_cert` - set this to customize the certificate file to verify the peer +- `cert_file` - path to the certificate that will be used for mTLS authentication +- `cert_key_file` - path to the file contains private key for mTLS certificate +- `cert_key_password` - string or function which returns password for decrypting the mTLS private key +- `connection_pool_maxsize` - set the number of connections to save that can be reused by urllib3 +- `auth_basic` - enable http basic authentication when talking to a InfluxDB 1.8.x without authentication but is accessed via reverse proxy with basic authentication (defaults to false) +- `profilers` - set the list of enabled [Flux profilers](https://docs.influxdata.com/influxdb/v2.0/reference/flux/stdlib/profiler/) + +``` python +self.client = InfluxDBClient.from_config_file("config.ini") +``` + +``` ini +[influx2] +url=http://localhost:8086 +org=my-org +token=my-token +timeout=6000 +verify_ssl=False +``` + +### Via Environment Properties + +A client can be configured via environment properties. + +Supported properties are: + +- `INFLUXDB_V2_URL` - the url to connect to InfluxDB +- `INFLUXDB_V2_ORG` - default destination organization for writes and queries +- `INFLUXDB_V2_TOKEN` - the token to use for the authorization +- `INFLUXDB_V2_TIMEOUT` - socket timeout in ms (default value is 10000) +- `INFLUXDB_V2_VERIFY_SSL` - set this to false to skip verifying SSL certificate when calling API from https server +- `INFLUXDB_V2_SSL_CA_CERT` - set this to customize the certificate file to verify the peer +- `INFLUXDB_V2_CERT_FILE` - path to the certificate that will be used for mTLS authentication +- `INFLUXDB_V2_CERT_KEY_FILE` - path to the file contains private key for mTLS certificate +- `INFLUXDB_V2_CERT_KEY_PASSWORD` - string or function which returns password for decrypting the mTLS private key +- `INFLUXDB_V2_CONNECTION_POOL_MAXSIZE` - set the number of connections to save that can be reused by urllib3 +- `INFLUXDB_V2_AUTH_BASIC` - enable http basic authentication when talking to a InfluxDB 1.8.x without authentication but is accessed via reverse proxy with basic authentication (defaults to false) +- `INFLUXDB_V2_PROFILERS` - set the list of enabled [Flux profilers](https://docs.influxdata.com/influxdb/v2.0/reference/flux/stdlib/profiler/) + +``` python +self.client = InfluxDBClient.from_env_properties() +``` + +### Profile query + +The [Flux Profiler package](https://docs.influxdata.com/influxdb/v2.0/reference/flux/stdlib/profiler/) provides performance profiling tools for Flux queries and operations. + +You can enable printing profiler information of the Flux query in client +library by: + +- set QueryOptions.profilers in QueryApi, +- set `INFLUXDB_V2_PROFILERS` environment variable, +- set `profilers` option in configuration file. + +When the profiler is enabled, the result of flux query contains additional tables "profiler/". In order to have consistent behaviour with enabled/disabled profiler, `FluxCSVParser` excludes "profiler/" measurements from result. + +Example how to enable profilers using API: + +``` python +q = ''' + from(bucket: stringParam) + |> range(start: -5m, stop: now()) + |> filter(fn: (r) => r._measurement == "mem") + |> filter(fn: (r) => r._field == "available" or r._field == "free" or r._field == "used") + |> aggregateWindow(every: 1m, fn: mean) + |> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value") +''' +p = { + "stringParam": "my-bucket", +} + +query_api = client.query_api(query_options=QueryOptions(profilers=["query", "operator"])) +csv_result = query_api.query(query=q, params=p) +``` + +Example of a profiler output: + +``` text +=============== +Profiler: query +=============== + +from(bucket: stringParam) + |> range(start: -5m, stop: now()) + |> filter(fn: (r) => r._measurement == "mem") + |> filter(fn: (r) => r._field == "available" or r._field == "free" or r._field == "used") + |> aggregateWindow(every: 1m, fn: mean) + |> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value") + +======================== +Profiler: profiler/query +======================== +result : _profiler +table : 0 +_measurement : profiler/query +TotalDuration : 8924700 +CompileDuration : 350900 +QueueDuration : 33800 +PlanDuration : 0 +RequeueDuration : 0 +ExecuteDuration : 8486500 +Concurrency : 0 +MaxAllocated : 2072 +TotalAllocated : 0 +flux/query-plan : + +digraph { + ReadWindowAggregateByTime11 + // every = 1m, aggregates = [mean], createEmpty = true, timeColumn = "_stop" + pivot8 + generated_yield + + ReadWindowAggregateByTime11 -> pivot8 + pivot8 -> generated_yield +} + + +influxdb/scanned-bytes: 0 +influxdb/scanned-values: 0 + +=========================== +Profiler: profiler/operator +=========================== +result : _profiler +table : 1 +_measurement : profiler/operator +Type : *universe.pivotTransformation +Label : pivot8 +Count : 3 +MinDuration : 32600 +MaxDuration : 126200 +DurationSum : 193400 +MeanDuration : 64466.666666666664 + +=========================== +Profiler: profiler/operator +=========================== +result : _profiler +table : 1 +_measurement : profiler/operator +Type : *influxdb.readWindowAggregateSource +Label : ReadWindowAggregateByTime11 +Count : 1 +MinDuration : 940500 +MaxDuration : 940500 +DurationSum : 940500 +MeanDuration : 940500.0 +``` + +You can also use callback function to get profilers output. Return value of this callback is type of FluxRecord. + +Example how to use profilers with callback: + +``` python +class ProfilersCallback(object): + def __init__(self): + self.records = [] + + def __call__(self, flux_record): + self.records.append(flux_record.values) + +callback = ProfilersCallback() + +query_api = client.query_api(query_options=QueryOptions(profilers=["query", "operator"], profiler_callback=callback)) +tables = query_api.query('from(bucket:"my-bucket") |> range(start: -10m)') + +for profiler in callback.records: + print(f'Custom processing of profiler result: {profiler}') +``` + +Example output of this callback: + +``` text +Custom processing of profiler result: {'result': '_profiler', 'table': 0, '_measurement': 'profiler/query', 'TotalDuration': 18843792, 'CompileDuration': 1078666, 'QueueDuration': 93375, 'PlanDuration': 0, 'RequeueDuration': 0, 'ExecuteDuration': 17371000, 'Concurrency': 0, 'MaxAllocated': 448, 'TotalAllocated': 0, 'RuntimeErrors': None, 'flux/query-plan': 'digraph {\r\n ReadRange2\r\n generated_yield\r\n\r\n ReadRange2 -> generated_yield\r\n}\r\n\r\n', 'influxdb/scanned-bytes': 0, 'influxdb/scanned-values': 0} +Custom processing of profiler result: {'result': '_profiler', 'table': 1, '_measurement': 'profiler/operator', 'Type': '*influxdb.readFilterSource', 'Label': 'ReadRange2', 'Count': 1, 'MinDuration': 3274084, 'MaxDuration': 3274084, 'DurationSum': 3274084, 'MeanDuration': 3274084.0} +``` + + + +## How to use + +### Writes + + + +The [WriteApi](https://github.com/influxdata/influxdb-client-python/blob/master/influxdb_client/client/write_api.py) supports synchronous, asynchronous and batching writes into InfluxDB 2.0. The data should be passed as a [InfluxDB Line Protocol](https://docs.influxdata.com/influxdb/latest/write_protocols/line_protocol_tutorial/), [Data Point](https://github.com/influxdata/influxdb-client-python/blob/master/influxdb_client/client/write/point.py) or Observable stream. + +:warning: + +> The `WriteApi` in batching mode (default mode) is supposed to run as a +singleton. To flush all your data you should wrap the execution using +`with client.write_api(...) as write_api:` statement or call +`write_api.close()` at the end of your script. + +*The default instance of WriteApi use batching.* + +#### The data could be written as + +1. `string` or `bytes` that is formatted as a InfluxDB's line protocol +2. [Data Point](https://github.com/influxdata/influxdb-client-python/blob/master/influxdb_client/client/write/point.py#L16) structure +3. Dictionary style mapping with keys: `measurement`, `tags`, `fields` and `time` or custom structure +4. [NamedTuple](https://docs.python.org/3/library/collections.html#collections.namedtuple) +5. [Data Classes](https://docs.python.org/3/library/dataclasses.html) +6. [Pandas DataFrame](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.html) +7. List of above items +8. A `batching` type of write also supports an `Observable` that produce one of an above item + +You can find write examples at GitHub: [influxdb-client-python/examples](https://github.com/influxdata/influxdb-client-python/tree/master/examples#writes). + +#### Batching + +The batching is configurable by `write_options`: + + + +| Property | Description | Default Value | +|----------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------| +| **batch_size** | the number of data point to collect in a batch | `1000` | +| **flush_interval** | the number of milliseconds before the batch is written | `1000` | +| **jitter_interval** | the number of milliseconds to increase the batch flush interval by a random amount | `0` | +| **retry_interval** | the number of milliseconds to retry first unsuccessful write. The next retry delay is computed using exponential random backoff. The retry interval is used when the InfluxDB server does not specify \"Retry-After\" header. | `5000` | +| **max_retry_time** | maximum total retry timeout in milliseconds. | `180_000` | +| **max_retries** | the number of max retries when write fails | `5` | +| **max_retry_delay** | the maximum delay between each retry attempt in milliseconds | `125_000` | +| **max_close_wait** | the maximum amount of time to wait for batches to flush when `.close()` is called | `300_000` | +| **exponential_base** | the base for the exponential retry delay, the next delay is computed using random exponential backoff as a random value within the interval `retry_interval * exponential_base^(attempts-1)` and `retry_interval * exponential_base^(attempts)`. Example for `retry_interval=5_000, exponential_base=2, max_retry_delay=125_000, total=5` Retry delays are random distributed values within the ranges of `[5_000-10_000, 10_000-20_000, 20_000-40_000, 40_000-80_000, 80_000-125_000]` | `2` | + +``` python +from datetime import datetime, timedelta + +import pandas as pd +import reactivex as rx +from reactivex import operators as ops + +from influxdb_client import InfluxDBClient, Point, WriteOptions + +with InfluxDBClient(url="http://localhost:8086", token="my-token", org="my-org") as _client: + + with _client.write_api(write_options=WriteOptions(batch_size=500, + flush_interval=10_000, + jitter_interval=2_000, + retry_interval=5_000, + max_retries=5, + max_retry_delay=30_000, + max_close_wait=300_000, + exponential_base=2)) as _write_client: + + """ + Write Line Protocol formatted as string + """ + _write_client.write("my-bucket", "my-org", "h2o_feet,location=coyote_creek water_level=1.0 1") + _write_client.write("my-bucket", "my-org", ["h2o_feet,location=coyote_creek water_level=2.0 2", + "h2o_feet,location=coyote_creek water_level=3.0 3"]) + + """ + Write Line Protocol formatted as byte array + """ + _write_client.write("my-bucket", "my-org", "h2o_feet,location=coyote_creek water_level=1.0 1".encode()) + _write_client.write("my-bucket", "my-org", ["h2o_feet,location=coyote_creek water_level=2.0 2".encode(), + "h2o_feet,location=coyote_creek water_level=3.0 3".encode()]) + + """ + Write Dictionary-style object + """ + _write_client.write("my-bucket", "my-org", {"measurement": "h2o_feet", "tags": {"location": "coyote_creek"}, + "fields": {"water_level": 1.0}, "time": 1}) + _write_client.write("my-bucket", "my-org", [{"measurement": "h2o_feet", "tags": {"location": "coyote_creek"}, + "fields": {"water_level": 2.0}, "time": 2}, + {"measurement": "h2o_feet", "tags": {"location": "coyote_creek"}, + "fields": {"water_level": 3.0}, "time": 3}]) + + """ + Write Data Point + """ + _write_client.write("my-bucket", "my-org", + Point("h2o_feet").tag("location", "coyote_creek").field("water_level", 4.0).time(4)) + _write_client.write("my-bucket", "my-org", + [Point("h2o_feet").tag("location", "coyote_creek").field("water_level", 5.0).time(5), + Point("h2o_feet").tag("location", "coyote_creek").field("water_level", 6.0).time(6)]) + + """ + Write Observable stream + """ + _data = rx \ + .range(7, 11) \ + .pipe(ops.map(lambda i: "h2o_feet,location=coyote_creek water_level={0}.0 {0}".format(i))) + + _write_client.write("my-bucket", "my-org", _data) + + """ + Write Pandas DataFrame + """ + _now = datetime.utcnow() + _data_frame = pd.DataFrame(data=[["coyote_creek", 1.0], ["coyote_creek", 2.0]], + index=[_now, _now + timedelta(hours=1)], + columns=["location", "water_level"]) + + _write_client.write("my-bucket", "my-org", record=_data_frame, data_frame_measurement_name='h2o_feet', + data_frame_tag_columns=['location']) +``` + +#### Default Tags + +Sometimes is useful to store same information in every measurement e.g. `hostname`, `location`, `customer`. The client is able to use static value or env property as a tag value. + +The expressions: + +- `California Miner` - static value +- `${env.hostname}` - environment property + +##### Via API + +``` python +point_settings = PointSettings() +point_settings.add_default_tag("id", "132-987-655") +point_settings.add_default_tag("customer", "California Miner") +point_settings.add_default_tag("data_center", "${env.data_center}") + +self.write_client = self.client.write_api(write_options=SYNCHRONOUS, point_settings=point_settings) +``` + +``` python +self.write_client = self.client.write_api(write_options=SYNCHRONOUS, + point_settings=PointSettings(**{"id": "132-987-655", + "customer": "California Miner"})) +``` + +##### Via Configuration file + +In an [init](https://docs.python.org/3/library/configparser.html) configuration file you are able to specify default tags by `tags` segment. + +``` python +self.client = InfluxDBClient.from_config_file("config.ini") +``` + +``` +[influx2] +url=http://localhost:8086 +org=my-org +token=my-token +timeout=6000 + +[tags] +id = 132-987-655 +customer = California Miner +data_center = ${env.data_center} +``` + +You can also use a [TOML](https://toml.io/en/) or a[JSON](https://www.json.org/json-en.html) format for the configuration file. + +##### Via Environment Properties + +You are able to specify default tags by environment properties with prefix `INFLUXDB_V2_TAG_`. + +Examples: + +- `INFLUXDB_V2_TAG_ID` +- `INFLUXDB_V2_TAG_HOSTNAME` + +``` python +self.client = InfluxDBClient.from_env_properties() +``` + +#### Synchronous client + +Data are writes in a synchronous HTTP request. + +``` python +from influxdb_client import InfluxDBClient, Point +from influxdb_client .client.write_api import SYNCHRONOUS + +client = InfluxDBClient(url="http://localhost:8086", token="my-token", org="my-org") +write_api = client.write_api(write_options=SYNCHRONOUS) + +_point1 = Point("my_measurement").tag("location", "Prague").field("temperature", 25.3) +_point2 = Point("my_measurement").tag("location", "New York").field("temperature", 24.3) + +write_api.write(bucket="my-bucket", record=[_point1, _point2]) + +client.close() +``` + + +### Queries + +The result retrieved by [QueryApi](https://github.com/influxdata/influxdb-client-python/blob/master/influxdb_client/client/query_api.py) could be formatted as a: + +1. Flux data structure: [FluxTable](https://github.com/influxdata/influxdb-client-python/blob/master/influxdb_client/client/flux_table.py#L5), [FluxColumn](https://github.com/influxdata/influxdb-client-python/blob/master/influxdb_client/client/flux_table.py#L22) and [FluxRecord](https://github.com/influxdata/influxdb-client-python/blob/master/influxdb_client/client/flux_table.py#L31) +2. `influxdb_client.client.flux_table.CSVIterator` which will iterate over CSV lines +3. Raw unprocessed results as a `str` iterator +4. [Pandas DataFrame](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.html) + +The API also support streaming `FluxRecord` via [query_stream](https://github.com/influxdata/influxdb-client-python/blob/master/influxdb_client/client/query_api.py#L77), see example below: + +``` python +from influxdb_client import InfluxDBClient, Point, Dialect +from influxdb_client.client.write_api import SYNCHRONOUS + +client = InfluxDBClient(url="http://localhost:8086", token="my-token", org="my-org") + +write_api = client.write_api(write_options=SYNCHRONOUS) +query_api = client.query_api() + +""" +Prepare data +""" + +_point1 = Point("my_measurement").tag("location", "Prague").field("temperature", 25.3) +_point2 = Point("my_measurement").tag("location", "New York").field("temperature", 24.3) + +write_api.write(bucket="my-bucket", record=[_point1, _point2]) + +""" +Query: using Table structure +""" +tables = query_api.query('from(bucket:"my-bucket") |> range(start: -10m)') + +for table in tables: + print(table) + for record in table.records: + print(record.values) + +print() +print() + +""" +Query: using Bind parameters +""" + +p = {"_start": datetime.timedelta(hours=-1), + "_location": "Prague", + "_desc": True, + "_floatParam": 25.1, + "_every": datetime.timedelta(minutes=5) + } + +tables = query_api.query(''' + from(bucket:"my-bucket") |> range(start: _start) + |> filter(fn: (r) => r["_measurement"] == "my_measurement") + |> filter(fn: (r) => r["_field"] == "temperature") + |> filter(fn: (r) => r["location"] == _location and r["_value"] > _floatParam) + |> aggregateWindow(every: _every, fn: mean, createEmpty: true) + |> sort(columns: ["_time"], desc: _desc) +''', params=p) + +for table in tables: + print(table) + for record in table.records: + print(str(record["_time"]) + " - " + record["location"] + ": " + str(record["_value"])) + +print() +print() + +""" +Query: using Stream +""" +records = query_api.query_stream('from(bucket:"my-bucket") |> range(start: -10m)') + +for record in records: + print(f'Temperature in {record["location"]} is {record["_value"]}') + +""" +Interrupt a stream after retrieve a required data +""" +large_stream = query_api.query_stream('from(bucket:"my-bucket") |> range(start: -100d)') +for record in large_stream: + if record["location"] == "New York": + print(f'New York temperature: {record["_value"]}') + break + +large_stream.close() + +print() +print() + +""" +Query: using csv library +""" +csv_result = query_api.query_csv('from(bucket:"my-bucket") |> range(start: -10m)', + dialect=Dialect(header=False, delimiter=",", comment_prefix="#", annotations=[], + date_time_format="RFC3339")) +for csv_line in csv_result: + if not len(csv_line) == 0: + print(f'Temperature in {csv_line[9]} is {csv_line[6]}') + +""" +Close client +""" +client.close() +``` + +#### Pandas DataFrame + + + +:warning: + +> For DataFrame querying you should install Pandas dependency via `pip install 'influxdb-client[extra]'`. + +:warning: + +> Note that if a query returns more then one table than the client generates a `DataFrame` for each of them. + +The `client` is able to retrieve data in [Pandas DataFrame](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.html) format thought `query_data_frame`: + +``` python +from influxdb_client import InfluxDBClient, Point, Dialect +from influxdb_client.client.write_api import SYNCHRONOUS + +client = InfluxDBClient(url="http://localhost:8086", token="my-token", org="my-org") + +write_api = client.write_api(write_options=SYNCHRONOUS) +query_api = client.query_api() + +""" +Prepare data +""" + +_point1 = Point("my_measurement").tag("location", "Prague").field("temperature", 25.3) +_point2 = Point("my_measurement").tag("location", "New York").field("temperature", 24.3) + +write_api.write(bucket="my-bucket", record=[_point1, _point2]) + +""" +Query: using Pandas DataFrame +""" +data_frame = query_api.query_data_frame('from(bucket:"my-bucket") ' + '|> range(start: -10m) ' + '|> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value") ' + '|> keep(columns: ["location", "temperature"])') +print(data_frame.to_string()) + +""" +Close client +""" +client.close() +``` + +Output: + +``` text +result table location temperature +0 _result 0 New York 24.3 +1 _result 1 Prague 25.3 +``` + + + +### Examples + + + +#### How to efficiently import large dataset + +The following example shows how to import dataset with a dozen megabytes. If you would like to import gigabytes of data then +use our multiprocessing example: [import_data_set_multiprocessing.py](https://github.com/influxdata/influxdb-client-python/blob/master/examples/import_data_set_multiprocessing.py) for use a full capability of your hardware. + +- sources - [import_data_set.py](https://github.com/influxdata/influxdb-client-python/blob/master/examples/import_data_set.py) + +``` python +""" +Import VIX - CBOE Volatility Index - from "vix-daily.csv" file into InfluxDB 2.0 + +https://datahub.io/core/finance-vix#data +""" + +from collections import OrderedDict +from csv import DictReader + +import reactivex as rx +from reactivex import operators as ops + +from influxdb_client import InfluxDBClient, Point, WriteOptions + +def parse_row(row: OrderedDict): + """Parse row of CSV file into Point with structure: + + financial-analysis,type=ily close=18.47,high=19.82,low=18.28,open=19.82 1198195200000000000 + + CSV format: + Date,VIX Open,VIX High,VIX Low,VIX Close\n + 2004-01-02,17.96,18.68,17.54,18.22\n + 2004-01-05,18.45,18.49,17.44,17.49\n + 2004-01-06,17.66,17.67,16.19,16.73\n + 2004-01-07,16.72,16.75,15.5,15.5\n + 2004-01-08,15.42,15.68,15.32,15.61\n + 2004-01-09,16.15,16.88,15.57,16.75\n + ... + + :param row: the row of CSV file + :return: Parsed csv row to [Point] + """ + + """ + For better performance is sometimes useful directly create a LineProtocol to avoid unnecessary escaping overhead: + """ + # from datetime import timezone + # import ciso8601 + # from influxdb_client.client.write.point import EPOCH + # + # time = (ciso8601.parse_datetime(row["Date"]).replace(tzinfo=timezone.utc) - EPOCH).total_seconds() * 1e9 + # return f"financial-analysis,type=vix-daily" \ + # f" close={float(row['VIX Close'])},high={float(row['VIX High'])},low={float(row['VIX Low'])},open={float(row['VIX Open'])} " \ + # f" {int(time)}" + + return Point("financial-analysis") \ + .tag("type", "vix-daily") \ + .field("open", float(row['VIX Open'])) \ + .field("high", float(row['VIX High'])) \ + .field("low", float(row['VIX Low'])) \ + .field("close", float(row['VIX Close'])) \ + .time(row['Date']) + + +""" +Converts vix-daily.csv into sequence of datad point +""" +data = rx \ + .from_iterable(DictReader(open('vix-daily.csv', 'r'))) \ + .pipe(ops.map(lambda row: parse_row(row))) + +client = InfluxDBClient(url="http://localhost:8086", token="my-token", org="my-org", debug=True) + +""" +Create client that writes data in batches with 50_000 items. +""" +write_api = client.write_api(write_options=WriteOptions(batch_size=50_000, flush_interval=10_000)) + +""" +Write data into InfluxDB +""" +write_api.write(bucket="my-bucket", record=data) +write_api.close() + +""" +Querying max value of CBOE Volatility Index +""" +query = 'from(bucket:"my-bucket")' \ + ' |> range(start: 0, stop: now())' \ + ' |> filter(fn: (r) => r._measurement == "financial-analysis")' \ + ' |> max()' +result = client.query_api().query(query=query) + +""" +Processing results +""" +print() +print("=== results ===") +print() +for table in result: + for record in table.records: + print('max {0:5} = {1}'.format(record.get_field(), record.get_value())) + +""" +Close client +""" +client.close() +``` + +#### Efficiency write data from IOT sensor + +- sources - [iot_sensor.py](https://github.com/influxdata/influxdb-client-python/blob/master/examples/iot_sensor.py) + +``` python +""" +Efficiency write data from IOT sensor - write changed temperature every minute +""" +import atexit +import platform +from datetime import timedelta + +import psutil as psutil +import reactivex as rx +from reactivex import operators as ops + +from influxdb_client import InfluxDBClient, WriteApi, WriteOptions + +def on_exit(db_client: InfluxDBClient, write_api: WriteApi): + """Close clients after terminate a script. + + :param db_client: InfluxDB client + :param write_api: WriteApi + :return: nothing + """ + write_api.close() + db_client.close() + + +def sensor_temperature(): + """Read a CPU temperature. The [psutil] doesn't support MacOS so we use [sysctl]. + + :return: actual CPU temperature + """ + os_name = platform.system() + if os_name == 'Darwin': + from subprocess import check_output + output = check_output(["sysctl", "machdep.xcpm.cpu_thermal_level"]) + import re + return re.findall(r'\d+', str(output))[0] + else: + return psutil.sensors_temperatures()["coretemp"][0] + + +def line_protocol(temperature): + """Create a InfluxDB line protocol with structure: + + iot_sensor,hostname=mine_sensor_12,type=temperature value=68 + + :param temperature: the sensor temperature + :return: Line protocol to write into InfluxDB + """ + + import socket + return 'iot_sensor,hostname={},type=temperature value={}'.format(socket.gethostname(), temperature) + + +""" +Read temperature every minute; distinct_until_changed - produce only if temperature change +""" +data = rx\ + .interval(period=timedelta(seconds=60))\ + .pipe(ops.map(lambda t: sensor_temperature()), + ops.distinct_until_changed(), + ops.map(lambda temperature: line_protocol(temperature))) + +_db_client = InfluxDBClient(url="http://localhost:8086", token="my-token", org="my-org", debug=True) + +""" +Create client that writes data into InfluxDB +""" +_write_api = _db_client.write_api(write_options=WriteOptions(batch_size=1)) +_write_api.write(bucket="my-bucket", record=data) + + +""" +Call after terminate a script +""" +atexit.register(on_exit, _db_client, _write_api) + +input() +``` + +#### Connect to InfluxDB Cloud + +The following example demonstrate the simplest way how to write and query date with the InfluxDB Cloud. + +At first point you should create an authentication token as is described [here](https://v2.docs.influxdata.com/v2.0/security/tokens/create-token/). + +After that you should configure properties: `influx_cloud_url`,`influx_cloud_token`, `bucket` and `org` in a `influx_cloud.py` example. + +The last step is run a python script via: `python3 influx_cloud.py`. + +- sources - [influx_cloud.py](https://github.com/influxdata/influxdb-client-python/blob/master/examples/influx_cloud.py) + +``` python +""" +Connect to InfluxDB 2.0 - write data and query them +""" + +from datetime import datetime + +from influxdb_client import Point, InfluxDBClient +from influxdb_client.client.write_api import SYNCHRONOUS + +""" +Configure credentials +""" +influx_cloud_url = 'https://us-west-2-1.aws.cloud2.influxdata.com' +influx_cloud_token = '...' +bucket = '...' +org = '...' + +client = InfluxDBClient(url=influx_cloud_url, token=influx_cloud_token) +try: + kind = 'temperature' + host = 'host1' + device = 'opt-123' + + """ + Write data by Point structure + """ + point = Point(kind).tag('host', host).tag('device', device).field('value', 25.3).time(time=datetime.utcnow()) + + print(f'Writing to InfluxDB cloud: {point.to_line_protocol()} ...') + + write_api = client.write_api(write_options=SYNCHRONOUS) + write_api.write(bucket=bucket, org=org, record=point) + + print() + print('success') + print() + print() + + """ + Query written data + """ + query = f'from(bucket: "{bucket}") |> range(start: -1d) |> filter(fn: (r) => r._measurement == "{kind}")' + print(f'Querying from InfluxDB cloud: "{query}" ...') + print() + + query_api = client.query_api() + tables = query_api.query(query=query, org=org) + + for table in tables: + for row in table.records: + print(f'{row.values["_time"]}: host={row.values["host"]},device={row.values["device"]} ' + f'{row.values["_value"]} °C') + + print() + print('success') + +except Exception as e: + print(e) +finally: + client.close() +``` + +#### How to use Jupyter + Pandas + InfluxDB 2 + +The first example shows how to use client capabilities to predict stock price via [Keras](https://keras.io), [TensorFlow](https://www.tensorflow.org), [sklearn](https://scikit-learn.org/stable/): + +The example is taken from [Kaggle](https://www.kaggle.com/chaitanyacc4/predicting-stock-prices-of-apple-inc). + +- sources - [stock-predictions.ipynb](notebooks/stock-predictions.ipynb) + +![image](https://raw.githubusercontent.com/influxdata/influxdb-client-python/master/docs/images/stock-price-prediction.gif) + +Result: + +![image](https://raw.githubusercontent.com/influxdata/influxdb-client-python/master/docs/images/stock-price-prediction-results.png) + +The second example shows how to use client capabilities to realtime visualization via [hvPlot](https://hvplot.pyviz.org), [Streamz](https://streamz.readthedocs.io/en/latest/), [RxPY](https://rxpy.readthedocs.io/en/latest/): + +- sources - [realtime-stream.ipynb](notebooks/realtime-stream.ipynb) + +![image](https://raw.githubusercontent.com/influxdata/influxdb-client-python/master/docs/images/realtime-result.gif) + +#### Other examples + +You can find all examples at GitHub: [influxdb-client-python/examples](https://github.com/influxdata/influxdb-client-python/tree/master/examples#examples). + + + +## Advanced Usage + +### Gzip support + + + +`InfluxDBClient` does not enable gzip compression for http requests by default. If you want to enable gzip to reduce transfer data's size, you can call: + +``` python +from influxdb_client import InfluxDBClient + +_db_client = InfluxDBClient(url="http://localhost:8086", token="my-token", org="my-org", enable_gzip=True) +``` + + +### Authenticate to the InfluxDB + + + +`InfluxDBClient` supports three options how to authorize a connection: + +- _Token_ +- _Username & Password_ +- _HTTP Basic_ + +#### Token + +Use the `token` to authenticate to the InfluxDB API. In your API requests, an _Authorization_ header will be sent. The header value, provide the word _Token_ followed by a space and an InfluxDB API token. The word _token_ is case-sensitive. + +``` python +from influxdb_client import InfluxDBClient + +with InfluxDBClient(url="http://localhost:8086", token="my-token") as client +``` + +:warning: + +> Note that this is a preferred way how to authenticate to InfluxDB API. + + +#### Username & Password + +Authenticates via username and password credentials. If successful, creates a new session for the user. + +``` python +from influxdb_client import InfluxDBClient + +with InfluxDBClient(url="http://localhost:8086", username="my-user", password="my-password") as client +``` + +:warning: + +> The `username/password` auth is based on the HTTP "Basic" authentication. The authorization expires when the [time-to-live (TTL)](https://docs.influxdata.com/influxdb/latest/reference/config-options/#session-length) (default 60 minutes) is reached and client produces `unauthorized exception`. + +#### HTTP Basic + +Use this to enable basic authentication when talking to a InfluxDB 1.8.x that does not use auth-enabled but is protected by a reverse proxy with basic authentication. + +``` python +from influxdb_client import InfluxDBClient + +with InfluxDBClient(url="http://localhost:8086", auth_basic=True, token="my-proxy-secret") as client +``` + +:warning: + +> Don't use this when directly talking to InfluxDB 2. + + + +### Proxy configuration + + + +You can configure the client to tunnel requests through an HTTP proxy. The following proxy options are supported: + +- `proxy` - Set this to configure the http proxy to be used, ex. `http://localhost:3128` +- `proxy_headers` - A dictionary containing headers that will be sent to the proxy. Could be used for proxy authentication. + +``` python +from influxdb_client import InfluxDBClient + +with InfluxDBClient(url="http://localhost:8086", + token="my-token", + org="my-org", + proxy="http://localhost:3128") as client: +``` + +If your proxy notify the client with permanent redirect (`HTTP 301`) to **different host**. The client removes `Authorization` header, because otherwise the contents of `Authorization` is sent to third parties which is a security vulnerability. + +You can change this behaviour by: + +``` python +from urllib3 import Retry +Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset() +Retry.DEFAULT.remove_headers_on_redirect = Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT +``` + + +### Delete data + + + +The [delete_api.py](influxdb_client/client/delete_api.py) supports deletes [points](https://v2.docs.influxdata.com/v2.0/reference/glossary/#point) from an InfluxDB bucket. + +``` python +from influxdb_client import InfluxDBClient + +client = InfluxDBClient(url="http://localhost:8086", token="my-token") + +delete_api = client.delete_api() + +""" +Delete Data +""" +start = "1970-01-01T00:00:00Z" +stop = "2021-02-01T00:00:00Z" +delete_api.delete(start, stop, '_measurement="my_measurement"', bucket='my-bucket', org='my-org') + +""" +Close client +""" +client.close() +``` + + +### InfluxDB 1.8 API compatibility + +[InfluxDB 1.8.0 introduced forward compatibility APIs](https://docs.influxdata.com/influxdb/v1.8/tools/api/#influxdb-2-0-api-compatibility-endpoints) for InfluxDB 2.0. This allows you to easily move from InfluxDB 1.x to InfluxDB 2.0 Cloud or open source. + +The following forward compatible APIs are available: + + | API | Endpoint | Description | + |-----------------------------------------------------|------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| + | [query_api.py](influxdb_client/client/query_api.py) | [/api/v2/query](https://docs.influxdata.com/influxdb/v1.8/tools/api/#apiv2query-http-endpoint) | Query data in InfluxDB 1.8.0+ using the InfluxDB 2.0 API and [Flux](https://docs.influxdata.com/flux/latest/) (endpoint should be enabled by [flux-enabled option](https://docs.influxdata.com/influxdb/v1.8/administration/config/#flux-enabled-false)) | + | [write_api.py](influxdb_client/client/write_api.py) | [/api/v2/write](https://docs.influxdata.com/influxdb/v1.8/tools/api/#apiv2write-http-endpoint) | Write data to InfluxDB 1.8.0+ using the InfluxDB 2.0 API | + | [ping()](influxdb_client/client/influxdb_client.py) | [/ping](https://docs.influxdata.com/influxdb/v1.8/tools/api/#ping-http-endpoint) | Check the status of your InfluxDB instance | + +For detail info see [InfluxDB 1.8 example](examples/influxdb_18_example.py). + +### Handling Errors + + + +Errors happen, and it's important that your code is prepared for them. All client related exceptions are delivered from `InfluxDBError`. +If the exception cannot be recovered in the client it is returned to the application. These exceptions are left for the developer to handle. + +Almost all APIs directly return unrecoverable exceptions to be handled this way: + +``` python +from influxdb_client import InfluxDBClient +from influxdb_client.client.exceptions import InfluxDBError +from influxdb_client.client.write_api import SYNCHRONOUS + +with InfluxDBClient(url="http://localhost:8086", token="my-token", org="my-org") as client: + try: + client.write_api(write_options=SYNCHRONOUS).write("my-bucket", record="mem,tag=a value=86") + except InfluxDBError as e: + if e.response.status == 401: + raise Exception(f"Insufficient write permissions to 'my-bucket'.") from e + raise +``` + +The only exception is **batching** `WriteAPI` (for more info see [Batching](#batching)) where you need to register custom callbacks to handle batch events. +This is because this API runs in the `background` in a `separate` thread and isn't possible to directly return underlying exceptions. + +``` python +from influxdb_client import InfluxDBClient +from influxdb_client.client.exceptions import InfluxDBError + + +class BatchingCallback(object): + + def success(self, conf: (str, str, str), data: str): + print(f"Written batch: {conf}, data: {data}") + + def error(self, conf: (str, str, str), data: str, exception: InfluxDBError): + print(f"Cannot write batch: {conf}, data: {data} due: {exception}") + + def retry(self, conf: (str, str, str), data: str, exception: InfluxDBError): + print(f"Retryable error occurs for batch: {conf}, data: {data} retry: {exception}") + + +with InfluxDBClient(url="http://localhost:8086", token="my-token", org="my-org") as client: + callback = BatchingCallback() + with client.write_api(success_callback=callback.success, + error_callback=callback.error, + retry_callback=callback.retry) as write_api: + pass +``` + +#### HTTP Retry Strategy + +By default, the client uses a retry strategy only for batching writes (for more info see [Batching](#batching)). +For other HTTP requests there is no one retry strategy, but it could be configured by `retries` parameter of `InfluxDBClient`. + +For more info about how configure HTTP retry see details in [urllib3 documentation](https://urllib3.readthedocs.io/en/latest/reference/index.html?highlight=retry#urllib3.Retry). + +``` python +from urllib3 import Retry + +from influxdb_client import InfluxDBClient + +retries = Retry(connect=5, read=2, redirect=5) +client = InfluxDBClient(url="http://localhost:8086", token="my-token", org="my-org", retries=retries) +``` + + + +### Nanosecond precision + + + +The Python's [datetime](https://docs.python.org/3/library/datetime.html) doesn't support precision with nanoseconds so the library during writes and queries ignores everything after microseconds. + +If you would like to use `datetime` with nanosecond precision you should use [pandas.Timestamp](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.Timestamp.html#pandas.Timestamp) that is replacement for python `datetime.datetime` object, and also you should set a proper `DateTimeHelper` to the client. + +- sources - [nanosecond_precision.py](https://github.com/influxdata/influxdb-client-python/blob/master/examples/nanosecond_precision.py) + +``` python +from influxdb_client import Point, InfluxDBClient +from influxdb_client.client.util.date_utils_pandas import PandasDateTimeHelper +from influxdb_client.client.write_api import SYNCHRONOUS + +""" +Set PandasDate helper which supports nanoseconds. +""" +import influxdb_client.client.util.date_utils as date_utils + +date_utils.date_helper = PandasDateTimeHelper() + +""" +Prepare client. +""" +client = InfluxDBClient(url="http://localhost:8086", token="my-token", org="my-org") + +write_api = client.write_api(write_options=SYNCHRONOUS) +query_api = client.query_api() + +""" +Prepare data +""" + +point = Point("h2o_feet") \ + .field("water_level", 10) \ + .tag("location", "pacific") \ + .time('1996-02-25T21:20:00.001001231Z') + +print(f'Time serialized with nanosecond precision: {point.to_line_protocol()}') +print() + +write_api.write(bucket="my-bucket", record=point) + +""" +Query: using Stream +""" +query = ''' +from(bucket:"my-bucket") + |> range(start: 0, stop: now()) + |> filter(fn: (r) => r._measurement == "h2o_feet") +''' +records = query_api.query_stream(query) + +for record in records: + print(f'Temperature in {record["location"]} is {record["_value"]} at time: {record["_time"]}') + +""" +Close client +""" +client.close() +``` + + +### How to use Asyncio + + + +Starting from version 1.27.0 for Python 3.7+ the `influxdb-client` package supports `async/await` based on [asyncio](https://docs.python.org/3/library/asyncio.html), [aiohttp](https://docs.aiohttp.org) and [aiocsv](https://pypi.org/project/aiocsv/). +You can install `aiohttp` and `aiocsv` directly: + +> ``` bash +> $ python -m pip install influxdb-client aiohttp aiocsv +> ``` + +or use the `[async]` extra: + +> ``` bash +> $ python -m pip install influxdb-client[async] +> ``` + +:warning: + +> The `InfluxDBClientAsync` should be initialised inside `async coroutine` otherwise there can be unexpected behaviour. For more info see: [Why is creating a ClientSession outside an event loop dangerous?](https://docs.aiohttp.org/en/stable/faq.html#why-is-creating-a-clientsession-outside-of-an-event-loop-dangerous). + +#### Async APIs + +All async APIs are available via `influxdb_client.client.influxdb_client_async.InfluxDBClientAsync`. The `async` version of the client supports following asynchronous APIs: + +- `influxdb_client.client.write_api_async.WriteApiAsync` +- `influxdb_client.client.query_api_async.QueryApiAsync` +- `influxdb_client.client.delete_api_async.DeleteApiAsync` +- Management services into `influxdb_client.service` supports async + operation + +and also check to readiness of the InfluxDB via `/ping` endpoint: + +> ``` python +> import asyncio +> +> from influxdb_client.client.influxdb_client_async import InfluxDBClientAsync +> +> +> async def main(): +> async with InfluxDBClientAsync(url="http://localhost:8086", token="my-token", org="my-org") as client: +> ready = await client.ping() +> print(f"InfluxDB: {ready}") +> +> +> if __name__ == "__main__": +> asyncio.run(main()) +> ``` + +#### Async Write API + +The `influxdb_client.client.write_api_async.WriteApiAsync` supports ingesting data as: + +- `string` or `bytes` that is formatted as a InfluxDB\'s line protocol +- [Data Point](https://github.com/influxdata/influxdb-client-python/blob/master/influxdb_client/client/write/point.py#L16) structure +- Dictionary style mapping with keys: `measurement`, `tags`, `fields` and `time` or custom structure +- [NamedTuple](https://docs.python.org/3/library/collections.html#collections.namedtuple) +- [Data Classes](https://docs.python.org/3/library/dataclasses.html) +- [Pandas DataFrame](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.html) +- List of above items + +> ``` python +> import asyncio +> +> from influxdb_client import Point +> from influxdb_client.client.influxdb_client_async import InfluxDBClientAsync +> +> +> async def main(): +> async with InfluxDBClientAsync(url="http://localhost:8086", token="my-token", org="my-org") as client: +> +> write_api = client.write_api() +> +> _point1 = Point("async_m").tag("location", "Prague").field("temperature", 25.3) +> _point2 = Point("async_m").tag("location", "New York").field("temperature", 24.3) +> +> successfully = await write_api.write(bucket="my-bucket", record=[_point1, _point2]) +> +> print(f" > successfully: {successfully}") +> +> +> if __name__ == "__main__": +> asyncio.run(main()) +> ``` + +#### Async Query API + +The `influxdb_client.client.query_api_async.QueryApiAsync` supports retrieve data as: + +- List of `influxdb_client.client.flux_table.FluxTable` +- Stream of `influxdb_client.client.flux_table.FluxRecord` via `typing.AsyncGenerator` +- [Pandas DataFrame](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.html) +- Stream of [Pandas DataFrame](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.html) via `typing.AsyncGenerator` +- Raw `str` output + +> ``` python +> import asyncio +> +> from influxdb_client.client.influxdb_client_async import InfluxDBClientAsync +> +> +> async def main(): +> async with InfluxDBClientAsync(url="http://localhost:8086", token="my-token", org="my-org") as client: +> # Stream of FluxRecords +> query_api = client.query_api() +> records = await query_api.query_stream('from(bucket:"my-bucket") ' +> '|> range(start: -10m) ' +> '|> filter(fn: (r) => r["_measurement"] == "async_m")') +> async for record in records: +> print(record) +> +> +> if __name__ == "__main__": +> asyncio.run(main()) +> ``` + +#### Async Delete API + +> ``` python +> import asyncio +> from datetime import datetime +> +> from influxdb_client.client.influxdb_client_async import InfluxDBClientAsync +> +> +> async def main(): +> async with InfluxDBClientAsync(url="http://localhost:8086", token="my-token", org="my-org") as client: +> start = datetime.utcfromtimestamp(0) +> stop = datetime.now() +> # Delete data with location = 'Prague' +> successfully = await client.delete_api().delete(start=start, stop=stop, bucket="my-bucket", +> predicate="location = \"Prague\"") +> print(f" > successfully: {successfully}") +> +> +> if __name__ == "__main__": +> asyncio.run(main()) +> ``` + +#### Management API + +> ``` python +> import asyncio +> +> from influxdb_client import OrganizationsService +> from influxdb_client.client.influxdb_client_async import InfluxDBClientAsync +> +> +> async def main(): +> async with InfluxDBClientAsync(url='http://localhost:8086', token='my-token', org='my-org') as client: +> # Initialize async OrganizationsService +> organizations_service = OrganizationsService(api_client=client.api_client) +> +> # Find organization with name 'my-org' +> organizations = await organizations_service.get_orgs(org='my-org') +> for organization in organizations.orgs: +> print(f'name: {organization.name}, id: {organization.id}') +> +> +> if __name__ == "__main__": +> asyncio.run(main()) +> ``` + +#### Proxy and redirects + +You can configure the client to tunnel requests through an HTTP proxy. +The following proxy options are supported: + +- `proxy` - Set this to configure the http proxy to be used, ex. `http://localhost:3128` +- `proxy_headers` - A dictionary containing headers that will be sent to the proxy. Could be used for proxy authentication. + +``` python +from influxdb_client.client.influxdb_client_async import InfluxDBClientAsync + + +async with InfluxDBClientAsync(url="http://localhost:8086", + token="my-token", + org="my-org", + proxy="http://localhost:3128") as client: +``` + +If your proxy notify the client with permanent redirect (`HTTP 301`) to **different host**. +The client removes `Authorization` header, because otherwise the contents of `Authorization` is sent to third parties which is a security vulnerability. + +Client automatically follows HTTP redirects. The default redirect policy is to follow up to `10` consecutive requests. +The redirects can be configured via: + +- `allow_redirects` - If set to `False`, do not follow HTTP redirects. + `True` by default. +- `max_redirects` - Maximum number of HTTP redirects to follow. `10` + by default. + + + +### Logging + + + +The client uses Python's [logging](https://docs.python.org/3/library/logging.html) facility for logging the library activity. The following logger categories are +exposed: + +- `influxdb_client.client.influxdb_client` +- `influxdb_client.client.influxdb_client_async` +- `influxdb_client.client.write_api` +- `influxdb_client.client.write_api_async` +- `influxdb_client.client.write.retry` +- `influxdb_client.client.write.dataframe_serializer` +- `influxdb_client.client.util.multiprocessing_helper` +- `influxdb_client.client.http` +- `influxdb_client.client.exceptions` + +The default logging level is `warning` without configured logger output. You can use the standard logger interface to change the log level and handler: + +``` python +import logging +import sys + +from influxdb_client import InfluxDBClient + +with InfluxDBClient(url="http://localhost:8086", token="my-token", org="my-org") as client: + for _, logger in client.conf.loggers.items(): + logger.setLevel(logging.DEBUG) + logger.addHandler(logging.StreamHandler(sys.stdout)) +``` + +#### Debugging + +For debug purpose you can enable verbose logging of HTTP requests and set the `debug` level to all client's logger categories by: + +``` python +client = InfluxDBClient(url="http://localhost:8086", token="my-token", debug=True) +``` + +Both HTTP request headers and body will be logged to standard output. + + + +## Local tests + +``` console +# start/restart InfluxDB2 on local machine using docker +./scripts/influxdb-restart.sh + +# install requirements +pip install -e . --user +pip install -e .\[extra\] --user +pip install -e .\[test\] --user + +# run unit & integration tests +pytest tests +``` + +## Contributing + +Bug reports and pull requests are welcome on GitHub at . + +## License + +The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT). diff --git a/README.rst b/README.rst deleted file mode 100644 index e22f8eab..00000000 --- a/README.rst +++ /dev/null @@ -1,1657 +0,0 @@ -influxdb-client-python -====================== - -.. marker-index-start - -.. image:: https://circleci.com/gh/influxdata/influxdb-client-python.svg?style=svg - :target: https://circleci.com/gh/influxdata/influxdb-client-python - :alt: CircleCI - - -.. image:: https://codecov.io/gh/influxdata/influxdb-client-python/branch/master/graph/badge.svg - :target: https://codecov.io/gh/influxdata/influxdb-client-python - :alt: codecov - -.. image:: https://img.shields.io/circleci/project/github/influxdata/influxdb-client-python/master.svg - :target: https://circleci.com/gh/influxdata/influxdb-client-python - :alt: CI status - -.. image:: https://img.shields.io/pypi/v/influxdb-client.svg - :target: https://pypi.org/project/influxdb-client/ - :alt: PyPI package - -.. image:: https://anaconda.org/influxdata/influxdb_client/badges/version.svg - :target: https://anaconda.org/influxdata/influxdb_client - :alt: Anaconda.org package - -.. image:: https://img.shields.io/pypi/pyversions/influxdb-client.svg - :target: https://pypi.python.org/pypi/influxdb-client - :alt: Supported Python versions - -.. image:: https://readthedocs.org/projects/influxdb-client/badge/?version=stable - :target: https://influxdb-client.readthedocs.io/en/stable/ - :alt: Documentation status - -.. image:: https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social - :target: https://www.influxdata.com/slack - :alt: Slack Status - -This repository contains the Python client library for use with InfluxDB 2.x and Flux. InfluxDB 3.x users should instead use the lightweight `v3 client library `_. InfluxDB 1.x users should use the `v1 client library `_. - -For ease of migration and a consistent query and write experience, v2 users should consider using InfluxQL and the `v1 client library `_. - -The API of the **influxdb-client-python** is not the backwards-compatible with the old one - **influxdb-python**. - -Documentation -------------- - -This section contains links to the client library documentation. - -* `Product documentation `_, `Getting Started <#getting-started>`_ -* `Examples `_ -* `API Reference `_ -* `Changelog `_ - -InfluxDB 2.0 client features ----------------------------- - -- Querying data - - using the Flux language - - into csv, raw data, `flux_table `_ structure, `Pandas DataFrame `_ - - `How to queries <#queries>`_ -- Writing data using - - `Line Protocol `_ - - `Data Point `__ - - `RxPY `__ Observable - - `Pandas DataFrame `_ - - `How to writes <#writes>`_ -- `InfluxDB 2.0 API `_ client for management - - the client is generated from the `swagger `_ by using the `openapi-generator `_ - - organizations & users management - - buckets management - - tasks management - - authorizations - - health check - - ... -- `InfluxDB 1.8 API compatibility`_ -- Examples - - `Connect to InfluxDB Cloud`_ - - `How to efficiently import large dataset`_ - - `Efficiency write data from IOT sensor`_ - - `How to use Jupyter + Pandas + InfluxDB 2`_ -- `Advanced Usage`_ - - `Gzip support`_ - - `Proxy configuration`_ - - `Nanosecond precision`_ - - `Delete data`_ - - `Handling Errors`_ - - `Logging`_ - -Installation ------------- -.. marker-install-start - -InfluxDB python library uses `RxPY `__ - The Reactive Extensions for Python (RxPY). - -**Python 3.7** or later is required. - -.. note:: - - It is recommended to use ``ciso8601`` with client for parsing dates. ``ciso8601`` is much faster than built-in Python datetime. Since it's written as a ``C`` module the best way is build it from sources: - - **Windows**: - - You have to install `Visual C++ Build Tools 2015 `_ to build ``ciso8601`` by ``pip``. - - **conda**: - - Install from sources: ``conda install -c conda-forge/label/cf202003 ciso8601``. - -pip install -^^^^^^^^^^^ - -The python package is hosted on `PyPI `_, you can install latest version directly: - -.. code-block:: sh - - pip install 'influxdb-client[ciso]' - -Then import the package: - -.. code-block:: python - - import influxdb_client - -If your application uses async/await in Python you can install with the ``async`` extra:: - - $ pip install influxdb-client[async] - -For more info see `How to use Asyncio`. - -Setuptools -^^^^^^^^^^ - -Install via `Setuptools `_. - -.. code-block:: sh - - python setup.py install --user - -(or ``sudo python setup.py install`` to install the package for all users) - -.. marker-install-end - -Getting Started ---------------- - -Please follow the `Installation`_ and then run the following: - -.. marker-query-start - -.. code-block:: python - - from influxdb_client import InfluxDBClient, Point - from influxdb_client.client.write_api import SYNCHRONOUS - - bucket = "my-bucket" - - client = InfluxDBClient(url="http://localhost:8086", token="my-token", org="my-org") - - write_api = client.write_api(write_options=SYNCHRONOUS) - query_api = client.query_api() - - p = Point("my_measurement").tag("location", "Prague").field("temperature", 25.3) - - write_api.write(bucket=bucket, record=p) - - ## using Table structure - tables = query_api.query('from(bucket:"my-bucket") |> range(start: -10m)') - - for table in tables: - print(table) - for row in table.records: - print (row.values) - - - ## using csv library - csv_result = query_api.query_csv('from(bucket:"my-bucket") |> range(start: -10m)') - val_count = 0 - for row in csv_result: - for cell in row: - val_count += 1 - - -.. marker-query-end - -Client configuration --------------------- - -Via File -^^^^^^^^ -A client can be configured via ``*.ini`` file in segment ``influx2``. - -The following options are supported: - -- ``url`` - the url to connect to InfluxDB -- ``org`` - default destination organization for writes and queries -- ``token`` - the token to use for the authorization -- ``timeout`` - socket timeout in ms (default value is 10000) -- ``verify_ssl`` - set this to false to skip verifying SSL certificate when calling API from https server -- ``ssl_ca_cert`` - set this to customize the certificate file to verify the peer -- ``cert_file`` - path to the certificate that will be used for mTLS authentication -- ``cert_key_file`` - path to the file contains private key for mTLS certificate -- ``cert_key_password`` - string or function which returns password for decrypting the mTLS private key -- ``connection_pool_maxsize`` - set the number of connections to save that can be reused by urllib3 -- ``auth_basic`` - enable http basic authentication when talking to a InfluxDB 1.8.x without authentication but is accessed via reverse proxy with basic authentication (defaults to false) -- ``profilers`` - set the list of enabled `Flux profilers `_ - -.. code-block:: python - - self.client = InfluxDBClient.from_config_file("config.ini") - -.. code-block:: ini - - [influx2] - url=http://localhost:8086 - org=my-org - token=my-token - timeout=6000 - verify_ssl=False - -Via Environment Properties -^^^^^^^^^^^^^^^^^^^^^^^^^^ -A client can be configured via environment properties. - -Supported properties are: - -- ``INFLUXDB_V2_URL`` - the url to connect to InfluxDB -- ``INFLUXDB_V2_ORG`` - default destination organization for writes and queries -- ``INFLUXDB_V2_TOKEN`` - the token to use for the authorization -- ``INFLUXDB_V2_TIMEOUT`` - socket timeout in ms (default value is 10000) -- ``INFLUXDB_V2_VERIFY_SSL`` - set this to false to skip verifying SSL certificate when calling API from https server -- ``INFLUXDB_V2_SSL_CA_CERT`` - set this to customize the certificate file to verify the peer -- ``INFLUXDB_V2_CERT_FILE`` - path to the certificate that will be used for mTLS authentication -- ``INFLUXDB_V2_CERT_KEY_FILE`` - path to the file contains private key for mTLS certificate -- ``INFLUXDB_V2_CERT_KEY_PASSWORD`` - string or function which returns password for decrypting the mTLS private key -- ``INFLUXDB_V2_CONNECTION_POOL_MAXSIZE`` - set the number of connections to save that can be reused by urllib3 -- ``INFLUXDB_V2_AUTH_BASIC`` - enable http basic authentication when talking to a InfluxDB 1.8.x without authentication but is accessed via reverse proxy with basic authentication (defaults to false) -- ``INFLUXDB_V2_PROFILERS`` - set the list of enabled `Flux profilers `_ - -.. code-block:: python - - self.client = InfluxDBClient.from_env_properties() - -Profile query -^^^^^^^^^^^^^ - -The `Flux Profiler package `_ provides -performance profiling tools for Flux queries and operations. - -You can enable printing profiler information of the Flux query in client library by: - -- set QueryOptions.profilers in QueryApi, -- set ``INFLUXDB_V2_PROFILERS`` environment variable, -- set ``profilers`` option in configuration file. - -When the profiler is enabled, the result of flux query contains additional tables "profiler/\*". -In order to have consistent behaviour with enabled/disabled profiler, ``FluxCSVParser`` excludes "profiler/\*" measurements -from result. - -Example how to enable profilers using API: - -.. code-block:: python - - q = ''' - from(bucket: stringParam) - |> range(start: -5m, stop: now()) - |> filter(fn: (r) => r._measurement == "mem") - |> filter(fn: (r) => r._field == "available" or r._field == "free" or r._field == "used") - |> aggregateWindow(every: 1m, fn: mean) - |> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value") - ''' - p = { - "stringParam": "my-bucket", - } - - query_api = client.query_api(query_options=QueryOptions(profilers=["query", "operator"])) - csv_result = query_api.query(query=q, params=p) - - -Example of a profiler output: - -.. code-block:: text - - =============== - Profiler: query - =============== - - from(bucket: stringParam) - |> range(start: -5m, stop: now()) - |> filter(fn: (r) => r._measurement == "mem") - |> filter(fn: (r) => r._field == "available" or r._field == "free" or r._field == "used") - |> aggregateWindow(every: 1m, fn: mean) - |> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value") - - ======================== - Profiler: profiler/query - ======================== - result : _profiler - table : 0 - _measurement : profiler/query - TotalDuration : 8924700 - CompileDuration : 350900 - QueueDuration : 33800 - PlanDuration : 0 - RequeueDuration : 0 - ExecuteDuration : 8486500 - Concurrency : 0 - MaxAllocated : 2072 - TotalAllocated : 0 - flux/query-plan : - - digraph { - ReadWindowAggregateByTime11 - // every = 1m, aggregates = [mean], createEmpty = true, timeColumn = "_stop" - pivot8 - generated_yield - - ReadWindowAggregateByTime11 -> pivot8 - pivot8 -> generated_yield - } - - - influxdb/scanned-bytes: 0 - influxdb/scanned-values: 0 - - =========================== - Profiler: profiler/operator - =========================== - result : _profiler - table : 1 - _measurement : profiler/operator - Type : *universe.pivotTransformation - Label : pivot8 - Count : 3 - MinDuration : 32600 - MaxDuration : 126200 - DurationSum : 193400 - MeanDuration : 64466.666666666664 - - =========================== - Profiler: profiler/operator - =========================== - result : _profiler - table : 1 - _measurement : profiler/operator - Type : *influxdb.readWindowAggregateSource - Label : ReadWindowAggregateByTime11 - Count : 1 - MinDuration : 940500 - MaxDuration : 940500 - DurationSum : 940500 - MeanDuration : 940500.0 - -You can also use callback function to get profilers output. -Return value of this callback is type of FluxRecord. - -Example how to use profilers with callback: - -.. code-block:: python - - class ProfilersCallback(object): - def __init__(self): - self.records = [] - - def __call__(self, flux_record): - self.records.append(flux_record.values) - - callback = ProfilersCallback() - - query_api = client.query_api(query_options=QueryOptions(profilers=["query", "operator"], profiler_callback=callback)) - tables = query_api.query('from(bucket:"my-bucket") |> range(start: -10m)') - - for profiler in callback.records: - print(f'Custom processing of profiler result: {profiler}') - -Example output of this callback: - -.. code-block:: text - - Custom processing of profiler result: {'result': '_profiler', 'table': 0, '_measurement': 'profiler/query', 'TotalDuration': 18843792, 'CompileDuration': 1078666, 'QueueDuration': 93375, 'PlanDuration': 0, 'RequeueDuration': 0, 'ExecuteDuration': 17371000, 'Concurrency': 0, 'MaxAllocated': 448, 'TotalAllocated': 0, 'RuntimeErrors': None, 'flux/query-plan': 'digraph {\r\n ReadRange2\r\n generated_yield\r\n\r\n ReadRange2 -> generated_yield\r\n}\r\n\r\n', 'influxdb/scanned-bytes': 0, 'influxdb/scanned-values': 0} - Custom processing of profiler result: {'result': '_profiler', 'table': 1, '_measurement': 'profiler/operator', 'Type': '*influxdb.readFilterSource', 'Label': 'ReadRange2', 'Count': 1, 'MinDuration': 3274084, 'MaxDuration': 3274084, 'DurationSum': 3274084, 'MeanDuration': 3274084.0} - - -.. marker-index-end - - -How to use ----------- - -Writes -^^^^^^ -.. marker-writes-start - -The `WriteApi `_ supports synchronous, asynchronous and batching writes into InfluxDB 2.0. -The data should be passed as a `InfluxDB Line Protocol `_\ , `Data Point `_ or Observable stream. - -.. warning:: - - The ``WriteApi`` in batching mode (default mode) is suppose to run as a singleton. - To flush all your data you should wrap the execution using ``with client.write_api(...) as write_api:`` statement - or call ``write_api.close()`` at the end of your script. - -*The default instance of WriteApi use batching.* - -The data could be written as -"""""""""""""""""""""""""""" - -1. ``string`` or ``bytes`` that is formatted as a InfluxDB's line protocol -2. `Data Point `__ structure -3. Dictionary style mapping with keys: ``measurement``, ``tags``, ``fields`` and ``time`` or custom structure -4. `NamedTuple `_ -5. `Data Classes `_ -6. `Pandas DataFrame `_ -7. List of above items -8. A ``batching`` type of write also supports an ``Observable`` that produce one of an above item - -You can find write examples at GitHub: `influxdb-client-python/examples `__. - -Batching -"""""""" - -The batching is configurable by ``write_options``\ : - -.. list-table:: - :header-rows: 1 - - * - Property - - Description - - Default Value - * - **batch_size** - - the number of data point to collect in a batch - - ``1000`` - * - **flush_interval** - - the number of milliseconds before the batch is written - - ``1000`` - * - **jitter_interval** - - the number of milliseconds to increase the batch flush interval by a random amount - - ``0`` - * - **retry_interval** - - the number of milliseconds to retry first unsuccessful write. The next retry delay is computed using exponential random backoff. The retry interval is used when the InfluxDB server does not specify "Retry-After" header. - - ``5000`` - * - **max_retry_time** - - maximum total retry timeout in milliseconds. - - ``180_000`` - * - **max_retries** - - the number of max retries when write fails - - ``5`` - * - **max_retry_delay** - - the maximum delay between each retry attempt in milliseconds - - ``125_000`` - * - **max_close_wait** - - the maximum amount of time to wait for batches to flush when `.close()` is called - - ``300_000`` - * - **exponential_base** - - the base for the exponential retry delay, the next delay is computed using random exponential backoff as a random value within the interval ``retry_interval * exponential_base^(attempts-1)`` and ``retry_interval * exponential_base^(attempts)``. Example for ``retry_interval=5_000, exponential_base=2, max_retry_delay=125_000, total=5`` Retry delays are random distributed values within the ranges of ``[5_000-10_000, 10_000-20_000, 20_000-40_000, 40_000-80_000, 80_000-125_000]`` - - ``2`` - - -.. code-block:: python - - from datetime import datetime, timedelta - - import pandas as pd - import reactivex as rx - from reactivex import operators as ops - - from influxdb_client import InfluxDBClient, Point, WriteOptions - - with InfluxDBClient(url="http://localhost:8086", token="my-token", org="my-org") as _client: - - with _client.write_api(write_options=WriteOptions(batch_size=500, - flush_interval=10_000, - jitter_interval=2_000, - retry_interval=5_000, - max_retries=5, - max_retry_delay=30_000, - max_close_wait=300_000, - exponential_base=2)) as _write_client: - - """ - Write Line Protocol formatted as string - """ - _write_client.write("my-bucket", "my-org", "h2o_feet,location=coyote_creek water_level=1.0 1") - _write_client.write("my-bucket", "my-org", ["h2o_feet,location=coyote_creek water_level=2.0 2", - "h2o_feet,location=coyote_creek water_level=3.0 3"]) - - """ - Write Line Protocol formatted as byte array - """ - _write_client.write("my-bucket", "my-org", "h2o_feet,location=coyote_creek water_level=1.0 1".encode()) - _write_client.write("my-bucket", "my-org", ["h2o_feet,location=coyote_creek water_level=2.0 2".encode(), - "h2o_feet,location=coyote_creek water_level=3.0 3".encode()]) - - """ - Write Dictionary-style object - """ - _write_client.write("my-bucket", "my-org", {"measurement": "h2o_feet", "tags": {"location": "coyote_creek"}, - "fields": {"water_level": 1.0}, "time": 1}) - _write_client.write("my-bucket", "my-org", [{"measurement": "h2o_feet", "tags": {"location": "coyote_creek"}, - "fields": {"water_level": 2.0}, "time": 2}, - {"measurement": "h2o_feet", "tags": {"location": "coyote_creek"}, - "fields": {"water_level": 3.0}, "time": 3}]) - - """ - Write Data Point - """ - _write_client.write("my-bucket", "my-org", - Point("h2o_feet").tag("location", "coyote_creek").field("water_level", 4.0).time(4)) - _write_client.write("my-bucket", "my-org", - [Point("h2o_feet").tag("location", "coyote_creek").field("water_level", 5.0).time(5), - Point("h2o_feet").tag("location", "coyote_creek").field("water_level", 6.0).time(6)]) - - """ - Write Observable stream - """ - _data = rx \ - .range(7, 11) \ - .pipe(ops.map(lambda i: "h2o_feet,location=coyote_creek water_level={0}.0 {0}".format(i))) - - _write_client.write("my-bucket", "my-org", _data) - - """ - Write Pandas DataFrame - """ - _now = datetime.utcnow() - _data_frame = pd.DataFrame(data=[["coyote_creek", 1.0], ["coyote_creek", 2.0]], - index=[_now, _now + timedelta(hours=1)], - columns=["location", "water_level"]) - - _write_client.write("my-bucket", "my-org", record=_data_frame, data_frame_measurement_name='h2o_feet', - data_frame_tag_columns=['location']) - - - -Default Tags -"""""""""""" - -Sometimes is useful to store same information in every measurement e.g. ``hostname``, ``location``, ``customer``. -The client is able to use static value or env property as a tag value. - -The expressions: - -- ``California Miner`` - static value -- ``${env.hostname}`` - environment property - -Via API -_______ - -.. code-block:: python - - point_settings = PointSettings() - point_settings.add_default_tag("id", "132-987-655") - point_settings.add_default_tag("customer", "California Miner") - point_settings.add_default_tag("data_center", "${env.data_center}") - - self.write_client = self.client.write_api(write_options=SYNCHRONOUS, point_settings=point_settings) - -.. code-block:: python - - self.write_client = self.client.write_api(write_options=SYNCHRONOUS, - point_settings=PointSettings(**{"id": "132-987-655", - "customer": "California Miner"})) - -Via Configuration file -______________________ - -In a `init `_ configuration file you are able to specify default tags by ``tags`` segment. - -.. code-block:: python - - self.client = InfluxDBClient.from_config_file("config.ini") - -.. code-block:: - - [influx2] - url=http://localhost:8086 - org=my-org - token=my-token - timeout=6000 - - [tags] - id = 132-987-655 - customer = California Miner - data_center = ${env.data_center} - -You can also use a `TOML `_ or a `JSON `_ format for the configuration file. - -Via Environment Properties -__________________________ -You are able to specify default tags by environment properties with prefix ``INFLUXDB_V2_TAG_``. - -Examples: - -- ``INFLUXDB_V2_TAG_ID`` -- ``INFLUXDB_V2_TAG_HOSTNAME`` - -.. code-block:: python - - self.client = InfluxDBClient.from_env_properties() - -Synchronous client -"""""""""""""""""" - -Data are writes in a synchronous HTTP request. - -.. code-block:: python - - from influxdb_client import InfluxDBClient, Point - from influxdb_client .client.write_api import SYNCHRONOUS - - client = InfluxDBClient(url="http://localhost:8086", token="my-token", org="my-org") - write_api = client.write_api(write_options=SYNCHRONOUS) - - _point1 = Point("my_measurement").tag("location", "Prague").field("temperature", 25.3) - _point2 = Point("my_measurement").tag("location", "New York").field("temperature", 24.3) - - write_api.write(bucket="my-bucket", record=[_point1, _point2]) - - client.close() - -.. marker-writes-end - -Queries -^^^^^^^ - -The result retrieved by `QueryApi `_ could be formatted as a: - -1. Flux data structure: `FluxTable `_, `FluxColumn `_ and `FluxRecord `_ -2. :code:`influxdb_client.client.flux_table.CSVIterator` which will iterate over CSV lines -3. Raw unprocessed results as a ``str`` iterator -4. `Pandas DataFrame `_ - -The API also support streaming ``FluxRecord`` via `query_stream `_, see example below: - -.. code-block:: python - - from influxdb_client import InfluxDBClient, Point, Dialect - from influxdb_client.client.write_api import SYNCHRONOUS - - client = InfluxDBClient(url="http://localhost:8086", token="my-token", org="my-org") - - write_api = client.write_api(write_options=SYNCHRONOUS) - query_api = client.query_api() - - """ - Prepare data - """ - - _point1 = Point("my_measurement").tag("location", "Prague").field("temperature", 25.3) - _point2 = Point("my_measurement").tag("location", "New York").field("temperature", 24.3) - - write_api.write(bucket="my-bucket", record=[_point1, _point2]) - - """ - Query: using Table structure - """ - tables = query_api.query('from(bucket:"my-bucket") |> range(start: -10m)') - - for table in tables: - print(table) - for record in table.records: - print(record.values) - - print() - print() - - """ - Query: using Bind parameters - """ - - p = {"_start": datetime.timedelta(hours=-1), - "_location": "Prague", - "_desc": True, - "_floatParam": 25.1, - "_every": datetime.timedelta(minutes=5) - } - - tables = query_api.query(''' - from(bucket:"my-bucket") |> range(start: _start) - |> filter(fn: (r) => r["_measurement"] == "my_measurement") - |> filter(fn: (r) => r["_field"] == "temperature") - |> filter(fn: (r) => r["location"] == _location and r["_value"] > _floatParam) - |> aggregateWindow(every: _every, fn: mean, createEmpty: true) - |> sort(columns: ["_time"], desc: _desc) - ''', params=p) - - for table in tables: - print(table) - for record in table.records: - print(str(record["_time"]) + " - " + record["location"] + ": " + str(record["_value"])) - - print() - print() - - """ - Query: using Stream - """ - records = query_api.query_stream('from(bucket:"my-bucket") |> range(start: -10m)') - - for record in records: - print(f'Temperature in {record["location"]} is {record["_value"]}') - - """ - Interrupt a stream after retrieve a required data - """ - large_stream = query_api.query_stream('from(bucket:"my-bucket") |> range(start: -100d)') - for record in large_stream: - if record["location"] == "New York": - print(f'New York temperature: {record["_value"]}') - break - - large_stream.close() - - print() - print() - - """ - Query: using csv library - """ - csv_result = query_api.query_csv('from(bucket:"my-bucket") |> range(start: -10m)', - dialect=Dialect(header=False, delimiter=",", comment_prefix="#", annotations=[], - date_time_format="RFC3339")) - for csv_line in csv_result: - if not len(csv_line) == 0: - print(f'Temperature in {csv_line[9]} is {csv_line[6]}') - - """ - Close client - """ - client.close() - -Pandas DataFrame -"""""""""""""""" -.. marker-pandas-start - -.. note:: For DataFrame querying you should install Pandas dependency via ``pip install 'influxdb-client[extra]'``. - -.. note:: Note that if a query returns more then one table then the client generates a ``DataFrame`` for each of them. - -The ``client`` is able to retrieve data in `Pandas DataFrame `_ format thought ``query_data_frame``: - -.. code-block:: python - - from influxdb_client import InfluxDBClient, Point, Dialect - from influxdb_client.client.write_api import SYNCHRONOUS - - client = InfluxDBClient(url="http://localhost:8086", token="my-token", org="my-org") - - write_api = client.write_api(write_options=SYNCHRONOUS) - query_api = client.query_api() - - """ - Prepare data - """ - - _point1 = Point("my_measurement").tag("location", "Prague").field("temperature", 25.3) - _point2 = Point("my_measurement").tag("location", "New York").field("temperature", 24.3) - - write_api.write(bucket="my-bucket", record=[_point1, _point2]) - - """ - Query: using Pandas DataFrame - """ - data_frame = query_api.query_data_frame('from(bucket:"my-bucket") ' - '|> range(start: -10m) ' - '|> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value") ' - '|> keep(columns: ["location", "temperature"])') - print(data_frame.to_string()) - - """ - Close client - """ - client.close() - -Output: - -.. code-block:: text - - result table location temperature - 0 _result 0 New York 24.3 - 1 _result 1 Prague 25.3 - -.. marker-pandas-end - -Examples -^^^^^^^^ - -.. marker-examples-start - -How to efficiently import large dataset -""""""""""""""""""""""""""""""""""""""" - -The following example shows how to import dataset with dozen megabytes. -If you would like to import gigabytes of data then use our multiprocessing example: `import_data_set_multiprocessing.py `_ for use a full capability of your hardware. - -* sources - `import_data_set.py `_ - -.. code-block:: python - - """ - Import VIX - CBOE Volatility Index - from "vix-daily.csv" file into InfluxDB 2.0 - - https://datahub.io/core/finance-vix#data - """ - - from collections import OrderedDict - from csv import DictReader - - import reactivex as rx - from reactivex import operators as ops - - from influxdb_client import InfluxDBClient, Point, WriteOptions - - def parse_row(row: OrderedDict): - """Parse row of CSV file into Point with structure: - - financial-analysis,type=ily close=18.47,high=19.82,low=18.28,open=19.82 1198195200000000000 - - CSV format: - Date,VIX Open,VIX High,VIX Low,VIX Close\n - 2004-01-02,17.96,18.68,17.54,18.22\n - 2004-01-05,18.45,18.49,17.44,17.49\n - 2004-01-06,17.66,17.67,16.19,16.73\n - 2004-01-07,16.72,16.75,15.5,15.5\n - 2004-01-08,15.42,15.68,15.32,15.61\n - 2004-01-09,16.15,16.88,15.57,16.75\n - ... - - :param row: the row of CSV file - :return: Parsed csv row to [Point] - """ - - """ - For better performance is sometimes useful directly create a LineProtocol to avoid unnecessary escaping overhead: - """ - # from datetime import timezone - # import ciso8601 - # from influxdb_client.client.write.point import EPOCH - # - # time = (ciso8601.parse_datetime(row["Date"]).replace(tzinfo=timezone.utc) - EPOCH).total_seconds() * 1e9 - # return f"financial-analysis,type=vix-daily" \ - # f" close={float(row['VIX Close'])},high={float(row['VIX High'])},low={float(row['VIX Low'])},open={float(row['VIX Open'])} " \ - # f" {int(time)}" - - return Point("financial-analysis") \ - .tag("type", "vix-daily") \ - .field("open", float(row['VIX Open'])) \ - .field("high", float(row['VIX High'])) \ - .field("low", float(row['VIX Low'])) \ - .field("close", float(row['VIX Close'])) \ - .time(row['Date']) - - - """ - Converts vix-daily.csv into sequence of datad point - """ - data = rx \ - .from_iterable(DictReader(open('vix-daily.csv', 'r'))) \ - .pipe(ops.map(lambda row: parse_row(row))) - - client = InfluxDBClient(url="http://localhost:8086", token="my-token", org="my-org", debug=True) - - """ - Create client that writes data in batches with 50_000 items. - """ - write_api = client.write_api(write_options=WriteOptions(batch_size=50_000, flush_interval=10_000)) - - """ - Write data into InfluxDB - """ - write_api.write(bucket="my-bucket", record=data) - write_api.close() - - """ - Querying max value of CBOE Volatility Index - """ - query = 'from(bucket:"my-bucket")' \ - ' |> range(start: 0, stop: now())' \ - ' |> filter(fn: (r) => r._measurement == "financial-analysis")' \ - ' |> max()' - result = client.query_api().query(query=query) - - """ - Processing results - """ - print() - print("=== results ===") - print() - for table in result: - for record in table.records: - print('max {0:5} = {1}'.format(record.get_field(), record.get_value())) - - """ - Close client - """ - client.close() - -Efficiency write data from IOT sensor -""""""""""""""""""""""""""""""""""""" - -* sources - `iot_sensor.py `_ - -.. code-block:: python - - """ - Efficiency write data from IOT sensor - write changed temperature every minute - """ - import atexit - import platform - from datetime import timedelta - - import psutil as psutil - import reactivex as rx - from reactivex import operators as ops - - from influxdb_client import InfluxDBClient, WriteApi, WriteOptions - - def on_exit(db_client: InfluxDBClient, write_api: WriteApi): - """Close clients after terminate a script. - - :param db_client: InfluxDB client - :param write_api: WriteApi - :return: nothing - """ - write_api.close() - db_client.close() - - - def sensor_temperature(): - """Read a CPU temperature. The [psutil] doesn't support MacOS so we use [sysctl]. - - :return: actual CPU temperature - """ - os_name = platform.system() - if os_name == 'Darwin': - from subprocess import check_output - output = check_output(["sysctl", "machdep.xcpm.cpu_thermal_level"]) - import re - return re.findall(r'\d+', str(output))[0] - else: - return psutil.sensors_temperatures()["coretemp"][0] - - - def line_protocol(temperature): - """Create a InfluxDB line protocol with structure: - - iot_sensor,hostname=mine_sensor_12,type=temperature value=68 - - :param temperature: the sensor temperature - :return: Line protocol to write into InfluxDB - """ - - import socket - return 'iot_sensor,hostname={},type=temperature value={}'.format(socket.gethostname(), temperature) - - - """ - Read temperature every minute; distinct_until_changed - produce only if temperature change - """ - data = rx\ - .interval(period=timedelta(seconds=60))\ - .pipe(ops.map(lambda t: sensor_temperature()), - ops.distinct_until_changed(), - ops.map(lambda temperature: line_protocol(temperature))) - - _db_client = InfluxDBClient(url="http://localhost:8086", token="my-token", org="my-org", debug=True) - - """ - Create client that writes data into InfluxDB - """ - _write_api = _db_client.write_api(write_options=WriteOptions(batch_size=1)) - _write_api.write(bucket="my-bucket", record=data) - - - """ - Call after terminate a script - """ - atexit.register(on_exit, _db_client, _write_api) - - input() - -Connect to InfluxDB Cloud -""""""""""""""""""""""""" -The following example demonstrate a simplest way how to write and query date with the InfluxDB Cloud. - -At first point you should create an authentication token as is described `here `_. - -After that you should configure properties: ``influx_cloud_url``, ``influx_cloud_token``, ``bucket`` and ``org`` in a ``influx_cloud.py`` example. - -The last step is run a python script via: ``python3 influx_cloud.py``. - -* sources - `influx_cloud.py `_ - -.. code-block:: python - - """ - Connect to InfluxDB 2.0 - write data and query them - """ - - from datetime import datetime - - from influxdb_client import Point, InfluxDBClient - from influxdb_client.client.write_api import SYNCHRONOUS - - """ - Configure credentials - """ - influx_cloud_url = 'https://us-west-2-1.aws.cloud2.influxdata.com' - influx_cloud_token = '...' - bucket = '...' - org = '...' - - client = InfluxDBClient(url=influx_cloud_url, token=influx_cloud_token) - try: - kind = 'temperature' - host = 'host1' - device = 'opt-123' - - """ - Write data by Point structure - """ - point = Point(kind).tag('host', host).tag('device', device).field('value', 25.3).time(time=datetime.utcnow()) - - print(f'Writing to InfluxDB cloud: {point.to_line_protocol()} ...') - - write_api = client.write_api(write_options=SYNCHRONOUS) - write_api.write(bucket=bucket, org=org, record=point) - - print() - print('success') - print() - print() - - """ - Query written data - """ - query = f'from(bucket: "{bucket}") |> range(start: -1d) |> filter(fn: (r) => r._measurement == "{kind}")' - print(f'Querying from InfluxDB cloud: "{query}" ...') - print() - - query_api = client.query_api() - tables = query_api.query(query=query, org=org) - - for table in tables: - for row in table.records: - print(f'{row.values["_time"]}: host={row.values["host"]},device={row.values["device"]} ' - f'{row.values["_value"]} °C') - - print() - print('success') - - except Exception as e: - print(e) - finally: - client.close() - -How to use Jupyter + Pandas + InfluxDB 2 -"""""""""""""""""""""""""""""""""""""""" -The first example shows how to use client capabilities to predict stock price via `Keras `_, `TensorFlow `_, `sklearn `_: - -The example is taken from `Kaggle `_. - -* sources - `stock-predictions.ipynb `_ - -.. image:: https://raw.githubusercontent.com/influxdata/influxdb-client-python/master/docs/images/stock-price-prediction.gif - -Result: - -.. image:: https://raw.githubusercontent.com/influxdata/influxdb-client-python/master/docs/images/stock-price-prediction-results.png - -The second example shows how to use client capabilities to realtime visualization via `hvPlot `_, `Streamz `_, `RxPY `_: - -* sources - `realtime-stream.ipynb `_ - -.. image:: https://raw.githubusercontent.com/influxdata/influxdb-client-python/master/docs/images/realtime-result.gif - -Other examples -"""""""""""""" - -You can find all examples at GitHub: `influxdb-client-python/examples `__. - -.. marker-examples-end - -Advanced Usage --------------- - -Gzip support -^^^^^^^^^^^^ -.. marker-gzip-start - -``InfluxDBClient`` does not enable gzip compression for http requests by default. If you want to enable gzip to reduce transfer data's size, you can call: - -.. code-block:: python - - from influxdb_client import InfluxDBClient - - _db_client = InfluxDBClient(url="http://localhost:8086", token="my-token", org="my-org", enable_gzip=True) - -.. marker-gzip-end - -Authenticate to the InfluxDB -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. marker-authenticate-start - -``InfluxDBClient`` supports three options how to authorize a connection: - -- `Token` -- `Username & Password` -- `HTTP Basic` - -Token -""""" - -Use the ``token`` to authenticate to the InfluxDB API. In your API requests, an `Authorization` header will be send. -The header value, provide the word `Token` followed by a space and an InfluxDB API token. The word `token`` is case-sensitive. - -.. code-block:: python - - from influxdb_client import InfluxDBClient - - with InfluxDBClient(url="http://localhost:8086", token="my-token") as client - -.. note:: Note that this is a preferred way how to authenticate to InfluxDB API. - -Username & Password -""""""""""""""""""" - -Authenticates via username and password credentials. If successful, creates a new session for the user. - -.. code-block:: python - - from influxdb_client import InfluxDBClient - - with InfluxDBClient(url="http://localhost:8086", username="my-user", password="my-password") as client - -.. warning:: - - The ``username/password`` auth is based on the HTTP "Basic" authentication. - The authorization expires when the `time-to-live (TTL) `__ - (default 60 minutes) is reached and client produces ``unauthorized exception``. - -HTTP Basic -"""""""""" - -Use this to enable basic authentication when talking to a InfluxDB 1.8.x that does not use auth-enabled -but is protected by a reverse proxy with basic authentication. - -.. code-block:: python - - from influxdb_client import InfluxDBClient - - with InfluxDBClient(url="http://localhost:8086", auth_basic=True, token="my-proxy-secret") as client - - -.. warning:: Don't use this when directly talking to InfluxDB 2. - -.. marker-authenticate-end - -Proxy configuration -^^^^^^^^^^^^^^^^^^^ -.. marker-proxy-start - -You can configure the client to tunnel requests through an HTTP proxy. -The following proxy options are supported: - -- ``proxy`` - Set this to configure the http proxy to be used, ex. ``http://localhost:3128`` -- ``proxy_headers`` - A dictionary containing headers that will be sent to the proxy. Could be used for proxy authentication. - -.. code-block:: python - - from influxdb_client import InfluxDBClient - - with InfluxDBClient(url="http://localhost:8086", - token="my-token", - org="my-org", - proxy="http://localhost:3128") as client: - -.. note:: - - If your proxy notify the client with permanent redirect (``HTTP 301``) to **different host**. - The client removes ``Authorization`` header, because otherwise the contents of ``Authorization`` is sent to third parties - which is a security vulnerability. - - You can change this behaviour by: - - .. code-block:: python - - from urllib3 import Retry - Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset() - Retry.DEFAULT.remove_headers_on_redirect = Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT - -.. marker-proxy-end - -Delete data -^^^^^^^^^^^ -.. marker-delete-start - -The `delete_api.py `_ supports deletes `points `_ from an InfluxDB bucket. - -.. code-block:: python - - from influxdb_client import InfluxDBClient - - client = InfluxDBClient(url="http://localhost:8086", token="my-token") - - delete_api = client.delete_api() - - """ - Delete Data - """ - start = "1970-01-01T00:00:00Z" - stop = "2021-02-01T00:00:00Z" - delete_api.delete(start, stop, '_measurement="my_measurement"', bucket='my-bucket', org='my-org') - - """ - Close client - """ - client.close() - -.. marker-delete-end - -InfluxDB 1.8 API compatibility -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -`InfluxDB 1.8.0 introduced forward compatibility APIs `_ for InfluxDB 2.0. This allow you to easily move from InfluxDB 1.x to InfluxDB 2.0 Cloud or open source. - -The following forward compatible APIs are available: - -======================================================= ==================================================================================================== ======= - API Endpoint Description -======================================================= ==================================================================================================== ======= -`query_api.py `_ `/api/v2/query `_ Query data in InfluxDB 1.8.0+ using the InfluxDB 2.0 API and `Flux `_ (endpoint should be enabled by `flux-enabled option `_) -`write_api.py `_ `/api/v2/write `_ Write data to InfluxDB 1.8.0+ using the InfluxDB 2.0 API -`ping() `_ `/ping `_ Check the status of your InfluxDB instance -======================================================= ==================================================================================================== ======= - -For detail info see `InfluxDB 1.8 example `_. - -Handling Errors -^^^^^^^^^^^^^^^ -.. marker-handling-errors-start - -Errors happen and it's important that your code is prepared for them. All client related exceptions are delivered from -``InfluxDBError``. If the exception cannot be recovered in the client it is returned to the application. -These exceptions are left for the developer to handle. - -Almost all APIs directly return unrecoverable exceptions to be handled this way: - -.. code-block:: python - - from influxdb_client import InfluxDBClient - from influxdb_client.client.exceptions import InfluxDBError - from influxdb_client.client.write_api import SYNCHRONOUS - - with InfluxDBClient(url="http://localhost:8086", token="my-token", org="my-org") as client: - try: - client.write_api(write_options=SYNCHRONOUS).write("my-bucket", record="mem,tag=a value=86") - except InfluxDBError as e: - if e.response.status == 401: - raise Exception(f"Insufficient write permissions to 'my-bucket'.") from e - raise - - -The only exception is **batching** ``WriteAPI`` (for more info see `Batching`_). where you need to register custom callbacks to handle batch events. -This is because this API runs in the ``background`` in a ``separate`` thread and isn't possible to directly -return underlying exceptions. - -.. code-block:: python - - from influxdb_client import InfluxDBClient - from influxdb_client.client.exceptions import InfluxDBError - - - class BatchingCallback(object): - - def success(self, conf: (str, str, str), data: str): - print(f"Written batch: {conf}, data: {data}") - - def error(self, conf: (str, str, str), data: str, exception: InfluxDBError): - print(f"Cannot write batch: {conf}, data: {data} due: {exception}") - - def retry(self, conf: (str, str, str), data: str, exception: InfluxDBError): - print(f"Retryable error occurs for batch: {conf}, data: {data} retry: {exception}") - - - with InfluxDBClient(url="http://localhost:8086", token="my-token", org="my-org") as client: - callback = BatchingCallback() - with client.write_api(success_callback=callback.success, - error_callback=callback.error, - retry_callback=callback.retry) as write_api: - pass - -HTTP Retry Strategy -""""""""""""""""""" -By default the client uses a retry strategy only for batching writes (for more info see `Batching`_). -For other HTTP requests there is no one retry strategy, but it could be configured by ``retries`` -parameter of ``InfluxDBClient``. - -For more info about how configure HTTP retry see details in `urllib3 documentation `_. - -.. code-block:: python - - from urllib3 import Retry - - from influxdb_client import InfluxDBClient - - retries = Retry(connect=5, read=2, redirect=5) - client = InfluxDBClient(url="http://localhost:8086", token="my-token", org="my-org", retries=retries) - -.. marker-handling-errors-end - -Nanosecond precision -^^^^^^^^^^^^^^^^^^^^ -.. marker-nanosecond-start - -The Python's `datetime `_ doesn't support precision with nanoseconds -so the library during writes and queries ignores everything after microseconds. - -If you would like to use ``datetime`` with nanosecond precision you should use -`pandas.Timestamp `_ -that is replacement for python ``datetime.datetime`` object and also you should set a proper ``DateTimeHelper`` to the client. - -* sources - `nanosecond_precision.py `_ - -.. code-block:: python - - from influxdb_client import Point, InfluxDBClient - from influxdb_client.client.util.date_utils_pandas import PandasDateTimeHelper - from influxdb_client.client.write_api import SYNCHRONOUS - - """ - Set PandasDate helper which supports nanoseconds. - """ - import influxdb_client.client.util.date_utils as date_utils - - date_utils.date_helper = PandasDateTimeHelper() - - """ - Prepare client. - """ - client = InfluxDBClient(url="http://localhost:8086", token="my-token", org="my-org") - - write_api = client.write_api(write_options=SYNCHRONOUS) - query_api = client.query_api() - - """ - Prepare data - """ - - point = Point("h2o_feet") \ - .field("water_level", 10) \ - .tag("location", "pacific") \ - .time('1996-02-25T21:20:00.001001231Z') - - print(f'Time serialized with nanosecond precision: {point.to_line_protocol()}') - print() - - write_api.write(bucket="my-bucket", record=point) - - """ - Query: using Stream - """ - query = ''' - from(bucket:"my-bucket") - |> range(start: 0, stop: now()) - |> filter(fn: (r) => r._measurement == "h2o_feet") - ''' - records = query_api.query_stream(query) - - for record in records: - print(f'Temperature in {record["location"]} is {record["_value"]} at time: {record["_time"]}') - - """ - Close client - """ - client.close() - -.. marker-nanosecond-end - -How to use Asyncio -^^^^^^^^^^^^^^^^^^ -.. marker-asyncio-start - -Starting from version 1.27.0 for Python 3.7+ the ``influxdb-client`` package supports ``async/await`` based on -`asyncio `_, `aiohttp `_ and `aiocsv `_. -You can install ``aiohttp`` and ``aiocsv`` directly: - - .. code-block:: bash - - $ python -m pip install influxdb-client aiohttp aiocsv - -or use the ``[async]`` extra: - - .. code-block:: bash - - $ python -m pip install influxdb-client[async] - -.. warning:: - - The ``InfluxDBClientAsync`` should be initialised inside ``async coroutine`` - otherwise there can be unexpected behaviour. - For more info see: `Why is creating a ClientSession outside of an event loop dangerous? `__. - -Async APIs -"""""""""" -All async APIs are available via :code:`influxdb_client.client.influxdb_client_async.InfluxDBClientAsync`. -The ``async`` version of the client supports following asynchronous APIs: - -* :code:`influxdb_client.client.write_api_async.WriteApiAsync` -* :code:`influxdb_client.client.query_api_async.QueryApiAsync` -* :code:`influxdb_client.client.delete_api_async.DeleteApiAsync` -* Management services into ``influxdb_client.service`` supports async operation - -and also check to readiness of the InfluxDB via ``/ping`` endpoint: - - .. code-block:: python - - import asyncio - - from influxdb_client.client.influxdb_client_async import InfluxDBClientAsync - - - async def main(): - async with InfluxDBClientAsync(url="http://localhost:8086", token="my-token", org="my-org") as client: - ready = await client.ping() - print(f"InfluxDB: {ready}") - - - if __name__ == "__main__": - asyncio.run(main()) - -Async Write API -""""""""""""""" - -The :code:`influxdb_client.client.write_api_async.WriteApiAsync` supports ingesting data as: - -* ``string`` or ``bytes`` that is formatted as a InfluxDB's line protocol -* `Data Point `__ structure -* Dictionary style mapping with keys: ``measurement``, ``tags``, ``fields`` and ``time`` or custom structure -* `NamedTuple `_ -* `Data Classes `_ -* `Pandas DataFrame `_ -* List of above items - - .. code-block:: python - - import asyncio - - from influxdb_client import Point - from influxdb_client.client.influxdb_client_async import InfluxDBClientAsync - - - async def main(): - async with InfluxDBClientAsync(url="http://localhost:8086", token="my-token", org="my-org") as client: - - write_api = client.write_api() - - _point1 = Point("async_m").tag("location", "Prague").field("temperature", 25.3) - _point2 = Point("async_m").tag("location", "New York").field("temperature", 24.3) - - successfully = await write_api.write(bucket="my-bucket", record=[_point1, _point2]) - - print(f" > successfully: {successfully}") - - - if __name__ == "__main__": - asyncio.run(main()) - - -Async Query API -""""""""""""""" - -The :code:`influxdb_client.client.query_api_async.QueryApiAsync` supports retrieve data as: - -* List of :code:`influxdb_client.client.flux_table.FluxTable` -* Stream of :code:`influxdb_client.client.flux_table.FluxRecord` via :code:`typing.AsyncGenerator` -* `Pandas DataFrame `_ -* Stream of `Pandas DataFrame `_ via :code:`typing.AsyncGenerator` -* Raw :code:`str` output - - .. code-block:: python - - import asyncio - - from influxdb_client.client.influxdb_client_async import InfluxDBClientAsync - - - async def main(): - async with InfluxDBClientAsync(url="http://localhost:8086", token="my-token", org="my-org") as client: - # Stream of FluxRecords - query_api = client.query_api() - records = await query_api.query_stream('from(bucket:"my-bucket") ' - '|> range(start: -10m) ' - '|> filter(fn: (r) => r["_measurement"] == "async_m")') - async for record in records: - print(record) - - - if __name__ == "__main__": - asyncio.run(main()) - - -Async Delete API -"""""""""""""""" - - .. code-block:: python - - import asyncio - from datetime import datetime - - from influxdb_client.client.influxdb_client_async import InfluxDBClientAsync - - - async def main(): - async with InfluxDBClientAsync(url="http://localhost:8086", token="my-token", org="my-org") as client: - start = datetime.utcfromtimestamp(0) - stop = datetime.now() - # Delete data with location = 'Prague' - successfully = await client.delete_api().delete(start=start, stop=stop, bucket="my-bucket", - predicate="location = \"Prague\"") - print(f" > successfully: {successfully}") - - - if __name__ == "__main__": - asyncio.run(main()) - - -Management API -"""""""""""""" - - .. code-block:: python - - import asyncio - - from influxdb_client import OrganizationsService - from influxdb_client.client.influxdb_client_async import InfluxDBClientAsync - - - async def main(): - async with InfluxDBClientAsync(url='http://localhost:8086', token='my-token', org='my-org') as client: - # Initialize async OrganizationsService - organizations_service = OrganizationsService(api_client=client.api_client) - - # Find organization with name 'my-org' - organizations = await organizations_service.get_orgs(org='my-org') - for organization in organizations.orgs: - print(f'name: {organization.name}, id: {organization.id}') - - - if __name__ == "__main__": - asyncio.run(main()) - - -Proxy and redirects -""""""""""""""""""" - -You can configure the client to tunnel requests through an HTTP proxy. -The following proxy options are supported: - -- ``proxy`` - Set this to configure the http proxy to be used, ex. ``http://localhost:3128`` -- ``proxy_headers`` - A dictionary containing headers that will be sent to the proxy. Could be used for proxy authentication. - -.. code-block:: python - - from influxdb_client.client.influxdb_client_async import InfluxDBClientAsync - - - async with InfluxDBClientAsync(url="http://localhost:8086", - token="my-token", - org="my-org", - proxy="http://localhost:3128") as client: - -.. note:: - - If your proxy notify the client with permanent redirect (``HTTP 301``) to **different host**. - The client removes ``Authorization`` header, because otherwise the contents of ``Authorization`` is sent to third parties - which is a security vulnerability. - -Client automatically follows HTTP redirects. The default redirect policy is to follow up to ``10`` consecutive requests. The redirects can be configured via: - -- ``allow_redirects`` - If set to ``False``, do not follow HTTP redirects. ``True`` by default. -- ``max_redirects`` - Maximum number of HTTP redirects to follow. ``10`` by default. - - -.. marker-asyncio-end - -Logging -^^^^^^^ -.. marker-logging-start - -The client uses Python's `logging `__ facility for logging the library activity. The following logger categories are exposed: - -- ``influxdb_client.client.influxdb_client`` -- ``influxdb_client.client.influxdb_client_async`` -- ``influxdb_client.client.write_api`` -- ``influxdb_client.client.write_api_async`` -- ``influxdb_client.client.write.retry`` -- ``influxdb_client.client.write.dataframe_serializer`` -- ``influxdb_client.client.util.multiprocessing_helper`` -- ``influxdb_client.client.http`` -- ``influxdb_client.client.exceptions`` - -The default logging level is `warning` without configured logger output. You can use the standard logger interface to change the log level and handler: - -.. code-block:: python - - import logging - import sys - - from influxdb_client import InfluxDBClient - - with InfluxDBClient(url="http://localhost:8086", token="my-token", org="my-org") as client: - for _, logger in client.conf.loggers.items(): - logger.setLevel(logging.DEBUG) - logger.addHandler(logging.StreamHandler(sys.stdout)) - -Debugging -""""""""" - -For debug purpose you can enable verbose logging of HTTP requests and set the ``debug`` level to all client's logger categories by: - -.. code-block:: python - - client = InfluxDBClient(url="http://localhost:8086", token="my-token", debug=True) - -.. note:: - - Both HTTP request headers and body will be logged to standard output. - -.. marker-logging-end - -Local tests ------------ - -.. code-block:: console - - # start/restart InfluxDB2 on local machine using docker - ./scripts/influxdb-restart.sh - - # install requirements - pip install -e . --user - pip install -e .\[extra\] --user - pip install -e .\[test\] --user - - # run unit & integration tests - pytest tests - - -Contributing ------------- - -Bug reports and pull requests are welcome on GitHub at `https://github.com/influxdata/influxdb-client-python `_. - -License -------- - -The gem is available as open source under the terms of the `MIT License `_. diff --git a/docs/conf.py b/docs/conf.py index 79e29693..7ee13777 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -55,6 +55,7 @@ 'sphinx.ext.todo', 'sphinx.ext.viewcode', 'sphinx_rtd_theme', + 'myst_parser' # 'sphinx_autodoc_typehints' ] @@ -64,8 +65,7 @@ # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # -# source_suffix = ['.rst', '.md'] -source_suffix = '.rst' +source_suffix = ['.rst', '.md'] # The master toctree document. master_doc = 'index' diff --git a/docs/development.rst b/docs/development.rst index afe24c37..97e7df8e 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -21,8 +21,8 @@ tl;dr # run lint and tests make lint test -Getting Started -^^^^^^^^^^^^^^^ +Getting Started With Development +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 1. Install Python @@ -111,8 +111,8 @@ and see a full report for code coverage across the whole project. Clicking on a specific file will show a line-by-line report of what lines were or were not covered. -Documentation -^^^^^^^^^^^^^ +Documentation Building +^^^^^^^^^^^^^^^^^^^^^^^^ The docs are built using Sphinx. To build all the docs run: diff --git a/docs/index.rst b/docs/index.rst index 89bf462f..6c9eb602 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -12,9 +12,10 @@ InfluxDB 2.0 python client migration development -.. include:: ../README.rst - :start-after: marker-index-start - :end-before: marker-index-end +.. include:: ../README.md + :parser: myst_parser.sphinx_ + :start-after: + :end-before: Indices and tables ================== diff --git a/docs/requirements.txt b/docs/requirements.txt index fbbe6da5..1ea73cd5 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,3 @@ sphinx>=5.0.0 -sphinx_rtd_theme==1.3.0 \ No newline at end of file +sphinx_rtd_theme==1.3.0 +myst_parser>=0.19.2 diff --git a/docs/usage.rst b/docs/usage.rst index cd269e74..6b5f5b38 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -6,73 +6,85 @@ User Guide Query ^^^^^ -.. include:: ../README.rst - :start-after: marker-query-start - :end-before: marker-query-end +.. include:: ../README.md + :parser: myst_parser.sphinx_ + :start-after: + :end-before: Write ^^^^^ -.. include:: ../README.rst - :start-after: marker-writes-start - :end-before: marker-writes-end +.. include:: ../README.md + :parser: myst_parser.sphinx_ + :start-after: + :end-before: Delete data ^^^^^^^^^^^ -.. include:: ../README.rst - :start-after: marker-delete-start - :end-before: marker-delete-end +.. include:: ../README.md + :parser: myst_parser.sphinx_ + :start-after: + :end-before: Pandas DataFrame ^^^^^^^^^^^^^^^^ -.. include:: ../README.rst - :start-after: marker-pandas-start - :end-before: marker-pandas-end +.. include:: ../README.md + :parser: myst_parser.sphinx_ + :start-after: + :end-before: How to use Asyncio ^^^^^^^^^^^^^^^^^^ -.. include:: ../README.rst - :start-after: marker-asyncio-start - :end-before: marker-asyncio-end +.. include:: ../README.md + :parser: myst_parser.sphinx_ + :start-after: + :end-before: Gzip support ^^^^^^^^^^^^ -.. include:: ../README.rst - :start-after: marker-gzip-start - :end-before: marker-gzip-end +.. include:: ../README.md + :parser: myst_parser.sphinx_ + :start-after: + :end-before: Proxy configuration ^^^^^^^^^^^^^^^^^^^ -.. include:: ../README.rst - :start-after: marker-proxy-start - :end-before: marker-proxy-end +.. include:: ../README.md + :parser: myst_parser.sphinx_ + :start-after: + :end-before: Authentication ^^^^^^^^^^^^^^ -.. include:: ../README.rst - :start-after: marker-authenticate-start - :end-before: marker-authenticate-end +.. include:: ../README.md + :parser: myst_parser.sphinx_ + :start-after: + :end-before: Nanosecond precision ^^^^^^^^^^^^^^^^^^^^ -.. include:: ../README.rst - :start-after: marker-nanosecond-start - :end-before: marker-nanosecond-end +.. include:: ../README.md + :parser: myst_parser.sphinx_ + :start-after: + :end-before: Handling Errors ^^^^^^^^^^^^^^^ -.. include:: ../README.rst - :start-after: marker-handling-errors-start - :end-before: marker-handling-errors-end +.. include:: ../README.md + :parser: myst_parser.sphinx_ + :start-after: + :end-before: Logging ^^^^^^^ -.. include:: ../README.rst - :start-after: marker-logging-start - :end-before: marker-logging-end +.. include:: ../README.md + :parser: myst_parser.sphinx_ + :start-after: + :end-before: Examples ^^^^^^^^ -.. include:: ../README.rst - :start-after: marker-examples-start - :end-before: marker-examples-end \ No newline at end of file +.. include:: ../README.md + :parser: myst_parser.sphinx_ + :start-after: + :end-before: diff --git a/setup.py b/setup.py index a2ed2886..e15b963e 100644 --- a/setup.py +++ b/setup.py @@ -44,13 +44,9 @@ 'aiocsv>=1.2.2' ] -with open('README.rst', 'r') as f: - # Remove `class` text role as it's not allowed on PyPI - lines = [] - for line in f: - lines.append(line.replace(":class:`~", "`")) - - readme = "".join(lines) +from pathlib import Path +this_directory = Path(__file__).parent +long_description = (this_directory / "README.md").read_text() NAME = "influxdb_client" @@ -62,13 +58,13 @@ name=NAME, version=meta['VERSION'], description="InfluxDB 2.0 Python client library", - long_description=readme, + long_description=long_description, url="https://github.com/influxdata/influxdb-client-python", keywords=["InfluxDB", "InfluxDB Python Client"], tests_require=test_requires, install_requires=requires, extras_require={'extra': extra_requires, 'ciso': ciso_requires, 'async': async_requires, 'test': test_requires}, - long_description_content_type="text/x-rst", + long_description_content_type="text/markdown", packages=find_packages(exclude=('tests*',)), test_suite='tests', python_requires='>=3.7', From ed89701687cc03b86df6135d1b554e6676160030 Mon Sep 17 00:00:00 2001 From: Lars Nilse Date: Tue, 6 Feb 2024 07:49:01 +0100 Subject: [PATCH 12/51] fix: Pandas 3.0 warning for inplace method (#642) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * silence pandas 3.0 warning * add to changelog * docs: Update CHANGELOG.md --------- Co-authored-by: Jakub Bednář --- CHANGELOG.md | 2 +- influxdb_client/client/write/dataframe_serializer.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d5b10565..e542413a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ### Bug Fixes 1. [#636](https://github.com/influxdata/influxdb-client-python/pull/636): Handle missing data in data frames -2. [#638](https://github.com/influxdata/influxdb-client-python/pull/638): Refactor DataFrame operations to avoid chained assignment and resolve FutureWarning in pandas, ensuring compatibility with pandas 3.0. +2. [#638](https://github.com/influxdata/influxdb-client-python/pull/638), [#642](https://github.com/influxdata/influxdb-client-python/pull/642): Refactor DataFrame operations to avoid chained assignment and resolve FutureWarning in pandas, ensuring compatibility with pandas 3.0. ### Documentation 1. [#639](https://github.com/influxdata/influxdb-client-python/pull/639): Use Markdown for `README` diff --git a/influxdb_client/client/write/dataframe_serializer.py b/influxdb_client/client/write/dataframe_serializer.py index 98262526..6121171f 100644 --- a/influxdb_client/client/write/dataframe_serializer.py +++ b/influxdb_client/client/write/dataframe_serializer.py @@ -234,7 +234,7 @@ def __init__(self, data_frame, point_settings, precision=DEFAULT_WRITE_PRECISION for k, v in dict(data_frame.dtypes).items(): if k in data_frame_tag_columns: - data_frame.replace({k: ''}, np.nan, inplace=True) + data_frame = data_frame.replace({k: ''}, np.nan) self.data_frame = data_frame self.f = f From 2661e4db2014ac269d875291fa991c8cc1057fbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Bedn=C3=A1=C5=99?= Date: Thu, 29 Feb 2024 09:29:16 +0100 Subject: [PATCH 13/51] fix: dispose window scheduler (#641) --- CHANGELOG.md | 3 ++- influxdb_client/client/write_api.py | 9 ++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e542413a..a4d62793 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,8 @@ ### Bug Fixes 1. [#636](https://github.com/influxdata/influxdb-client-python/pull/636): Handle missing data in data frames -2. [#638](https://github.com/influxdata/influxdb-client-python/pull/638), [#642](https://github.com/influxdata/influxdb-client-python/pull/642): Refactor DataFrame operations to avoid chained assignment and resolve FutureWarning in pandas, ensuring compatibility with pandas 3.0. +1. [#638](https://github.com/influxdata/influxdb-client-python/pull/638), [#642](https://github.com/influxdata/influxdb-client-python/pull/642): Refactor DataFrame operations to avoid chained assignment and resolve FutureWarning in pandas, ensuring compatibility with pandas 3.0. +1. [#641](https://github.com/influxdata/influxdb-client-python/pull/641): Correctly dispose ThreadPoolScheduler in WriteApi ### Documentation 1. [#639](https://github.com/influxdata/influxdb-client-python/pull/639): Use Markdown for `README` diff --git a/influxdb_client/client/write_api.py b/influxdb_client/client/write_api.py index 050a7a5c..3b3db68f 100644 --- a/influxdb_client/client/write_api.py +++ b/influxdb_client/client/write_api.py @@ -250,16 +250,18 @@ def __init__(self, self._success_callback = kwargs.get('success_callback', None) self._error_callback = kwargs.get('error_callback', None) self._retry_callback = kwargs.get('retry_callback', None) + self._window_scheduler = None if self._write_options.write_type is WriteType.batching: # Define Subject that listen incoming data and produces writes into InfluxDB self._subject = Subject() + self._window_scheduler = ThreadPoolScheduler(1) self._disposable = self._subject.pipe( # Split incoming data to windows by batch_size or flush_interval ops.window_with_time_or_count(count=write_options.batch_size, timespan=timedelta(milliseconds=write_options.flush_interval), - scheduler=ThreadPoolScheduler(1)), + scheduler=self._window_scheduler), # Map window into groups defined by 'organization', 'bucket' and 'precision' ops.flat_map(lambda window: window.pipe( # Group window by 'organization', 'bucket' and 'precision' @@ -440,6 +442,10 @@ def __del__(self): ) break + if self._window_scheduler: + self._window_scheduler.executor.shutdown(wait=False) + self._window_scheduler = None + if self._disposable: self._disposable = None pass @@ -565,6 +571,7 @@ def __getstate__(self): # Remove rx del state['_subject'] del state['_disposable'] + del state['_window_scheduler'] del state['_write_service'] return state From e82b016f3ea90b901c5207a559287aa3322d137a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 29 Feb 2024 09:30:47 +0100 Subject: [PATCH 14/51] chore(deps): bump sphinx-rtd-theme from 1.3.0 to 2.0.0 (#618) Bumps [sphinx-rtd-theme](https://github.com/readthedocs/sphinx_rtd_theme) from 1.3.0 to 2.0.0. - [Changelog](https://github.com/readthedocs/sphinx_rtd_theme/blob/master/docs/changelog.rst) - [Commits](https://github.com/readthedocs/sphinx_rtd_theme/compare/1.3.0...2.0.0) --- updated-dependencies: - dependency-name: sphinx-rtd-theme dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 1ea73cd5..dc6dddec 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,3 @@ sphinx>=5.0.0 -sphinx_rtd_theme==1.3.0 +sphinx_rtd_theme==2.0.0 myst_parser>=0.19.2 From c430620bf9daa9ab8158b038c3e5f9a4df720c30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Bedn=C3=A1=C5=99?= Date: Thu, 29 Feb 2024 14:09:14 +0100 Subject: [PATCH 15/51] chore: add support for Python 3.12 (#643) --- .circleci/config.yml | 3 +++ CHANGELOG.md | 3 +++ scripts/ci-test.sh | 10 +++++----- setup.py | 1 + tests/test_InfluxDBClient.py | 5 +++-- tests/test_QueryApiDataFrame.py | 6 +++--- tests/test_Warnings.py | 1 + tests/test_WriteApiDataFrame.py | 8 ++++---- 8 files changed, 23 insertions(+), 14 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e471dfbb..a47bde06 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -202,6 +202,9 @@ workflows: - tests-python: name: test-3.11 python-image: "cimg/python:3.11" + - tests-python: + name: test-3.12 + python-image: "cimg/python:3.12" nightly: when: diff --git a/CHANGELOG.md b/CHANGELOG.md index a4d62793..3f2ccac9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ ## 1.41.0 [unreleased] +### Features +1. [#643](https://github.com/influxdata/influxdb-client-python/pull/643): Add a support for Python 3.12 + ### Bug Fixes 1. [#636](https://github.com/influxdata/influxdb-client-python/pull/636): Handle missing data in data frames 1. [#638](https://github.com/influxdata/influxdb-client-python/pull/638), [#642](https://github.com/influxdata/influxdb-client-python/pull/642): Refactor DataFrame operations to avoid chained assignment and resolve FutureWarning in pandas, ensuring compatibility with pandas 3.0. diff --git a/scripts/ci-test.sh b/scripts/ci-test.sh index 8e2cdd4e..dc7c9b59 100755 --- a/scripts/ci-test.sh +++ b/scripts/ci-test.sh @@ -8,13 +8,13 @@ ENABLED_CISO_8601="${ENABLED_CISO_8601:-true}" # Install requirements # python --version -pip install -e . --user -pip install -e .\[extra\] --user -pip install -e .\[test\] --user -pip install -e .\[async\] --user +pip install . --user +pip install .\[extra\] --user +pip install .\[test\] --user +pip install .\[async\] --user if [ "$ENABLED_CISO_8601" = true ] ; then echo "ciso8601 is enabled" - pip install -e .\[ciso\] --user + pip install .\[ciso\] --user else echo "ciso8601 is disabled" fi diff --git a/setup.py b/setup.py index e15b963e..ac1154d3 100644 --- a/setup.py +++ b/setup.py @@ -78,6 +78,7 @@ 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', 'Topic :: Database', 'Topic :: Software Development :: Libraries', 'Topic :: Software Development :: Libraries :: Python Modules', diff --git a/tests/test_InfluxDBClient.py b/tests/test_InfluxDBClient.py index c2f9b0a5..7fdf834f 100644 --- a/tests/test_InfluxDBClient.py +++ b/tests/test_InfluxDBClient.py @@ -248,8 +248,9 @@ def _start_http_server(self): urllib3.disable_warnings() # Configure HTTP server self.httpd = http.server.HTTPServer(('localhost', 0), ServerWithSelfSingedSSL) - self.httpd.socket = ssl.wrap_socket(self.httpd.socket, certfile=f'{os.path.dirname(__file__)}/server.pem', - server_side=True) + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(f'{os.path.dirname(__file__)}/server.pem') + self.httpd.socket = context.wrap_socket(self.httpd.socket, server_side=True) # Start server at background self.httpd_thread = threading.Thread(target=self.httpd.serve_forever) self.httpd_thread.start() diff --git a/tests/test_QueryApiDataFrame.py b/tests/test_QueryApiDataFrame.py index 670074bc..31396be6 100644 --- a/tests/test_QueryApiDataFrame.py +++ b/tests/test_QueryApiDataFrame.py @@ -268,7 +268,7 @@ def test_query_with_warning(self): '|> range(start: -5s, stop: now()) ' '|> filter(fn: (r) => r._measurement == "mem") ' "my-org") - self.assertEqual(1, len(warnings)) + self.assertEqual(1, len([w for w in warnings if w.category == MissingPivotFunction])) def test_query_without_warning(self): httpretty.register_uri(httpretty.POST, uri="http://localhost/api/v2/query", status=200, body='\n') @@ -284,7 +284,7 @@ def test_query_without_warning(self): '|> filter(fn: (r) => r._measurement == "mem") ' '|> schema.fieldsAsCols() ' "my-org") - self.assertEqual(0, len(warns)) + self.assertEqual(0, len([w for w in warns if w.category == MissingPivotFunction])) with warnings.catch_warnings(record=True) as warns: self.client.query_api().query_data_frame( @@ -293,7 +293,7 @@ def test_query_without_warning(self): '|> filter(fn: (r) => r._measurement == "mem") ' '|> pivot(rowKey: ["_time"], columnKey: ["_field"], valueColumn: "_value")' "my-org") - self.assertEqual(0, len(warns)) + self.assertEqual(0, len([w for w in warns if w.category == MissingPivotFunction])) def test_pivoted_data(self): query_response = \ diff --git a/tests/test_Warnings.py b/tests/test_Warnings.py index 9d32d368..f3bc3f20 100644 --- a/tests/test_Warnings.py +++ b/tests/test_Warnings.py @@ -27,4 +27,5 @@ def test_cloud_only_warning(self): with InfluxDBClient(url="http://localhost", token="my-token", org="my-org") as client: service = BucketSchemasService(api_client=client.api_client) service.get_measurement_schemas(bucket_id="01010101") + warnings = [w for w in warnings if w.category == CloudOnlyWarning] self.assertEqual(1, len(warnings)) diff --git a/tests/test_WriteApiDataFrame.py b/tests/test_WriteApiDataFrame.py index 3675519a..6ea7a98b 100644 --- a/tests/test_WriteApiDataFrame.py +++ b/tests/test_WriteApiDataFrame.py @@ -313,7 +313,7 @@ def test_with_period_index(self): data_frame = pd.DataFrame(data={ 'value': [1, 2], }, - index=pd.period_range(start='2020-04-05 01:00', freq='H', periods=2)) + index=pd.period_range(start='2020-04-05 01:00', freq='h', periods=2)) points = data_frame_to_list_of_points(data_frame=data_frame, point_settings=PointSettings(), @@ -498,7 +498,7 @@ def test_specify_timezone_period_time_index(self): data_frame = pd.DataFrame(data={ 'value1': [10, 20], 'value2': [30, 40], - }, index=pd.period_range(start='2020-05-24 10:00', freq='H', periods=2)) + }, index=pd.period_range(start='2020-05-24 10:00', freq='h', periods=2)) print(data_frame.to_string()) @@ -519,7 +519,7 @@ def test_serialization_for_nan_in_columns_starting_with_digits(self): '2value': [30.0, np.nan, np.nan, np.nan, np.nan], '3value': [30.0, 30.0, 30.0, np.nan, np.nan], 'avalue': [30.0, 30.0, 30.0, 30.0, 30.0] - }, index=pd.period_range('2020-05-24 10:00', freq='H', periods=5)) + }, index=pd.period_range('2020-05-24 10:00', freq='h', periods=5)) points = data_frame_to_list_of_points(data_frame, PointSettings(), @@ -536,7 +536,7 @@ def test_serialization_for_nan_in_columns_starting_with_digits(self): '1value': [np.nan], 'avalue': [30.0], 'bvalue': [30.0] - }, index=pd.period_range('2020-05-24 10:00', freq='H', periods=1)) + }, index=pd.period_range('2020-05-24 10:00', freq='h', periods=1)) points = data_frame_to_list_of_points(data_frame, PointSettings(), From 45c105f3681fc84a8466f8a057f03f8a4fd8918d Mon Sep 17 00:00:00 2001 From: Jakub Bednar Date: Fri, 1 Mar 2024 09:24:03 +0100 Subject: [PATCH 16/51] chore(release): release version 1.41.0 [skip CI] --- CHANGELOG.md | 2 +- influxdb_client/version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f2ccac9..d8315ded 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## 1.41.0 [unreleased] +## 1.41.0 [2024-03-01] ### Features 1. [#643](https://github.com/influxdata/influxdb-client-python/pull/643): Add a support for Python 3.12 diff --git a/influxdb_client/version.py b/influxdb_client/version.py index 7a63f210..d9a0312b 100644 --- a/influxdb_client/version.py +++ b/influxdb_client/version.py @@ -1,3 +1,3 @@ """Version of the Client that is used in User-Agent header.""" -VERSION = '1.41.0dev0' +VERSION = '1.41.0' From a645ea99e0b518e78f112c5dee3fe81dfb31a2cd Mon Sep 17 00:00:00 2001 From: Jakub Bednar Date: Fri, 1 Mar 2024 09:29:58 +0100 Subject: [PATCH 17/51] chore(release): prepare for next development iteration --- CHANGELOG.md | 2 ++ conda/meta.yaml | 6 +++--- influxdb_client/version.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8315ded..27fbe50f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +## 1.42.0 [unreleased] + ## 1.41.0 [2024-03-01] ### Features diff --git a/conda/meta.yaml b/conda/meta.yaml index da95cce4..8cd787cf 100644 --- a/conda/meta.yaml +++ b/conda/meta.yaml @@ -1,5 +1,5 @@ {% set name = "influxdb_client" %} -{% set version = "1.40.0" %} +{% set version = "1.41.0" %} package: @@ -7,8 +7,8 @@ package: version: {{ version }} source: - url: https://files.pythonhosted.org/packages/63/11/07ed82352a28e4e8b623a487337befec77d5bd18293dcc940d769e633f82/influxdb_client-1.40.0.tar.gz - sha256: 027f970af1518479d8806f1cdf5ba20280f943e1b621c2acdbf9ca8dc9bdf1cb + url: https://files.pythonhosted.org/packages/bf/80/8f5dab7bfe55e65181cbc1e268806ae1ddc935a1b2ae612ed7f41d8bd0c1/influxdb_client-1.41.0.tar.gz + sha256: 4b85bad3991f3de24818366c87c8868a64917fea2d21bbcc2b579fbe5d904990 build: number: 0 diff --git a/influxdb_client/version.py b/influxdb_client/version.py index d9a0312b..3ea7c0e4 100644 --- a/influxdb_client/version.py +++ b/influxdb_client/version.py @@ -1,3 +1,3 @@ """Version of the Client that is used in User-Agent header.""" -VERSION = '1.41.0' +VERSION = '1.42.0dev0' From e2cb42e05e5a7a86222b0bdd723e52abb557237e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Bedn=C3=A1=C5=99?= Date: Tue, 16 Apr 2024 08:01:03 +0200 Subject: [PATCH 18/51] fix: serialize Pandas NaN values into LineProtocol (#648) --- CHANGELOG.md | 3 ++ .../client/write/dataframe_serializer.py | 33 ++++++++----------- tests/test_WriteApiDataFrame.py | 26 +++++++++++++++ 3 files changed, 42 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 27fbe50f..d4f8c925 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ ## 1.42.0 [unreleased] +### Bug Fixes +1. [#648](https://github.com/influxdata/influxdb-client-python/pull/648): Fix `DataFrame` serialization with `NaN` values + ## 1.41.0 [2024-03-01] ### Features diff --git a/influxdb_client/client/write/dataframe_serializer.py b/influxdb_client/client/write/dataframe_serializer.py index 6121171f..ccc198ac 100644 --- a/influxdb_client/client/write/dataframe_serializer.py +++ b/influxdb_client/client/write/dataframe_serializer.py @@ -19,14 +19,6 @@ def _itertuples(data_frame): return zip(data_frame.index, *cols) -def _not_nan(x): - return x == x - - -def _any_not_nan(p, indexes): - return any(map(lambda x: _not_nan(p[x]), indexes)) - - class DataframeSerializer: """Serialize DataFrame into LineProtocols.""" @@ -77,7 +69,7 @@ def __init__(self, data_frame, point_settings, precision=DEFAULT_WRITE_PRECISION # When NaNs are present, the expression looks like this (split # across two lines to satisfy the code-style checker) # - # lambda p: f"""{measurement_name} {"" if math.isnan(p[1]) + # lambda p: f"""{measurement_name} {"" if pd.isna(p[1]) # else f"{keys[0]}={p[1]}"},{keys[1]}={p[2]}i {p[0].value}""" # # When there's a NaN value in column a, we'll end up with a comma at the start of the @@ -175,7 +167,7 @@ def __init__(self, data_frame, point_settings, precision=DEFAULT_WRITE_PRECISION # This column is a tag column. if null_columns.iloc[index]: key_value = f"""{{ - '' if {val_format} == '' or type({val_format}) == float and math.isnan({val_format}) else + '' if {val_format} == '' or pd.isna({val_format}) else f',{key_format}={{str({val_format}).translate(_ESCAPE_STRING)}}' }}""" else: @@ -192,19 +184,16 @@ def __init__(self, data_frame, point_settings, precision=DEFAULT_WRITE_PRECISION # field column has no nulls, we don't run the comma-removal # regexp substitution step. sep = '' if len(field_indexes) == 0 else ',' - if issubclass(value.type, np.integer): - field_value = f"{sep}{key_format}={{{val_format}}}i" - elif issubclass(value.type, np.bool_): - field_value = f'{sep}{key_format}={{{val_format}}}' - elif issubclass(value.type, np.floating): + if issubclass(value.type, np.integer) or issubclass(value.type, np.floating) or issubclass(value.type, np.bool_): # noqa: E501 + suffix = 'i' if issubclass(value.type, np.integer) else '' if null_columns.iloc[index]: - field_value = f"""{{"" if math.isnan({val_format}) else f"{sep}{key_format}={{{val_format}}}"}}""" + field_value = f"""{{"" if pd.isna({val_format}) else f"{sep}{key_format}={{{val_format}}}{suffix}"}}""" # noqa: E501 else: - field_value = f'{sep}{key_format}={{{val_format}}}' + field_value = f"{sep}{key_format}={{{val_format}}}{suffix}" else: if null_columns.iloc[index]: field_value = f"""{{ - '' if type({val_format}) == float and math.isnan({val_format}) else + '' if pd.isna({val_format}) else f'{sep}{key_format}="{{str({val_format}).translate(_ESCAPE_STRING)}}"' }}""" else: @@ -229,17 +218,21 @@ def __init__(self, data_frame, point_settings, precision=DEFAULT_WRITE_PRECISION '_ESCAPE_KEY': _ESCAPE_KEY, '_ESCAPE_STRING': _ESCAPE_STRING, 'keys': keys, - 'math': math, + 'pd': pd, }) for k, v in dict(data_frame.dtypes).items(): if k in data_frame_tag_columns: data_frame = data_frame.replace({k: ''}, np.nan) + def _any_not_nan(p, indexes): + return any(map(lambda x: not pd.isna(p[x]), indexes)) + self.data_frame = data_frame self.f = f self.field_indexes = field_indexes self.first_field_maybe_null = null_columns.iloc[field_indexes[0] - 1] + self._any_not_nan = _any_not_nan # # prepare chunks @@ -266,7 +259,7 @@ def serialize(self, chunk_idx: int = None): # When the first field is null (None/NaN), we'll have # a spurious leading comma which needs to be removed. lp = (re.sub('^(( |[^ ])* ),([a-zA-Z0-9])(.*)', '\\1\\3\\4', self.f(p)) - for p in filter(lambda x: _any_not_nan(x, self.field_indexes), _itertuples(chunk))) + for p in filter(lambda x: self._any_not_nan(x, self.field_indexes), _itertuples(chunk))) return list(lp) else: return list(map(self.f, _itertuples(chunk))) diff --git a/tests/test_WriteApiDataFrame.py b/tests/test_WriteApiDataFrame.py index 6ea7a98b..1e1f0ad3 100644 --- a/tests/test_WriteApiDataFrame.py +++ b/tests/test_WriteApiDataFrame.py @@ -159,6 +159,32 @@ def test_write_object_field_nan(self): self.assertEqual("measurement val=2i 1586046600000000000", points[1]) + def test_write_missing_values(self): + from influxdb_client.extras import pd + + data_frame = pd.DataFrame({ + "a_bool": [True, None, False], + "b_int": [None, 1, 2], + "c_float": [1.0, 2.0, None], + "d_str": ["a", "b", None], + }) + + data_frame['a_bool'] = data_frame['a_bool'].astype(pd.BooleanDtype()) + data_frame['b_int'] = data_frame['b_int'].astype(pd.Int64Dtype()) + data_frame['c_float'] = data_frame['c_float'].astype(pd.Float64Dtype()) + data_frame['d_str'] = data_frame['d_str'].astype(pd.StringDtype()) + + print(data_frame) + points = data_frame_to_list_of_points( + data_frame=data_frame, + point_settings=PointSettings(), + data_frame_measurement_name='measurement') + + self.assertEqual(3, len(points)) + self.assertEqual("measurement a_bool=True,c_float=1.0,d_str=\"a\" 0", points[0]) + self.assertEqual("measurement b_int=1i,c_float=2.0,d_str=\"b\" 1", points[1]) + self.assertEqual("measurement a_bool=False,b_int=2i 2", points[2]) + def test_write_field_bool(self): from influxdb_client.extras import pd From 131ac06b9c9e8275c59964558f9f10cd3f00d457 Mon Sep 17 00:00:00 2001 From: Jakub Bednar Date: Wed, 17 Apr 2024 08:04:17 +0200 Subject: [PATCH 19/51] chore(release): release version 1.42.0 [skip CI] --- CHANGELOG.md | 2 +- influxdb_client/version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d4f8c925..551d7bdd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## 1.42.0 [unreleased] +## 1.42.0 [2024-04-17] ### Bug Fixes 1. [#648](https://github.com/influxdata/influxdb-client-python/pull/648): Fix `DataFrame` serialization with `NaN` values diff --git a/influxdb_client/version.py b/influxdb_client/version.py index 3ea7c0e4..86f24fc9 100644 --- a/influxdb_client/version.py +++ b/influxdb_client/version.py @@ -1,3 +1,3 @@ """Version of the Client that is used in User-Agent header.""" -VERSION = '1.42.0dev0' +VERSION = '1.42.0' From 550ff26d3a4a3461a10396f30e55f8561a2cbf06 Mon Sep 17 00:00:00 2001 From: Jakub Bednar Date: Wed, 17 Apr 2024 08:09:38 +0200 Subject: [PATCH 20/51] chore(release): prepare for next development iteration --- CHANGELOG.md | 2 ++ conda/meta.yaml | 6 +++--- influxdb_client/version.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 551d7bdd..6c4465cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +## 1.43.0 [unreleased] + ## 1.42.0 [2024-04-17] ### Bug Fixes diff --git a/conda/meta.yaml b/conda/meta.yaml index 8cd787cf..97710c15 100644 --- a/conda/meta.yaml +++ b/conda/meta.yaml @@ -1,5 +1,5 @@ {% set name = "influxdb_client" %} -{% set version = "1.41.0" %} +{% set version = "1.42.0" %} package: @@ -7,8 +7,8 @@ package: version: {{ version }} source: - url: https://files.pythonhosted.org/packages/bf/80/8f5dab7bfe55e65181cbc1e268806ae1ddc935a1b2ae612ed7f41d8bd0c1/influxdb_client-1.41.0.tar.gz - sha256: 4b85bad3991f3de24818366c87c8868a64917fea2d21bbcc2b579fbe5d904990 + url: https://files.pythonhosted.org/packages/58/84/e8746501977f66fe7546d721971d2cc268a30762abab1f91ab73bd95163d/influxdb_client-1.42.0.tar.gz + sha256: f5e877feb671eda41e2b5c98ed1dc8ec3327fd8991360dc614822119cda06491 build: number: 0 diff --git a/influxdb_client/version.py b/influxdb_client/version.py index 86f24fc9..a8523e0d 100644 --- a/influxdb_client/version.py +++ b/influxdb_client/version.py @@ -1,3 +1,3 @@ """Version of the Client that is used in User-Agent header.""" -VERSION = '1.42.0' +VERSION = '1.43.0dev0' From 0047fa95cf852bb2415d6dff3603e362faa8e53c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Bedn=C3=A1=C5=99?= Date: Thu, 9 May 2024 09:01:39 +0200 Subject: [PATCH 21/51] chore(deps): update jinja2 (#653) --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ac1154d3..5596713f 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ 'aioresponses>=0.7.3', 'sphinx==1.8.5', 'sphinx_rtd_theme', - 'jinja2==3.1.3' + 'jinja2>=3.1.4' ] extra_requires = [ From 08acb17b60e74947296b6be2d1904ad22b72e5bf Mon Sep 17 00:00:00 2001 From: alespour <42931850+alespour@users.noreply.github.com> Date: Tue, 14 May 2024 14:14:53 +0200 Subject: [PATCH 22/51] fix: deprecated urllib calls (#655) --- CHANGELOG.md | 3 +++ influxdb_client/client/exceptions.py | 5 ++++- influxdb_client/rest.py | 7 +++++-- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c4465cb..9391f964 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ ## 1.43.0 [unreleased] +### Bug Fixes +1. [#655](https://github.com/influxdata/influxdb-client-python/pull/655): Replace deprecated `urllib` calls `HTTPResponse.getheaders()` and `HTTPResponse.getheader()`. + ## 1.42.0 [2024-04-17] ### Bug Fixes diff --git a/influxdb_client/client/exceptions.py b/influxdb_client/client/exceptions.py index 48681add..2ca235c8 100644 --- a/influxdb_client/client/exceptions.py +++ b/influxdb_client/client/exceptions.py @@ -15,7 +15,10 @@ def __init__(self, response: HTTPResponse = None, message: str = None): if response is not None: self.response = response self.message = self._get_message(response) - self.retry_after = response.getheader('Retry-After') + if isinstance(response, HTTPResponse): # response is HTTPResponse + self.retry_after = response.headers.get('Retry-After') + else: # response is RESTResponse + self.retry_after = response.getheader('Retry-After') else: self.response = None self.message = message or 'no response' diff --git a/influxdb_client/rest.py b/influxdb_client/rest.py index 8f50e51a..cd4dbff4 100644 --- a/influxdb_client/rest.py +++ b/influxdb_client/rest.py @@ -13,7 +13,7 @@ import logging from typing import Dict - +from urllib3 import HTTPResponse from influxdb_client.client.exceptions import InfluxDBError from influxdb_client.configuration import Configuration @@ -34,7 +34,10 @@ def __init__(self, status=None, reason=None, http_resp=None): self.status = http_resp.status self.reason = http_resp.reason self.body = http_resp.data - self.headers = http_resp.getheaders() + if isinstance(http_resp, HTTPResponse): # response is HTTPResponse + self.headers = http_resp.headers + else: # response is RESTResponse + self.headers = http_resp.getheaders() else: self.status = status self.reason = reason From 51d6fa45160920ae7f1903ce53abb1b7081ffff8 Mon Sep 17 00:00:00 2001 From: alespour <42931850+alespour@users.noreply.github.com> Date: Wed, 15 May 2024 09:20:05 +0200 Subject: [PATCH 23/51] fix: packaging type info (#654) --- CHANGELOG.md | 3 +++ influxdb_client/py.typed | 0 2 files changed, 3 insertions(+) create mode 100644 influxdb_client/py.typed diff --git a/CHANGELOG.md b/CHANGELOG.md index 9391f964..c74c323f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,9 @@ ### Bug Fixes 1. [#655](https://github.com/influxdata/influxdb-client-python/pull/655): Replace deprecated `urllib` calls `HTTPResponse.getheaders()` and `HTTPResponse.getheader()`. +### Others +1. [#654](https://github.com/influxdata/influxdb-client-python/pull/654): Enable packaging type information - `py.typed` + ## 1.42.0 [2024-04-17] ### Bug Fixes diff --git a/influxdb_client/py.typed b/influxdb_client/py.typed new file mode 100644 index 00000000..e69de29b From 83322a3215b7d054d9555682e9c240f7eda322db Mon Sep 17 00:00:00 2001 From: Jakub Bednar Date: Fri, 17 May 2024 11:29:35 +0200 Subject: [PATCH 24/51] chore(release): release version 1.43.0 [skip CI] --- CHANGELOG.md | 2 +- influxdb_client/version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c74c323f..612978b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## 1.43.0 [unreleased] +## 1.43.0 [2024-05-17] ### Bug Fixes 1. [#655](https://github.com/influxdata/influxdb-client-python/pull/655): Replace deprecated `urllib` calls `HTTPResponse.getheaders()` and `HTTPResponse.getheader()`. diff --git a/influxdb_client/version.py b/influxdb_client/version.py index a8523e0d..bad85a7a 100644 --- a/influxdb_client/version.py +++ b/influxdb_client/version.py @@ -1,3 +1,3 @@ """Version of the Client that is used in User-Agent header.""" -VERSION = '1.43.0dev0' +VERSION = '1.43.0' From d2393e0a8773a043d89780fdfea18d2449e14ae0 Mon Sep 17 00:00:00 2001 From: Jakub Bednar Date: Fri, 17 May 2024 11:34:00 +0200 Subject: [PATCH 25/51] chore(release): prepare for next development iteration --- CHANGELOG.md | 2 ++ conda/meta.yaml | 6 +++--- influxdb_client/version.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 612978b1..0eb128ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +## 1.44.0 [unreleased] + ## 1.43.0 [2024-05-17] ### Bug Fixes diff --git a/conda/meta.yaml b/conda/meta.yaml index 97710c15..186640f0 100644 --- a/conda/meta.yaml +++ b/conda/meta.yaml @@ -1,5 +1,5 @@ {% set name = "influxdb_client" %} -{% set version = "1.42.0" %} +{% set version = "1.43.0" %} package: @@ -7,8 +7,8 @@ package: version: {{ version }} source: - url: https://files.pythonhosted.org/packages/58/84/e8746501977f66fe7546d721971d2cc268a30762abab1f91ab73bd95163d/influxdb_client-1.42.0.tar.gz - sha256: f5e877feb671eda41e2b5c98ed1dc8ec3327fd8991360dc614822119cda06491 + url: https://files.pythonhosted.org/packages/3a/1f/d610ac86af1204bb12698a4d9ac4bd743141e01c13dc44d2e5a8bcf9c556/influxdb_client-1.43.0.tar.gz + sha256: ae2614d891baed52c0ae8f6194a04ee5b1c6422f6061318a3639fe63b7671b25 build: number: 0 diff --git a/influxdb_client/version.py b/influxdb_client/version.py index bad85a7a..a445d0b6 100644 --- a/influxdb_client/version.py +++ b/influxdb_client/version.py @@ -1,3 +1,3 @@ """Version of the Client that is used in User-Agent header.""" -VERSION = '1.43.0' +VERSION = '1.44.0dev0' From 73849e7baa1d386f086c3877f2c268d63d8a038c Mon Sep 17 00:00:00 2001 From: Nicolas Thumann Date: Mon, 20 May 2024 12:49:46 +0200 Subject: [PATCH 26/51] perf: Prefer datetime.fromisoformat over dateutil.parse in Python 3.11+ (#657) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * perf: Prefer datetime.fromisoformat over dateutil.parse in Python 3.11+ * docs: Add PR to changelog * docs: Update CHANGELOG.md --------- Co-authored-by: Jakub Bednář --- CHANGELOG.md | 3 +++ influxdb_client/client/util/date_utils.py | 9 +++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0eb128ef..d7a3a503 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ ## 1.44.0 [unreleased] +### Features +1. [#657](https://github.com/influxdata/influxdb-client-python/pull/657): Prefer datetime.fromisoformat over dateutil.parse in Python 3.11+ + ## 1.43.0 [2024-05-17] ### Bug Fixes diff --git a/influxdb_client/client/util/date_utils.py b/influxdb_client/client/util/date_utils.py index 11baecb5..7b6750c8 100644 --- a/influxdb_client/client/util/date_utils.py +++ b/influxdb_client/client/util/date_utils.py @@ -1,5 +1,6 @@ """Utils to get right Date parsing function.""" import datetime +from sys import version_info import threading from datetime import timezone as tz @@ -78,7 +79,8 @@ def get_date_helper() -> DateHelper: """ Return DateHelper with proper implementation. - If there is a 'ciso8601' than use 'ciso8601.parse_datetime' else use 'dateutil.parse'. + If there is a 'ciso8601' than use 'ciso8601.parse_datetime' else + use 'datetime.fromisoformat' (Python >= 3.11) or 'dateutil.parse' (Python < 3.11). """ global date_helper if date_helper is None: @@ -90,7 +92,10 @@ def get_date_helper() -> DateHelper: import ciso8601 _date_helper.parse_date = ciso8601.parse_datetime except ModuleNotFoundError: - _date_helper.parse_date = parser.parse + if (version_info.major, version_info.minor) >= (3, 11): + _date_helper.parse_date = datetime.datetime.fromisoformat + else: + _date_helper.parse_date = parser.parse date_helper = _date_helper return date_helper From 6798be4bf07c85c17fa635aa6256e085e251fc93 Mon Sep 17 00:00:00 2001 From: Mehdi Ben Abdallah Date: Tue, 28 May 2024 13:08:27 +0200 Subject: [PATCH 27/51] feat: bucket pagination (#658) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: support buckets pagination * docs: update example * docs: add changelog entry * Update CHANGELOG.md Co-authored-by: Jakub Bednář * encapsulate pagination --------- Co-authored-by: Mehdi BEN ABDALLAH <@mbenabda> Co-authored-by: Jakub Bednář --- CHANGELOG.md | 1 + examples/buckets_management.py | 2 +- influxdb_client/client/_pages.py | 66 ++++++++++++++++++++++++++++ influxdb_client/client/bucket_api.py | 13 ++++++ influxdb_client/client/tasks_api.py | 52 +--------------------- tests/test_BucketsApi.py | 47 ++++++++++++++++++-- 6 files changed, 126 insertions(+), 55 deletions(-) create mode 100644 influxdb_client/client/_pages.py diff --git a/CHANGELOG.md b/CHANGELOG.md index d7a3a503..2c6efa56 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ### Features 1. [#657](https://github.com/influxdata/influxdb-client-python/pull/657): Prefer datetime.fromisoformat over dateutil.parse in Python 3.11+ +1. [#658](https://github.com/influxdata/influxdb-client-python/pull/658): Add `find_buckets_iter` function that allow iterate through all pages of buckets. ## 1.43.0 [2024-05-17] diff --git a/examples/buckets_management.py b/examples/buckets_management.py index cc81b58f..c2a24092 100644 --- a/examples/buckets_management.py +++ b/examples/buckets_management.py @@ -36,7 +36,7 @@ List all Buckets """ print(f"\n------- List -------\n") - buckets = buckets_api.find_buckets().buckets + buckets = buckets_api.find_buckets_iter() print("\n".join([f" ---\n ID: {bucket.id}\n Name: {bucket.name}\n Retention: {bucket.retention_rules}" for bucket in buckets])) print("---") diff --git a/influxdb_client/client/_pages.py b/influxdb_client/client/_pages.py new file mode 100644 index 00000000..5e418427 --- /dev/null +++ b/influxdb_client/client/_pages.py @@ -0,0 +1,66 @@ + + +class _Page: + def __init__(self, values, has_next, next_after): + self.has_next = has_next + self.values = values + self.next_after = next_after + + @staticmethod + def empty(): + return _Page([], False, None) + + @staticmethod + def initial(after): + return _Page([], True, after) + + +class _PageIterator: + def __init__(self, page: _Page, get_next_page): + self.page = page + self.get_next_page = get_next_page + + def __iter__(self): + return self + + def __next__(self): + if not self.page.values: + if self.page.has_next: + self.page = self.get_next_page(self.page) + if not self.page.values: + raise StopIteration + return self.page.values.pop(0) + + +class _Paginated: + def __init__(self, paginated_getter, pluck_page_resources_from_response): + self.paginated_getter = paginated_getter + self.pluck_page_resources_from_response = pluck_page_resources_from_response + + def find_iter(self, **kwargs): + """Iterate over resources with pagination. + + :key str org: The organization name. + :key str org_id: The organization ID. + :key str after: The last resource ID from which to seek from (but not including). + :key int limit: the maximum number of items per page + :return: resources iterator + """ + + def get_next_page(page: _Page): + return self._find_next_page(page, **kwargs) + + return iter(_PageIterator(_Page.initial(kwargs.get('after')), get_next_page)) + + def _find_next_page(self, page: _Page, **kwargs): + if not page.has_next: + return _Page.empty() + + kw_args = {**kwargs, 'after': page.next_after} if page.next_after is not None else kwargs + response = self.paginated_getter(**kw_args) + + resources = self.pluck_page_resources_from_response(response) + has_next = response.links.next is not None + last_id = resources[-1].id if resources else None + + return _Page(resources, has_next, last_id) diff --git a/influxdb_client/client/bucket_api.py b/influxdb_client/client/bucket_api.py index 47763bee..684da767 100644 --- a/influxdb_client/client/bucket_api.py +++ b/influxdb_client/client/bucket_api.py @@ -8,6 +8,7 @@ from influxdb_client import BucketsService, Bucket, PostBucketRequest, PatchBucketRequest from influxdb_client.client.util.helpers import get_org_query_param +from influxdb_client.client._pages import _Paginated class BucketsApi(object): @@ -117,3 +118,15 @@ def find_buckets(self, **kwargs): :return: Buckets """ return self._buckets_service.get_buckets(**kwargs) + + def find_buckets_iter(self, **kwargs): + """Iterate over all buckets with pagination. + + :key str name: Only returns buckets with the specified name + :key str org: The organization name. + :key str org_id: The organization ID. + :key str after: The last resource ID from which to seek from (but not including). + :key int limit: the maximum number of buckets in one page + :return: Buckets iterator + """ + return _Paginated(self._buckets_service.get_buckets, lambda response: response.buckets).find_iter(**kwargs) diff --git a/influxdb_client/client/tasks_api.py b/influxdb_client/client/tasks_api.py index 9edb2ec9..5ca18fbd 100644 --- a/influxdb_client/client/tasks_api.py +++ b/influxdb_client/client/tasks_api.py @@ -9,38 +9,7 @@ from influxdb_client import TasksService, Task, TaskCreateRequest, TaskUpdateRequest, LabelResponse, LabelMapping, \ AddResourceMemberRequestBody, RunManually, Run, LogEvent - - -class _Page: - def __init__(self, values, has_next, next_after): - self.has_next = has_next - self.values = values - self.next_after = next_after - - @staticmethod - def empty(): - return _Page([], False, None) - - @staticmethod - def initial(after): - return _Page([], True, after) - - -class _PageIterator: - def __init__(self, page: _Page, get_next_page): - self.page = page - self.get_next_page = get_next_page - - def __iter__(self): - return self - - def __next__(self): - if not self.page.values: - if self.page.has_next: - self.page = self.get_next_page(self.page) - if not self.page.values: - raise StopIteration - return self.page.values.pop(0) +from influxdb_client.client._pages import _Paginated class TasksApi(object): @@ -80,11 +49,7 @@ def find_tasks_iter(self, **kwargs): :key int limit: the number of tasks in one page :return: Tasks iterator """ - - def get_next_page(page: _Page): - return self._find_tasks_next_page(page, **kwargs) - - return iter(_PageIterator(_Page.initial(kwargs.get('after')), get_next_page)) + return _Paginated(self._service.get_tasks, lambda response: response.tasks).find_iter(**kwargs) def create_task(self, task: Task = None, task_create_request: TaskCreateRequest = None) -> Task: """Create a new task.""" @@ -259,16 +224,3 @@ def get_logs(self, task_id: str) -> List['LogEvent']: def find_tasks_by_user(self, task_user_id): """List all tasks by user.""" return self.find_tasks(user=task_user_id) - - def _find_tasks_next_page(self, page: _Page, **kwargs): - if not page.has_next: - return _Page.empty() - - args = {**kwargs, 'after': page.next_after} if page.next_after is not None else kwargs - tasks_response = self._service.get_tasks(**args) - - tasks = tasks_response.tasks - has_next = tasks_response.links.next is not None - last_id = tasks[-1].id if tasks else None - - return _Page(tasks, has_next, last_id) diff --git a/tests/test_BucketsApi.py b/tests/test_BucketsApi.py index db7e28d1..58bbd280 100644 --- a/tests/test_BucketsApi.py +++ b/tests/test_BucketsApi.py @@ -83,26 +83,65 @@ def test_create_bucket_retention_list(self): self.delete_test_bucket(my_bucket) - def test_pagination(self): + def test_find_buckets(self): my_org = self.find_my_org() - buckets = self.buckets_api.find_buckets().buckets + buckets = self.buckets_api.find_buckets(limit=100).buckets size = len(buckets) # create 2 buckets self.buckets_api.create_bucket(bucket_name=generate_bucket_name(), org=my_org) self.buckets_api.create_bucket(bucket_name=generate_bucket_name(), org=my_org) - buckets = self.buckets_api.find_buckets().buckets + buckets = self.buckets_api.find_buckets(limit=size + 2).buckets self.assertEqual(size + 2, len(buckets)) # offset 1 - buckets = self.buckets_api.find_buckets(offset=1).buckets + buckets = self.buckets_api.find_buckets(offset=1, limit=size + 2).buckets self.assertEqual(size + 1, len(buckets)) # count 1 buckets = self.buckets_api.find_buckets(limit=1).buckets self.assertEqual(1, len(buckets)) + def test_find_buckets_iter(self): + def count_unique_ids(items): + return len(set(map(lambda item: item.id, items))) + + my_org = self.find_my_org() + more_buckets = 10 + num_of_buckets = count_unique_ids(self.buckets_api.find_buckets_iter()) + more_buckets + + a_bucket_name = None + for _ in range(more_buckets): + bucket_name = self.generate_name("it find_buckets_iter") + self.buckets_api.create_bucket(bucket_name=bucket_name, org=my_org) + a_bucket_name = bucket_name + + # get no buckets + buckets = self.buckets_api.find_buckets_iter(name=a_bucket_name + "blah") + self.assertEqual(count_unique_ids(buckets), 0) + + # get bucket by name + buckets = self.buckets_api.find_buckets_iter(name=a_bucket_name) + self.assertEqual(count_unique_ids(buckets), 1) + + # get buckets in 3-4 batches + buckets = self.buckets_api.find_buckets_iter(limit=num_of_buckets // 3) + self.assertEqual(count_unique_ids(buckets), num_of_buckets) + + # get buckets in one batch + buckets = self.buckets_api.find_buckets_iter(limit=num_of_buckets) + self.assertEqual(count_unique_ids(buckets), num_of_buckets) + + # get buckets in one batch, requesting too much + buckets = self.buckets_api.find_buckets_iter(limit=num_of_buckets + 1) + self.assertEqual(count_unique_ids(buckets), num_of_buckets) + + # skip some buckets + *_, skip_bucket = self.buckets_api.find_buckets(limit=num_of_buckets // 3).buckets + buckets = self.buckets_api.find_buckets_iter(after=skip_bucket.id) + self.assertEqual(count_unique_ids(buckets), num_of_buckets - num_of_buckets // 3) + def test_update_bucket(self): my_org = self.find_my_org() From bd0eff2183a829ba2d741a237520680e6e3c1423 Mon Sep 17 00:00:00 2001 From: Jakub Bednar Date: Mon, 24 Jun 2024 11:47:12 +0200 Subject: [PATCH 28/51] chore(release): release version 1.44.0 [skip CI] --- CHANGELOG.md | 2 +- influxdb_client/version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c6efa56..eb9cd952 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## 1.44.0 [unreleased] +## 1.44.0 [2024-06-24] ### Features 1. [#657](https://github.com/influxdata/influxdb-client-python/pull/657): Prefer datetime.fromisoformat over dateutil.parse in Python 3.11+ diff --git a/influxdb_client/version.py b/influxdb_client/version.py index a445d0b6..e40ba040 100644 --- a/influxdb_client/version.py +++ b/influxdb_client/version.py @@ -1,3 +1,3 @@ """Version of the Client that is used in User-Agent header.""" -VERSION = '1.44.0dev0' +VERSION = '1.44.0' From b844a823b504a80c692376cc189050eed6067e1a Mon Sep 17 00:00:00 2001 From: Jakub Bednar Date: Mon, 24 Jun 2024 11:51:46 +0200 Subject: [PATCH 29/51] chore(release): prepare for next development iteration --- CHANGELOG.md | 2 ++ conda/meta.yaml | 6 +++--- influxdb_client/version.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb9cd952..83a29b5d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +## 1.45.0 [unreleased] + ## 1.44.0 [2024-06-24] ### Features diff --git a/conda/meta.yaml b/conda/meta.yaml index 186640f0..b250ec14 100644 --- a/conda/meta.yaml +++ b/conda/meta.yaml @@ -1,5 +1,5 @@ {% set name = "influxdb_client" %} -{% set version = "1.43.0" %} +{% set version = "1.44.0" %} package: @@ -7,8 +7,8 @@ package: version: {{ version }} source: - url: https://files.pythonhosted.org/packages/3a/1f/d610ac86af1204bb12698a4d9ac4bd743141e01c13dc44d2e5a8bcf9c556/influxdb_client-1.43.0.tar.gz - sha256: ae2614d891baed52c0ae8f6194a04ee5b1c6422f6061318a3639fe63b7671b25 + url: https://files.pythonhosted.org/packages/9e/a1/ab4f2a3b90334c2e7cb795fbc85483a30134078b1bad0a165a34cb827aa7/influxdb_client-1.44.0.tar.gz + sha256: da9bc0cc49de4a0ac844d833c1efa65227ec5a2254e63cdbe07b5d532c0c37f8 build: number: 0 diff --git a/influxdb_client/version.py b/influxdb_client/version.py index e40ba040..237fe182 100644 --- a/influxdb_client/version.py +++ b/influxdb_client/version.py @@ -1,3 +1,3 @@ """Version of the Client that is used in User-Agent header.""" -VERSION = '1.44.0' +VERSION = '1.45.0dev0' From 653af4657265755ff718c2f03339616d036fea3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Bedn=C3=A1=C5=99?= Date: Wed, 26 Jun 2024 15:00:00 +0200 Subject: [PATCH 30/51] refactor: to timezone specific datetime helper to avoid use deprecated functions (#652) --- CHANGELOG.md | 3 +++ README.md | 10 +++++----- examples/asynchronous.py | 2 +- examples/example.py | 6 +++--- examples/influx_cloud.py | 5 +++-- examples/logging_handler.py | 2 +- examples/write_structured_data.py | 4 ++-- influxdb_client/client/write/point.py | 2 +- tests/test_InfluxDBClientAsync.py | 11 ++++++----- tests/test_MultiprocessingWriter.py | 6 +++--- tests/test_PandasDateTimeHelper.py | 2 +- 11 files changed, 29 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 83a29b5d..8f9df7e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ ## 1.45.0 [unreleased] +### Bug Fixes +1. [#652](https://github.com/influxdata/influxdb-client-python/pull/652): Refactor to `timezone` specific `datetime` helpers to avoid use deprecated functions + ## 1.44.0 [2024-06-24] ### Features diff --git a/README.md b/README.md index ce78bd00..ef4eff86 100644 --- a/README.md +++ b/README.md @@ -392,7 +392,7 @@ The batching is configurable by `write_options`: | **exponential_base** | the base for the exponential retry delay, the next delay is computed using random exponential backoff as a random value within the interval `retry_interval * exponential_base^(attempts-1)` and `retry_interval * exponential_base^(attempts)`. Example for `retry_interval=5_000, exponential_base=2, max_retry_delay=125_000, total=5` Retry delays are random distributed values within the ranges of `[5_000-10_000, 10_000-20_000, 20_000-40_000, 40_000-80_000, 80_000-125_000]` | `2` | ``` python -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone import pandas as pd import reactivex as rx @@ -456,7 +456,7 @@ with InfluxDBClient(url="http://localhost:8086", token="my-token", org="my-org") """ Write Pandas DataFrame """ - _now = datetime.utcnow() + _now = datetime.now(tz=timezone.utc) _data_frame = pd.DataFrame(data=[["coyote_creek", 1.0], ["coyote_creek", 2.0]], index=[_now, _now + timedelta(hours=1)], columns=["location", "water_level"]) @@ -923,7 +923,7 @@ The last step is run a python script via: `python3 influx_cloud.py`. Connect to InfluxDB 2.0 - write data and query them """ -from datetime import datetime +from datetime import datetime, timezone from influxdb_client import Point, InfluxDBClient from influxdb_client.client.write_api import SYNCHRONOUS @@ -945,7 +945,7 @@ try: """ Write data by Point structure """ - point = Point(kind).tag('host', host).tag('device', device).field('value', 25.3).time(time=datetime.utcnow()) + point = Point(kind).tag('host', host).tag('device', device).field('value', 25.3).time(time=datetime.now(tz=timezone.utc)) print(f'Writing to InfluxDB cloud: {point.to_line_protocol()} ...') @@ -1407,7 +1407,7 @@ The `influxdb_client.client.query_api_async.QueryApiAsync` supports retrieve dat > > async def main(): > async with InfluxDBClientAsync(url="http://localhost:8086", token="my-token", org="my-org") as client: -> start = datetime.utcfromtimestamp(0) +> start = datetime.fromtimestamp(0) > stop = datetime.now() > # Delete data with location = 'Prague' > successfully = await client.delete_api().delete(start=start, stop=stop, bucket="my-bucket", diff --git a/examples/asynchronous.py b/examples/asynchronous.py index 4205d461..ad0b876c 100644 --- a/examples/asynchronous.py +++ b/examples/asynchronous.py @@ -76,7 +76,7 @@ async def main(): Delete data """ print(f"\n------- Delete data with location = 'Prague' -------\n") - successfully = await client.delete_api().delete(start=datetime.utcfromtimestamp(0), stop=datetime.now(), + successfully = await client.delete_api().delete(start=datetime.fromtimestamp(0), stop=datetime.now(), predicate="location = \"Prague\"", bucket="my-bucket") print(f" > successfully: {successfully}") diff --git a/examples/example.py b/examples/example.py index 0082ade1..f6ac61f6 100644 --- a/examples/example.py +++ b/examples/example.py @@ -1,5 +1,5 @@ import codecs -from datetime import datetime +from datetime import datetime, timezone from influxdb_client import WritePrecision, InfluxDBClient, Point from influxdb_client.client.write_api import SYNCHRONOUS @@ -7,8 +7,8 @@ with InfluxDBClient(url="http://localhost:8086", token="my-token", org="my-org", debug=False) as client: query_api = client.query_api() - p = Point("my_measurement").tag("location", "Prague").field("temperature", 25.3).time(datetime.utcnow(), - WritePrecision.MS) + p = Point("my_measurement").tag("location", "Prague").field("temperature", 25.3) \ + .time(datetime.now(tz=timezone.utc), WritePrecision.MS) write_api = client.write_api(write_options=SYNCHRONOUS) # write using point structure diff --git a/examples/influx_cloud.py b/examples/influx_cloud.py index 6c8ed6f2..96b0fc3c 100644 --- a/examples/influx_cloud.py +++ b/examples/influx_cloud.py @@ -2,7 +2,7 @@ Connect to InfluxDB 2.0 - write data and query them """ -from datetime import datetime +from datetime import datetime, timezone from influxdb_client import Point, InfluxDBClient from influxdb_client.client.write_api import SYNCHRONOUS @@ -23,7 +23,8 @@ """ Write data by Point structure """ - point = Point(kind).tag('host', host).tag('device', device).field('value', 25.3).time(time=datetime.utcnow()) + point = Point(kind).tag('host', host).tag('device', device).field('value', 25.3) \ + .time(time=datetime.now(tz=timezone.utc)) print(f'Writing to InfluxDB cloud: {point.to_line_protocol()} ...') diff --git a/examples/logging_handler.py b/examples/logging_handler.py index 08f2ae05..6f875f7b 100644 --- a/examples/logging_handler.py +++ b/examples/logging_handler.py @@ -45,7 +45,7 @@ def use_logger(): Point('my-measurement') .tag('host', 'host1') .field('temperature', 25.3) - .time(datetime.datetime.utcnow(), WritePrecision.MS) + .time(datetime.datetime.now(tz=datetime.timezone.utc), WritePrecision.MS) ) diff --git a/examples/write_structured_data.py b/examples/write_structured_data.py index 26a904f3..14a4e8ae 100644 --- a/examples/write_structured_data.py +++ b/examples/write_structured_data.py @@ -1,6 +1,6 @@ from collections import namedtuple from dataclasses import dataclass -from datetime import datetime +from datetime import datetime, timezone from influxdb_client import InfluxDBClient from influxdb_client.client.write_api import SYNCHRONOUS @@ -37,7 +37,7 @@ class Car: version="2021.06.05.5874", pressure=125, temperature=10, - timestamp=datetime.utcnow()) + timestamp=datetime.now(tz=timezone.utc)) print(sensor) """ diff --git a/influxdb_client/client/write/point.py b/influxdb_client/client/write/point.py index 31d44d5c..cc95d204 100644 --- a/influxdb_client/client/write/point.py +++ b/influxdb_client/client/write/point.py @@ -10,7 +10,7 @@ from influxdb_client.client.util.date_utils import get_date_helper from influxdb_client.domain.write_precision import WritePrecision -EPOCH = datetime.utcfromtimestamp(0).replace(tzinfo=timezone.utc) +EPOCH = datetime.fromtimestamp(0, tz=timezone.utc) DEFAULT_WRITE_PRECISION = WritePrecision.NS diff --git a/tests/test_InfluxDBClientAsync.py b/tests/test_InfluxDBClientAsync.py index 123967a7..20eabd7d 100644 --- a/tests/test_InfluxDBClientAsync.py +++ b/tests/test_InfluxDBClientAsync.py @@ -2,7 +2,7 @@ import logging import unittest import os -from datetime import datetime +from datetime import datetime, timezone from io import StringIO import pytest @@ -202,11 +202,11 @@ async def test_write_empty_data(self): async def test_write_points_different_precision(self): measurement = generate_name("measurement") _point1 = Point(measurement).tag("location", "Prague").field("temperature", 25.3) \ - .time(datetime.utcfromtimestamp(0), write_precision=WritePrecision.S) + .time(datetime.fromtimestamp(0, tz=timezone.utc), write_precision=WritePrecision.S) _point2 = Point(measurement).tag("location", "New York").field("temperature", 24.3) \ - .time(datetime.utcfromtimestamp(1), write_precision=WritePrecision.MS) + .time(datetime.fromtimestamp(1, tz=timezone.utc), write_precision=WritePrecision.MS) _point3 = Point(measurement).tag("location", "Berlin").field("temperature", 24.3) \ - .time(datetime.utcfromtimestamp(2), write_precision=WritePrecision.NS) + .time(datetime.fromtimestamp(2, tz=timezone.utc), write_precision=WritePrecision.NS) await self.client.write_api().write(bucket="my-bucket", record=[_point1, _point2, _point3], write_precision=WritePrecision.NS) query = f''' @@ -228,7 +228,8 @@ async def test_delete_api(self): measurement = generate_name("measurement") await self._prepare_data(measurement) - successfully = await self.client.delete_api().delete(start=datetime.utcfromtimestamp(0), stop=datetime.utcnow(), + successfully = await self.client.delete_api().delete(start=datetime.fromtimestamp(0), + stop=datetime.now(tz=timezone.utc), predicate="location = \"Prague\"", bucket="my-bucket") self.assertEqual(True, successfully) query = f''' diff --git a/tests/test_MultiprocessingWriter.py b/tests/test_MultiprocessingWriter.py index 940ae6ec..e7996b5f 100644 --- a/tests/test_MultiprocessingWriter.py +++ b/tests/test_MultiprocessingWriter.py @@ -1,6 +1,6 @@ import os import unittest -from datetime import datetime +from datetime import datetime, timezone from influxdb_client import WritePrecision, InfluxDBClient from influxdb_client.client.util.date_utils import get_date_helper @@ -53,7 +53,7 @@ def test_use_context_manager(self): self.assertIsNotNone(writer) def test_pass_parameters(self): - unique = get_date_helper().to_nanoseconds(datetime.utcnow() - datetime.utcfromtimestamp(0)) + unique = get_date_helper().to_nanoseconds(datetime.now(tz=timezone.utc) - datetime.fromtimestamp(0, tz=timezone.utc)) # write data with MultiprocessingWriter(url=self.url, token=self.token, org=self.org, write_options=SYNCHRONOUS) as writer: @@ -69,4 +69,4 @@ def test_pass_parameters(self): self.assertIsNotNone(record) self.assertEqual("a", record["tag"]) self.assertEqual(5, record["_value"]) - self.assertEqual(get_date_helper().to_utc(datetime.utcfromtimestamp(10)), record["_time"]) + self.assertEqual(get_date_helper().to_utc(datetime.fromtimestamp(10, tz=timezone.utc)), record["_time"]) diff --git a/tests/test_PandasDateTimeHelper.py b/tests/test_PandasDateTimeHelper.py index 60017172..2c7e4ce5 100644 --- a/tests/test_PandasDateTimeHelper.py +++ b/tests/test_PandasDateTimeHelper.py @@ -23,7 +23,7 @@ def test_parse_date(self): def test_to_nanoseconds(self): date = self.helper.parse_date('2020-08-07T06:21:57.331249158Z').replace(tzinfo=timezone.utc) - nanoseconds = self.helper.to_nanoseconds(date - datetime.utcfromtimestamp(0).replace(tzinfo=timezone.utc)) + nanoseconds = self.helper.to_nanoseconds(date - datetime.fromtimestamp(0, tz=timezone.utc)) self.assertEqual(nanoseconds, 1596781317331249158) From 74a0fbf95518bec3981f1bd24930154af9433162 Mon Sep 17 00:00:00 2001 From: Jacob Marble Date: Thu, 8 Aug 2024 22:02:21 -0700 Subject: [PATCH 31/51] fix(write-api): accept 201 response to write (#663) InfluxDB v3 will soon return 201 or 204, in cases where InfluxDB v1 and v2 only return 204. --- CHANGELOG.md | 1 + influxdb_client/client/write_api_async.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f9df7e5..05c37a17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ### Bug Fixes 1. [#652](https://github.com/influxdata/influxdb-client-python/pull/652): Refactor to `timezone` specific `datetime` helpers to avoid use deprecated functions +1. [#663](https://github.com/influxdata/influxdb-client-python/pull/663): Accept HTTP 201 response to write request ## 1.44.0 [2024-06-24] diff --git a/influxdb_client/client/write_api_async.py b/influxdb_client/client/write_api_async.py index 2f32802f..e9e2018b 100644 --- a/influxdb_client/client/write_api_async.py +++ b/influxdb_client/client/write_api_async.py @@ -122,4 +122,4 @@ async def write(self, bucket: str, org: str = None, precision=write_precision, async_req=False, _return_http_data_only=False, content_type="text/plain; charset=utf-8") - return response[1] == 204 + return response[1] in (201, 204) From db1630d52ecd29f2f1008ca25c742fa26ee3777c Mon Sep 17 00:00:00 2001 From: Jakub Bednar Date: Mon, 12 Aug 2024 10:15:34 +0200 Subject: [PATCH 32/51] chore(release): release version 1.45.0 [skip CI] --- CHANGELOG.md | 2 +- influxdb_client/version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 05c37a17..a07e311e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## 1.45.0 [unreleased] +## 1.45.0 [2024-08-12] ### Bug Fixes 1. [#652](https://github.com/influxdata/influxdb-client-python/pull/652): Refactor to `timezone` specific `datetime` helpers to avoid use deprecated functions diff --git a/influxdb_client/version.py b/influxdb_client/version.py index 237fe182..cc536e49 100644 --- a/influxdb_client/version.py +++ b/influxdb_client/version.py @@ -1,3 +1,3 @@ """Version of the Client that is used in User-Agent header.""" -VERSION = '1.45.0dev0' +VERSION = '1.45.0' From 0dcc35bc62a3a2cbed3d32d6e50480a88961a819 Mon Sep 17 00:00:00 2001 From: Jakub Bednar Date: Mon, 12 Aug 2024 10:20:01 +0200 Subject: [PATCH 33/51] chore(release): prepare for next development iteration --- CHANGELOG.md | 2 ++ conda/meta.yaml | 6 +++--- influxdb_client/version.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a07e311e..9dab2d69 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +## 1.46.0 [unreleased] + ## 1.45.0 [2024-08-12] ### Bug Fixes diff --git a/conda/meta.yaml b/conda/meta.yaml index b250ec14..0c626075 100644 --- a/conda/meta.yaml +++ b/conda/meta.yaml @@ -1,5 +1,5 @@ {% set name = "influxdb_client" %} -{% set version = "1.44.0" %} +{% set version = "1.45.0" %} package: @@ -7,8 +7,8 @@ package: version: {{ version }} source: - url: https://files.pythonhosted.org/packages/9e/a1/ab4f2a3b90334c2e7cb795fbc85483a30134078b1bad0a165a34cb827aa7/influxdb_client-1.44.0.tar.gz - sha256: da9bc0cc49de4a0ac844d833c1efa65227ec5a2254e63cdbe07b5d532c0c37f8 + url: https://files.pythonhosted.org/packages/71/cd/a016f327d0669074526b36ae7c1bb84760e3c0d29911f6e8e4046a217f32/influxdb_client-1.45.0.tar.gz + sha256: e24aa0a838f58487b2382c654fa8183fb5ca504af70438a42ca20dd79669a2be build: number: 0 diff --git a/influxdb_client/version.py b/influxdb_client/version.py index cc536e49..4a0a5c92 100644 --- a/influxdb_client/version.py +++ b/influxdb_client/version.py @@ -1,3 +1,3 @@ """Version of the Client that is used in User-Agent header.""" -VERSION = '1.45.0' +VERSION = '1.46.0dev0' From 45e6607a94efbdb23fc607ca7540c74ff0749009 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Bedn=C3=A1=C5=99?= Date: Mon, 12 Aug 2024 21:23:37 +0200 Subject: [PATCH 34/51] fix: multiprocessing example (#664) * fix: multiprocessing example * docs: update CHANGELOG.md --- CHANGELOG.md | 3 + examples/import_data_set_multiprocessing.py | 129 ++++++++++---------- 2 files changed, 68 insertions(+), 64 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9dab2d69..923317ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ ## 1.46.0 [unreleased] +### Examples: +1. [#664](https://github.com/influxdata/influxdb-client-python/pull/664/): Multiprocessing example uses new source of data + ## 1.45.0 [2024-08-12] ### Bug Fixes diff --git a/examples/import_data_set_multiprocessing.py b/examples/import_data_set_multiprocessing.py index 60de64c5..b20b6174 100644 --- a/examples/import_data_set_multiprocessing.py +++ b/examples/import_data_set_multiprocessing.py @@ -4,6 +4,7 @@ https://github.com/toddwschneider/nyc-taxi-data """ import concurrent.futures +import gzip import io import multiprocessing from collections import OrderedDict @@ -92,10 +93,10 @@ def parse_row(row: OrderedDict): return Point("taxi-trip-data") \ .tag("dispatching_base_num", row['dispatching_base_num']) \ - .tag("PULocationID", row['PULocationID']) \ - .tag("DOLocationID", row['DOLocationID']) \ + .tag("PULocationID", row['PUlocationID']) \ + .tag("DOLocationID", row['DOlocationID']) \ .tag("SR_Flag", row['SR_Flag']) \ - .field("dropoff_datetime", row['dropoff_datetime']) \ + .field("dropoff_datetime", row['dropOff_datetime']) \ .time(row['pickup_datetime']) \ .to_line_protocol() @@ -113,7 +114,7 @@ def parse_rows(rows, total_size): counter_.value += len(_parsed_rows) if counter_.value % 10_000 == 0: print('{0:8}{1}'.format(counter_.value, ' - {0:.2f} %' - .format(100 * float(progress_.value) / float(int(total_size))) if total_size else "")) + .format(float(progress_.value) / float(int(total_size))) if total_size else "")) pass queue_.put(_parsed_rows) @@ -141,80 +142,80 @@ def init_counter(counter, progress, queue): progress_ = Value('i', 0) startTime = datetime.now() - url = "https://s3.amazonaws.com/nyc-tlc/trip+data/fhv_tripdata_2019-01.csv" - # url = "file:///Users/bednar/Developer/influxdata/influxdb-client-python/examples/fhv_tripdata_2019-01.csv" + url = "https://github.com/DataTalksClub/nyc-tlc-data/releases/download/fhv/fhv_tripdata_2019-01.csv.gz" """ Open URL and for stream data """ response = urlopen(url) - if response.headers: - content_length = response.headers['Content-length'] - io_wrapper = ProgressTextIOWrapper(response) - io_wrapper.progress = progress_ + # we can't get content length from response because the gzip stream content length is unknown + # so we set it to this value, just for progress display + content_length = 23143223 """ - Start writer as a new process + Open GZIP stream """ - writer = InfluxDBWriter(queue_) - writer.start() + with gzip.open(response, 'rb') as stream: + io_wrapper = ProgressTextIOWrapper(stream, encoding='utf-8') + io_wrapper.progress = progress_ - """ - Create process pool for parallel encoding into LineProtocol - """ - cpu_count = multiprocessing.cpu_count() - with concurrent.futures.ProcessPoolExecutor(cpu_count, initializer=init_counter, - initargs=(counter_, progress_, queue_)) as executor: """ - Converts incoming HTTP stream into sequence of LineProtocol + Start writer as a new process """ - data = rx \ - .from_iterable(DictReader(io_wrapper)) \ - .pipe(ops.buffer_with_count(10_000), - # Parse 10_000 rows into LineProtocol on subprocess - ops.flat_map(lambda rows: executor.submit(parse_rows, rows, content_length))) + writer = InfluxDBWriter(queue_) + writer.start() """ - Write data into InfluxDB + Create process pool for parallel encoding into LineProtocol """ - data.subscribe(on_next=lambda x: None, on_error=lambda ex: print(f'Unexpected error: {ex}')) - - """ - Terminate Writer - """ - queue_.put(None) - queue_.join() + cpu_count = multiprocessing.cpu_count() + with concurrent.futures.ProcessPoolExecutor(cpu_count, initializer=init_counter, + initargs=(counter_, progress_, queue_)) as executor: + """ + Converts incoming HTTP stream into sequence of LineProtocol + """ + data = rx \ + .from_iterable(DictReader(io_wrapper)) \ + .pipe(ops.buffer_with_count(10_000), + # Parse 10_000 rows into LineProtocol on subprocess + ops.map(lambda rows: executor.submit(parse_rows, rows, content_length))) + + """ + Write data into InfluxDB + """ + data.subscribe(on_next=lambda x: None, on_error=lambda ex: print(f'Unexpected error: {ex}')) - print() - print(f'Import finished in: {datetime.now() - startTime}') - print() - - """ - Querying 10 pickups from dispatching 'B00008' - """ - query = 'from(bucket:"my-bucket")' \ - '|> range(start: 2019-01-01T00:00:00Z, stop: now()) ' \ - '|> filter(fn: (r) => r._measurement == "taxi-trip-data")' \ - '|> filter(fn: (r) => r.dispatching_base_num == "B00008")' \ - '|> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value")' \ - '|> rename(columns: {_time: "pickup_datetime"})' \ - '|> drop(columns: ["_start", "_stop"])|> limit(n:10, offset: 0)' - - client = InfluxDBClient(url="http://localhost:8086", token="my-token", org="my-org", debug=False) - result = client.query_api().query(query=query) + """ + Terminate Writer + """ + queue_.put(None) + queue_.join() - """ - Processing results - """ - print() - print("=== Querying 10 pickups from dispatching 'B00008' ===") - print() - for table in result: - for record in table.records: - print( - f'Dispatching: {record["dispatching_base_num"]} pickup: {record["pickup_datetime"]} dropoff: {record["dropoff_datetime"]}') + print() + print(f'Import finished in: {datetime.now() - startTime}') + print() - """ - Close client - """ - client.close() + """ + Querying 10 pickups from dispatching 'B00008' + """ + query = 'from(bucket:"my-bucket")' \ + '|> range(start: 2019-01-01T00:00:00Z, stop: now()) ' \ + '|> filter(fn: (r) => r._measurement == "taxi-trip-data")' \ + '|> filter(fn: (r) => r.dispatching_base_num == "B00008")' \ + '|> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value")' \ + '|> rename(columns: {_time: "pickup_datetime"})' \ + '|> drop(columns: ["_start", "_stop"])|> limit(n:10, offset: 0)' + + with InfluxDBClient(url="http://localhost:8086", token="my-token", org="my-org", debug=False) as client: + result = client.query_api().query(query=query) + + """ + Processing results + """ + print() + print("=== Querying 10 pickups from dispatching 'B00008' ===") + print() + for table in result: + for record in table.records: + print( + f'Dispatching: {record["dispatching_base_num"]} pickup: {record["pickup_datetime"]} dropoff: {record["dropoff_datetime"]}') From 7ad95339bfb08f25e7bf8e4da2b1bd379667b9ec Mon Sep 17 00:00:00 2001 From: karel-rehor Date: Wed, 21 Aug 2024 09:36:23 +0200 Subject: [PATCH 35/51] feat: add `headers` field to `InfluxDBError` and add example of use (#665) --- CHANGELOG.md | 1 + examples/README.md | 1 + examples/http_error_handling.py | 126 +++++++++++++++++++++++++++ influxdb_client/client/exceptions.py | 2 + tests/test_InfluxDBClientAsync.py | 19 ++++ tests/test_WriteApi.py | 37 ++++++++ 6 files changed, 186 insertions(+) create mode 100644 examples/http_error_handling.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 923317ab..e22bf238 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ### Examples: 1. [#664](https://github.com/influxdata/influxdb-client-python/pull/664/): Multiprocessing example uses new source of data +1. [#665](https://github.com/influxdata/influxdb-client-python/pull/665): Shows how to leverage header fields in errors returned on write. ## 1.45.0 [2024-08-12] diff --git a/examples/README.md b/examples/README.md index 1678d00e..2b42ffd7 100644 --- a/examples/README.md +++ b/examples/README.md @@ -15,6 +15,7 @@ - manually download [NYC TLC Trip Record Data](https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page) - install Apache Arrow `pip install pyarrow` dependency - [write_batching_by_bytes_count.py](write_batching_by_bytes_count.py) - How to use RxPY to prepare batches by maximum bytes count. +- [http_error_handling.py](http_error_handling.py) - How to leverage HttpHeader information when errors are returned on write. ## Queries - [query.py](query.py) - How to query data into `FluxTable`s, `Stream` and `CSV` diff --git a/examples/http_error_handling.py b/examples/http_error_handling.py new file mode 100644 index 00000000..c125a7ff --- /dev/null +++ b/examples/http_error_handling.py @@ -0,0 +1,126 @@ +""" +Illustrates getting header values from Errors that may occur on write. + +To test against cloud set the following environment variables: + INFLUX_URL + INFLUX_TOKEN + INFLUX_DATABASE + INFLUX_ORG + +...otherwise will run against a standard OSS endpoint. +""" +import asyncio +import os +from typing import MutableMapping + +from influxdb_client import InfluxDBClient +from influxdb_client.client.exceptions import InfluxDBError +from influxdb_client.client.influxdb_client_async import InfluxDBClientAsync +from influxdb_client.client.write_api import SYNCHRONOUS +from influxdb_client.rest import ApiException + + +def get_envar(key, default): + try: + return os.environ[key] + except: + return default + + +class Config(object): + + def __init__(self): + self.url = get_envar("INFLUX_URL", "http://localhost:8086") + self.token = get_envar("INFLUX_TOKEN", "my-token") + self.bucket = get_envar("INFLUX_DATABASE", "my-bucket") + self.org = get_envar("INFLUX_ORG", "my-org") + + def __str__(self): + return (f"config:\n" + f" url: {self.url}\n" + f" token: ****redacted*****\n" + f" bucket: {self.bucket}\n" + f" org: {self.org}\n" + ) + + +# To encapsulate functions used in batch writing +class BatchCB(object): + + def success(self, conf: (str, str, str), data: str): + print(f"Write success: {conf}, data: {data}") + + def error(self, conf: (str, str, str), data: str, exception: InfluxDBError): + print(f"\nBatch -> Write failed: {conf}, data: {data}, error: {exception.message}") + report_headers(exception.headers) + + def retry(self, conf: (str, str, str), data: str, exception: InfluxDBError): + print(f"Write failed but retryable: {conf}, data: {data}, error: {exception}") + + +# simple reporter that server is available +def report_ping(ping: bool): + if not ping: + raise ValueError("InfluxDB: Failed to ping server") + else: + print("InfluxDB: ready") + + +# report some useful expected header fields +def report_headers(headers: MutableMapping[str, str]): + print(" Date: ", headers.get("Date")) + print(" X-Influxdb-Build: ", headers.get("X-Influxdb-Build")) + print(" X-Influxdb-Version: ", headers.get("X-Influxdb-Version")) # OSS version, Cloud should be None + print(" X-Platform-Error-Code: ", headers.get("X-Platform-Error-Code")) # OSS invalid, Cloud should be None + print(" Retry-After: ", headers.get("Retry-After")) # Should be None + print(" Trace-Id: ", headers.get("Trace-Id")) # OSS should be None, Cloud should return value + + +# try a write using a synchronous call +def use_sync(conf: Config): + print("Using sync") + with InfluxDBClient(url=conf.url, token=conf.token, org=conf.org) as client: + report_ping(client.ping()) + try: + client.write_api(write_options=SYNCHRONOUS).write(bucket=conf.bucket, record="cpu,location=G4 usage=") + except ApiException as ae: + print("\nSync -> Caught ApiException: ", ae.message) + report_headers(ae.headers) + + print("Sync write done") + + +# try a write using batch API +def use_batch(conf: Config): + print("Using batch") + with InfluxDBClient(url=conf.url, token=conf.token, org=conf.org) as client: + cb = BatchCB() + with client.write_api(success_callback=cb.success, + error_callback=cb.error, + retry_callback=cb.retry) as write_api: + write_api.write(bucket=conf.bucket, record="cpu,location=G9 usage=") + print("Batch write sent") + print("Batch write done") + + +# try a write using async.io +async def use_async(conf: Config): + print("Using async") + async with InfluxDBClientAsync(url=conf.url, token=conf.token, org=conf.org) as client: + report_ping(await client.ping()) + try: + await client.write_api().write(bucket=conf.bucket, record="cpu,location=G7 usage=") + except InfluxDBError as ie: + print("\nAsync -> Caught InfluxDBError: ", ie.message) + report_headers(ie.headers) + print("Async write done") + + +if __name__ == "__main__": + conf = Config() + print(conf) + use_sync(conf) + print("\n Continuing...\n") + use_batch(conf) + print("\n Continuing...\n") + asyncio.run(use_async(conf)) diff --git a/influxdb_client/client/exceptions.py b/influxdb_client/client/exceptions.py index 2ca235c8..bfa453e2 100644 --- a/influxdb_client/client/exceptions.py +++ b/influxdb_client/client/exceptions.py @@ -16,8 +16,10 @@ def __init__(self, response: HTTPResponse = None, message: str = None): self.response = response self.message = self._get_message(response) if isinstance(response, HTTPResponse): # response is HTTPResponse + self.headers = response.headers self.retry_after = response.headers.get('Retry-After') else: # response is RESTResponse + self.headers = response.getheaders() self.retry_after = response.getheader('Retry-After') else: self.response = None diff --git a/tests/test_InfluxDBClientAsync.py b/tests/test_InfluxDBClientAsync.py index 20eabd7d..7f8c6214 100644 --- a/tests/test_InfluxDBClientAsync.py +++ b/tests/test_InfluxDBClientAsync.py @@ -1,5 +1,6 @@ import asyncio import logging +import re import unittest import os from datetime import datetime, timezone @@ -390,6 +391,24 @@ async def test_query_exception_propagation(self): await self.client.query_api().query("buckets()", "my-org") self.assertEqual("unauthorized access", e.value.message) + @async_test + async def test_write_exception_propagation(self): + await self.client.close() + self.client = InfluxDBClientAsync(url="http://localhost:8086", token="wrong", org="my-org") + + with pytest.raises(InfluxDBError) as e: + await self.client.write_api().write(bucket="my_bucket", + record="temperature,location=hic cels=") + self.assertEqual("unauthorized access", e.value.message) + headers = e.value.headers + self.assertIsNotNone(headers) + self.assertIsNotNone(headers.get("Content-Length")) + self.assertIsNotNone(headers.get("Date")) + self.assertIsNotNone(headers.get("X-Platform-Error-Code")) + self.assertIn("application/json", headers.get("Content-Type")) + self.assertTrue(re.compile("^v.*").match(headers.get("X-Influxdb-Version"))) + self.assertEqual("OSS", headers.get("X-Influxdb-Build")) + @async_test @aioresponses() async def test_parse_utf8_two_bytes_character(self, mocked): diff --git a/tests/test_WriteApi.py b/tests/test_WriteApi.py index 474bf394..b2cc7ca7 100644 --- a/tests/test_WriteApi.py +++ b/tests/test_WriteApi.py @@ -3,12 +3,16 @@ from __future__ import absolute_import import datetime +import json +import logging import os +import re import sys import unittest from collections import namedtuple from datetime import timedelta from multiprocessing.pool import ApplyResult +from types import SimpleNamespace import httpretty import pytest @@ -190,6 +194,17 @@ def test_write_error(self): self.assertEqual(400, exception.status) self.assertEqual("Bad Request", exception.reason) + # assert headers + self.assertIsNotNone(exception.headers) + self.assertIsNotNone(exception.headers.get("Content-Length")) + self.assertIsNotNone(exception.headers.get("Date")) + self.assertIsNotNone(exception.headers.get("X-Platform-Error-Code")) + self.assertIn("application/json", exception.headers.get("Content-Type")) + self.assertTrue(re.compile("^v.*").match(exception.headers.get("X-Influxdb-Version"))) + self.assertEqual("OSS", exception.headers.get("X-Influxdb-Build")) + # assert body + b = json.loads(exception.body, object_hook=lambda d: SimpleNamespace(**d)) + self.assertTrue(re.compile("^unable to parse.*invalid field format").match(b.message)) def test_write_dictionary(self): _bucket = self.create_test_bucket() @@ -609,6 +624,28 @@ def test_write_result(self): self.assertEqual(None, result.get()) self.delete_test_bucket(_bucket) + def test_write_error(self): + _bucket = self.create_test_bucket() + + _record = "h2o_feet,location=coyote_creek level\\ water_level=" + result = self.write_client.write(_bucket.name, self.org, _record) + + with self.assertRaises(ApiException) as cm: + result.get() + self.assertEqual(400, cm.exception.status) + self.assertEqual("Bad Request", cm.exception.reason) + # assert headers + self.assertIsNotNone(cm.exception.headers) + self.assertIsNotNone(cm.exception.headers.get("Content-Length")) + self.assertIsNotNone(cm.exception.headers.get("Date")) + self.assertIsNotNone(cm.exception.headers.get("X-Platform-Error-Code")) + self.assertIn("application/json", cm.exception.headers.get("Content-Type")) + self.assertTrue(re.compile("^v.*").match(cm.exception.headers.get("X-Influxdb-Version"))) + self.assertEqual("OSS", cm.exception.headers.get("X-Influxdb-Build")) + # assert body + b = json.loads(cm.exception.body, object_hook=lambda d: SimpleNamespace(**d)) + self.assertTrue(re.compile("^unable to parse.*missing field value").match(b.message)) + def test_write_dictionaries(self): bucket = self.create_test_bucket() From 63949b53efd8c19ad954dca82c0fd09fe26af1e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Bedn=C3=A1=C5=99?= Date: Mon, 2 Sep 2024 12:49:43 +0200 Subject: [PATCH 36/51] fix: add py.typed to the package definition (#667) --- CHANGELOG.md | 3 +++ setup.py | 1 + 2 files changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e22bf238..8006956a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ ## 1.46.0 [unreleased] +### Bug Fixes +1. [#667](https://github.com/influxdata/influxdb-client-python/pull/667): Missing `py.typed` in distribution package + ### Examples: 1. [#664](https://github.com/influxdata/influxdb-client-python/pull/664/): Multiprocessing example uses new source of data 1. [#665](https://github.com/influxdata/influxdb-client-python/pull/665): Shows how to leverage header fields in errors returned on write. diff --git a/setup.py b/setup.py index 5596713f..6d4a34cb 100644 --- a/setup.py +++ b/setup.py @@ -66,6 +66,7 @@ extras_require={'extra': extra_requires, 'ciso': ciso_requires, 'async': async_requires, 'test': test_requires}, long_description_content_type="text/markdown", packages=find_packages(exclude=('tests*',)), + package_data={'influxdb_client': ['py.typed']}, test_suite='tests', python_requires='>=3.7', include_package_data=True, From dfd815d92dd26817396b9594c79c6c6fd977bd36 Mon Sep 17 00:00:00 2001 From: Jakub Bednar Date: Fri, 13 Sep 2024 08:57:42 +0200 Subject: [PATCH 37/51] chore(release): release version 1.46.0 [skip CI] --- CHANGELOG.md | 2 +- influxdb_client/version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8006956a..742634f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## 1.46.0 [unreleased] +## 1.46.0 [2024-09-13] ### Bug Fixes 1. [#667](https://github.com/influxdata/influxdb-client-python/pull/667): Missing `py.typed` in distribution package diff --git a/influxdb_client/version.py b/influxdb_client/version.py index 4a0a5c92..413edc0d 100644 --- a/influxdb_client/version.py +++ b/influxdb_client/version.py @@ -1,3 +1,3 @@ """Version of the Client that is used in User-Agent header.""" -VERSION = '1.46.0dev0' +VERSION = '1.46.0' From 70ce8cb765ec742a96b665e6d41c97e7cd59ba8a Mon Sep 17 00:00:00 2001 From: Jakub Bednar Date: Fri, 13 Sep 2024 09:01:22 +0200 Subject: [PATCH 38/51] chore(release): prepare for next development iteration --- CHANGELOG.md | 2 ++ conda/meta.yaml | 6 +++--- influxdb_client/version.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 742634f7..ebaccef3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +## 1.47.0 [unreleased] + ## 1.46.0 [2024-09-13] ### Bug Fixes diff --git a/conda/meta.yaml b/conda/meta.yaml index 0c626075..2595c51e 100644 --- a/conda/meta.yaml +++ b/conda/meta.yaml @@ -1,5 +1,5 @@ {% set name = "influxdb_client" %} -{% set version = "1.45.0" %} +{% set version = "1.46.0" %} package: @@ -7,8 +7,8 @@ package: version: {{ version }} source: - url: https://files.pythonhosted.org/packages/71/cd/a016f327d0669074526b36ae7c1bb84760e3c0d29911f6e8e4046a217f32/influxdb_client-1.45.0.tar.gz - sha256: e24aa0a838f58487b2382c654fa8183fb5ca504af70438a42ca20dd79669a2be + url: https://files.pythonhosted.org/packages/53/9e/4bd499eff06eab47f7995178623d508703d2b4fedab1a3544b04ef06fb0c/influxdb_client-1.46.0.tar.gz + sha256: d5b5f3787db8ad75e64bf069fdc4d441e43b1a1d57f2c11082af309ef0b9722c build: number: 0 diff --git a/influxdb_client/version.py b/influxdb_client/version.py index 413edc0d..76745307 100644 --- a/influxdb_client/version.py +++ b/influxdb_client/version.py @@ -1,3 +1,3 @@ """Version of the Client that is used in User-Agent header.""" -VERSION = '1.46.0' +VERSION = '1.47.0dev0' From 3d70dbd66e434d5ac58e8d018e566d3089223db3 Mon Sep 17 00:00:00 2001 From: Vitaly Chait Date: Tue, 24 Sep 2024 15:24:49 +0300 Subject: [PATCH 39/51] fix: url attribute type validation (#672) --- CHANGELOG.md | 3 +++ influxdb_client/client/_base.py | 2 ++ tests/test_InfluxDBClient.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 34 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ebaccef3..6937e242 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ ## 1.47.0 [unreleased] +### Bug Fixes +1. [#672](https://github.com/influxdata/influxdb-client-python/pull/672): Adding type validation to url attribute in client object + ## 1.46.0 [2024-09-13] ### Bug Fixes diff --git a/influxdb_client/client/_base.py b/influxdb_client/client/_base.py index 8dcf75e9..d4f17901 100644 --- a/influxdb_client/client/_base.py +++ b/influxdb_client/client/_base.py @@ -53,6 +53,8 @@ def __init__(self, url, token, debug=None, timeout=10_000, enable_gzip=False, or self.default_tags = default_tags self.conf = _Configuration() + if not isinstance(self.url, str): + raise ValueError('"url" attribute is not str instance') if self.url.endswith("/"): self.conf.host = self.url[:-1] else: diff --git a/tests/test_InfluxDBClient.py b/tests/test_InfluxDBClient.py index 7fdf834f..228f391b 100644 --- a/tests/test_InfluxDBClient.py +++ b/tests/test_InfluxDBClient.py @@ -323,6 +323,35 @@ def test_version(self): version = self.client.version() self.assertTrue(len(version) > 0) + def test_url_attribute(self): + # Wrong URL attribute + wrong_types = [ + None, + True, False, + 123, 123.5, + dict({"url" : "http://localhost:8086"}), + list(["http://localhost:8086"]), + tuple(("http://localhost:8086")) + ] + correct_types = [ + "http://localhost:8086" + ] + for url_type in wrong_types: + try: + client_not_running = InfluxDBClient(url=url_type, token="my-token", debug=True) + status = True + except ValueError as e: + status = False + self.assertFalse(status) + for url_type in correct_types: + try: + client_not_running = InfluxDBClient(url=url_type, token="my-token", debug=True) + status = True + except ValueError as e: + status = False + self.assertTrue(status) + + def test_build(self): build = self.client.build() self.assertEqual('oss', build.lower()) From 28a4a048345f18d7dcc64ed85e5b9fe91899f0a9 Mon Sep 17 00:00:00 2001 From: Daniel <158782574+youarecode@users.noreply.github.com> Date: Tue, 8 Oct 2024 05:44:24 -0300 Subject: [PATCH 40/51] fix: type linting at client.flux_table.FluxTable (#677) --- CHANGELOG.md | 3 ++- influxdb_client/client/flux_table.py | 4 ++-- setup.py | 1 - 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6937e242..1511f9f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,8 @@ ## 1.47.0 [unreleased] ### Bug Fixes -1. [#672](https://github.com/influxdata/influxdb-client-python/pull/672): Adding type validation to url attribute in client object +1. [#672](https://github.com/influxdata/influxdb-client-python/pull/672): Add type validation to url attribute in client object +1. [#674](https://github.com/influxdata/influxdb-client-python/pull/674): Add type linting to client.flux_table.FluxTable, remove duplicated `from pathlib import Path` at setup.py ## 1.46.0 [2024-09-13] diff --git a/influxdb_client/client/flux_table.py b/influxdb_client/client/flux_table.py index 98a83159..5fd9a061 100644 --- a/influxdb_client/client/flux_table.py +++ b/influxdb_client/client/flux_table.py @@ -46,8 +46,8 @@ class FluxTable(FluxStructure): def __init__(self) -> None: """Initialize defaults.""" - self.columns = [] - self.records = [] + self.columns: List[FluxColumn] = [] + self.records: List[FluxRecord] = [] def get_group_key(self): """ diff --git a/setup.py b/setup.py index 6d4a34cb..cda0d087 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,6 @@ 'aiocsv>=1.2.2' ] -from pathlib import Path this_directory = Path(__file__).parent long_description = (this_directory / "README.md").read_text() From 7e01edbe6ce61e4c4bdd18addf8c3a64f32385a3 Mon Sep 17 00:00:00 2001 From: karel-rehor Date: Wed, 9 Oct 2024 09:57:10 +0200 Subject: [PATCH 41/51] fix: async write prec where DEFAULT_PRECISION should not be used (#675) * fix: (WIP) issue 669 write precision to default in async API * chore: fix lint issues * docs: update CHANGELOG.md * chore: improve indexing of range --- CHANGELOG.md | 6 +- influxdb_client/client/write_api_async.py | 27 ++-- tests/test_InfluxDBClientAsync.py | 149 ++++++++++++++++++++-- 3 files changed, 159 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1511f9f9..e7d08c81 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,10 @@ ## 1.47.0 [unreleased] ### Bug Fixes -1. [#672](https://github.com/influxdata/influxdb-client-python/pull/672): Add type validation to url attribute in client object -1. [#674](https://github.com/influxdata/influxdb-client-python/pull/674): Add type linting to client.flux_table.FluxTable, remove duplicated `from pathlib import Path` at setup.py + +1. [#672](https://github.com/influxdata/influxdb-client-python/pull/672): Adding type validation to url attribute in client object +2. [#674](https://github.com/influxdata/influxdb-client-python/pull/674): Add type linting to client.flux_table.FluxTable, remove duplicated `from pathlib import Path` at setup.py +3. [#675](https://github.com/influxdata/influxdb-client-python/pull/675): Ensures WritePrecision in Point is preferred to `DEFAULT_PRECISION` ## 1.46.0 [2024-09-13] diff --git a/influxdb_client/client/write_api_async.py b/influxdb_client/client/write_api_async.py index e9e2018b..38937eca 100644 --- a/influxdb_client/client/write_api_async.py +++ b/influxdb_client/client/write_api_async.py @@ -1,5 +1,6 @@ """Collect and async write time series data to InfluxDB Cloud or InfluxDB OSS.""" import logging +from asyncio import ensure_future, gather from collections import defaultdict from typing import Union, Iterable, NamedTuple @@ -114,12 +115,20 @@ async def write(self, bucket: str, org: str = None, self._append_default_tags(record) payloads = defaultdict(list) - self._serialize(record, write_precision, payloads, precision_from_point=False, **kwargs) - - # joint list by \n - body = b'\n'.join(payloads[write_precision]) - response = await self._write_service.post_write_async(org=org, bucket=bucket, body=body, - precision=write_precision, async_req=False, - _return_http_data_only=False, - content_type="text/plain; charset=utf-8") - return response[1] in (201, 204) + self._serialize(record, write_precision, payloads, precision_from_point=True, **kwargs) + + futures = [] + for payload_precision, payload_line in payloads.items(): + futures.append(ensure_future + (self._write_service.post_write_async(org=org, bucket=bucket, + body=b'\n'.join(payload_line), + precision=payload_precision, async_req=False, + _return_http_data_only=False, + content_type="text/plain; charset=utf-8"))) + + results = await gather(*futures, return_exceptions=True) + for result in results: + if isinstance(result, Exception): + raise result + + return False not in [re[1] in (201, 204) for re in results] diff --git a/tests/test_InfluxDBClientAsync.py b/tests/test_InfluxDBClientAsync.py index 7f8c6214..cb0586b9 100644 --- a/tests/test_InfluxDBClientAsync.py +++ b/tests/test_InfluxDBClientAsync.py @@ -1,11 +1,15 @@ import asyncio +import dateutil.parser import logging +import math import re +import time import unittest import os from datetime import datetime, timezone from io import StringIO +import pandas import pytest import warnings from aioresponses import aioresponses @@ -199,30 +203,151 @@ async def test_write_empty_data(self): self.assertEqual(True, response) + def gen_fractional_utc(self, nano, precision) -> str: + raw_sec = nano / 1_000_000_000 + if precision == WritePrecision.NS: + rem = f"{nano % 1_000_000_000}".rjust(9,"0").rstrip("0") + return (datetime.fromtimestamp(math.floor(raw_sec), tz=timezone.utc) + .isoformat() + .replace("+00:00", "") + f".{rem}Z") + #f".{rem}Z")) + elif precision == WritePrecision.US: + # rem = f"{round(nano / 1_000) % 1_000_000}"#.ljust(6,"0") + return (datetime.fromtimestamp(round(raw_sec,6), tz=timezone.utc) + .isoformat() + .replace("+00:00","") + .strip("0") + "Z" + ) + elif precision == WritePrecision.MS: + #rem = f"{round(nano / 1_000_000) % 1_000}".rjust(3, "0") + return (datetime.fromtimestamp(round(raw_sec,3), tz=timezone.utc) + .isoformat() + .replace("+00:00","") + .strip("0") + "Z" + ) + elif precision == WritePrecision.S: + return (datetime.fromtimestamp(round(raw_sec), tz=timezone.utc) + .isoformat() + .replace("+00:00","Z")) + else: + raise ValueError(f"Unknown precision: {precision}") + + @async_test async def test_write_points_different_precision(self): + now_ns = time.time_ns() + now_us = now_ns / 1_000 + now_ms = now_us / 1_000 + now_s = now_ms / 1_000 + + now_date_s = self.gen_fractional_utc(now_ns, WritePrecision.S) + now_date_ms = self.gen_fractional_utc(now_ns, WritePrecision.MS) + now_date_us = self.gen_fractional_utc(now_ns, WritePrecision.US) + now_date_ns = self.gen_fractional_utc(now_ns, WritePrecision.NS) + + points = { + WritePrecision.S: [], + WritePrecision.MS: [], + WritePrecision.US: [], + WritePrecision.NS: [] + } + + expected = {} + measurement = generate_name("measurement") - _point1 = Point(measurement).tag("location", "Prague").field("temperature", 25.3) \ - .time(datetime.fromtimestamp(0, tz=timezone.utc), write_precision=WritePrecision.S) - _point2 = Point(measurement).tag("location", "New York").field("temperature", 24.3) \ - .time(datetime.fromtimestamp(1, tz=timezone.utc), write_precision=WritePrecision.MS) - _point3 = Point(measurement).tag("location", "Berlin").field("temperature", 24.3) \ - .time(datetime.fromtimestamp(2, tz=timezone.utc), write_precision=WritePrecision.NS) - await self.client.write_api().write(bucket="my-bucket", record=[_point1, _point2, _point3], + # basic date-time value + points[WritePrecision.S].append(Point(measurement).tag("method", "SecDateTime").field("temperature", 25.3) \ + .time(datetime.fromtimestamp(round(now_s), tz=timezone.utc), write_precision=WritePrecision.S)) + expected['SecDateTime'] = now_date_s + points[WritePrecision.MS].append(Point(measurement).tag("method", "MilDateTime").field("temperature", 24.3) \ + .time(datetime.fromtimestamp(round(now_s,3), tz=timezone.utc), write_precision=WritePrecision.MS)) + expected['MilDateTime'] = now_date_ms + points[WritePrecision.US].append(Point(measurement).tag("method", "MicDateTime").field("temperature", 24.3) \ + .time(datetime.fromtimestamp(round(now_s,6), tz=timezone.utc), write_precision=WritePrecision.US)) + expected['MicDateTime'] = now_date_us + # N.B. datetime does not handle nanoseconds +# points[WritePrecision.NS].append(Point(measurement).tag("method", "NanDateTime").field("temperature", 24.3) \ +# .time(datetime.fromtimestamp(now_s, tz=timezone.utc), write_precision=WritePrecision.NS)) + + # long timestamps based on POSIX time + points[WritePrecision.S].append(Point(measurement).tag("method", "SecPosix").field("temperature", 24.3) \ + .time(round(now_s), write_precision=WritePrecision.S)) + expected['SecPosix'] = now_date_s + points[WritePrecision.MS].append(Point(measurement).tag("method", "MilPosix").field("temperature", 24.3) \ + .time(round(now_ms), write_precision=WritePrecision.MS)) + expected['MilPosix'] = now_date_ms + points[WritePrecision.US].append(Point(measurement).tag("method", "MicPosix").field("temperature", 24.3) \ + .time(round(now_us), write_precision=WritePrecision.US)) + expected['MicPosix'] = now_date_us + points[WritePrecision.NS].append(Point(measurement).tag("method", "NanPosix").field("temperature", 24.3) \ + .time(now_ns, write_precision=WritePrecision.NS)) + expected['NanPosix'] = now_date_ns + + # ISO Zulu datetime with ms, us and ns e.g. "2024-09-27T13:17:16.412399728Z" + points[WritePrecision.S].append(Point(measurement).tag("method", "SecDTZulu").field("temperature", 24.3) \ + .time(now_date_s, write_precision=WritePrecision.S)) + expected['SecDTZulu'] = now_date_s + points[WritePrecision.MS].append(Point(measurement).tag("method", "MilDTZulu").field("temperature", 24.3) \ + .time(now_date_ms, write_precision=WritePrecision.MS)) + expected['MilDTZulu'] = now_date_ms + points[WritePrecision.US].append(Point(measurement).tag("method", "MicDTZulu").field("temperature", 24.3) \ + .time(now_date_us, write_precision=WritePrecision.US)) + expected['MicDTZulu'] = now_date_us + # This keeps resulting in micro second resolution in response +# points[WritePrecision.NS].append(Point(measurement).tag("method", "NanDTZulu").field("temperature", 24.3) \ +# .time(now_date_ns, write_precision=WritePrecision.NS)) + + recs = [x for x in [v for v in points.values()]] + + await self.client.write_api().write(bucket="my-bucket", record=recs, write_precision=WritePrecision.NS) query = f''' from(bucket:"my-bucket") |> range(start: 0) |> filter(fn: (r) => r["_measurement"] == "{measurement}") - |> keep(columns: ["_time"]) + |> keep(columns: ["method","_time"]) ''' query_api = self.client.query_api() + # ensure calls fully processed on server + await asyncio.sleep(1) + raw = await query_api.query_raw(query) - self.assertEqual(8, len(raw.splitlines())) - self.assertEqual(',,0,1970-01-01T00:00:02Z', raw.splitlines()[4]) - self.assertEqual(',,0,1970-01-01T00:00:01Z', raw.splitlines()[5]) - self.assertEqual(',,0,1970-01-01T00:00:00Z', raw.splitlines()[6]) + linesRaw = raw.splitlines()[4:] + + lines = [] + for lnr in linesRaw: + lines.append(lnr[2:].split(",")) + + def get_time_for_method(lines, method): + for l in lines: + if l[2] == method: + return l[1] + return "" + + self.assertEqual(15, len(raw.splitlines())) + + for key in expected: + t = get_time_for_method(lines,key) + comp_time = dateutil.parser.isoparse(get_time_for_method(lines,key)) + target_time = dateutil.parser.isoparse(expected[key]) + self.assertEqual(target_time.date(), comp_time.date()) + self.assertEqual(target_time.hour, comp_time.hour) + self.assertEqual(target_time.second,comp_time.second) + dif = abs(target_time.microsecond - comp_time.microsecond) + if key[:3] == "Sec": + # Already tested + pass + elif key[:3] == "Mil": + # may be slight rounding differences + self.assertLess(dif, 1500, f"failed to match timestamp for {key} {target_time} != {comp_time}") + elif key[:3] == "Mic": + # may be slight rounding differences + self.assertLess(dif, 150, f"failed to match timestamp for {key} {target_time} != {comp_time}") + elif key[:3] == "Nan": + self.assertEqual(expected[key], get_time_for_method(lines, key)) + else: + raise Exception(f"Unhandled key {key}") @async_test async def test_delete_api(self): From 06b71146b20d2f3e7d40eac4fe5d2d81e4b02c62 Mon Sep 17 00:00:00 2001 From: Jakub Bednar Date: Tue, 22 Oct 2024 07:56:31 +0200 Subject: [PATCH 42/51] chore(release): release version 1.47.0 [skip CI] --- CHANGELOG.md | 2 +- influxdb_client/version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e7d08c81..727c6ea1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## 1.47.0 [unreleased] +## 1.47.0 [2024-10-22] ### Bug Fixes diff --git a/influxdb_client/version.py b/influxdb_client/version.py index 76745307..19339327 100644 --- a/influxdb_client/version.py +++ b/influxdb_client/version.py @@ -1,3 +1,3 @@ """Version of the Client that is used in User-Agent header.""" -VERSION = '1.47.0dev0' +VERSION = '1.47.0' From 012c50aae5a72160dfe381f4002c40934a4ab880 Mon Sep 17 00:00:00 2001 From: Jakub Bednar Date: Tue, 22 Oct 2024 08:02:30 +0200 Subject: [PATCH 43/51] chore(release): prepare for next development iteration --- CHANGELOG.md | 2 ++ conda/meta.yaml | 6 +++--- influxdb_client/version.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 727c6ea1..c7e0c837 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +## 1.48.0 [unreleased] + ## 1.47.0 [2024-10-22] ### Bug Fixes diff --git a/conda/meta.yaml b/conda/meta.yaml index 2595c51e..c7598938 100644 --- a/conda/meta.yaml +++ b/conda/meta.yaml @@ -1,5 +1,5 @@ {% set name = "influxdb_client" %} -{% set version = "1.46.0" %} +{% set version = "1.47.0" %} package: @@ -7,8 +7,8 @@ package: version: {{ version }} source: - url: https://files.pythonhosted.org/packages/53/9e/4bd499eff06eab47f7995178623d508703d2b4fedab1a3544b04ef06fb0c/influxdb_client-1.46.0.tar.gz - sha256: d5b5f3787db8ad75e64bf069fdc4d441e43b1a1d57f2c11082af309ef0b9722c + url: https://files.pythonhosted.org/packages/f0/d7/07b6d9c02b975ba7961427af5a40c910871a97f543b4f5762112084cea48/influxdb_client-1.47.0.tar.gz + sha256: 549f2c0ad458bbf79de1291ad5b07b823d80a3bcdbe77b4f0b436461aa008e2b build: number: 0 diff --git a/influxdb_client/version.py b/influxdb_client/version.py index 19339327..0cde499d 100644 --- a/influxdb_client/version.py +++ b/influxdb_client/version.py @@ -1,3 +1,3 @@ """Version of the Client that is used in User-Agent header.""" -VERSION = '1.47.0' +VERSION = '1.48.0dev0' From 44dd00eef8da9416ff9eebf6cb536dfbc4bcc5dc Mon Sep 17 00:00:00 2001 From: karel-rehor Date: Mon, 25 Nov 2024 14:49:34 +0100 Subject: [PATCH 44/51] fix: catch CancelledError and TimeoutError and add note about timeout (#679) * chore: add explanatory note to CancelledError. * docs: update CHANGELOG.md and README.md * chore: remove import of asyncio in non-async API --- CHANGELOG.md | 4 ++++ README.md | 7 +++++++ influxdb_client/client/flux_csv_parser.py | 6 +++++- 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c7e0c837..35a88ac3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ ## 1.48.0 [unreleased] +### Bug Fixes + +1. [#679](https://github.com/influxdata/influxdb-client-python/pull/679): Add note to caught errors about need to check client timeout. + ## 1.47.0 [2024-10-22] ### Bug Fixes diff --git a/README.md b/README.md index ef4eff86..5b541dcf 100644 --- a/README.md +++ b/README.md @@ -1313,6 +1313,13 @@ All async APIs are available via `influxdb_client.client.influxdb_client_async.I and also check to readiness of the InfluxDB via `/ping` endpoint: +The `InfluxDBClientAsync` constructor accepts a number of __configuration properties__. Most useful among these are: + +* `connection_pool_maxsize` - The total number of simultaneous connections. Defaults to `multiprocessing.cpu_count() * 5`. +* `enable_gzip` - enable gzip compression during `write` and `query` calls. Defaults to `false`. +* `proxy` - URL of an HTTP proxy to be used. +* `timeout` - The maximum number of milliseconds for handling HTTP requests from initial handshake to handling response data. This is passed directly to the underlying transport library. If large amounts of data are anticipated, for example from `query_api.query_stream(...)`, this should be increased to avoid `TimeoutError` or `CancelledError`. Defaults to 10_000 ms. + > ``` python > import asyncio > diff --git a/influxdb_client/client/flux_csv_parser.py b/influxdb_client/client/flux_csv_parser.py index 7a73e3f8..99e68094 100644 --- a/influxdb_client/client/flux_csv_parser.py +++ b/influxdb_client/client/flux_csv_parser.py @@ -1,6 +1,5 @@ """Parsing response from InfluxDB to FluxStructures or DataFrame.""" - import base64 import codecs import csv as csv_parser @@ -147,6 +146,11 @@ async def _parse_flux_response_async(self): df = self._prepare_data_frame() if not self._is_profiler_table(metadata.table): yield df + except BaseException as e: + e_type = type(e).__name__ + if "CancelledError" in e_type or "TimeoutError" in e_type: + e.add_note("Stream cancelled during read. Recommended: Check Influxdb client `timeout` setting.") + raise finally: self._close() From c8d806f1bb3357dec5bf62b0fe2eb84e1789b485 Mon Sep 17 00:00:00 2001 From: Jakub Bednar Date: Wed, 27 Nov 2024 09:24:29 +0100 Subject: [PATCH 45/51] chore(release): release version 1.48.0 [skip CI] --- CHANGELOG.md | 2 +- influxdb_client/version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 35a88ac3..7b8b3c58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## 1.48.0 [unreleased] +## 1.48.0 [2024-11-27] ### Bug Fixes diff --git a/influxdb_client/version.py b/influxdb_client/version.py index 0cde499d..2008b9ce 100644 --- a/influxdb_client/version.py +++ b/influxdb_client/version.py @@ -1,3 +1,3 @@ """Version of the Client that is used in User-Agent header.""" -VERSION = '1.48.0dev0' +VERSION = '1.48.0' From 2d0adb9f1b73587643bd49f4eb323e7cd413d1c1 Mon Sep 17 00:00:00 2001 From: Jakub Bednar Date: Wed, 27 Nov 2024 09:29:29 +0100 Subject: [PATCH 46/51] chore(release): prepare for next development iteration --- CHANGELOG.md | 2 ++ conda/meta.yaml | 6 +++--- influxdb_client/version.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b8b3c58..ffb5f768 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +## 1.49.0 [unreleased] + ## 1.48.0 [2024-11-27] ### Bug Fixes diff --git a/conda/meta.yaml b/conda/meta.yaml index c7598938..af5027bd 100644 --- a/conda/meta.yaml +++ b/conda/meta.yaml @@ -1,5 +1,5 @@ {% set name = "influxdb_client" %} -{% set version = "1.47.0" %} +{% set version = "1.48.0" %} package: @@ -7,8 +7,8 @@ package: version: {{ version }} source: - url: https://files.pythonhosted.org/packages/f0/d7/07b6d9c02b975ba7961427af5a40c910871a97f543b4f5762112084cea48/influxdb_client-1.47.0.tar.gz - sha256: 549f2c0ad458bbf79de1291ad5b07b823d80a3bcdbe77b4f0b436461aa008e2b + url: https://files.pythonhosted.org/packages/11/47/b756380917cb4b968bd871fc006128e2cc9897fb1ab4bcf7d108f9601e78/influxdb_client-1.48.0.tar.gz + sha256: 414d5b5eff7d2b6b453f33e2826ea9872ea04a11996ba9c8604b0c1df57c8559 build: number: 0 diff --git a/influxdb_client/version.py b/influxdb_client/version.py index 2008b9ce..a4ac1780 100644 --- a/influxdb_client/version.py +++ b/influxdb_client/version.py @@ -1,3 +1,3 @@ """Version of the Client that is used in User-Agent header.""" -VERSION = '1.48.0' +VERSION = '1.49.0dev0' From ab16384b7e9931da8b74f9a19af89939c0a3b673 Mon Sep 17 00:00:00 2001 From: karel-rehor Date: Wed, 4 Dec 2024 11:19:19 +0100 Subject: [PATCH 47/51] chore: add type checks for Authorization with new example. (#682) * chore: add type checks for Authorization with new example. * docs: update CHANGELOG.md and examples/README.md --- CHANGELOG.md | 8 ++ examples/README.md | 1 + examples/authorizations.py | 103 +++++++++++++++++++ influxdb_client/client/authorizations_api.py | 4 +- influxdb_client/domain/authorization.py | 4 + tests/test_AuthorizationApi.py | 19 ++++ 6 files changed, 138 insertions(+), 1 deletion(-) create mode 100644 examples/authorizations.py diff --git a/CHANGELOG.md b/CHANGELOG.md index ffb5f768..3470d909 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ ## 1.49.0 [unreleased] +### Bug Fixes + +1. [#682](https://github.com/influxdata/influxdb-client-python/pull/682): Check core types when creating Authentication instances. + +### Examples + +1. [#682](https://github.com/influxdata/influxdb-client-python/pull/682): New example for working with Authentication API. + ## 1.48.0 [2024-11-27] ### Bug Fixes diff --git a/examples/README.md b/examples/README.md index 2b42ffd7..7d3a5eea 100644 --- a/examples/README.md +++ b/examples/README.md @@ -28,6 +28,7 @@ - [monitoring_and_alerting.py](monitoring_and_alerting.py) - How to create the Check with Slack notification. - [task_example.py](task_example.py) - How to create a Task by API - [templates_management.py](templates_management.py) - How to use Templates and Stack API +- [authorizations.py](authorizations.py) - How to create and use authorizations. ## InfluxDB Cloud diff --git a/examples/authorizations.py b/examples/authorizations.py new file mode 100644 index 00000000..5857f624 --- /dev/null +++ b/examples/authorizations.py @@ -0,0 +1,103 @@ +import os + +from influxdb_client import InfluxDBClient, BucketRetentionRules, PermissionResource, Permission, Authorization, \ + WriteOptions +from influxdb_client.client.write_api import WriteType +from influxdb_client.rest import ApiException + +HOST_URL = os.environ.get("INFLUX_HOST") if os.environ.get("INFLUX_HOST") is not None else "http://localhost:8086" +TOKEN = os.environ.get("INFLUX_TOKEN") if os.environ.get("INFLUX_TOKEN") is not None else "my-token" +ORG = os.environ.get("INFLUX_ORG") if os.environ.get("INFLUX_ORG") is not None else "my-org" +SYS_BUCKET = os.environ.get("INFLUX_DB") if os.environ.get("INFLUX_DB") is not None else "my-bucket" +BUCKET = "special-bucket" + + +def create_auths(): + # Create authorizations with an initial client using all-access permissions + with InfluxDBClient(url=HOST_URL, token=TOKEN, org=ORG, debug=False) as globalClient: + bucket_rules = BucketRetentionRules(type="expire", every_seconds=3600) + bucket = globalClient.buckets_api().create_bucket(bucket_name=BUCKET, + retention_rules=bucket_rules, + org=ORG) + + bucket_permission_resource_r = PermissionResource(org=ORG, + org_id=bucket.org_id, + type="buckets", + id=bucket.id) + bucket_permission_resource_w = PermissionResource(org=ORG, + org_id=bucket.org_id, + type="buckets", + id=bucket.id) + read_bucket = Permission(action="read", resource=bucket_permission_resource_r) + write_bucket = Permission(action="write", resource=bucket_permission_resource_w) + permissions = [read_bucket, write_bucket] + auth_payload = Authorization(org_id=bucket.org_id, + permissions=permissions, + description="Shared bucket auth from Authorization object", + id="auth1_base") + auth_api = globalClient.authorizations_api() + # use keyword arguments + auth1 = auth_api.create_authorization(authorization=auth_payload) + # or use positional arguments + auth2 = auth_api.create_authorization(bucket.org_id, permissions) + + return auth1, auth2 + + +def try_sys_bucket(client): + print("starting to write") + + w_api = client.write_api(write_options=WriteOptions(write_type=WriteType.synchronous)) + try: + w_api.write(bucket=SYS_BUCKET, record="cpu,host=r2d2 use=3.14") + except ApiException as ae: + print(f"Write to {SYS_BUCKET} failed (as expected) due to:") + print(ae) + + +def try_restricted_bucket(client): + print("starting to write") + w_api = client.write_api(write_options=WriteOptions(write_type=WriteType.synchronous)) + + w_api.write(bucket=BUCKET, record="cpu,host=r2d2 usage=3.14") + print("written") + print("now query") + q_api = client.query_api() + query = f''' + from(bucket:"{BUCKET}") + |> range(start: -5m) + |> filter(fn: (r) => r["_measurement"] == "cpu")''' + + tables = q_api.query(query=query, org=ORG) + for table in tables: + for record in table.records: + print(record["_time"].isoformat(sep="T") + " | " + record["host"] + " | " + record["_field"] + "=" + str(record["_value"])) + + +def main(): + """ + a1 is generated using a local Authorization instance + a2 is generated using local permissions and an internally created Authorization + :return: void + """ + print("=== Setting up authorizations ===") + a1, a2 = create_auths() + + print("=== Using a1 authorization ===") + client1 = InfluxDBClient(url=HOST_URL, token=a1.token, org=ORG, debug=False) + print(" --- Try System Bucket ---") + try_sys_bucket(client1) + print(" --- Try Special Bucket ---") + try_restricted_bucket(client1) + print() + + print("=== Using a2 authorization ===") + client2 = InfluxDBClient(url=HOST_URL, token=a2.token, org=ORG, debug=False) + print(" --- Try System Bucket ---") + try_sys_bucket(client2) + print(" --- Try Special Bucket ---") + try_restricted_bucket(client2) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/influxdb_client/client/authorizations_api.py b/influxdb_client/client/authorizations_api.py index b7179b62..05be6ecd 100644 --- a/influxdb_client/client/authorizations_api.py +++ b/influxdb_client/client/authorizations_api.py @@ -11,7 +11,7 @@ def __init__(self, influxdb_client): self._influxdb_client = influxdb_client self._authorizations_service = AuthorizationsService(influxdb_client.api_client) - def create_authorization(self, org_id=None, permissions: list = None, + def create_authorization(self, org_id: str = None, permissions: list = None, authorization: Authorization = None) -> Authorization: """ Create an authorization. @@ -23,6 +23,8 @@ def create_authorization(self, org_id=None, permissions: list = None, """ if authorization is not None: + if not isinstance(authorization, Authorization): + raise TypeError(f"Attempt to use non-Authorization value for authorization: {authorization}") return self._authorizations_service.post_authorizations(authorization_post_request=authorization) # if org_id is not None and permissions is not None: diff --git a/influxdb_client/domain/authorization.py b/influxdb_client/domain/authorization.py index 67a0bfd3..aef38d9c 100644 --- a/influxdb_client/domain/authorization.py +++ b/influxdb_client/domain/authorization.py @@ -82,8 +82,12 @@ def __init__(self, created_at=None, updated_at=None, org_id=None, permissions=No if updated_at is not None: self.updated_at = updated_at if org_id is not None: + if not isinstance(org_id, str): + raise TypeError("org_id must be a string.") self.org_id = org_id if permissions is not None: + if not isinstance(permissions, list): + raise TypeError("permissions must be a list.") self.permissions = permissions if id is not None: self.id = id diff --git a/tests/test_AuthorizationApi.py b/tests/test_AuthorizationApi.py index 8b1850d9..036f0d60 100644 --- a/tests/test_AuthorizationApi.py +++ b/tests/test_AuthorizationApi.py @@ -45,6 +45,25 @@ def test_createAuthorization(self): self.assertEqual(authorization.links["user"], "/api/v2/users/" + self.user.id) + def test_AuthorizationTypeAssert(self): + self.assertRaisesRegex(TypeError, "org_id must be a string.", Authorization, org_id={}) + self.assertRaisesRegex(TypeError, "permissions must be a list.", Authorization, permissions={}) + + def test_createAuthorizationWrongTypes(self): + user_resource = PermissionResource(org_id=self.organization.id, type="users") + read_users = Permission(action="read", resource=user_resource) + + org_resource = PermissionResource(org_id=self.organization.id, type="orgs") + write_organizations = Permission(action="write", resource=org_resource) + + permissions = [read_users, write_organizations] + self.assertRaisesRegex(TypeError, "org_id must be a string.", + self.authorizations_api.create_authorization, permissions) + self.assertRaisesRegex(TypeError, "permissions must be a list", + self.authorizations_api.create_authorization, "123456789ABCDEF0", "Foo") + self.assertRaisesRegex(TypeError, "Attempt to use non-Authorization value for authorization: Foo", + self.authorizations_api.create_authorization, "123456789ABCDEF0", permissions, "Foo") + def test_authorizationDescription(self): organization = self.my_organization From 74013566a9df3e41dc1ef67cda0cbd0f6b83c733 Mon Sep 17 00:00:00 2001 From: Daniel O'Connor Date: Sat, 15 Feb 2025 18:48:39 +1030 Subject: [PATCH 48/51] docs: minor docstring typo/grammar correction (#687) --- influxdb_client/client/influxdb_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/influxdb_client/client/influxdb_client.py b/influxdb_client/client/influxdb_client.py index 6079aac0..cbae75a9 100644 --- a/influxdb_client/client/influxdb_client.py +++ b/influxdb_client/client/influxdb_client.py @@ -265,7 +265,7 @@ def retry(self, conf: (str, str, str), data: str, exception: InfluxDBError): :param write_options: Write API configuration :param point_settings: settings to store default tags - :key success_callback: The callable ``callback`` to run after successfully writen a batch. + :key success_callback: The callable ``callback`` to run after having successfully written a batch. The callable must accept two arguments: - `Tuple`: ``(bucket, organization, precision)`` @@ -273,7 +273,7 @@ def retry(self, conf: (str, str, str), data: str, exception: InfluxDBError): **[batching mode]** - :key error_callback: The callable ``callback`` to run after unsuccessfully writen a batch. + :key error_callback: The callable ``callback`` to run after having unsuccessfully written a batch. The callable must accept three arguments: - `Tuple`: ``(bucket, organization, precision)`` From 9001fea11a6908d2f0590fde58cf9caa459bcb76 Mon Sep 17 00:00:00 2001 From: Jakub Bednar Date: Thu, 22 May 2025 11:28:21 +0200 Subject: [PATCH 49/51] chore(release): release version 1.49.0 [skip CI] --- CHANGELOG.md | 2 +- influxdb_client/version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3470d909..9eea3a41 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## 1.49.0 [unreleased] +## 1.49.0 [2025-05-22] ### Bug Fixes diff --git a/influxdb_client/version.py b/influxdb_client/version.py index a4ac1780..9c626c0c 100644 --- a/influxdb_client/version.py +++ b/influxdb_client/version.py @@ -1,3 +1,3 @@ """Version of the Client that is used in User-Agent header.""" -VERSION = '1.49.0dev0' +VERSION = '1.49.0' From 4ce3746cffbcad51cd2a16214b47b4579f86333e Mon Sep 17 00:00:00 2001 From: Jakub Bednar Date: Thu, 22 May 2025 13:52:04 +0200 Subject: [PATCH 50/51] chore(release): prepare for next development iteration --- CHANGELOG.md | 2 ++ conda/meta.yaml | 6 +++--- influxdb_client/version.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9eea3a41..af872d37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +## 1.50.0 [unreleased] + ## 1.49.0 [2025-05-22] ### Bug Fixes diff --git a/conda/meta.yaml b/conda/meta.yaml index af5027bd..33a0c26c 100644 --- a/conda/meta.yaml +++ b/conda/meta.yaml @@ -1,5 +1,5 @@ {% set name = "influxdb_client" %} -{% set version = "1.48.0" %} +{% set version = "1.49.0" %} package: @@ -7,8 +7,8 @@ package: version: {{ version }} source: - url: https://files.pythonhosted.org/packages/11/47/b756380917cb4b968bd871fc006128e2cc9897fb1ab4bcf7d108f9601e78/influxdb_client-1.48.0.tar.gz - sha256: 414d5b5eff7d2b6b453f33e2826ea9872ea04a11996ba9c8604b0c1df57c8559 + url: https://files.pythonhosted.org/packages/2a/f3/9c418215cf399529175ed5b198d15a21c2e29f28d90932107634b375c9ee/influxdb_client-1.49.0.tar.gz + sha256: 4a53a218adef6ac9458bfbd31fa08c76194f70310c6b4e01f53d804bd2c48e03 build: number: 0 diff --git a/influxdb_client/version.py b/influxdb_client/version.py index 9c626c0c..03ca288e 100644 --- a/influxdb_client/version.py +++ b/influxdb_client/version.py @@ -1,3 +1,3 @@ """Version of the Client that is used in User-Agent header.""" -VERSION = '1.49.0' +VERSION = '1.50.0dev0' From feb97eef067013881e798b322f90a83e27d07366 Mon Sep 17 00:00:00 2001 From: sonnh <46211823+NguyenHoangSon96@users.noreply.github.com> Date: Tue, 3 Jun 2025 20:12:37 +0700 Subject: [PATCH 51/51] feat: move setuptools to build dependency (#696) --- CHANGELOG.md | 4 ++++ pyproject.toml | 3 +++ setup.py | 1 - 3 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 pyproject.toml diff --git a/CHANGELOG.md b/CHANGELOG.md index af872d37..3da5e8ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ ## 1.50.0 [unreleased] +### Features + +1. [696](https://github.com/influxdata/influxdb-client-python/pull/696): Move "setuptools" package to build dependency. + ## 1.49.0 [2025-05-22] ### Bug Fixes diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..20c12656 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools>=21.0.0"] +build-backend = "setuptools.build_meta" \ No newline at end of file diff --git a/setup.py b/setup.py index cda0d087..76c2748c 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,6 @@ 'reactivex >= 4.0.4', 'certifi >= 14.05.14', 'python_dateutil >= 2.5.3', - 'setuptools >= 21.0.0', 'urllib3 >= 1.26.0' ]