From c6334f8cbb13586d0aa53135f340e0d3edf162c8 Mon Sep 17 00:00:00 2001 From: "tzonghao.chen" Date: Fri, 9 Sep 2016 11:54:14 -0400 Subject: [PATCH 1/2] Fix DataFrameClient empty tag processing Empty or None tags should be left out, otherwise it would break line protocol. --- influxdb/_dataframe_client.py | 14 ++++++++++--- influxdb/tests/dataframe_client_test.py | 27 ++++++++++++++++--------- 2 files changed, 29 insertions(+), 12 deletions(-) diff --git a/influxdb/_dataframe_client.py b/influxdb/_dataframe_client.py index 0341a41d..9eec9254 100644 --- a/influxdb/_dataframe_client.py +++ b/influxdb/_dataframe_client.py @@ -279,13 +279,21 @@ def _convert_dataframe_to_lines(self, # If tag columns exist, make an array of formatted tag keys and values if tag_columns: tag_df = dataframe[tag_columns] + tag_df = tag_df.fillna('') # replace NA with empty string tag_df = tag_df.sort_index(axis=1) tag_df = self._stringify_dataframe( tag_df, numeric_precision, datatype='tag') - tags = (',' + ( - (tag_df.columns.values + '=').tolist() + tag_df)).sum(axis=1) - del tag_df + # prepend tag keys + tag_df = tag_df.apply( + lambda s: s.apply( + lambda v, l: l + '=' + v if v else None, l=s.name)) + + # join tags, but leave out None values + tags = tag_df.apply( + lambda r: ',' + ','.join(r.dropna()), axis=1) + + del tag_df else: tags = '' diff --git a/influxdb/tests/dataframe_client_test.py b/influxdb/tests/dataframe_client_test.py index 782e5c82..eb2e292c 100644 --- a/influxdb/tests/dataframe_client_test.py +++ b/influxdb/tests/dataframe_client_test.py @@ -201,20 +201,29 @@ def test_write_points_from_dataframe_with_tag_cols_and_defaults(self): def test_write_points_from_dataframe_with_tag_escaped(self): now = pd.Timestamp('1970-01-01 00:00+00:00') dataframe = pd.DataFrame( - data=[['blue', 1, "1", 1, 1.0, 'hot'], - ['red,green=orange', 0, "2", 2, 2.0, 'cold']], - index=[now, now + timedelta(hours=1)], - columns=["tag_one", "tag_two", "column_one", - "column_two", "column_three", - "tag_three"]) + data=[ + ['blue orange', "1", 1, 'hot=cold'], # space, equal + ['red,green', "2", 2, r'cold\fire'], # comma, backslash + ['some', "2", 2, ''], # skip empty + ['some', "2", 2, None], # skip None + ], + index=pd.period_range(now, freq='H', periods=4), + columns=["tag_one", "column_one", "column_two", "tag_three"] + ) expected_escaped_tags = ( - b"foo,tag_one=blue " + b"foo,tag_one=blue\\ orange,tag_three=hot\\=cold " b"column_one=\"1\",column_two=1i " b"0\n" - b"foo,tag_one=red\\,green\\=orange " + b"foo,tag_one=red\\,green,tag_three=cold\\\\fire " b"column_one=\"2\",column_two=2i " b"3600000000000\n" + b"foo,tag_one=some " + b"column_one=\"2\",column_two=2i " + b"7200000000000\n" + b"foo,tag_one=some " + b"column_one=\"2\",column_two=2i " + b"10800000000000\n" ) with requests_mock.Mocker() as m: @@ -224,7 +233,7 @@ def test_write_points_from_dataframe_with_tag_escaped(self): cli = DataFrameClient(database='db') cli.write_points(dataframe, 'foo', field_columns=['column_one', 'column_two'], - tag_columns=['tag_one']) + tag_columns=['tag_one', 'tag_three']) self.assertEqual(m.last_request.body, expected_escaped_tags) def test_write_points_from_dataframe_with_numeric_column_names(self): From a7c64405af3e819f6d4b864d5eb7b38a1395bea0 Mon Sep 17 00:00:00 2001 From: "tzonghao.chen" Date: Fri, 9 Sep 2016 12:12:08 -0400 Subject: [PATCH 2/2] Handle all-empty tags --- influxdb/_dataframe_client.py | 5 ++++- influxdb/tests/dataframe_client_test.py | 6 +++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/influxdb/_dataframe_client.py b/influxdb/_dataframe_client.py index 9eec9254..f854f82c 100644 --- a/influxdb/_dataframe_client.py +++ b/influxdb/_dataframe_client.py @@ -291,7 +291,10 @@ def _convert_dataframe_to_lines(self, # join tags, but leave out None values tags = tag_df.apply( - lambda r: ',' + ','.join(r.dropna()), axis=1) + lambda r: ','.join(r.dropna()), axis=1) + + # prepend comma + tags = tags.apply(lambda v: ',' + v if v else '') del tag_df else: diff --git a/influxdb/tests/dataframe_client_test.py b/influxdb/tests/dataframe_client_test.py index eb2e292c..3667b0fc 100644 --- a/influxdb/tests/dataframe_client_test.py +++ b/influxdb/tests/dataframe_client_test.py @@ -206,8 +206,9 @@ def test_write_points_from_dataframe_with_tag_escaped(self): ['red,green', "2", 2, r'cold\fire'], # comma, backslash ['some', "2", 2, ''], # skip empty ['some', "2", 2, None], # skip None + ['', "2", 2, None], # all tags empty ], - index=pd.period_range(now, freq='H', periods=4), + index=pd.period_range(now, freq='H', periods=5), columns=["tag_one", "column_one", "column_two", "tag_three"] ) @@ -224,6 +225,9 @@ def test_write_points_from_dataframe_with_tag_escaped(self): b"foo,tag_one=some " b"column_one=\"2\",column_two=2i " b"10800000000000\n" + b"foo " + b"column_one=\"2\",column_two=2i " + b"14400000000000\n" ) with requests_mock.Mocker() as m: