Skip to content

Commit 391e371

Browse files
nmariznmariz
committed
BUG32623479: The X DevAPI returns str for binary types values
In the X DevAPI implementation of MySQL Connector/Python tries to convert binary types to str. This patch fix the issue and improves efficiency of converting protobuf values to Python. Tests were added for regression.
1 parent 9c9dc69 commit 391e371

File tree

6 files changed

+174
-38
lines changed

6 files changed

+174
-38
lines changed

‎CHANGES.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ v8.0.26
1414
- WL#14542: Deprecate TLS 1.0 and 1.1
1515
- WL#14440: Support for authentication_kerberos_client authentication plugin
1616
- WL#14237: Support query attributes
17+
- BUG#32623479: The X DevAPI returns str for binary types values
1718

1819
v8.0.25
1920
=======

‎lib/mysqlx/crud.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright (c) 2016, 2020, Oracle and/or its affiliates.
1+
# Copyright (c) 2016, 2021, Oracle and/or its affiliates.
22
#
33
# This program is free software; you can redistribute it and/or modify
44
# it under the terms of the GNU General Public License, version 2.0, as
@@ -61,7 +61,7 @@ class DatabaseObject(object):
6161
"""
6262
def __init__(self, schema, name):
6363
self._schema = schema
64-
self._name = name
64+
self._name = name.decode() if isinstance(name, bytes) else name
6565
self._session = self._schema.get_session()
6666
self._connection = self._session.get_connection()
6767

‎lib/mysqlx/result.py

Lines changed: 27 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright (c) 2016, 2020, Oracle and/or its affiliates.
1+
# Copyright (c) 2016, 2021, Oracle and/or its affiliates.
22
#
33
# This program is free software; you can redistribute it and/or modify
44
# it under the terms of the GNU General Public License, version 2.0, as
@@ -40,24 +40,17 @@
4040

4141

4242
# pylint: disable=C0111
43-
def from_protobuf(col_type, payload):
43+
def from_protobuf(column, payload):
4444
if len(payload) == 0:
4545
return None
4646

47+
if column.get_type() == ColumnType.STRING:
48+
return decode_from_bytes(payload[:-1]) # Strip trailing char
49+
4750
try:
48-
return {
49-
ColumnProtoType.SINT: varsint_from_protobuf,
50-
ColumnProtoType.UINT: varint_from_protobuf,
51-
ColumnProtoType.BYTES: bytes_from_protobuf,
52-
ColumnProtoType.DATETIME: datetime_from_protobuf,
53-
ColumnProtoType.TIME: time_from_protobuf,
54-
ColumnProtoType.FLOAT: float_from_protobuf,
55-
ColumnProtoType.DOUBLE: double_from_protobuf,
56-
ColumnProtoType.BIT: varint_from_protobuf,
57-
ColumnProtoType.SET: set_from_protobuf,
58-
ColumnProtoType.ENUM: bytes_from_protobuf,
59-
ColumnProtoType.DECIMAL: decimal_from_protobuf,
60-
}[col_type](payload)
51+
return ColumnProtoType.converter_map[
52+
column.get_proto_type()
53+
](payload)
6154
except KeyError as err:
6255
sys.stderr.write("{0}".format(err))
6356
sys.stderr.write("{0}".format(payload.encode("hex")))
@@ -66,17 +59,17 @@ def from_protobuf(col_type, payload):
6659

6760
def bytes_from_protobuf(payload):
6861
# Strip trailing char
69-
return decode_from_bytes(payload[:-1])
62+
return payload[:-1]
7063

7164

7265
def float_from_protobuf(payload):
7366
assert len(payload) == 4
74-
return struct.unpack("<f", payload)
67+
return struct.unpack("<f", payload)[0]
7568

7669

7770
def double_from_protobuf(payload):
7871
assert len(payload) == 8
79-
return struct.unpack("<d", payload)
72+
return struct.unpack("<d", payload)[0]
8073

8174

8275
def varint_from_protobuf_stream(payload):
@@ -328,6 +321,20 @@ class ColumnProtoType(object):
328321
BIT = 17
329322
DECIMAL = 18
330323

324+
converter_map = {
325+
SINT: varsint_from_protobuf,
326+
UINT: varint_from_protobuf,
327+
BYTES: bytes_from_protobuf,
328+
DATETIME: datetime_from_protobuf,
329+
TIME: time_from_protobuf,
330+
FLOAT: float_from_protobuf,
331+
DOUBLE: double_from_protobuf,
332+
BIT: varint_from_protobuf,
333+
SET: set_from_protobuf,
334+
ENUM: bytes_from_protobuf,
335+
DECIMAL: decimal_from_protobuf,
336+
}
337+
331338

332339
class Flags(object):
333340
def __init__(self, value):
@@ -961,9 +968,8 @@ def _read_item(self, dumping):
961968
item = [None] * len(row["field"])
962969
if not dumping:
963970
for key in range(len(row["field"])):
964-
col = self._columns[key]
965-
item[key] = from_protobuf(col.get_proto_type(),
966-
row["field"][key])
971+
column = self._columns[key]
972+
item[key] = from_protobuf(column, row["field"][key])
967973
return Row(self, item)
968974

969975
def _page_in_items(self):

‎tests/test_mysqlx_connection.py

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# -*- coding: utf-8 -*-
22

3-
# Copyright (c) 2016, 2020, Oracle and/or its affiliates.
3+
# Copyright (c) 2016, 2021, Oracle and/or its affiliates.
44
#
55
# This program is free software; you can redistribute it and/or modify
66
# it under the terms of the GNU General Public License, version 2.0, as
@@ -1519,12 +1519,16 @@ def test_connection_attributes(self):
15191519
# Note that for an empty string "" value the server stores a Null value
15201520
expected_attrs["quash"] = "None"
15211521
for row in rows:
1522-
self.assertEqual(expected_attrs[row.get_string('ATTR_NAME')],
1523-
row.get_string('ATTR_VALUE'),
1522+
attr_name, attr_value = (
1523+
row["ATTR_NAME"].decode(),
1524+
row["ATTR_VALUE"].decode() if row['ATTR_VALUE'] else "None",
1525+
)
1526+
self.assertEqual(expected_attrs[attr_name],
1527+
attr_value,
15241528
"Attribute {} with value {} differs of {}".format(
1525-
row.get_string('ATTR_NAME'),
1526-
row.get_string('ATTR_VALUE'),
1527-
expected_attrs[row.get_string('ATTR_NAME')]))
1529+
attr_name,
1530+
attr_value,
1531+
expected_attrs[attr_name]))
15281532

15291533
# Verify connection-attributes can be skiped to be set on server
15301534
# by URI as "connection_attributes"=false

‎tests/test_mysqlx_crud.py

Lines changed: 124 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# -*- coding: utf-8 -*-
22

3-
# Copyright (c) 2016, 2020, Oracle and/or its affiliates.
3+
# Copyright (c) 2016, 2021, Oracle and/or its affiliates.
44
#
55
# This program is free software; you can redistribute it and/or modify
66
# it under the terms of the GNU General Public License, version 2.0, as
@@ -31,17 +31,21 @@
3131
"""Unittests for mysqlx.crud
3232
"""
3333

34+
import datetime
35+
import decimal
3436
import gc
3537
import json
3638
import logging
37-
import unittest
39+
import sys
3840
import threading
3941
import time
40-
42+
import unittest
4143
import tests
44+
4245
import mysqlx
4346

4447
LOGGER = logging.getLogger(tests.LOGGER_NAME)
48+
ARCH_64BIT = sys.maxsize > 2**32 and sys.platform != "win32"
4549

4650
_CREATE_TEST_TABLE_QUERY = "CREATE TABLE `{0}`.`{1}` (id INT)"
4751
_INSERT_TEST_TABLE_QUERY = "INSERT INTO `{0}`.`{1}` VALUES ({2})"
@@ -2907,6 +2911,123 @@ def test_column_metadata(self):
29072911

29082912
drop_table(self.schema, "test")
29092913

2914+
def test_column_type(self):
2915+
self.session.use_pure = True
2916+
table_name = "column_types"
2917+
columns_names = (
2918+
"my_null",
2919+
"my_bit",
2920+
"my_tinyint",
2921+
"my_smallint",
2922+
"my_mediumint",
2923+
"my_int",
2924+
"my_bigint",
2925+
"my_decimal",
2926+
"my_float",
2927+
"my_double",
2928+
"my_date",
2929+
"my_time",
2930+
"my_datetime",
2931+
"my_year",
2932+
"my_char",
2933+
"my_varchar",
2934+
"my_varbinary",
2935+
"my_enum",
2936+
"my_geometry",
2937+
"my_blob",
2938+
)
2939+
create_table_stmt = (
2940+
f"CREATE TABLE {table_name} ("
2941+
"id INT NOT NULL AUTO_INCREMENT, "
2942+
"my_null INT, "
2943+
"my_bit BIT(7), "
2944+
"my_tinyint TINYINT, "
2945+
"my_smallint SMALLINT, "
2946+
"my_mediumint MEDIUMINT, "
2947+
"my_int INT, "
2948+
"my_bigint BIGINT, "
2949+
"my_decimal DECIMAL(20,10), "
2950+
"my_float FLOAT, "
2951+
"my_double DOUBLE, "
2952+
"my_date DATE, "
2953+
"my_time TIME, "
2954+
"my_datetime DATETIME, "
2955+
"my_year YEAR, "
2956+
"my_char CHAR(100), "
2957+
"my_varchar VARCHAR(100), "
2958+
"my_varbinary VARBINARY(100), "
2959+
"my_enum ENUM('x-small', 'small', 'medium', 'large', 'x-large'), "
2960+
"my_geometry GEOMETRY, "
2961+
"my_blob BLOB, "
2962+
"PRIMARY KEY (id))"
2963+
)
2964+
self.session.sql(create_table_stmt).execute()
2965+
table = self.schema.get_table(table_name)
2966+
table.insert(
2967+
*columns_names
2968+
).values(
2969+
None,
2970+
124,
2971+
127,
2972+
32767,
2973+
8388607,
2974+
2147483647,
2975+
4294967295 if ARCH_64BIT else 2147483647,
2976+
"1.2",
2977+
3.14,
2978+
4.28,
2979+
"2018-12-31 00:00:00",
2980+
"12:13:14",
2981+
"2019-02-04 10:36:03",
2982+
2019,
2983+
"abc",
2984+
"abc",
2985+
"MySQL 🐬",
2986+
"x-large",
2987+
mysqlx.expr("ST_GeomFromText('POINT(21 34)')"),
2988+
"random blob data",
2989+
).execute()
2990+
2991+
exp = [
2992+
None,
2993+
124,
2994+
127,
2995+
32767,
2996+
8388607,
2997+
2147483647,
2998+
4294967295 if ARCH_64BIT else 2147483647,
2999+
decimal.Decimal("1.2000000000"),
3000+
3.140000104904175,
3001+
4.28,
3002+
datetime.datetime(2018, 12, 31),
3003+
datetime.timedelta(0, 43994),
3004+
datetime.datetime(2019, 2, 4, 10, 36, 3),
3005+
2019,
3006+
"abc",
3007+
"abc",
3008+
b"MySQL \xf0\x9f\x90\xac",
3009+
b"x-large",
3010+
b"\x00\x00\x00\x00\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x005"
3011+
b"@\x00\x00\x00\x00\x00\x00A@",
3012+
b"random blob data",
3013+
]
3014+
3015+
row = table.select(*columns_names).execute().fetch_one()
3016+
self.assertEqual([row[col] for col in columns_names], exp)
3017+
3018+
row = self.session.sql(
3019+
"SELECT {} FROM {}.{}".format(
3020+
",".join(columns_names),
3021+
self.schema.name,
3022+
table_name,
3023+
)
3024+
).execute().fetch_one()
3025+
self.assertEqual([row[col] for col in columns_names], exp)
3026+
3027+
self.session.sql(
3028+
f"DROP TABLE IF EXISTS {self.schema.name}.{table_name}"
3029+
).execute()
3030+
29103031
def test_is_view(self):
29113032
table_name = "table_test"
29123033
view_name = "view_test"

‎tests/test_mysqlx_pooling.py

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# -*- coding: utf-8 -*-
22

3-
# Copyright (c) 2018, 2020, Oracle and/or its affiliates.
3+
# Copyright (c) 2018, 2021, Oracle and/or its affiliates.
44
#
55
# This program is free software; you can redistribute it and/or modify
66
# it under the terms of the GNU General Public License, version 2.0, as
@@ -1051,12 +1051,16 @@ def test_connection_attributes(self):
10511051
# Note that for an empty string "" value the server stores a Null value
10521052
expected_attrs["quash"] = "None"
10531053
for row in rows:
1054-
self.assertEqual(expected_attrs[row.get_string('ATTR_NAME')],
1055-
row.get_string('ATTR_VALUE'),
1054+
attr_name, attr_value = (
1055+
row["ATTR_NAME"].decode(),
1056+
row["ATTR_VALUE"].decode() if row['ATTR_VALUE'] else "None",
1057+
)
1058+
self.assertEqual(expected_attrs[attr_name],
1059+
attr_value,
10561060
"Attribute {} with value {} differs of {}".format(
1057-
row.get_string('ATTR_NAME'),
1058-
row.get_string('ATTR_VALUE'),
1059-
expected_attrs[row.get_string('ATTR_NAME')]))
1061+
attr_name,
1062+
attr_value,
1063+
expected_attrs[attr_name]))
10601064

10611065

10621066
@unittest.skipIf(tests.MYSQL_VERSION < (5, 7, 14), "XPlugin not compatible")

0 commit comments

Comments
 (0)