Skip to content

Commit 01a9bd5

Browse files
nmariznmariz
committed
WL10452: Add Protobuf C++ extension for Linux variants and Mac OSX
This patch replaces the pure Python implementation of Protobuf by a C++ extension. With this change, the Connector/Python implementation of MySQL X Protobuf now supports Python 2 and 3 versions, on Linux variants and Mac OSX Changes were made in unittests and building/installing setup.
1 parent f38063d commit 01a9bd5

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+3715
-4299
lines changed

‎.gitignore

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,5 +12,5 @@ cpy_server*/
1212
*_output.txt
1313
tests_*.log
1414
dev*.py
15-
/lib/mysqlx/expr_backup.py
16-
/lib/test.py
15+
src/mysqlxpb/mysqlx/mysqlx*
16+

‎.gitmodules

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
[submodule "cpyint"]
22
path = cpyint
33
url = ../connector-python-internal.git
4-
branch = master
4+
branch = master-wl10198

‎MANIFEST.in

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ include MANIFEST.in
99
recursive-include examples *.py
1010
recursive-include lib *.py
1111
recursive-include tests *.py *.csv *.pem *.cnf
12-
recursive-include src *.c *.h *.cc
12+
recursive-include src *.c *.h *.cc *.proto
1313

1414
include docs/README_DOCS.txt
1515

‎cpyint

‎lib/cpy_distutils.py

Lines changed: 194 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# MySQL Connector/Python - MySQL driver written in Python.
2-
# Copyright (c) 2014, 2016, Oracle and/or its affiliates. All rights reserved.
2+
# Copyright (c) 2014, 2017, Oracle and/or its affiliates. All rights reserved.
33

44
# MySQL Connector/Python is licensed under the terms of the GPLv2
55
# <http://www.gnu.org/licenses/old-licenses/gpl-2.0.html>, like most
@@ -35,16 +35,26 @@
3535
import os
3636
import shlex
3737
import struct
38-
from subprocess import Popen, PIPE, STDOUT
38+
from subprocess import Popen, PIPE, STDOUT, check_call
3939
import sys
4040
import platform
41+
import shutil
42+
4143

4244
ARCH_64BIT = sys.maxsize > 2**32 # Works with Python 2.6 and greater
4345
py_arch = '64-bit' if ARCH_64BIT else '32-bit'
4446

4547
CEXT_OPTIONS = [
4648
('with-mysql-capi=', None,
4749
"Location of MySQL C API installation or path to mysql_config"),
50+
('with-protobuf-include-dir=', None,
51+
"Location of Protobuf include directory"),
52+
('with-protobuf-lib-dir=', None,
53+
"Location of Protobuf library directory"),
54+
('with-protoc=', None,
55+
"Location of Protobuf protoc binary"),
56+
('extra-compile-args=', None,
57+
"Extra compile args")
4858
]
4959

5060
CEXT_STATIC_OPTIONS = [
@@ -273,7 +283,11 @@ class BuildExtDynamic(build_ext):
273283

274284
def initialize_options(self):
275285
build_ext.initialize_options(self)
286+
self.extra_compile_args = None
276287
self.with_mysql_capi = None
288+
self.with_protobuf_include_dir = None
289+
self.with_protobuf_lib_dir = None
290+
self.with_protoc = None
277291

278292
def _finalize_connector_c(self, connc_loc):
279293
"""Finalize the --with-connector-c command line argument
@@ -381,9 +395,6 @@ def _finalize_connector_c(self, connc_loc):
381395

382396
# We try to offer a nice message when the architecture of Python
383397
# is not the same as MySQL Connector/C binaries.
384-
py_arch = '64-bit' if ARCH_64BIT else '32-bit'
385-
print("# Python architecture: {0}".format(py_arch))
386-
print("# Python ARCH_64BIT: {0}".format(ARCH_64BIT))
387398
print("# self.arch: {0}".format(self.arch))
388399
if ARCH_64BIT != connc_64bit:
389400
log.error("Python is {0}, but does not "
@@ -395,14 +406,62 @@ def _finalize_connector_c(self, connc_loc):
395406
sys.exit(1)
396407

397408
def finalize_options(self):
398-
self.set_undefined_options('install',
399-
('with_mysql_capi', 'with_mysql_capi'))
409+
self.set_undefined_options(
410+
'install',
411+
('extra_compile_args', 'extra_compile_args'),
412+
('with_mysql_capi', 'with_mysql_capi'),
413+
('with_protobuf_include_dir', 'with_protobuf_include_dir'),
414+
('with_protobuf_lib_dir', 'with_protobuf_lib_dir'),
415+
('with_protoc', 'with_protoc'))
400416

401417
build_ext.finalize_options(self)
402418

419+
print("# Python architecture: {0}".format(py_arch))
420+
print("# Python ARCH_64BIT: {0}".format(ARCH_64BIT))
421+
403422
if self.with_mysql_capi:
404423
self._finalize_connector_c(self.with_mysql_capi)
405424

425+
if not self.with_protobuf_include_dir:
426+
self.with_protobuf_include_dir = \
427+
os.environ.get("MYSQLXPB_PROTOBUF_INCLUDE_DIR")
428+
429+
if not self.with_protobuf_lib_dir:
430+
self.with_protobuf_lib_dir = \
431+
os.environ.get("MYSQLXPB_PROTOBUF_LIB_DIR")
432+
433+
if not self.with_protoc:
434+
self.with_protoc = os.environ.get("MYSQLXPB_PROTOC")
435+
436+
def run_protoc(self):
437+
if self.with_protobuf_include_dir:
438+
print("# Protobuf include directory: {0}"
439+
"".format(self.with_protobuf_include_dir))
440+
else:
441+
log.error("Unable to find Protobuf include directory.")
442+
sys.exit(1)
443+
444+
if self.with_protobuf_lib_dir:
445+
print("# Protobuf library directory: {0}"
446+
"".format(self.with_protobuf_lib_dir))
447+
else:
448+
log.error("Unable to find Protobuf library directory.")
449+
sys.exit(1)
450+
451+
if self.with_protoc:
452+
print("# Protobuf protoc binary: {0}".format(self.with_protoc))
453+
else:
454+
log.error("Unable to find Protobuf protoc binary.")
455+
sys.exit(1)
456+
457+
base_path = os.path.join(os.getcwd(), "src", "mysqlxpb", "mysqlx")
458+
command = [self.with_protoc, "-I"]
459+
command.append(os.path.join(base_path, "protocol"))
460+
command.extend(glob(os.path.join(base_path, "protocol", "*.proto")))
461+
command.append("--cpp_out={0}".format(base_path))
462+
log.info("# Running protoc command: {0}".format(" ".join(command)))
463+
check_call(command)
464+
406465
def fix_compiler(self):
407466
platform = get_platform()
408467

@@ -444,6 +503,18 @@ def fix_compiler(self):
444503
# Add system headers to Extensions extra_compile_args
445504
sysheaders = [ '-isystem' + dir for dir in cc.include_dirs]
446505
for ext in self.extensions:
506+
# Add Protobuf include and library dirs
507+
if ext.name == "_mysqlxpb":
508+
ext.include_dirs.append(self.with_protobuf_include_dir)
509+
ext.library_dirs.append(self.with_protobuf_lib_dir)
510+
if os.name == 'nt':
511+
ext.libraries.append("libprotobuf")
512+
else:
513+
ext.libraries.append("protobuf")
514+
# Add extra compile args
515+
if self.extra_compile_args:
516+
ext.extra_compile_args.append(self.extra_compile_args)
517+
# Add system headers
447518
for sysheader in sysheaders:
448519
if sysheader not in ext.extra_compile_args:
449520
ext.extra_compile_args.append(sysheader)
@@ -454,16 +525,26 @@ def fix_compiler(self):
454525

455526
def run(self):
456527
"""Run the command"""
457-
if not self.with_mysql_capi:
458-
return
459-
460528
if os.name == 'nt':
529+
for ext in self.extensions:
530+
# Add Protobuf include and library dirs
531+
if ext.name == "_mysqlxpb":
532+
ext.include_dirs.append(self.with_protobuf_include_dir)
533+
ext.library_dirs.append(self.with_protobuf_lib_dir)
534+
ext.libraries.append("libprotobuf")
535+
# Use the multithread, static version of the run-time library
536+
ext.extra_compile_args.append("/MT")
537+
# Add extra compile args
538+
if self.extra_compile_args:
539+
ext.extra_compile_args.extend(self.extra_compile_args)
540+
self.run_protoc()
461541
build_ext.run(self)
462542
else:
463543
self.real_build_extensions = self.build_extensions
464544
self.build_extensions = lambda: None
465545
build_ext.run(self)
466546
self.fix_compiler()
547+
self.run_protoc()
467548
self.real_build_extensions()
468549

469550

@@ -474,17 +555,66 @@ class BuildExtStatic(BuildExtDynamic):
474555
user_options = build_ext.user_options + CEXT_OPTIONS
475556

476557
def finalize_options(self):
558+
options_pairs = []
477559
if not self.with_mysql_capi:
478-
self.set_undefined_options('install',
479-
('with_mysql_capi', 'with_mysql_capi'))
560+
options_pairs.append(('with_mysql_capi', 'with_mysql_capi'))
561+
if not self.with_protobuf_include_dir:
562+
options_pairs.append(('with_protobuf_include_dir',
563+
'with_protobuf_include_dir'))
564+
if not self.with_protobuf_lib_dir:
565+
options_pairs.append(('with_protobuf_lib_dir',
566+
'with_protobuf_lib_dir'))
567+
if not self.with_protoc:
568+
options_pairs.append(('with_protoc', 'with_protoc'))
569+
if options_pairs:
570+
self.set_undefined_options('install', *options_pairs)
480571

481572
build_ext.finalize_options(self)
573+
574+
print("# Python architecture: {0}".format(py_arch))
575+
print("# Python ARCH_64BIT: {0}".format(ARCH_64BIT))
576+
482577
self.connc_lib = os.path.join(self.build_temp, 'connc', 'lib')
483578
self.connc_include = os.path.join(self.build_temp, 'connc', 'include')
579+
self.protobuf_lib = os.path.join(self.build_temp, 'protobuf', 'lib')
580+
self.protobuf_include = os.path.join(self.build_temp, 'protobuf', 'include')
484581

485582
if self.with_mysql_capi:
486583
self._finalize_connector_c(self.with_mysql_capi)
487584

585+
if not self.with_protobuf_include_dir:
586+
self.with_protobuf_include_dir = \
587+
os.environ.get("MYSQLXPB_PROTOBUF_INCLUDE_DIR")
588+
589+
if not self.with_protobuf_lib_dir:
590+
self.with_protobuf_lib_dir = \
591+
os.environ.get("MYSQLXPB_PROTOBUF_LIB_DIR")
592+
593+
if not self.with_protoc:
594+
self.with_protoc = os.environ.get("MYSQLXPB_PROTOC")
595+
596+
if self.with_protobuf_include_dir:
597+
print("# Protobuf include directory: {0}"
598+
"".format(self.with_protobuf_include_dir))
599+
else:
600+
log.error("Unable to find Protobuf include directory.")
601+
sys.exit(1)
602+
603+
if self.with_protobuf_lib_dir:
604+
print("# Protobuf library directory: {0}"
605+
"".format(self.with_protobuf_lib_dir))
606+
else:
607+
log.error("Unable to find Protobuf library directory.")
608+
sys.exit(1)
609+
610+
if self.with_protoc:
611+
print("# Protobuf protoc binary: {0}".format(self.with_protoc))
612+
else:
613+
log.error("Unable to find Protobuf protoc binary.")
614+
sys.exit(1)
615+
616+
self._finalize_protobuf()
617+
488618
def _finalize_connector_c(self, connc_loc):
489619
if not os.path.isdir(connc_loc):
490620
log.error("MySQL C API should be a directory")
@@ -503,6 +633,39 @@ def _finalize_connector_c(self, connc_loc):
503633
if os.path.isfile(lib_file_path) and not lib_file.endswith('.a'):
504634
os.unlink(os.path.join(self.connc_lib, lib_file))
505635

636+
def _finalize_protobuf(self):
637+
if not os.path.isdir(self.with_protobuf_lib_dir):
638+
log.error("Protobuf library dir should be a directory")
639+
sys.exit(1)
640+
641+
if not os.path.isdir(self.with_protobuf_include_dir):
642+
log.error("Protobuf include dir should be a directory")
643+
sys.exit(1)
644+
645+
if not os.path.exists(self.protobuf_lib):
646+
os.makedirs(self.protobuf_lib)
647+
648+
if not os.path.exists(self.protobuf_include):
649+
os.makedirs(self.protobuf_include)
650+
651+
log.info("Copying Protobuf libraries")
652+
lib_files = glob(os.path.join(self.with_protobuf_lib_dir, "libprotobuf*"))
653+
for lib_file in lib_files:
654+
if os.path.isfile(lib_file):
655+
log.info("copying {0} -> {1}".format(lib_file, self.protobuf_lib))
656+
shutil.copy2(lib_file, self.protobuf_lib)
657+
658+
log.info("Copying Protobuf header files")
659+
copy_tree(self.with_protobuf_include_dir, self.protobuf_include)
660+
661+
# Remove all but static libraries to force static linking
662+
if os.name == "posix":
663+
log.info("Removing non-static Protobuf libraries from {0}"
664+
"".format(self.protobuf_lib))
665+
for lib_file in os.listdir(self.protobuf_lib):
666+
lib_file_path = os.path.join(self.protobuf_lib, lib_file)
667+
if os.path.isfile(lib_file_path) and not lib_file.endswith(".a"):
668+
os.unlink(os.path.join(self.protobuf_lib, lib_file))
506669

507670
def fix_compiler(self):
508671
BuildExtDynamic.fix_compiler(self)
@@ -514,7 +677,8 @@ def fix_compiler(self):
514677
if os.name == 'posix':
515678
include_dirs.append(self.connc_include)
516679
library_dirs.append(self.connc_lib)
517-
libraries.append("mysqlclient")
680+
if self.with_mysql_capi:
681+
libraries.append("mysqlclient")
518682

519683
# As we statically link and the "libmysqlclient.a" library
520684
# carry no information what it depends on, we need to
@@ -523,9 +687,16 @@ def fix_compiler(self):
523687
libraries.append("rt")
524688

525689
for ext in self.extensions:
526-
ext.include_dirs.extend(include_dirs)
527-
ext.library_dirs.extend(library_dirs)
528-
ext.libraries.extend(libraries)
690+
if ext.name == "_mysql_connector":
691+
ext.include_dirs.extend(include_dirs)
692+
ext.library_dirs.extend(library_dirs)
693+
ext.libraries.extend(libraries)
694+
elif ext.name == "_mysqlxpb" \
695+
and platform.system() not in ["Darwin", "Windows"]:
696+
ext.libraries.append("rt")
697+
# Add extra compile args
698+
if self.extra_compile_args:
699+
ext.extra_compile_args.append(self.extra_compile_args)
529700

530701

531702
class InstallLib(install_lib):
@@ -574,13 +745,17 @@ class Install(install):
574745

575746
def initialize_options(self):
576747
install.initialize_options(self)
748+
self.extra_compile_args = None
577749
self.with_mysql_capi = None
750+
self.with_protobuf_include_dir = None
751+
self.with_protobuf_lib_dir = None
752+
self.with_protoc = None
578753
self.byte_code_only = None
579754
self.static = None
580755

581756
def finalize_options(self):
582757
if self.static:
583-
log.info("Linking CExtension statically with MySQL libraries")
758+
log.info("Linking C Extension statically with libraries")
584759
self.distribution.cmdclass['build_ext'] = BuildExtStatic
585760

586761
if self.byte_code_only is None:
@@ -600,7 +775,7 @@ def finalize_options(self):
600775

601776
def run(self):
602777
if not self.need_ext:
603-
log.info("Not Installing C Extension")
778+
log.info("Not Installing MySQL C Extension")
604779
else:
605-
log.info("Installing C Extension")
780+
log.info("Installing MySQL C Extension")
606781
install.run(self)

‎lib/mysql/connector/fabric/connection.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# MySQL Connector/Python - MySQL driver written in Python.
2-
# Copyright (c) 2013, 2016, Oracle and/or its affiliates. All rights reserved.
2+
# Copyright (c) 2013, 2017, Oracle and/or its affiliates. All rights reserved.
33

44
# MySQL Connector/Python is licensed under the terms of the GPLv2
55
# <http://www.gnu.org/licenses/old-licenses/gpl-2.0.html>, like most
@@ -68,7 +68,7 @@
6868

6969
if sys.version_info[0] == 2:
7070
try:
71-
from httplib import HTTPSConnection
71+
from httplib import HTTPSConnection # pylint: disable=C0412
7272
except ImportError:
7373
HAVE_SSL = False
7474
else:
@@ -461,7 +461,7 @@ def __init__(self, ssl_config): #pylint: disable=E1002
461461
if PY2:
462462
urllib2.HTTPSHandler.__init__(self)
463463
else:
464-
super().__init__() # pylint: disable=W0104
464+
super().__init__() # pylint: disable=W0104,E1004
465465
self._ssl_config = ssl_config
466466

467467
def https_open(self, req):
@@ -489,7 +489,7 @@ def __init__(self, username, password, #pylint: disable=E1002
489489
if PY2:
490490
Transport.__init__(self, use_datetime=False)
491491
else:
492-
super().__init__(use_datetime=False)
492+
super().__init__(use_datetime=False) # pylint: disable=E1004
493493
self._username = username
494494
self._password = password
495495
self._use_datetime = use_datetime

0 commit comments

Comments
 (0)