From cd4d0ec11dd19c719226aa0f99699233101e6bce Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Sat, 24 Mar 2018 15:41:37 +0100 Subject: [PATCH 1/2] bpo-33127: Compatibility patch for LibreSSL 2.7.0 (GH-6210) LibreSSL 2.7 introduced OpenSSL 1.1.0 API. The ssl module now detects LibreSSL 2.7 and only provides API shims for OpenSSL < 1.1.0 and LibreSSL < 2.7. Documentation updates and fixes for failing tests will be provided in another patch set. Signed-off-by: Christian Heimes --- .../2018-03-24-15-08-24.bpo-33127.olJmHv.rst | 1 + Modules/_ssl.c | 25 +- Tools/ssl/multissltests.py | 442 ++++++++++++++++++ 3 files changed, 460 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2018-03-24-15-08-24.bpo-33127.olJmHv.rst create mode 100755 Tools/ssl/multissltests.py diff --git a/Misc/NEWS.d/next/Library/2018-03-24-15-08-24.bpo-33127.olJmHv.rst b/Misc/NEWS.d/next/Library/2018-03-24-15-08-24.bpo-33127.olJmHv.rst new file mode 100644 index 00000000000000..635aabbde0311b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-24-15-08-24.bpo-33127.olJmHv.rst @@ -0,0 +1 @@ +The ssl module now compiles with LibreSSL 2.7.1. diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 86482677ae7a3e..056d7e7083e2c0 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -101,6 +101,12 @@ struct py_ssl_library_code { #if (OPENSSL_VERSION_NUMBER >= 0x10100000L) && !defined(LIBRESSL_VERSION_NUMBER) # define OPENSSL_VERSION_1_1 1 +# define PY_OPENSSL_1_1_API 1 +#endif + +/* LibreSSL 2.7.0 provides necessary OpenSSL 1.1.0 APIs */ +#if defined(LIBRESSL_VERSION_NUMBER) && LIBRESSL_VERSION_NUMBER >= 0x2070000fL +# define PY_OPENSSL_1_1_API 1 #endif /* Openssl comes with TLSv1.1 and TLSv1.2 between 1.0.0h and 1.0.1 @@ -129,16 +135,18 @@ struct py_ssl_library_code { #define INVALID_SOCKET (-1) #endif -#ifdef OPENSSL_VERSION_1_1 -/* OpenSSL 1.1.0+ */ -#ifndef OPENSSL_NO_SSL2 -#define OPENSSL_NO_SSL2 -#endif -#else /* OpenSSL < 1.1.0 */ -#if defined(WITH_THREAD) +/* OpenSSL 1.0.2 and LibreSSL needs extra code for locking */ +#ifndef OPENSSL_VERSION_1_1 #define HAVE_OPENSSL_CRYPTO_LOCK #endif +#if defined(OPENSSL_VERSION_1_1) && !defined(OPENSSL_NO_SSL2) +#define OPENSSL_NO_SSL2 +#endif + +#ifndef PY_OPENSSL_1_1_API +/* OpenSSL 1.1 API shims for OpenSSL < 1.1.0 and LibreSSL < 2.7.0 */ + #define TLS_method SSLv23_method static int X509_NAME_ENTRY_set(const X509_NAME_ENTRY *ne) @@ -187,7 +195,8 @@ static X509_VERIFY_PARAM *X509_STORE_get0_param(X509_STORE *store) { return store->param; } -#endif /* OpenSSL < 1.1.0 or LibreSSL */ + +#endif /* OpenSSL < 1.1.0 or LibreSSL < 2.7.0 */ enum py_ssl_error { diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py new file mode 100755 index 00000000000000..a51566c5069dd9 --- /dev/null +++ b/Tools/ssl/multissltests.py @@ -0,0 +1,442 @@ +#!./python +"""Run Python tests against multiple installations of OpenSSL and LibreSSL + +The script + + (1) downloads OpenSSL / LibreSSL tar bundle + (2) extracts it to ./src + (3) compiles OpenSSL / LibreSSL + (4) installs OpenSSL / LibreSSL into ../multissl/$LIB/$VERSION/ + (5) forces a recompilation of Python modules using the + header and library files from ../multissl/$LIB/$VERSION/ + (6) runs Python's test suite + +The script must be run with Python's build directory as current working +directory. + +The script uses LD_RUN_PATH, LD_LIBRARY_PATH, CPPFLAGS and LDFLAGS to bend +search paths for header files and shared libraries. It's known to work on +Linux with GCC and clang. + +Please keep this script compatible with Python 2.7, and 3.4 to 3.7. + +(c) 2013-2017 Christian Heimes +""" +from __future__ import print_function + +import argparse +from datetime import datetime +import logging +import os +try: + from urllib.request import urlopen +except ImportError: + from urllib2 import urlopen +import subprocess +import shutil +import sys +import tarfile + + +log = logging.getLogger("multissl") + +OPENSSL_OLD_VERSIONS = [ + "1.0.2", +] + +OPENSSL_RECENT_VERSIONS = [ + "1.0.2n", + "1.1.0g", + "1.1.1-pre1", +] + +LIBRESSL_OLD_VERSIONS = [ +] + +LIBRESSL_RECENT_VERSIONS = [ + "2.7.1", +] + +# store files in ../multissl +HERE = os.path.dirname(os.path.abspath(__file__)) +PYTHONROOT = os.path.abspath(os.path.join(HERE, '..', '..')) +MULTISSL_DIR = os.path.abspath(os.path.join(PYTHONROOT, '..', 'multissl')) + + +parser = argparse.ArgumentParser( + prog='multissl', + description=( + "Run CPython tests with multiple OpenSSL and LibreSSL " + "versions." + ) +) +parser.add_argument( + '--debug', + action='store_true', + help="Enable debug mode", +) +parser.add_argument( + '--disable-ancient', + action='store_true', + help="Don't test OpenSSL < 1.0.2 and LibreSSL < 2.5.3.", +) +parser.add_argument( + '--openssl', + nargs='+', + default=(), + help=( + "OpenSSL versions, defaults to '{}' (ancient: '{}') if no " + "OpenSSL and LibreSSL versions are given." + ).format(OPENSSL_RECENT_VERSIONS, OPENSSL_OLD_VERSIONS) +) +parser.add_argument( + '--libressl', + nargs='+', + default=(), + help=( + "LibreSSL versions, defaults to '{}' (ancient: '{}') if no " + "OpenSSL and LibreSSL versions are given." + ).format(LIBRESSL_RECENT_VERSIONS, LIBRESSL_OLD_VERSIONS) +) +parser.add_argument( + '--tests', + nargs='*', + default=(), + help="Python tests to run, defaults to all SSL related tests.", +) +parser.add_argument( + '--base-directory', + default=MULTISSL_DIR, + help="Base directory for OpenSSL / LibreSSL sources and builds." +) +parser.add_argument( + '--no-network', + action='store_false', + dest='network', + help="Disable network tests." +) +parser.add_argument( + '--steps', + choices=['library', 'modules', 'tests'], + default='tests', + help=( + "Which steps to perform. 'library' downloads and compiles OpenSSL " + "or LibreSSL. 'module' also compiles Python modules. 'tests' builds " + "all and runs the test suite." + ) +) + + +class AbstractBuilder(object): + library = None + url_template = None + src_template = None + build_template = None + install_target = 'install' + + module_files = ("Modules/_ssl.c", + "Modules/_hashopenssl.c") + module_libs = ("_ssl", "_hashlib") + + def __init__(self, version, args): + self.version = version + self.args = args + # installation directory + self.install_dir = os.path.join( + os.path.join(args.base_directory, self.library.lower()), version + ) + # source file + self.src_dir = os.path.join(args.base_directory, 'src') + self.src_file = os.path.join( + self.src_dir, self.src_template.format(version)) + # build directory (removed after install) + self.build_dir = os.path.join( + self.src_dir, self.build_template.format(version)) + + def __str__(self): + return "<{0.__class__.__name__} for {0.version}>".format(self) + + def __eq__(self, other): + if not isinstance(other, AbstractBuilder): + return NotImplemented + return ( + self.library == other.library + and self.version == other.version + ) + + def __hash__(self): + return hash((self.library, self.version)) + + @property + def openssl_cli(self): + """openssl CLI binary""" + return os.path.join(self.install_dir, "bin", "openssl") + + @property + def openssl_version(self): + """output of 'bin/openssl version'""" + cmd = [self.openssl_cli, "version"] + return self._subprocess_output(cmd) + + @property + def pyssl_version(self): + """Value of ssl.OPENSSL_VERSION""" + cmd = [ + sys.executable, + '-c', 'import ssl; print(ssl.OPENSSL_VERSION)' + ] + return self._subprocess_output(cmd) + + @property + def include_dir(self): + return os.path.join(self.install_dir, "include") + + @property + def lib_dir(self): + return os.path.join(self.install_dir, "lib") + + @property + def has_openssl(self): + return os.path.isfile(self.openssl_cli) + + @property + def has_src(self): + return os.path.isfile(self.src_file) + + def _subprocess_call(self, cmd, env=None, **kwargs): + log.debug("Call '{}'".format(" ".join(cmd))) + return subprocess.check_call(cmd, env=env, **kwargs) + + def _subprocess_output(self, cmd, env=None, **kwargs): + log.debug("Call '{}'".format(" ".join(cmd))) + if env is None: + env = os.environ.copy() + env["LD_LIBRARY_PATH"] = self.lib_dir + out = subprocess.check_output(cmd, env=env, **kwargs) + return out.strip().decode("utf-8") + + def _download_src(self): + """Download sources""" + src_dir = os.path.dirname(self.src_file) + if not os.path.isdir(src_dir): + os.makedirs(src_dir) + url = self.url_template.format(self.version) + log.info("Downloading from {}".format(url)) + req = urlopen(url) + # KISS, read all, write all + data = req.read() + log.info("Storing {}".format(self.src_file)) + with open(self.src_file, "wb") as f: + f.write(data) + + def _unpack_src(self): + """Unpack tar.gz bundle""" + # cleanup + if os.path.isdir(self.build_dir): + shutil.rmtree(self.build_dir) + os.makedirs(self.build_dir) + + tf = tarfile.open(self.src_file) + name = self.build_template.format(self.version) + base = name + '/' + # force extraction into build dir + members = tf.getmembers() + for member in list(members): + if member.name == name: + members.remove(member) + elif not member.name.startswith(base): + raise ValueError(member.name, base) + member.name = member.name[len(base):].lstrip('/') + log.info("Unpacking files to {}".format(self.build_dir)) + tf.extractall(self.build_dir, members) + + def _build_src(self): + """Now build openssl""" + log.info("Running build in {}".format(self.build_dir)) + cwd = self.build_dir + cmd = ["./config", "shared", "--prefix={}".format(self.install_dir)] + self._subprocess_call(cmd, cwd=cwd) + # Old OpenSSL versions do not support parallel builds. + self._subprocess_call(["make", "-j1"], cwd=cwd) + + def _make_install(self, remove=True): + self._subprocess_call( + ["make", "-j1", self.install_target], + cwd=self.build_dir + ) + if remove: + shutil.rmtree(self.build_dir) + + def install(self): + log.info(self.openssl_cli) + if not self.has_openssl: + if not self.has_src: + self._download_src() + else: + log.debug("Already has src {}".format(self.src_file)) + self._unpack_src() + self._build_src() + self._make_install() + else: + log.info("Already has installation {}".format(self.install_dir)) + # validate installation + version = self.openssl_version + if self.version not in version: + raise ValueError(version) + + def recompile_pymods(self): + log.warning("Using build from {}".format(self.build_dir)) + # force a rebuild of all modules that use OpenSSL APIs + for fname in self.module_files: + os.utime(fname, None) + # remove all build artefacts + for root, dirs, files in os.walk('build'): + for filename in files: + if filename.startswith(self.module_libs): + os.unlink(os.path.join(root, filename)) + + # overwrite header and library search paths + env = os.environ.copy() + env["CPPFLAGS"] = "-I{}".format(self.include_dir) + env["LDFLAGS"] = "-L{}".format(self.lib_dir) + # set rpath + env["LD_RUN_PATH"] = self.lib_dir + + log.info("Rebuilding Python modules") + cmd = [sys.executable, "setup.py", "build"] + self._subprocess_call(cmd, env=env) + self.check_imports() + + def check_imports(self): + cmd = [sys.executable, "-c", "import _ssl; import _hashlib"] + self._subprocess_call(cmd) + + def check_pyssl(self): + version = self.pyssl_version + if self.version not in version: + raise ValueError(version) + + def run_python_tests(self, tests, network=True): + if not tests: + cmd = [sys.executable, 'Lib/test/ssltests.py', '-j0'] + elif sys.version_info < (3, 3): + cmd = [sys.executable, '-m', 'test.regrtest'] + else: + cmd = [sys.executable, '-m', 'test', '-j0'] + if network: + cmd.extend(['-u', 'network', '-u', 'urlfetch']) + cmd.extend(['-w', '-r']) + cmd.extend(tests) + self._subprocess_call(cmd, stdout=None) + + +class BuildOpenSSL(AbstractBuilder): + library = "OpenSSL" + url_template = "https://www.openssl.org/source/openssl-{}.tar.gz" + src_template = "openssl-{}.tar.gz" + build_template = "openssl-{}" + # only install software, skip docs + install_target = 'install_sw' + + +class BuildLibreSSL(AbstractBuilder): + library = "LibreSSL" + url_template = ( + "https://ftp.openbsd.org/pub/OpenBSD/LibreSSL/libressl-{}.tar.gz") + src_template = "libressl-{}.tar.gz" + build_template = "libressl-{}" + + +def configure_make(): + if not os.path.isfile('Makefile'): + log.info('Running ./configure') + subprocess.check_call([ + './configure', '--config-cache', '--quiet', + '--with-pydebug' + ]) + + log.info('Running make') + subprocess.check_call(['make', '--quiet']) + + +def main(): + args = parser.parse_args() + if not args.openssl and not args.libressl: + args.openssl = list(OPENSSL_RECENT_VERSIONS) + args.libressl = list(LIBRESSL_RECENT_VERSIONS) + if not args.disable_ancient: + args.openssl.extend(OPENSSL_OLD_VERSIONS) + args.libressl.extend(LIBRESSL_OLD_VERSIONS) + + logging.basicConfig( + level=logging.DEBUG if args.debug else logging.INFO, + format="*** %(levelname)s %(message)s" + ) + + start = datetime.now() + + if args.steps in {'modules', 'tests'}: + for name in ['setup.py', 'Modules/_ssl.c']: + if not os.path.isfile(os.path.join(PYTHONROOT, name)): + parser.error( + "Must be executed from CPython build dir" + ) + if not os.path.samefile('python', sys.executable): + parser.error( + "Must be executed with ./python from CPython build dir" + ) + # check for configure and run make + configure_make() + + # download and register builder + builds = [] + + for version in args.openssl: + build = BuildOpenSSL( + version, + args + ) + build.install() + builds.append(build) + + for version in args.libressl: + build = BuildLibreSSL( + version, + args + ) + build.install() + builds.append(build) + + if args.steps in {'modules', 'tests'}: + for build in builds: + try: + build.recompile_pymods() + build.check_pyssl() + if args.steps == 'tests': + build.run_python_tests( + tests=args.tests, + network=args.network, + ) + except Exception as e: + log.exception("%s failed", build) + print("{} failed: {}".format(build, e), file=sys.stderr) + sys.exit(2) + + log.info("\n{} finished in {}".format( + args.steps.capitalize(), + datetime.now() - start + )) + print('Python: ', sys.version) + if args.steps == 'tests': + if args.tests: + print('Executed Tests:', ' '.join(args.tests)) + else: + print('Executed all SSL tests.') + + print('OpenSSL / LibreSSL versions:') + for build in builds: + print(" * {0.library} {0.version}".format(build)) + + +if __name__ == "__main__": + main() From 09455012dd67650bf7017a68753e6fa3f9c7deb9 Mon Sep 17 00:00:00 2001 From: larryhastings Date: Thu, 28 Feb 2019 23:15:20 -0800 Subject: [PATCH 2/2] Delete multissltests.py --- Tools/ssl/multissltests.py | 442 ------------------------------------- 1 file changed, 442 deletions(-) delete mode 100755 Tools/ssl/multissltests.py diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py deleted file mode 100755 index a51566c5069dd9..00000000000000 --- a/Tools/ssl/multissltests.py +++ /dev/null @@ -1,442 +0,0 @@ -#!./python -"""Run Python tests against multiple installations of OpenSSL and LibreSSL - -The script - - (1) downloads OpenSSL / LibreSSL tar bundle - (2) extracts it to ./src - (3) compiles OpenSSL / LibreSSL - (4) installs OpenSSL / LibreSSL into ../multissl/$LIB/$VERSION/ - (5) forces a recompilation of Python modules using the - header and library files from ../multissl/$LIB/$VERSION/ - (6) runs Python's test suite - -The script must be run with Python's build directory as current working -directory. - -The script uses LD_RUN_PATH, LD_LIBRARY_PATH, CPPFLAGS and LDFLAGS to bend -search paths for header files and shared libraries. It's known to work on -Linux with GCC and clang. - -Please keep this script compatible with Python 2.7, and 3.4 to 3.7. - -(c) 2013-2017 Christian Heimes -""" -from __future__ import print_function - -import argparse -from datetime import datetime -import logging -import os -try: - from urllib.request import urlopen -except ImportError: - from urllib2 import urlopen -import subprocess -import shutil -import sys -import tarfile - - -log = logging.getLogger("multissl") - -OPENSSL_OLD_VERSIONS = [ - "1.0.2", -] - -OPENSSL_RECENT_VERSIONS = [ - "1.0.2n", - "1.1.0g", - "1.1.1-pre1", -] - -LIBRESSL_OLD_VERSIONS = [ -] - -LIBRESSL_RECENT_VERSIONS = [ - "2.7.1", -] - -# store files in ../multissl -HERE = os.path.dirname(os.path.abspath(__file__)) -PYTHONROOT = os.path.abspath(os.path.join(HERE, '..', '..')) -MULTISSL_DIR = os.path.abspath(os.path.join(PYTHONROOT, '..', 'multissl')) - - -parser = argparse.ArgumentParser( - prog='multissl', - description=( - "Run CPython tests with multiple OpenSSL and LibreSSL " - "versions." - ) -) -parser.add_argument( - '--debug', - action='store_true', - help="Enable debug mode", -) -parser.add_argument( - '--disable-ancient', - action='store_true', - help="Don't test OpenSSL < 1.0.2 and LibreSSL < 2.5.3.", -) -parser.add_argument( - '--openssl', - nargs='+', - default=(), - help=( - "OpenSSL versions, defaults to '{}' (ancient: '{}') if no " - "OpenSSL and LibreSSL versions are given." - ).format(OPENSSL_RECENT_VERSIONS, OPENSSL_OLD_VERSIONS) -) -parser.add_argument( - '--libressl', - nargs='+', - default=(), - help=( - "LibreSSL versions, defaults to '{}' (ancient: '{}') if no " - "OpenSSL and LibreSSL versions are given." - ).format(LIBRESSL_RECENT_VERSIONS, LIBRESSL_OLD_VERSIONS) -) -parser.add_argument( - '--tests', - nargs='*', - default=(), - help="Python tests to run, defaults to all SSL related tests.", -) -parser.add_argument( - '--base-directory', - default=MULTISSL_DIR, - help="Base directory for OpenSSL / LibreSSL sources and builds." -) -parser.add_argument( - '--no-network', - action='store_false', - dest='network', - help="Disable network tests." -) -parser.add_argument( - '--steps', - choices=['library', 'modules', 'tests'], - default='tests', - help=( - "Which steps to perform. 'library' downloads and compiles OpenSSL " - "or LibreSSL. 'module' also compiles Python modules. 'tests' builds " - "all and runs the test suite." - ) -) - - -class AbstractBuilder(object): - library = None - url_template = None - src_template = None - build_template = None - install_target = 'install' - - module_files = ("Modules/_ssl.c", - "Modules/_hashopenssl.c") - module_libs = ("_ssl", "_hashlib") - - def __init__(self, version, args): - self.version = version - self.args = args - # installation directory - self.install_dir = os.path.join( - os.path.join(args.base_directory, self.library.lower()), version - ) - # source file - self.src_dir = os.path.join(args.base_directory, 'src') - self.src_file = os.path.join( - self.src_dir, self.src_template.format(version)) - # build directory (removed after install) - self.build_dir = os.path.join( - self.src_dir, self.build_template.format(version)) - - def __str__(self): - return "<{0.__class__.__name__} for {0.version}>".format(self) - - def __eq__(self, other): - if not isinstance(other, AbstractBuilder): - return NotImplemented - return ( - self.library == other.library - and self.version == other.version - ) - - def __hash__(self): - return hash((self.library, self.version)) - - @property - def openssl_cli(self): - """openssl CLI binary""" - return os.path.join(self.install_dir, "bin", "openssl") - - @property - def openssl_version(self): - """output of 'bin/openssl version'""" - cmd = [self.openssl_cli, "version"] - return self._subprocess_output(cmd) - - @property - def pyssl_version(self): - """Value of ssl.OPENSSL_VERSION""" - cmd = [ - sys.executable, - '-c', 'import ssl; print(ssl.OPENSSL_VERSION)' - ] - return self._subprocess_output(cmd) - - @property - def include_dir(self): - return os.path.join(self.install_dir, "include") - - @property - def lib_dir(self): - return os.path.join(self.install_dir, "lib") - - @property - def has_openssl(self): - return os.path.isfile(self.openssl_cli) - - @property - def has_src(self): - return os.path.isfile(self.src_file) - - def _subprocess_call(self, cmd, env=None, **kwargs): - log.debug("Call '{}'".format(" ".join(cmd))) - return subprocess.check_call(cmd, env=env, **kwargs) - - def _subprocess_output(self, cmd, env=None, **kwargs): - log.debug("Call '{}'".format(" ".join(cmd))) - if env is None: - env = os.environ.copy() - env["LD_LIBRARY_PATH"] = self.lib_dir - out = subprocess.check_output(cmd, env=env, **kwargs) - return out.strip().decode("utf-8") - - def _download_src(self): - """Download sources""" - src_dir = os.path.dirname(self.src_file) - if not os.path.isdir(src_dir): - os.makedirs(src_dir) - url = self.url_template.format(self.version) - log.info("Downloading from {}".format(url)) - req = urlopen(url) - # KISS, read all, write all - data = req.read() - log.info("Storing {}".format(self.src_file)) - with open(self.src_file, "wb") as f: - f.write(data) - - def _unpack_src(self): - """Unpack tar.gz bundle""" - # cleanup - if os.path.isdir(self.build_dir): - shutil.rmtree(self.build_dir) - os.makedirs(self.build_dir) - - tf = tarfile.open(self.src_file) - name = self.build_template.format(self.version) - base = name + '/' - # force extraction into build dir - members = tf.getmembers() - for member in list(members): - if member.name == name: - members.remove(member) - elif not member.name.startswith(base): - raise ValueError(member.name, base) - member.name = member.name[len(base):].lstrip('/') - log.info("Unpacking files to {}".format(self.build_dir)) - tf.extractall(self.build_dir, members) - - def _build_src(self): - """Now build openssl""" - log.info("Running build in {}".format(self.build_dir)) - cwd = self.build_dir - cmd = ["./config", "shared", "--prefix={}".format(self.install_dir)] - self._subprocess_call(cmd, cwd=cwd) - # Old OpenSSL versions do not support parallel builds. - self._subprocess_call(["make", "-j1"], cwd=cwd) - - def _make_install(self, remove=True): - self._subprocess_call( - ["make", "-j1", self.install_target], - cwd=self.build_dir - ) - if remove: - shutil.rmtree(self.build_dir) - - def install(self): - log.info(self.openssl_cli) - if not self.has_openssl: - if not self.has_src: - self._download_src() - else: - log.debug("Already has src {}".format(self.src_file)) - self._unpack_src() - self._build_src() - self._make_install() - else: - log.info("Already has installation {}".format(self.install_dir)) - # validate installation - version = self.openssl_version - if self.version not in version: - raise ValueError(version) - - def recompile_pymods(self): - log.warning("Using build from {}".format(self.build_dir)) - # force a rebuild of all modules that use OpenSSL APIs - for fname in self.module_files: - os.utime(fname, None) - # remove all build artefacts - for root, dirs, files in os.walk('build'): - for filename in files: - if filename.startswith(self.module_libs): - os.unlink(os.path.join(root, filename)) - - # overwrite header and library search paths - env = os.environ.copy() - env["CPPFLAGS"] = "-I{}".format(self.include_dir) - env["LDFLAGS"] = "-L{}".format(self.lib_dir) - # set rpath - env["LD_RUN_PATH"] = self.lib_dir - - log.info("Rebuilding Python modules") - cmd = [sys.executable, "setup.py", "build"] - self._subprocess_call(cmd, env=env) - self.check_imports() - - def check_imports(self): - cmd = [sys.executable, "-c", "import _ssl; import _hashlib"] - self._subprocess_call(cmd) - - def check_pyssl(self): - version = self.pyssl_version - if self.version not in version: - raise ValueError(version) - - def run_python_tests(self, tests, network=True): - if not tests: - cmd = [sys.executable, 'Lib/test/ssltests.py', '-j0'] - elif sys.version_info < (3, 3): - cmd = [sys.executable, '-m', 'test.regrtest'] - else: - cmd = [sys.executable, '-m', 'test', '-j0'] - if network: - cmd.extend(['-u', 'network', '-u', 'urlfetch']) - cmd.extend(['-w', '-r']) - cmd.extend(tests) - self._subprocess_call(cmd, stdout=None) - - -class BuildOpenSSL(AbstractBuilder): - library = "OpenSSL" - url_template = "https://www.openssl.org/source/openssl-{}.tar.gz" - src_template = "openssl-{}.tar.gz" - build_template = "openssl-{}" - # only install software, skip docs - install_target = 'install_sw' - - -class BuildLibreSSL(AbstractBuilder): - library = "LibreSSL" - url_template = ( - "https://ftp.openbsd.org/pub/OpenBSD/LibreSSL/libressl-{}.tar.gz") - src_template = "libressl-{}.tar.gz" - build_template = "libressl-{}" - - -def configure_make(): - if not os.path.isfile('Makefile'): - log.info('Running ./configure') - subprocess.check_call([ - './configure', '--config-cache', '--quiet', - '--with-pydebug' - ]) - - log.info('Running make') - subprocess.check_call(['make', '--quiet']) - - -def main(): - args = parser.parse_args() - if not args.openssl and not args.libressl: - args.openssl = list(OPENSSL_RECENT_VERSIONS) - args.libressl = list(LIBRESSL_RECENT_VERSIONS) - if not args.disable_ancient: - args.openssl.extend(OPENSSL_OLD_VERSIONS) - args.libressl.extend(LIBRESSL_OLD_VERSIONS) - - logging.basicConfig( - level=logging.DEBUG if args.debug else logging.INFO, - format="*** %(levelname)s %(message)s" - ) - - start = datetime.now() - - if args.steps in {'modules', 'tests'}: - for name in ['setup.py', 'Modules/_ssl.c']: - if not os.path.isfile(os.path.join(PYTHONROOT, name)): - parser.error( - "Must be executed from CPython build dir" - ) - if not os.path.samefile('python', sys.executable): - parser.error( - "Must be executed with ./python from CPython build dir" - ) - # check for configure and run make - configure_make() - - # download and register builder - builds = [] - - for version in args.openssl: - build = BuildOpenSSL( - version, - args - ) - build.install() - builds.append(build) - - for version in args.libressl: - build = BuildLibreSSL( - version, - args - ) - build.install() - builds.append(build) - - if args.steps in {'modules', 'tests'}: - for build in builds: - try: - build.recompile_pymods() - build.check_pyssl() - if args.steps == 'tests': - build.run_python_tests( - tests=args.tests, - network=args.network, - ) - except Exception as e: - log.exception("%s failed", build) - print("{} failed: {}".format(build, e), file=sys.stderr) - sys.exit(2) - - log.info("\n{} finished in {}".format( - args.steps.capitalize(), - datetime.now() - start - )) - print('Python: ', sys.version) - if args.steps == 'tests': - if args.tests: - print('Executed Tests:', ' '.join(args.tests)) - else: - print('Executed all SSL tests.') - - print('OpenSSL / LibreSSL versions:') - for build in builds: - print(" * {0.library} {0.version}".format(build)) - - -if __name__ == "__main__": - main()