From 6f5a5a1a8a065af443039e5c3420b6a115731564 Mon Sep 17 00:00:00 2001 From: Han Date: Wed, 14 Sep 2011 14:39:12 +0200 Subject: [PATCH 01/11] ENH: Generate custom MSCV runtime library for mingw32. --- numpy/distutils/mingw32ccompiler.py | 46 +++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/numpy/distutils/mingw32ccompiler.py b/numpy/distutils/mingw32ccompiler.py index 5b352c30055b..c21b66478e64 100644 --- a/numpy/distutils/mingw32ccompiler.py +++ b/numpy/distutils/mingw32ccompiler.py @@ -90,6 +90,11 @@ def __init__ (self, build_import_library() + # Check for custom msvc runtime library on Windows. Build if it doesn't exist. + if build_msvcr_library(): + # add preprocessor statement for using customized msvcr lib + self.define_macro('NPY_MINGW_USE_CUSTOM_MSVCR', '1') + # **changes: eric jones 4/11/01 # 2. increased optimization and turned off all warnings # 3. also added --driver-name g++ @@ -290,6 +295,47 @@ def generate_def(dll, dfile): d.write('%s\n' % s[1]) d.close() +def find_msvcr_dll(msvcr_dll_name): + for path in [sys.prefix] + os.environ['PATH'].split(';'): + filepath = os.path.join(path, msvcr_dll_name) + if os.path.exists(filepath): + return os.path.abspath(filepath) + return None + +def build_msvcr_library(debug=False): + if os.name != 'nt': + return False + + if debug: + log.warn('Cannot build msvcr-dbg library: not yet implemented') + return False + + msvcr_name = msvc_runtime_library() + msvcr_dll_name = msvcr_name + '.dll' + dll_file = find_msvcr_dll(msvcr_dll_name) + if not dll_file: + log.warn('Cannot build msvcr library: "%s" not found' % msvcr_dll_name) + return False + + out_name = "lib%s.a" % msvcr_name + out_file = os.path.join(sys.prefix, 'libs', out_name) + if os.path.isfile(out_file): + log.debug('Skip building msvcr library: "%s" exists' % (out_file)) + return True + + def_name = "lib%s.def" % msvcr_name + def_file = os.path.join(sys.prefix, 'libs', def_name) + + log.info('Building msvcr library: "%s" (from %s)' \ + % (out_file, dll_file)) + + generate_def(dll_file, def_file) + + cmd = ['dlltool', '-d', def_file, '-l', out_file] + retcode = subprocess.call(cmd) + + return (not retcode) + def build_import_library(): if os.name != 'nt': return From b7a35a172d5b83f557624621295b2adde36e0dd3 Mon Sep 17 00:00:00 2001 From: Han Date: Wed, 14 Sep 2011 14:40:53 +0200 Subject: [PATCH 02/11] ENH: Use customized MSVC runtime library if present. --- numpy/core/src/multiarray/datetime_strings.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/numpy/core/src/multiarray/datetime_strings.c b/numpy/core/src/multiarray/datetime_strings.c index 97d17023079a..7b236d202417 100644 --- a/numpy/core/src/multiarray/datetime_strings.c +++ b/numpy/core/src/multiarray/datetime_strings.c @@ -12,6 +12,12 @@ #include +#if defined(_WIN32) && \ + defined(__GNUC__) && \ + defined(NPY_MINGW_USE_CUSTOM_MSVCR) +#define time_t __time64_t +#endif + #define NPY_NO_DEPRECATED_API #define _MULTIARRAYMODULE #include From a41fe23a1f0baafd3b2c8c7fb178f304e7c8738c Mon Sep 17 00:00:00 2001 From: Han Date: Wed, 14 Sep 2011 14:56:31 +0200 Subject: [PATCH 03/11] ENH: Some commments. Skip customization for MSVC < 8.0. --- numpy/distutils/mingw32ccompiler.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/numpy/distutils/mingw32ccompiler.py b/numpy/distutils/mingw32ccompiler.py index c21b66478e64..5cdfd42794b7 100644 --- a/numpy/distutils/mingw32ccompiler.py +++ b/numpy/distutils/mingw32ccompiler.py @@ -295,9 +295,11 @@ def generate_def(dll, dfile): d.write('%s\n' % s[1]) d.close() -def find_msvcr_dll(msvcr_dll_name): +def find_dll(dll_name): + # First, look in the Python directory, then scan PATH for + # the given dll name. for path in [sys.prefix] + os.environ['PATH'].split(';'): - filepath = os.path.join(path, msvcr_dll_name) + filepath = os.path.join(path, dll_name) if os.path.exists(filepath): return os.path.abspath(filepath) return None @@ -311,8 +313,15 @@ def build_msvcr_library(debug=False): return False msvcr_name = msvc_runtime_library() + + # Skip using a custom library for versions < MSVC 8.0 + if int(msvcr_name[-2:]) < 80: + log.debug('Skip building msvcr library: custom functionality not present') + return False + + # Find the msvcr dll msvcr_dll_name = msvcr_name + '.dll' - dll_file = find_msvcr_dll(msvcr_dll_name) + dll_file = find_dll(msvcr_dll_name) if not dll_file: log.warn('Cannot build msvcr library: "%s" not found' % msvcr_dll_name) return False @@ -329,8 +338,10 @@ def build_msvcr_library(debug=False): log.info('Building msvcr library: "%s" (from %s)' \ % (out_file, dll_file)) + # Generate a symbol definition file from the msvcr dll generate_def(dll_file, def_file) + # Create a custom mingw library for the given symbol definitions cmd = ['dlltool', '-d', def_file, '-l', out_file] retcode = subprocess.call(cmd) From cc1c8b6b0b53843aa6ea3f5e3e479ea695a72972 Mon Sep 17 00:00:00 2001 From: Han Date: Wed, 14 Sep 2011 15:02:25 +0200 Subject: [PATCH 04/11] ENH: Clean up symbol definition file. --- numpy/distutils/mingw32ccompiler.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/numpy/distutils/mingw32ccompiler.py b/numpy/distutils/mingw32ccompiler.py index 5cdfd42794b7..cfb66c991e68 100644 --- a/numpy/distutils/mingw32ccompiler.py +++ b/numpy/distutils/mingw32ccompiler.py @@ -345,6 +345,9 @@ def build_msvcr_library(debug=False): cmd = ['dlltool', '-d', def_file, '-l', out_file] retcode = subprocess.call(cmd) + # Clean up symbol definitions + os.remove(def_file) + return (not retcode) def build_import_library(): From ba8c2a2316388d6e39b28b17a81cf7548b105587 Mon Sep 17 00:00:00 2001 From: Han Date: Wed, 14 Sep 2011 15:19:07 +0200 Subject: [PATCH 05/11] ENH: In debug mode, do not use customized MSVC runtime library. --- numpy/distutils/mingw32ccompiler.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/numpy/distutils/mingw32ccompiler.py b/numpy/distutils/mingw32ccompiler.py index cfb66c991e68..156db006a61b 100644 --- a/numpy/distutils/mingw32ccompiler.py +++ b/numpy/distutils/mingw32ccompiler.py @@ -175,6 +175,9 @@ def link(self, if not libraries: libraries = [] libraries.append(runtime_library) + if debug: + # There is no customized msvcr lib in debug mode + self.undefine_macro('NPY_MINGW_USE_CUSTOM_MSVCR') args = (self, target_desc, objects, From 28f7ae2f0424bbbf922b7d9463d856d727ec8095 Mon Sep 17 00:00:00 2001 From: 87 Date: Wed, 14 Sep 2011 23:15:25 +0200 Subject: [PATCH 06/11] BUG: Also build MSVC debug library if MINGW_USE_CUSTOM_MSVCR is defined (best effort). --- numpy/distutils/mingw32ccompiler.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/numpy/distutils/mingw32ccompiler.py b/numpy/distutils/mingw32ccompiler.py index 156db006a61b..6df7713787ab 100644 --- a/numpy/distutils/mingw32ccompiler.py +++ b/numpy/distutils/mingw32ccompiler.py @@ -91,9 +91,9 @@ def __init__ (self, build_import_library() # Check for custom msvc runtime library on Windows. Build if it doesn't exist. - if build_msvcr_library(): + if build_msvcr_library() | build_msvcr_library(debug=True): # add preprocessor statement for using customized msvcr lib - self.define_macro('NPY_MINGW_USE_CUSTOM_MSVCR', '1') + self.define_macro('NPY_MINGW_USE_CUSTOM_MSVCR') # **changes: eric jones 4/11/01 # 2. increased optimization and turned off all warnings @@ -175,9 +175,6 @@ def link(self, if not libraries: libraries = [] libraries.append(runtime_library) - if debug: - # There is no customized msvcr lib in debug mode - self.undefine_macro('NPY_MINGW_USE_CUSTOM_MSVCR') args = (self, target_desc, objects, @@ -311,17 +308,16 @@ def build_msvcr_library(debug=False): if os.name != 'nt': return False - if debug: - log.warn('Cannot build msvcr-dbg library: not yet implemented') - return False - msvcr_name = msvc_runtime_library() # Skip using a custom library for versions < MSVC 8.0 - if int(msvcr_name[-2:]) < 80: + if int(msvcr_name.lstrip('msvcr')) < 80: log.debug('Skip building msvcr library: custom functionality not present') return False + if debug: + msvcr_name += 'd' + # Find the msvcr dll msvcr_dll_name = msvcr_name + '.dll' dll_file = find_dll(msvcr_dll_name) From c862384b2449dbe00ddbd74189fceac03b7faced Mon Sep 17 00:00:00 2001 From: 87 Date: Wed, 14 Sep 2011 23:45:28 +0200 Subject: [PATCH 07/11] BUG: DLL finder should also look in WinSxS directory. --- numpy/distutils/mingw32ccompiler.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/numpy/distutils/mingw32ccompiler.py b/numpy/distutils/mingw32ccompiler.py index 6df7713787ab..d51df61468be 100644 --- a/numpy/distutils/mingw32ccompiler.py +++ b/numpy/distutils/mingw32ccompiler.py @@ -296,13 +296,24 @@ def generate_def(dll, dfile): d.close() def find_dll(dll_name): + + def _find_dll_in_winsxs(dll_name): + # Walk through the WinSxS directory to find the dll. + winsxs_path = os.path.join(os.environ['WINDIR'], 'winsxs') + if not os.path.exists(winsxs_path): + return None + for root, dirs, files in os.walk(winsxs_path): + if dll_name in files: + return os.path.join(root, dll_name) + return None + # First, look in the Python directory, then scan PATH for - # the given dll name. + # the given dll name. Lastly, look in the WinSxS directory. for path in [sys.prefix] + os.environ['PATH'].split(';'): filepath = os.path.join(path, dll_name) if os.path.exists(filepath): return os.path.abspath(filepath) - return None + return _find_dll_in_winsxs(dll_name) def build_msvcr_library(debug=False): if os.name != 'nt': From 4c24ce4635a68efffdc3b4532a4a623721659a9a Mon Sep 17 00:00:00 2001 From: 87 Date: Thu, 15 Sep 2011 09:32:41 +0200 Subject: [PATCH 08/11] BUG: Add another preprocessor statement for 64-bits fallback mechanism. --- numpy/core/src/multiarray/datetime_strings.c | 19 ++++++++++++++++++- numpy/distutils/mingw32ccompiler.py | 4 ++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/numpy/core/src/multiarray/datetime_strings.c b/numpy/core/src/multiarray/datetime_strings.c index 7b236d202417..3a3243c2773f 100644 --- a/numpy/core/src/multiarray/datetime_strings.c +++ b/numpy/core/src/multiarray/datetime_strings.c @@ -14,7 +14,8 @@ #if defined(_WIN32) && \ defined(__GNUC__) && \ - defined(NPY_MINGW_USE_CUSTOM_MSVCR) + (defined(NPY_MINGW_USE_CUSTOM_MSVCR) | \ + defined(NPY_MINGW_USE_64BIT_MSVCR)) #define time_t __time64_t #endif @@ -54,6 +55,14 @@ get_localtime(NPY_TIME_T *ts, struct tm *tms) func_name = "_localtime64_s"; goto fail; } + #elif defined(__GNUC__) && defined(NPY_MINGW_USE_64BIT_MSVCR) + struct tm *tms_tmp; + tms_tmp = _localtime64(ts); + if (tms_tmp == NULL) { + func_name = "_localtime64"; + goto fail; + } + memcpy((void*)tms, (void*)tms_tmp, sizeof(struct tm)); #else struct tm *tms_tmp; tms_tmp = localtime(ts); @@ -99,6 +108,14 @@ get_gmtime(NPY_TIME_T *ts, struct tm *tms) func_name = "_gmtime64_s"; goto fail; } + #elif defined(__GNUC__) && defined(NPY_MINGW_USE_64BIT_MSVCR) + struct tm *tms_tmp; + tms_tmp = _gmtime64(ts); + if (tms_tmp == NULL) { + func_name = "_gmtime64"; + goto fail; + } + memcpy((void*)tms, (void*)tms_tmp, sizeof(struct tm)); #else struct tm *tms_tmp; tms_tmp = gmtime(ts); diff --git a/numpy/distutils/mingw32ccompiler.py b/numpy/distutils/mingw32ccompiler.py index d51df61468be..cb51fea37a78 100644 --- a/numpy/distutils/mingw32ccompiler.py +++ b/numpy/distutils/mingw32ccompiler.py @@ -94,6 +94,10 @@ def __init__ (self, if build_msvcr_library() | build_msvcr_library(debug=True): # add preprocessor statement for using customized msvcr lib self.define_macro('NPY_MINGW_USE_CUSTOM_MSVCR') + # Else do a check if the current runtime version is >= VS2005 + elif int(msvc_runtime_library().lstrip('msvcr')) >= 80: + # add preprocessor statement to use 64-bit fallback + self.define_macro('NPY_MINGW_USE_64BIT_MSVCR') # **changes: eric jones 4/11/01 # 2. increased optimization and turned off all warnings From 98aa9f8a7b4bba8524f37fcc8eaa496fa7ea872a Mon Sep 17 00:00:00 2001 From: 87 Date: Sat, 17 Sep 2011 18:26:16 +0200 Subject: [PATCH 09/11] ENH: Incorporated review advice and changed macro hack to MinGW hint. When doing the typedef change, it conflicted with existing typedefs. I realized that MinGW did not know which MSVC runtime it would link with, but it had conditional definitions for MSVC 8 and up, which can be activated by defining __MSVCRT_VERSION__. So I added it to the compiler macros, which made the fallback hack and extra typedef unnecessary. --- numpy/core/src/multiarray/datetime_strings.c | 23 -------------------- numpy/distutils/mingw32ccompiler.py | 12 +++++----- 2 files changed, 7 insertions(+), 28 deletions(-) diff --git a/numpy/core/src/multiarray/datetime_strings.c b/numpy/core/src/multiarray/datetime_strings.c index 3a3243c2773f..97d17023079a 100644 --- a/numpy/core/src/multiarray/datetime_strings.c +++ b/numpy/core/src/multiarray/datetime_strings.c @@ -12,13 +12,6 @@ #include -#if defined(_WIN32) && \ - defined(__GNUC__) && \ - (defined(NPY_MINGW_USE_CUSTOM_MSVCR) | \ - defined(NPY_MINGW_USE_64BIT_MSVCR)) -#define time_t __time64_t -#endif - #define NPY_NO_DEPRECATED_API #define _MULTIARRAYMODULE #include @@ -55,14 +48,6 @@ get_localtime(NPY_TIME_T *ts, struct tm *tms) func_name = "_localtime64_s"; goto fail; } - #elif defined(__GNUC__) && defined(NPY_MINGW_USE_64BIT_MSVCR) - struct tm *tms_tmp; - tms_tmp = _localtime64(ts); - if (tms_tmp == NULL) { - func_name = "_localtime64"; - goto fail; - } - memcpy((void*)tms, (void*)tms_tmp, sizeof(struct tm)); #else struct tm *tms_tmp; tms_tmp = localtime(ts); @@ -108,14 +93,6 @@ get_gmtime(NPY_TIME_T *ts, struct tm *tms) func_name = "_gmtime64_s"; goto fail; } - #elif defined(__GNUC__) && defined(NPY_MINGW_USE_64BIT_MSVCR) - struct tm *tms_tmp; - tms_tmp = _gmtime64(ts); - if (tms_tmp == NULL) { - func_name = "_gmtime64"; - goto fail; - } - memcpy((void*)tms, (void*)tms_tmp, sizeof(struct tm)); #else struct tm *tms_tmp; tms_tmp = gmtime(ts); diff --git a/numpy/distutils/mingw32ccompiler.py b/numpy/distutils/mingw32ccompiler.py index cb51fea37a78..993672860dea 100644 --- a/numpy/distutils/mingw32ccompiler.py +++ b/numpy/distutils/mingw32ccompiler.py @@ -91,14 +91,16 @@ def __init__ (self, build_import_library() # Check for custom msvc runtime library on Windows. Build if it doesn't exist. - if build_msvcr_library() | build_msvcr_library(debug=True): + msvcr_success = build_msvcr_library() + msvcr_dbg_success = build_msvcr_library(debug=True) + if msvcr_success or msvcr_dbg_success: # add preprocessor statement for using customized msvcr lib self.define_macro('NPY_MINGW_USE_CUSTOM_MSVCR') - # Else do a check if the current runtime version is >= VS2005 - elif int(msvc_runtime_library().lstrip('msvcr')) >= 80: - # add preprocessor statement to use 64-bit fallback - self.define_macro('NPY_MINGW_USE_64BIT_MSVCR') + # Define the MSVC version as hint for MinGW + msvcr_version = '0x%03i0' % int(msvc_runtime_library().lstrip('msvcr')) + self.define_macro('__MSVCRT_VERSION__', msvcr_version) + # **changes: eric jones 4/11/01 # 2. increased optimization and turned off all warnings # 3. also added --driver-name g++ From 61438d7567edc578a9e5a1726ef4749b31ea796c Mon Sep 17 00:00:00 2001 From: 87 Date: Sat, 17 Sep 2011 20:09:23 +0200 Subject: [PATCH 10/11] BUG: Fixes for Python3 and some further enhancements. --- numpy/distutils/mingw32ccompiler.py | 36 +++++++++++++++-------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/numpy/distutils/mingw32ccompiler.py b/numpy/distutils/mingw32ccompiler.py index 993672860dea..42d834c6c963 100644 --- a/numpy/distutils/mingw32ccompiler.py +++ b/numpy/distutils/mingw32ccompiler.py @@ -274,15 +274,14 @@ def generate_def(dll, dfile): The .def file will be overwritten""" dump = dump_table(dll) for i in range(len(dump)): - if _START.match(dump[i]): + if _START.match(dump[i].decode()): break - - if i == len(dump): + else: raise ValueError("Symbol table not found") syms = [] for j in range(i+1, len(dump)): - m = _TABLE.match(dump[j]) + m = _TABLE.match(dump[j].decode()) if m: syms.append((int(m.group(1).strip()), m.group(2))) else: @@ -313,13 +312,15 @@ def _find_dll_in_winsxs(dll_name): return os.path.join(root, dll_name) return None - # First, look in the Python directory, then scan PATH for - # the given dll name. Lastly, look in the WinSxS directory. - for path in [sys.prefix] + os.environ['PATH'].split(';'): - filepath = os.path.join(path, dll_name) - if os.path.exists(filepath): - return os.path.abspath(filepath) - return _find_dll_in_winsxs(dll_name) + def _find_dll_in_path(dll_name): + # First, look in the Python directory, then scan PATH for + # the given dll name. + for path in [sys.prefix] + os.environ['PATH'].split(';'): + filepath = os.path.join(path, dll_name) + if os.path.exists(filepath): + return os.path.abspath(filepath) + + return _find_dll_in_winsxs(dll_name) or _find_dll_in_path(dll_name) def build_msvcr_library(debug=False): if os.name != 'nt': @@ -335,6 +336,13 @@ def build_msvcr_library(debug=False): if debug: msvcr_name += 'd' + # Skip if custom library already exists + out_name = "lib%s.a" % msvcr_name + out_file = os.path.join(sys.prefix, 'libs', out_name) + if os.path.isfile(out_file): + log.debug('Skip building msvcr library: "%s" exists' % (out_file)) + return True + # Find the msvcr dll msvcr_dll_name = msvcr_name + '.dll' dll_file = find_dll(msvcr_dll_name) @@ -342,12 +350,6 @@ def build_msvcr_library(debug=False): log.warn('Cannot build msvcr library: "%s" not found' % msvcr_dll_name) return False - out_name = "lib%s.a" % msvcr_name - out_file = os.path.join(sys.prefix, 'libs', out_name) - if os.path.isfile(out_file): - log.debug('Skip building msvcr library: "%s" exists' % (out_file)) - return True - def_name = "lib%s.def" % msvcr_name def_file = os.path.join(sys.prefix, 'libs', def_name) From f1d96d12d891864d0ba123bb00bffc3af5ff2867 Mon Sep 17 00:00:00 2001 From: Han Date: Tue, 4 Oct 2011 14:14:07 +0200 Subject: [PATCH 11/11] BUG: Fix version comparison bug on Py25. --- numpy/distutils/mingw32ccompiler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/numpy/distutils/mingw32ccompiler.py b/numpy/distutils/mingw32ccompiler.py index 42d834c6c963..e64cdf8b7b5f 100644 --- a/numpy/distutils/mingw32ccompiler.py +++ b/numpy/distutils/mingw32ccompiler.py @@ -115,7 +115,7 @@ def __init__ (self, # bad consequences, like using Py_ModuleInit4 instead of # Py_ModuleInit4_64, etc... So we add it here if get_build_architecture() == 'AMD64': - if self.gcc_version < "4.": + if self.gcc_version < "4.0": self.set_executables( compiler='gcc -g -DDEBUG -DMS_WIN64 -mno-cygwin -O0 -Wall', compiler_so='gcc -g -DDEBUG -DMS_WIN64 -mno-cygwin -O0 -Wall -Wstrict-prototypes', @@ -135,7 +135,7 @@ def __init__ (self, linker_exe='g++ -mno-cygwin', linker_so='%s -mno-cygwin -mdll -static %s' % (self.linker, entry_point)) - elif self.gcc_version < "4.": + elif self.gcc_version < "4.0": self.set_executables(compiler='gcc -mno-cygwin -O2 -Wall', compiler_so='gcc -mno-cygwin -O2 -Wall -Wstrict-prototypes', linker_exe='g++ -mno-cygwin',