python-cmake-buildsystem · fbordignon · Mar 19, 2018 · Mar 19, 2018 · Mar 21, 2018 · Mar 21, 2018
diff --git a/.circleci-matrix.yml b/.circleci-matrix.yml
@@ -1,12 +1,28 @@
 env:
 
-  - IMAGE=linux-x64 PY_VERSION=3.6.4
-  - IMAGE=linux-x64 PY_VERSION=3.5.5
-  - IMAGE=linux-x64 PY_VERSION=2.7.14
+  - IMAGE=linux-x64 PY_VERSION=3.8.5
+  - IMAGE=linux-x64 PY_VERSION=3.8.4
+  - IMAGE=linux-x64 PY_VERSION=3.8.3
+  - IMAGE=linux-x64 PY_VERSION=3.8.2
+  - IMAGE=linux-x64 PY_VERSION=3.8.1
+  - IMAGE=linux-x64 PY_VERSION=3.8.0
+  - IMAGE=linux-x64 PY_VERSION=3.7.7
+  - IMAGE=linux-x64 PY_VERSION=3.7.6
+  - IMAGE=linux-x64 PY_VERSION=3.6.9
+  - IMAGE=linux-x64 PY_VERSION=3.5.9
+  - IMAGE=linux-x64 PY_VERSION=2.7.17
 
-  - IMAGE=linux-x86 PY_VERSION=3.6.4
-  - IMAGE=linux-x86 PY_VERSION=3.5.5
-  - IMAGE=linux-x86 PY_VERSION=2.7.14
+  - IMAGE=linux-x86 PY_VERSION=3.8.5
+  - IMAGE=linux-x86 PY_VERSION=3.8.4
+  - IMAGE=linux-x86 PY_VERSION=3.8.3
+  - IMAGE=linux-x86 PY_VERSION=3.8.2
+  - IMAGE=linux-x86 PY_VERSION=3.8.1
+  - IMAGE=linux-x64 PY_VERSION=3.8.0
+  - IMAGE=linux-x86 PY_VERSION=3.7.7
+  - IMAGE=linux-x64 PY_VERSION=3.7.6
+  - IMAGE=linux-x86 PY_VERSION=3.6.9
+  - IMAGE=linux-x86 PY_VERSION=3.5.9
+  - IMAGE=linux-x86 PY_VERSION=2.7.17
 command:
   - |
     if [[ $STEP == "dependencies" ]]; then

diff --git a/.travis.yml b/.travis.yml
@@ -9,18 +9,47 @@ compiler:
  - clang
 
 env:
+ - PY_VERSION=3.8.5
+ - PY_VERSION=3.8.4
+ - PY_VERSION=3.8.3
+ - PY_VERSION=3.8.2
+ - PY_VERSION=3.8.1
+ - PY_VERSION=3.8.0
+ - PY_VERSION=3.7.7
+ - PY_VERSION=3.7.6
+ - PY_VERSION=3.7.5
+ - PY_VERSION=3.7.4
+ - PY_VERSION=3.7.4
+ - PY_VERSION=3.7.3
+ - PY_VERSION=3.7.2
+ - PY_VERSION=3.7.1
+   # - PY_VERSION=3.7.0
+
+ - PY_VERSION=3.6.10
+ - PY_VERSION=3.6.9
+ - PY_VERSION=3.6.8
+ - PY_VERSION=3.6.7
+ - PY_VERSION=3.6.6
+ - PY_VERSION=3.6.5
  - PY_VERSION=3.6.4
  - PY_VERSION=3.6.3
  - PY_VERSION=3.6.2
  - PY_VERSION=3.6.1
  - PY_VERSION=3.6.0
 
+ - PY_VERSION=3.5.9
+ - PY_VERSION=3.5.8
+ - PY_VERSION=3.5.7
+ - PY_VERSION=3.5.6
  - PY_VERSION=3.5.5
  - PY_VERSION=3.5.4
  - PY_VERSION=3.5.3
  - PY_VERSION=3.5.2
  - PY_VERSION=3.5.1
 
+ - PY_VERSION=2.7.17
+ - PY_VERSION=2.7.16
+ - PY_VERSION=2.7.15
  - PY_VERSION=2.7.14
  - PY_VERSION=2.7.13
  - PY_VERSION=2.7.12
@@ -38,7 +67,6 @@ sudo: false
 
 branches:
  only:
-  - master
   - /^ci.*$/
 
 addons:

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1,6 +1,17 @@
 cmake_minimum_required(VERSION 2.8.6)
 
-set(PYTHON_VERSION "3.6.7" CACHE STRING "The version of Python to build.")
+set(PYTHON_VERSION "3.10.0" CACHE STRING "The version of Python to build.")
+
+###############
+# ASI default settings
+#list(APPEND CMAKE_MODULE_PATH "$ENV{ASV_CMAKE}")
+#include(asv_cmake_defaults)
+#asv_cmake_defaults( )
+###############
+
+if(WIN32)
+    set( OPENSSL_ROOT_DIR i:/t/openssl)
+endif(WIN32)
 
 string(REPLACE "." ";" VERSION_LIST ${PYTHON_VERSION})
 list(GET VERSION_LIST 0 PY_VERSION_MAJOR)
@@ -42,7 +53,7 @@ include(CTest)
 
 # Options
 option(DOWNLOAD_SOURCES "Automatically download the Python sources" ON)
-set(_build_libpython_shared_default  0)
+set(_build_libpython_shared_default  1)
 set(_build_libpython_shared_desc "Build libpython as a shared library")
 
 if(DEFINED BUILD_SHARED OR DEFINED BUILD_STATIC)
@@ -67,7 +78,7 @@ option(BUILD_LIBPYTHON_SHARED ${_build_libpython_shared_desc} ${_build_libpython
 option(BUILD_EXTENSIONS_AS_BUILTIN "Default all modules as builtin to libpython" OFF)
 option(USE_LIB64 "Search for dependencies and install to prefix/lib64 instead of prefix/lib" OFF)
 if(WIN32)
-  option(INSTALL_WINDOWS_TRADITIONAL "Install the executable and extensions in the traditional location" OFF)
+  option(INSTALL_WINDOWS_TRADITIONAL "Install the executable and extensions in the traditional location" ON)
   set(_build_wininst_default ON)
   if(CMAKE_CROSSCOMPILING)
     set(_build_wininst_default OFF)
@@ -100,14 +111,15 @@ option(USE_SYSTEM_LIBRARIES "Use system libraries" ON)
 # Set platform dependent defaults
 set(_use_builtin_zlib_default OFF)
 set(_use_system_zlib_default ON)
-if(WIN32)
+if(WIN32 AND "${PY_VERSION}" VERSION_LESS "3.7.0")
   set(_use_builtin_zlib_default ON)
   set(_use_system_zlib_default OFF)
 endif()
 
 cmake_dependent_option(USE_SYSTEM_BZip2 "Use system BZip2" ON "USE_SYSTEM_LIBRARIES" OFF)
 cmake_dependent_option(USE_SYSTEM_Curses "Use system Curses" ON "USE_SYSTEM_LIBRARIES" OFF)
 cmake_dependent_option(USE_SYSTEM_EXPAT "Use system EXPAT" ON "USE_SYSTEM_LIBRARIES" OFF)
+cmake_dependent_option(USE_SYSTEM_FFI "Use system FFI" ON "USE_SYSTEM_LIBRARIES" OFF)
 cmake_dependent_option(USE_SYSTEM_OpenSSL "Use system OpenSSL" ON "USE_SYSTEM_LIBRARIES" OFF)
 cmake_dependent_option(USE_SYSTEM_TCL "Use system TCL" ON "USE_SYSTEM_LIBRARIES" OFF)
 cmake_dependent_option(USE_SYSTEM_ZLIB "Use system ZLIB" ${_use_system_zlib_default} "USE_SYSTEM_LIBRARIES" OFF)
@@ -126,7 +138,12 @@ cmake_dependent_option(USE_LIBEDIT "Use libedit instead of readline" OFF
 option(WITH_TSC "profile with the Pentium timestamp counter" OFF)
 option(ENABLE_IPV6 "Enable IP v6" ON)
 option(WITH_DOC_STRINGS "Enable if you want documentation strings in extension modules" ON)
-set(Py_DEBUG OFF) # TODO - Build debugging interpreter
+set(Py_DEBUG OFF) # Py_Debug turn on extra stuff
+if(CMAKE_BUILD_TYPE MATCHES Debug)
+    if(UNIX)
+        SET(CMAKE_DEBUG_POSTFIX "") # no _d on linux
+    endif(UNIX)
+endif(CMAKE_BUILD_TYPE MATCHES Debug)
 option(WITH_PYMALLOC "Define if you want to compile in Python-specific mallocs" ON)
 option(WITH_THREAD "Compile in rudimentary thread support" ON)
 if(UNIX)
@@ -174,7 +191,7 @@ get_filename_component(SRC_DIR "${SRC_DIR}" ABSOLUTE)
 
 # Download sources
 get_filename_component(_parent_dir ${CMAKE_CURRENT_BINARY_DIR} PATH)
-string(REGEX REPLACE "rc[1-9]$" "" _py_version_patch_no_rc ${PY_VERSION_PATCH})
+string(REGEX REPLACE "[a-z]+[1-9]$" "" _py_version_patch_no_rc ${PY_VERSION_PATCH})
 set(_py_version_no_rc "${PY_VERSION_MAJOR}.${PY_VERSION_MINOR}.${_py_version_patch_no_rc}")
 set(_download_link "https://www.python.org/ftp/python/${_py_version_no_rc}/Python-${PY_VERSION}.tgz")
 # Variable below represent the set of available python version.
@@ -192,12 +209,21 @@ set(_download_2.7.12_md5 "88d61f82e3616a4be952828b3694109d")
 set(_download_2.7.13_md5 "17add4bf0ad0ec2f08e0cae6d205c700")
 set(_download_2.7.14_md5 "cee2e4b33ad3750da77b2e85f2f8b724")
 set(_download_2.7.15_md5 "045fb3440219a1f6923fefdabde63342")
+if (UNIX OR APPLE)
+    # 2.7.16 and 2.7.17 - patch problems on VS
+    set(_download_2.7.16_md5 "f1a2ace631068444831d01485466ece0")
+    set(_download_2.7.17_md5 "27a7919fa8d1364bae766949aaa91a5b")
+endif()
 # 3.5.x
 set(_download_3.5.1_md5 "be78e48cdfc1a7ad90efff146dce6cfe")
 set(_download_3.5.2_md5 "3fe8434643a78630c61c6464fe2e7e72")
 set(_download_3.5.3_md5 "6192f0e45f02575590760e68c621a488")
 set(_download_3.5.4_md5 "2ed4802b7a2a7e40d2e797272bf388ec")
 set(_download_3.5.5_md5 "7c825b747d25c11e669e99b912398585")
+set(_download_3.5.6_md5 "99a7e803633a627b264a42ce976d8c19")
+set(_download_3.5.7_md5 "92f4c16c55429bf986f5ab45fe3a6659")
+set(_download_3.5.8_md5 "c52ac1fc37e5daebb08069ea0e27d293")
+set(_download_3.5.9_md5 "5a58675043bde569d235f41dadeada42")
 # 3.6.x
 set(_download_3.6.0_md5 "3f7062ccf8be76491884d0e47ac8b251")
 set(_download_3.6.1_md5 "2d0fc9f3a5940707590e07f03ecb08b9")
@@ -207,6 +233,36 @@ set(_download_3.6.4_md5 "9de6494314ea199e3633211696735f65")
 set(_download_3.6.5_md5 "ab25d24b1f8cc4990ade979f6dc37883")
 set(_download_3.6.6_md5 "9a080a86e1a8d85e45eee4b1cd0a18a2")
 set(_download_3.6.7_md5 "c83551d83bf015134b4b2249213f3f85")
+set(_download_3.6.8_md5 "48f393a04c2e66c77bfc114e589ec630")
+set(_download_3.6.9_md5 "ff7cdaef4846c89c1ec0d7b709bbd54d")
+set(_download_3.6.10_md5 "df5f494ef9fbb03a0264d1e9d406aada")
+# 3.7.x
+if (WIN32)
+    # 3.7.0 needs patch for
+    # In file included from /home/appsmith/asv/ports/build-64/Python-3.7.0/Modules/_ssl.c:38:
+    # Python-3.7.0/Modules/socketmodule.h:206:24: error: field ‘vm’ has incomplete type
+    #   206 |     struct sockaddr_vm vm;
+    set(_download_3.7.0_md5 "41b6595deb4147a1ed517a7d9a580271")
+endif()
+set(_download_3.7.1_md5 "99f78ecbfc766ea449c4d9e7eda19e83")
+set(_download_3.7.2_md5 "02a75015f7cd845e27b85192bb0ca4cb")
+set(_download_3.7.3_md5 "2ee10f25e3d1b14215d56c3882486fcf")
+set(_download_3.7.4_md5 "68111671e5b2db4aef7b9ab01bf0f9be")
+set(_download_3.7.5_md5 "1cd071f78ff6d9c7524c95303a3057aa")
+set(_download_3.7.6_md5 "3ef90f064506dd85b4b4ab87a7a83d44")
+set(_download_3.7.7_md5 "d348d978a5387512fbc7d7d52dd3a5ef")
+# 3.8.x
+set(_download_3.8.0_md5 "e18a9d1a0a6d858b9787e03fc6fdaa20")
+set(_download_3.8.1_md5 "f215fa2f55a78de739c1787ec56b2bcd")
+set(_download_3.8.2_md5 "f9f3768f757e34b342dbc06b41cbc844")
+set(_download_3.8.3_md5 "a7c10a2ac9d62de75a0ca5204e2e7d07")
+set(_download_3.8.4_md5 "387e63fe42c40a29e3408ce231315516")
+set(_download_3.8.5_md5 "e2f52bcf531c8cc94732c0b6ff933ff0")
+set(_download_3.8.6_md5 "ea132d6f449766623eee886966c7d41f")
+set(_download_3.9.0_md5 "e19e75ec81dd04de27797bf3f9d918fd")
+set(_download_3.10.0_md5 "729e36388ae9a832b01cf9138921b383")
+
+
 
 set(_extracted_dir "Python-${PY_VERSION}")
 
@@ -255,24 +311,28 @@ If you already downloaded the source, you could try to re-configure this project
 endif()
 message(STATUS "SRC_DIR: ${SRC_DIR}")
 
-# Extract version from python source (Copied from FindPythonLibs.cmake)
+# Extract version string from python source (Copied from FindPythonLibs.cmake)
 file(STRINGS "${SRC_DIR}/Include/patchlevel.h" python_version_str
     REGEX "^#define[ \t]+PY_VERSION[ \t]+\"[^\"]+\"")
 string(REGEX REPLACE "^#define[ \t]+PY_VERSION[ \t]+\"([^\"]+)\".*" "\\1"
-    _version "${python_version_str}")
-string(REGEX REPLACE "([0-9])\\..+" "\\1"
-    PY_VERSION_MAJOR ${_version})
-string(REGEX REPLACE "[0-9]\\.([0-9]+)\\..+" "\\1"
-    PY_VERSION_MINOR ${_version})
-string(REGEX REPLACE "[0-9]\\.[0-9]+\\.([0-9]+)[+]?" "\\1"
-    PY_VERSION_PATCH ${_version})
-set(PY_VERSION "${PY_VERSION_MAJOR}.${PY_VERSION_MINOR}.${PY_VERSION_PATCH}")
-message(STATUS "PY_VERSION: ${PY_VERSION}")
-if(NOT DEFINED _download_${PY_VERSION}_md5)
-    message(WARNING "warning: selected python version '${PY_VERSION}' is not tested. Tested versions match '2.7.[3-15]' or '3.5.[1-5]' or `3.6.[0-4]`")
-endif()
+      PY_VERSION_LONG "${python_version_str}")
 
-set(PYTHON_VERSION "${PY_VERSION}" CACHE STRING "The version of Python to build." FORCE)
+# Extract version info and set variables PY_VERSION, PY_VERSION_(MAJOR|MINOR|PATCH), PY_RELEASE_(LEVEL|SERIAL)
+include(cmake/PythonExtractVersionInfo.cmake)
+python_extract_version_info(
+  VERSION_STRING "${PY_VERSION_LONG}"
+  )
+message(STATUS "PY_VERSION     : ${PY_VERSION}")
+message(STATUS "PY_VERSION_LONG: ${PY_VERSION_LONG}")
+
+# Check version
+if(NOT DEFINED _download_${PY_VERSION_LONG}_md5)
+    message(WARNING "warning: selected python version '${PY_VERSION_LONG}' is not tested. Tested versions match '2.7.[3-14]' or '3.5.[1-5]' or `3.6.[0-4]`")
+endif()
+if(NOT "${PYTHON_VERSION}" STREQUAL "${PY_VERSION}")
+  message(STATUS "Overwriting PYTHON_VERSION with value read from 'patchlevel.h': ${PYTHON_VERSION} -> ${PY_VERSION}")
+  set(PYTHON_VERSION "${PY_VERSION}" CACHE STRING "The version of Python to build." FORCE)
+endif()
 
 # Apply patches
 option(PYTHON_APPLY_PATCHES "Apply patches" ON)
@@ -340,7 +400,7 @@ endif()
 # Install tree directory
 set(BIN_INSTALL_DIR bin)                         # Contains the python executable
 if(INSTALL_WINDOWS_TRADITIONAL)
-    set(BIN_INSTALL_DIR .)                       # Contains the python executable
+    set(BIN_INSTALL_DIR Scripts)                       # Contains the python executable
 endif()
 set(LD_VERSION ${LIBPYTHON_VERSION}${ABIFLAGS})
 set(CONFIG_INSTALL_DIR share/${LIBPYTHON})
@@ -404,6 +464,7 @@ endif()
 
 # Set include directories
 include_directories(${INCLUDE_BUILD_DIR})
+include_directories(${INCLUDE_BUILD_DIR}/internal)
 include_directories(${PYCONFIG_BUILD_DIR})
 include_directories(${SRC_DIR}/Python)
 
@@ -468,9 +529,6 @@ add_subdirectory(cmake/libpython CMakeBuild/libpython)
 add_subdirectory(cmake/python CMakeBuild/python)
 add_subdirectory(cmake/include CMakeBuild/include)
 add_subdirectory(cmake/lib CMakeBuild/lib)
-if(BUILD_WININST)
-    add_subdirectory(cmake/PC/bdist_wininst CMakeBuild/bdist_wininst)
-endif()
 
 # Add target to run "Argument Clinic" over all source files
 if(IS_PY3)
@@ -484,18 +542,20 @@ add_custom_target(clinic
 endif()
 
 # Add target to generate 'Include/graminit.h' and 'Python/graminit.c'
+# Makefile.pre regen-grammar
 add_custom_target(generate_graminit
-    COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $<TARGET_FILE:pgen>
+    COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $<TARGET_FILE:python> -m Parser.pgen
         ${SRC_DIR}/Grammar/Grammar
-        ${PROJECT_BINARY_DIR}/CMakeFiles/graminit.h
-        ${PROJECT_BINARY_DIR}/CMakeFiles/graminit.c
+        ${SRC_DIR}/Grammar/Grammar/Tokens
+        ${PROJECT_BINARY_DIR}/CMakeFiles/graminit.h.new
+        ${PROJECT_BINARY_DIR}/CMakeFiles/graminit.c.new
     COMMAND ${CMAKE_COMMAND} -E copy_if_different
         ${PROJECT_BINARY_DIR}/CMakeFiles/graminit.h
         ${SRC_DIR}/Include/graminit.h
     COMMAND ${CMAKE_COMMAND} -E copy_if_different
         ${PROJECT_BINARY_DIR}/CMakeFiles/graminit.c
         ${SRC_DIR}/Python/graminit.c
-    DEPENDS pgen
+    DEPENDS python
     WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
     COMMENT "Generating '${SRC_DIR}/Include/graminit.h' and '${SRC_DIR}/Python/graminit.c'."
     VERBATIM
@@ -621,14 +681,40 @@ if(BUILD_TESTING)
     endif()
     set(TESTPYTHON ${CMAKE_CROSSCOMPILING_EMULATOR} ${TESTPYTHON})
     include(cmake/UnitTests.cmake)
+    get_unittests(unittests)
     foreach(unittest ${unittests})
       set(EXTRATESTOPTS -v)
       # XXX Running 'test_doctest' in verbose mode always return 1
       if(unittest STREQUAL "test_doctest")
         set(EXTRATESTOPTS )
       endif()
-      add_test(NAME ${unittest} COMMAND ${PYTHON_WRAPPER_COMMAND} ${TESTPYTHON} ${TESTPROG} ${TESTOPTS} ${EXTRATESTOPTS} ${unittest})
+
+      # FIX - some tests time out on some versions
+      set(add_this_test TRUE)
+      if(${unittest} STREQUAL "test_ssl")
+          if (${PY_VERSION} VERSION_GREATER_EQUAL "3.7.4")
+              # test_ssl on python 3.7.4 and greater times out
+              set(add_this_test FALSE)
+          endif()
+      endif()
+
+      if (${add_this_test})
+          add_test(NAME ${unittest} COMMAND ${PYTHON_WRAPPER_COMMAND} ${TESTPYTHON} ${TESTPROG} ${TESTOPTS} ${EXTRATESTOPTS} ${unittest})
+      endif()
     endforeach()
+
+    function(add_cmakescript_test testname script)
+      add_test(cmake_${testname}_test ${CMAKE_COMMAND}
+        -DTEST_${testname}:BOOL=ON
+        -P ${CMAKE_SOURCE_DIR}/${script})
+      set_tests_properties(cmake_${testname}_test PROPERTIES
+        LABELS CMake
+        PASS_REGULAR_EXPRESSION "SUCCESS")
+    endfunction()
+    add_cmakescript_test(
+        python_extract_version_info
+        cmake/PythonExtractVersionInfo.cmake
+        )
 endif()
 
 
@@ -693,3 +779,6 @@ if(INSTALL_WINDOWS_TRADITIONAL)
   set(license_destination .)
 endif()
 install(FILES ${SRC_DIR}/LICENSE DESTINATION ${license_destination} COMPONENT Runtime RENAME LICENSE.txt)
+
+# run script last
+add_subdirectory(post-install)
diff --git a/README.rst b/README.rst
@@ -55,7 +55,7 @@ How to use this buildsystem:
 
 .. note::
 
-  By default, the build system will download the python 3.6.7 source from
+  By default, the build system will download the python 3.8.5 source from
   http://www.python.org/ftp/python/
 
 
@@ -180,6 +180,11 @@ options on the commandline with `-DOPTION=VALUE`, or use the "ccmake" gui.
     Associated python extensions are: ELEMENTTREE, PYEXPAT
     Following CMake variables can manually be set: EXPAT_LIBRARIES, EXPAT_INCLUDE_DIRS
 
+  USE_SYSTEM_FFI=ON|OFF       (defaults to ON)
+    If set to OFF, no attempt to detect libffi libraries will be done.
+    Associated python extensions are: CTYPES
+    Following CMake variables can manually be set: FFI_LIBRARY, FFI_INCLUDE_DIR
+
   USE_SYSTEM_OpenSSL=ON|OFF     (defaults to ON)
     If set to OFF, no attempt to detect OpenSSL libraries will be done.
     Associated python extensions are: HASHLIB, SSL, MD5, SHA, SHA256, SHA512