From f9be94491f5f675b636d48fc5a4c5ce926606a77 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 7 Dec 2022 18:22:37 +0100 Subject: [PATCH 1/3] gh-100086: Add build details to sys.version Add more build details about how Python was configured in sys.version, like "debug" or "release" build. Build details and related configure option: * "debug": --with-pydebug, "release" otherwise * "+assert": --with-assertions * "+tracerefs": --with-trace-refs * "+pystats": --enable-pystats * "asan": --with-address-sanitizer * "msan": --with-memory-sanitizer * "ubsan": --with-undefined-behavior-sanitizer * "lto+pgo": --with-lto --enable-optimizations * "framework": --enable-framework * "shared": --enable-shared --- Doc/whatsnew/3.12.rst | 4 ++ Makefile.pre.in | 4 +- ...-12-07-18-31-40.gh-issue-100086.oiz2PI.rst | 2 + Modules/getbuildinfo.c | 13 +++- configure | 68 +++++++++++++++++++ configure.ac | 67 ++++++++++++++++++ 6 files changed, 155 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2022-12-07-18-31-40.gh-issue-100086.oiz2PI.rst diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 73dc462f0b3303..9d90f931c11a9e 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -323,6 +323,10 @@ sys with contributions from Gregory P. Smith [Google] and Mark Shannon in :gh:`96123`.) +* Add more build details about how Python was configured in + :data:`sys.version`, like "debug" or "release" build. + (Contributed by Victor Stinner in :gh:`100086`.) + Optimizations ============= diff --git a/Makefile.pre.in b/Makefile.pre.in index f6df7a620deaed..53a5027e93e87f 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -123,7 +123,8 @@ PY_CORE_CFLAGS= $(PY_STDMODULE_CFLAGS) -DPy_BUILD_CORE PY_CORE_LDFLAGS=$(PY_LDFLAGS) $(PY_LDFLAGS_NODIST) # Strict or non-strict aliasing flags used to compile dtoa.c, see above CFLAGS_ALIASING=@CFLAGS_ALIASING@ - +# String describing the Python build used by sys.version (ex: "debug build") +PY_BUILD_STR=@PY_BUILD_STR@ # Machine-dependent subdirectories MACHDEP= @MACHDEP@ @@ -1245,6 +1246,7 @@ Modules/getbuildinfo.o: $(PARSER_OBJS) \ -DGITVERSION="\"`LC_ALL=C $(GITVERSION)`\"" \ -DGITTAG="\"`LC_ALL=C $(GITTAG)`\"" \ -DGITBRANCH="\"`LC_ALL=C $(GITBRANCH)`\"" \ + -DPY_BUILD_STR="\"$(PY_BUILD_STR)\"" \ -o $@ $(srcdir)/Modules/getbuildinfo.c Modules/getpath.o: $(srcdir)/Modules/getpath.c Python/frozen_modules/getpath.h Makefile $(PYTHON_HEADERS) diff --git a/Misc/NEWS.d/next/Library/2022-12-07-18-31-40.gh-issue-100086.oiz2PI.rst b/Misc/NEWS.d/next/Library/2022-12-07-18-31-40.gh-issue-100086.oiz2PI.rst new file mode 100644 index 00000000000000..94886a7e677da8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-07-18-31-40.gh-issue-100086.oiz2PI.rst @@ -0,0 +1,2 @@ +Add more build details about how Python was configured in :data:`sys.version`, +like "debug" or "release" build. Patch by Victor Stinner. diff --git a/Modules/getbuildinfo.c b/Modules/getbuildinfo.c index a24750b76c09bc..4217512f096d2f 100644 --- a/Modules/getbuildinfo.c +++ b/Modules/getbuildinfo.c @@ -30,6 +30,14 @@ #ifndef GITBRANCH #define GITBRANCH "" #endif +#ifndef PY_BUILD_STR + // On Windows, only provide basic info +# ifdef Py_DEBUG +# define PY_BUILD_STR "debug build" +# else +# define PY_BUILD_STR "release build" +# endif +#endif static int initialized = 0; static char buildinfo[50 + sizeof(GITVERSION) + @@ -50,8 +58,9 @@ Py_GetBuildInfo(void) gitid = "main"; } PyOS_snprintf(buildinfo, sizeof(buildinfo), - "%s%s%s, %.20s, %.9s", gitid, sep, revision, - DATE, TIME); + "%s%s%s, %.20s, %.9s, %s", + gitid, sep, revision, DATE, TIME, + PY_BUILD_STR); return buildinfo; } diff --git a/configure b/configure index 3f8daf9dad5fd8..c75b6150d05955 100755 --- a/configure +++ b/configure @@ -623,6 +623,7 @@ ac_includes_default="\ #endif" ac_subst_vars='LTLIBOBJS +PY_BUILD_STR MODULE_BLOCK MODULE_XXLIMITED_35_FALSE MODULE_XXLIMITED_35_TRUE @@ -11072,6 +11073,8 @@ $as_echo "no" >&6; } ;; esac +SANITIZERS="" + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-address-sanitizer" >&5 $as_echo_n "checking for --with-address-sanitizer... " >&6; } @@ -11080,6 +11083,7 @@ if test "${with_address_sanitizer+set}" = set; then : withval=$with_address_sanitizer; { $as_echo "$as_me:${as_lineno-$LINENO}: result: $withval" >&5 $as_echo "$withval" >&6; } +SANITIZERS="$SANITIZERS asan" BASECFLAGS="-fsanitize=address -fno-omit-frame-pointer $BASECFLAGS" LDFLAGS="-fsanitize=address $LDFLAGS" # ASan works by controlling memory allocation, our own malloc interferes. @@ -11130,6 +11134,7 @@ fi $as_echo "$ax_cv_check_cflags___fsanitize_memory" >&6; } if test "x$ax_cv_check_cflags___fsanitize_memory" = xyes; then : +SANITIZERS="$SANITIZERS msan" BASECFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer $BASECFLAGS" LDFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 $LDFLAGS" @@ -11154,6 +11159,7 @@ if test "${with_undefined_behavior_sanitizer+set}" = set; then : withval=$with_undefined_behavior_sanitizer; { $as_echo "$as_me:${as_lineno-$LINENO}: result: $withval" >&5 $as_echo "$withval" >&6; } +SANITIZERS="$SANITIZERS ubsan" BASECFLAGS="-fsanitize=undefined $BASECFLAGS" LDFLAGS="-fsanitize=undefined $LDFLAGS" with_ubsan="yes" @@ -11167,6 +11173,10 @@ with_ubsan="no" fi +# strip leading space +SANITIZERS=$(echo $SANITIZERS | sed 's/^ //') + + # Set info about shared libraries. @@ -28097,6 +28107,64 @@ $as_echo "$py_cv_module_xxlimited_35" >&6; } # substitute multiline block, must come after last PY_STDLIB_MOD() + +# String describing the Python build used by sys.version (ex: "debug build") +# Short string containing most important configure options. +# Examples: +# - "debug+tracerefs build" +# - "release+assert shared build" +# - "release lto+pgo build" + +if test "$Py_DEBUG" = 'true' ; then + PY_BUILD_STR="debug" +else + PY_BUILD_STR="release" + if test "$assertions" = 'true'; then + PY_BUILD_STR="$PY_BUILD_STR+assert" + fi +fi +if test "$with_trace_refs" = "yes"; then + PY_BUILD_STR="$PY_BUILD_STR+tracerefs" +fi +if test "$enable_pystats" = "yes"; then + PY_BUILD_STR="$PY_BUILD_STR+pystats" +fi +if test -n "${SANITIZERS}"; then + PY_BUILD_STR="$PY_BUILD_STR $SANITIZERS" +fi +if test "$Py_LTO" = 'true' ; then + LTO_BUILD_STR="lto" + if test "$Py_LTO_POLICY" = 'thin' ; then + LTO_BUILD_STR="lto=thin" + fi +else + LTO_BUILD_STR="" +fi +if test "$Py_OPT" = 'true' ; then + PGO_BUILD_STR="pgo" +else + PGO_BUILD_STR="" +fi +if test -n "${LTO_BUILD_STR}"; then + if test -n "${PGO_BUILD_STR}"; then + PY_BUILD_STR="$PY_BUILD_STR $LTO_BUILD_STR+$PGO_BUILD_STR" + else + PY_BUILD_STR="$PY_BUILD_STR $LTO_BUILD_STR" + fi +else + if test -n "${PGO_BUILD_STR}"; then + PY_BUILD_STR="$PY_BUILD_STR $PGO_BUILD_STR" + fi +fi +if test "$enable_framework"; then + PY_BUILD_STR="$PY_BUILD_STR framework" +fi +if test "$PY_ENABLE_SHARED" = 1; then + PY_BUILD_STR="$PY_BUILD_STR shared" +fi +PY_BUILD_STR="$PY_BUILD_STR build" + + # generate output files ac_config_files="$ac_config_files Makefile.pre Misc/python.pc Misc/python-embed.pc Misc/python-config.sh" diff --git a/configure.ac b/configure.ac index 734a4db8389915..2ce4ca2c841c14 100644 --- a/configure.ac +++ b/configure.ac @@ -3093,12 +3093,15 @@ case $ac_sys_system/$ac_sys_release in ;; esac +SANITIZERS="" + AC_MSG_CHECKING(for --with-address-sanitizer) AC_ARG_WITH(address_sanitizer, AS_HELP_STRING([--with-address-sanitizer], [enable AddressSanitizer memory error detector, 'asan' (default is no)]), [ AC_MSG_RESULT($withval) +SANITIZERS="$SANITIZERS asan" BASECFLAGS="-fsanitize=address -fno-omit-frame-pointer $BASECFLAGS" LDFLAGS="-fsanitize=address $LDFLAGS" # ASan works by controlling memory allocation, our own malloc interferes. @@ -3113,6 +3116,7 @@ AC_ARG_WITH(memory_sanitizer, [ AC_MSG_RESULT($withval) AX_CHECK_COMPILE_FLAG([-fsanitize=memory],[ +SANITIZERS="$SANITIZERS msan" BASECFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer $BASECFLAGS" LDFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 $LDFLAGS" ],[AC_MSG_ERROR([The selected compiler doesn't support memory sanitizer])]) @@ -3127,6 +3131,7 @@ AC_ARG_WITH(undefined_behavior_sanitizer, [enable UndefinedBehaviorSanitizer undefined behaviour detector, 'ubsan' (default is no)]), [ AC_MSG_RESULT($withval) +SANITIZERS="$SANITIZERS ubsan" BASECFLAGS="-fsanitize=undefined $BASECFLAGS" LDFLAGS="-fsanitize=undefined $LDFLAGS" with_ubsan="yes" @@ -3136,6 +3141,10 @@ AC_MSG_RESULT(no) with_ubsan="no" ]) +# strip leading space +SANITIZERS=$(echo $SANITIZERS | sed 's/^ //') + + # Set info about shared libraries. AC_SUBST(SHLIB_SUFFIX) AC_SUBST(LDSHARED) @@ -7374,6 +7383,64 @@ PY_STDLIB_MOD([xxlimited_35], [test "$with_trace_refs" = "no"], [test "$ac_cv_fu # substitute multiline block, must come after last PY_STDLIB_MOD() AC_SUBST([MODULE_BLOCK]) + +# String describing the Python build used by sys.version (ex: "debug build") +# Short string containing most important configure options. +# Examples: +# - "debug+tracerefs build" +# - "release+assert shared build" +# - "release lto+pgo build" +AC_SUBST(PY_BUILD_STR) +if test "$Py_DEBUG" = 'true' ; then + PY_BUILD_STR="debug" +else + PY_BUILD_STR="release" + if test "$assertions" = 'true'; then + PY_BUILD_STR="$PY_BUILD_STR+assert" + fi +fi +if test "$with_trace_refs" = "yes"; then + PY_BUILD_STR="$PY_BUILD_STR+tracerefs" +fi +if test "$enable_pystats" = "yes"; then + PY_BUILD_STR="$PY_BUILD_STR+pystats" +fi +if test -n "${SANITIZERS}"; then + PY_BUILD_STR="$PY_BUILD_STR $SANITIZERS" +fi +if test "$Py_LTO" = 'true' ; then + LTO_BUILD_STR="lto" + if test "$Py_LTO_POLICY" = 'thin' ; then + LTO_BUILD_STR="lto=thin" + fi +else + LTO_BUILD_STR="" +fi +if test "$Py_OPT" = 'true' ; then + PGO_BUILD_STR="pgo" +else + PGO_BUILD_STR="" +fi +if test -n "${LTO_BUILD_STR}"; then + if test -n "${PGO_BUILD_STR}"; then + PY_BUILD_STR="$PY_BUILD_STR $LTO_BUILD_STR+$PGO_BUILD_STR" + else + PY_BUILD_STR="$PY_BUILD_STR $LTO_BUILD_STR" + fi +else + if test -n "${PGO_BUILD_STR}"; then + PY_BUILD_STR="$PY_BUILD_STR $PGO_BUILD_STR" + fi +fi +if test "$enable_framework"; then + PY_BUILD_STR="$PY_BUILD_STR framework" +fi +if test "$PY_ENABLE_SHARED" = 1; then + PY_BUILD_STR="$PY_BUILD_STR shared" +fi +PY_BUILD_STR="$PY_BUILD_STR build" + + # generate output files AC_CONFIG_FILES(Makefile.pre Misc/python.pc Misc/python-embed.pc Misc/python-config.sh) AC_CONFIG_FILES([Modules/Setup.bootstrap Modules/Setup.stdlib]) From 94b93efeeda862cc78ec3989d4bd4623e64b21f1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 7 Dec 2022 22:12:22 +0100 Subject: [PATCH 2/3] Update platform._sys_version() --- Lib/platform.py | 8 +++++--- Lib/test/test_platform.py | 7 ++++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/Lib/platform.py b/Lib/platform.py index 6745321e31c279..d2d508ce121bd7 100755 --- a/Lib/platform.py +++ b/Lib/platform.py @@ -1035,7 +1035,9 @@ def processor(): r'([\w.+]+)\s*' # "version" r'\(#?([^,]+)' # "(#buildno" r'(?:,\s*([\w ]*)' # ", builddate" - r'(?:,\s*([\w :]*))?)?\)\s*' # ", buildtime)" + r'(?:,\s*([\w :]*))?)?' # ", buildtime" + r'(?:,\s*([\w :]*))?' # ", build details" + r'\)\s*' # ")" r'\[([^\]]+)\]?', re.ASCII) # "[compiler]" _ironpython_sys_version_parser = re.compile( @@ -1114,7 +1116,7 @@ def _sys_version(sys_version=None): raise ValueError( 'failed to parse Jython sys.version: %s' % repr(sys_version)) - version, buildno, builddate, buildtime, _ = match.groups() + version, buildno, builddate, buildtime, _, _ = match.groups() if builddate is None: builddate = '' compiler = sys.platform @@ -1136,7 +1138,7 @@ def _sys_version(sys_version=None): raise ValueError( 'failed to parse CPython sys.version: %s' % repr(sys_version)) - version, buildno, builddate, buildtime, compiler = \ + version, buildno, builddate, buildtime, _, compiler = \ match.groups() name = 'CPython' if builddate is None: diff --git a/Lib/test/test_platform.py b/Lib/test/test_platform.py index 3992faf8e5cd5b..a8296cad5e8378 100644 --- a/Lib/test/test_platform.py +++ b/Lib/test/test_platform.py @@ -119,7 +119,7 @@ def tearDown(self): sys.platform = self.save_platform def test_sys_version(self): - # Old test. + # Test platform._sys_version() for input, output in ( ('2.4.3 (#1, Jun 21 2006, 13:54:21) \n[GCC 3.3.4 (pre 3.3.5 20040809)]', ('CPython', '2.4.3', '', '', '1', 'Jun 21 2006 13:54:21', 'GCC 3.3.4 (pre 3.3.5 20040809)')), @@ -143,6 +143,11 @@ def test_sys_version(self): ('CPython', '2.4.3', '', '', 'truncation', '', 'GCC')), ('2.4.3 (truncation) \n[GCC]', ('CPython', '2.4.3', '', '', 'truncation', '', 'GCC')), + ('3.12.0a3+ (heads/py_build_str:f9be94491f, ' + 'Dec 7 2022, 21:57:04, debug build) ' + '[GCC 12.2.1 20221121 (Red Hat 12.2.1-4)]', + ('CPython', '3.12.0a3+', '', '', 'heads/py_build_str:f9be94491f', + 'Dec 7 2022 21:57:04', 'GCC 12.2.1 20221121 (Red Hat 12.2.1-4)')), ): # branch and revision are not "parsed", but fetched # from sys._git. Ignore them From 5fca7d39a0e118586610ea1d04ca4a8e039a1e3a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 7 Dec 2022 23:22:25 +0100 Subject: [PATCH 3/3] Fix buildinfo buffer size --- Modules/getbuildinfo.c | 6 +++++- configure | 3 +++ configure.ac | 3 +++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/Modules/getbuildinfo.c b/Modules/getbuildinfo.c index 4217512f096d2f..7b65fc634cba23 100644 --- a/Modules/getbuildinfo.c +++ b/Modules/getbuildinfo.c @@ -39,10 +39,14 @@ # endif #endif +// len(", release+assert+tracerefs+pystats asan msan ubsan lto=thin+pgo framework shared build") +#define PY_BUILD_STR_MAXLEN 86 + static int initialized = 0; static char buildinfo[50 + sizeof(GITVERSION) + ((sizeof(GITTAG) > sizeof(GITBRANCH)) ? - sizeof(GITTAG) : sizeof(GITBRANCH))]; + sizeof(GITTAG) : sizeof(GITBRANCH)) + + PY_BUILD_STR_MAXLEN]; const char * Py_GetBuildInfo(void) diff --git a/configure b/configure index c75b6150d05955..1a730c718bb16d 100755 --- a/configure +++ b/configure @@ -28114,6 +28114,9 @@ $as_echo "$py_cv_module_xxlimited_35" >&6; } # - "debug+tracerefs build" # - "release+assert shared build" # - "release lto+pgo build" +# +# If the PY_BUILD_STR format changes, PY_BUILD_STR_MAXLEN in +# Modules/getbuildinfo.c should be updated. if test "$Py_DEBUG" = 'true' ; then PY_BUILD_STR="debug" diff --git a/configure.ac b/configure.ac index 2ce4ca2c841c14..7811f840f4303a 100644 --- a/configure.ac +++ b/configure.ac @@ -7390,6 +7390,9 @@ AC_SUBST([MODULE_BLOCK]) # - "debug+tracerefs build" # - "release+assert shared build" # - "release lto+pgo build" +# +# If the PY_BUILD_STR format changes, PY_BUILD_STR_MAXLEN in +# Modules/getbuildinfo.c should be updated. AC_SUBST(PY_BUILD_STR) if test "$Py_DEBUG" = 'true' ; then PY_BUILD_STR="debug"