Skip to content

[WIP] bpo-35266: Add _PyPreConfig #10575

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 3 commits into from
Closed

[WIP] bpo-35266: Add _PyPreConfig #10575

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ad68ded
bpo-35266: Add _PyPreConfig
vstinner Nov 15, 2018
6f42d81
fix _freeze_importlib
vstinner Nov 16, 2018
5363c1a
Fix pymain_run_file() and test_cmd_line_script
vstinner Nov 16, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 43 additions & 8 deletions Include/coreconfig.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,13 +107,9 @@ typedef struct {
If set to -1 (default), inherit Py_UTF8Mode value. */
int utf8_mode;

wchar_t *pycache_prefix; /* PYTHONPYCACHEPREFIX, -X pycache_prefix=PATH */

wchar_t *program_name; /* Program name, see also Py_GetProgramName() */
int argc; /* Number of command line arguments,
-1 means unset */
wchar_t **argv; /* Command line arguments */
wchar_t *program; /* argv[0] or "" */

int nxoption; /* Number of -X options */
wchar_t **xoptions; /* -X options */
Expand Down Expand Up @@ -339,15 +335,54 @@ typedef struct {
/* Note: _PyCoreConfig_INIT sets other fields to 0/NULL */


typedef struct {
_PyCoreConfig core_config;
wchar_t *program_name; /* Program name, see also Py_GetProgramName() */
wchar_t *program; /* argv[0] or "" */
wchar_t *pycache_prefix; /* PYTHONPYCACHEPREFIX, -X pycache_prefix=PATH */
} _PyPreConfig;

#define _PyPreConfig_INIT \
(_PyPreConfig){.core_config = _PyCoreConfig_INIT}
/* Note: _PyPreConfig_INIT sets other fields to 0/NULL */


typedef struct {
_PyCoreConfig core_config;
PyObject *argv; /* sys.argv list, can be NULL */
PyObject *program_name; /* Program name, see also Py_GetProgramName() */
PyObject* program; /* argv[0] or "" */
PyObject *executable; /* sys.executable str */
PyObject *prefix; /* sys.prefix str */
PyObject *base_prefix; /* sys.base_prefix str, can be NULL */
PyObject *exec_prefix; /* sys.exec_prefix str */
PyObject *base_exec_prefix; /* sys.base_exec_prefix str, can be NULL */
PyObject *warnoptions; /* sys.warnoptions list, can be NULL */
PyObject *xoptions; /* sys._xoptions dict, can be NULL */
PyObject *module_search_path; /* sys.path list */
PyObject *pycache_prefix; /* sys.pycache_prefix str, can be NULL */
} _PyMainInterpreterConfig;

#define _PyMainInterpreterConfig_INIT \
(_PyMainInterpreterConfig){ \
.core_config = _PyCoreConfig_INIT}
/* Note: _PyMainInterpreterConfig_INIT sets other fields to 0/NULL */


#ifndef Py_LIMITED_API
PyAPI_FUNC(_PyInitError) _PyCoreConfig_Read(_PyCoreConfig *config);
PyAPI_FUNC(void) _PyPreConfig_Clear(_PyPreConfig *config);
PyAPI_FUNC(int) _PyPreConfig_Copy(_PyPreConfig *config,
const _PyPreConfig *config2);
PyAPI_FUNC(_PyInitError) _PyPreConfig_Read(_PyPreConfig *config);
PyAPI_FUNC(_PyInitError) _PyPreConfig_SetPathConfig(
const _PyPreConfig *config);

PyAPI_FUNC(void) _PyCoreConfig_Clear(_PyCoreConfig *);
PyAPI_FUNC(int) _PyCoreConfig_Copy(
_PyCoreConfig *config,
const _PyCoreConfig *config2);
PyAPI_FUNC(_PyInitError) _PyCoreConfig_InitPathConfig(_PyCoreConfig *config);
PyAPI_FUNC(_PyInitError) _PyCoreConfig_SetPathConfig(
const _PyCoreConfig *config);
PyAPI_FUNC(_PyInitError) _PyCoreConfig_InitPathConfig(
_PyPreConfig *preconfig);
PyAPI_FUNC(void) _PyCoreConfig_GetGlobalConfig(_PyCoreConfig *config);
PyAPI_FUNC(void) _PyCoreConfig_SetGlobalConfig(const _PyCoreConfig *config);
PyAPI_FUNC(const char*) _PyCoreConfig_GetEnv(
Expand Down
1 change: 1 addition & 0 deletions Include/internal/pycore_pathconfig.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ PyAPI_FUNC(_PyInitError) _PyPathConfig_SetGlobal(

PyAPI_FUNC(_PyInitError) _PyPathConfig_Calculate_impl(
_PyPathConfig *config,
const _PyPreConfig *pre_config,
const _PyCoreConfig *core_config);
PyAPI_FUNC(PyObject*) _PyPathConfig_ComputeArgv0(int argc, wchar_t **argv);
PyAPI_FUNC(int) _Py_FindEnvConfigValue(
Expand Down
9 changes: 3 additions & 6 deletions Include/pylifecycle.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,16 +23,12 @@ PyAPI_FUNC(int) Py_SetStandardStreamEncoding(const char *encoding,


#ifndef Py_LIMITED_API
/* PEP 432 Multi-phase initialization API (Private while provisional!) */
PyAPI_FUNC(_PyInitError) _Py_InitializeCore(
PyInterpreterState **interp,
const _PyCoreConfig *);
PyAPI_FUNC(int) _Py_IsCoreInitialized(void);


PyAPI_FUNC(_PyInitError) _PyMainInterpreterConfig_Read(
_PyMainInterpreterConfig *config,
const _PyCoreConfig *core_config);
const _PyPreConfig *preconfig);
PyAPI_FUNC(void) _PyMainInterpreterConfig_Clear(_PyMainInterpreterConfig *);
PyAPI_FUNC(int) _PyMainInterpreterConfig_Copy(
_PyMainInterpreterConfig *config,
Expand All @@ -51,7 +47,8 @@ PyAPI_FUNC(void) Py_Initialize(void);
PyAPI_FUNC(void) Py_InitializeEx(int);
#ifndef Py_LIMITED_API
PyAPI_FUNC(_PyInitError) _Py_InitializeFromConfig(
const _PyCoreConfig *config);
const _PyPreConfig *config,
PyInterpreterState **interp_p);
PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalInitError(_PyInitError err);
#endif
PyAPI_FUNC(void) Py_Finalize(void);
Expand Down
23 changes: 0 additions & 23 deletions Include/pystate.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,28 +25,6 @@ typedef struct _is PyInterpreterState;
#else
typedef PyObject* (*_PyFrameEvalFunction)(struct _frame *, int);

/* Placeholders while working on the new configuration API
*
* See PEP 432 for final anticipated contents
*/
typedef struct {
int install_signal_handlers; /* Install signal handlers? -1 means unset */
PyObject *argv; /* sys.argv list, can be NULL */
PyObject *executable; /* sys.executable str */
PyObject *prefix; /* sys.prefix str */
PyObject *base_prefix; /* sys.base_prefix str, can be NULL */
PyObject *exec_prefix; /* sys.exec_prefix str */
PyObject *base_exec_prefix; /* sys.base_exec_prefix str, can be NULL */
PyObject *warnoptions; /* sys.warnoptions list, can be NULL */
PyObject *xoptions; /* sys._xoptions dict, can be NULL */
PyObject *module_search_path; /* sys.path list */
PyObject *pycache_prefix; /* sys.pycache_prefix str, can be NULL */
} _PyMainInterpreterConfig;

#define _PyMainInterpreterConfig_INIT \
(_PyMainInterpreterConfig){.install_signal_handlers = -1}
/* Note: _PyMainInterpreterConfig_INIT sets other fields to 0/NULL */

typedef struct _is {

struct _is *next;
Expand Down Expand Up @@ -79,7 +57,6 @@ typedef struct _is {
int codecs_initialized;
int fscodec_initialized;

_PyCoreConfig core_config;
_PyMainInterpreterConfig config;
#ifdef HAVE_DLOPEN
int dlopenflags;
Expand Down
59 changes: 34 additions & 25 deletions Lib/test/test_embed.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -288,10 +288,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
'coerce_c_locale': 0,
'coerce_c_locale_warn': 0,

'pycache_prefix': None,
'program_name': './_testembed',
'argv': [],
'program': None,

'xoptions': [],
'warnoptions': [],
Expand Down Expand Up @@ -331,16 +328,19 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
})

# main config
DEFAULT_MAIN_CONFIG = {
'program_name': './_testembed',
'program': None,
'pycache_prefix': None,
}
COPY_MAIN_CONFIG = (
# Copy core config to main config for expected values
'argv',
'base_exec_prefix',
'base_prefix',
'exec_prefix',
'executable',
'install_signal_handlers',
'prefix',
'pycache_prefix',
'warnoptions',
# xoptions is created from core_config in check_main_config().
# 'module_search_paths' is copied to 'module_search_path'.
Expand Down Expand Up @@ -418,17 +418,20 @@ def main_xoptions(self, xoptions_list):
xoptions[opt] = True
return xoptions

def check_main_config(self, config):
def check_main_config(self, config, expected=None):
core_config = config['core_config']
main_config = config['main_config']

# main config
expected_main = {}
if expected is not None:
expected = dict(self.DEFAULT_MAIN_CONFIG, **expected)
else:
expected = dict(self.DEFAULT_MAIN_CONFIG)
for key in self.COPY_MAIN_CONFIG:
expected_main[key] = core_config[key]
expected_main['module_search_path'] = core_config['module_search_paths']
expected_main['xoptions'] = self.main_xoptions(core_config['xoptions'])
self.assertEqual(main_config, expected_main)
expected[key] = core_config[key]
expected['module_search_path'] = core_config['module_search_paths']
expected['xoptions'] = self.main_xoptions(core_config['xoptions'])
self.assertEqual(main_config, expected)

def check_core_config(self, config, expected, env):
if expected['stdio_encoding'] is None or expected['stdio_errors'] is None:
Expand Down Expand Up @@ -463,8 +466,8 @@ def check_global_config(self, config):

self.assertEqual(config['global_config'], expected_global)

def check_config(self, testname, expected):
expected = dict(self.DEFAULT_CORE_CONFIG, **expected)
def check_config(self, testname, expected_core, expected_main=None):
expected_core = dict(self.DEFAULT_CORE_CONFIG, **expected_core)

env = dict(os.environ)
# Remove PYTHON* environment variables to get deterministic environment
Expand All @@ -480,16 +483,15 @@ def check_config(self, testname, expected):
# Ignore err
config = json.loads(out)

self.check_core_config(config, expected, env)
self.check_main_config(config)
self.check_core_config(config, expected_core, env)
self.check_main_config(config, expected_main)
self.check_global_config(config)

def test_init_default_config(self):
self.check_config("init_default_config", {})

def test_init_global_config(self):
config = {
'program_name': './globalvar',
core_config = {
'site_import': 0,
'bytes_warning': 1,
'inspect': 1,
Expand All @@ -508,7 +510,10 @@ def test_init_global_config(self):
'user_site_directory': 0,
'_frozen': 1,
}
self.check_config("init_global_config", config)
main_config = {
'program_name': './globalvar',
}
self.check_config("init_global_config", core_config, main_config)

def test_init_from_config(self):
config = {
Expand All @@ -528,10 +533,7 @@ def test_init_from_config(self):
'filesystem_encoding': 'utf-8',
'filesystem_errors': self.UTF8_MODE_ERRORS,

'pycache_prefix': 'conf_pycache_prefix',
'program_name': './conf_program_name',
'argv': ['-c', 'pass'],
'program': 'conf_program',
'xoptions': ['core_xoption1=3', 'core_xoption2=', 'core_xoption3'],
'warnoptions': ['default', 'error::ResourceWarning'],

Expand All @@ -550,10 +552,15 @@ def test_init_from_config(self):
'_check_hash_pycs_mode': 'always',
'_frozen': 1,
}
self.check_config("init_from_config", config)
main_config = {
'program_name': './conf_program_name',
'program': 'conf_program',
'pycache_prefix': 'conf_pycache_prefix',
}
self.check_config("init_from_config", config, main_config)

def test_init_env(self):
config = {
core_config = {
'use_hash_seed': 1,
'hash_seed': 42,
'allocator': 'malloc_debug',
Expand All @@ -565,7 +572,6 @@ def test_init_env(self):
'filesystem_errors': self.UTF8_MODE_ERRORS,
'inspect': 1,
'optimization_level': 2,
'pycache_prefix': 'env_pycache_prefix',
'write_bytecode': 0,
'verbose': 1,
'buffered_stdio': 0,
Expand All @@ -575,7 +581,10 @@ def test_init_env(self):
'faulthandler': 1,
'dev_mode': 1,
}
self.check_config("init_env", config)
main_config = {
'pycache_prefix': 'env_pycache_prefix',
}
self.check_config("init_env", core_config, main_config)

def test_init_dev_mode(self):
config = {
Expand Down
2 changes: 1 addition & 1 deletion Modules/_testcapimodule.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4704,7 +4704,7 @@ static PyObject *
get_core_config(PyObject *self, PyObject *Py_UNUSED(args))
{
PyInterpreterState *interp = _PyInterpreterState_Get();
const _PyCoreConfig *config = &interp->core_config;
const _PyCoreConfig *config = &interp->config.core_config;
return _PyCoreConfig_AsDict(config);
}

Expand Down
21 changes: 13 additions & 8 deletions Modules/getpath.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -561,7 +561,8 @@ calculate_reduce_exec_prefix(PyCalculatePath *calculate, wchar_t *exec_prefix)


static _PyInitError
calculate_program_full_path(const _PyCoreConfig *core_config,
calculate_program_full_path(const _PyPreConfig *pre_config,
const _PyCoreConfig *core_config,
PyCalculatePath *calculate, _PyPathConfig *config)
{
wchar_t program_full_path[MAXPATHLEN+1];
Expand All @@ -581,8 +582,8 @@ calculate_program_full_path(const _PyCoreConfig *core_config,
* other way to find a directory to start the search from. If
* $PATH isn't exported, you lose.
*/
if (wcschr(core_config->program_name, SEP)) {
wcsncpy(program_full_path, core_config->program_name, MAXPATHLEN);
if (wcschr(pre_config->program_name, SEP)) {
wcsncpy(program_full_path, pre_config->program_name, MAXPATHLEN);
}
#ifdef __APPLE__
/* On Mac OS X, if a script uses an interpreter of the form
Expand Down Expand Up @@ -624,7 +625,7 @@ calculate_program_full_path(const _PyCoreConfig *core_config,
wcsncpy(program_full_path, path, MAXPATHLEN);
}

joinpath(program_full_path, core_config->program_name);
joinpath(program_full_path, pre_config->program_name);
if (isxfile(program_full_path)) {
break;
}
Expand Down Expand Up @@ -934,12 +935,13 @@ calculate_free(PyCalculatePath *calculate)


static _PyInitError
calculate_path_impl(const _PyCoreConfig *core_config,
calculate_path_impl(const _PyPreConfig *pre_config,
const _PyCoreConfig *core_config,
PyCalculatePath *calculate, _PyPathConfig *config)
{
_PyInitError err;

err = calculate_program_full_path(core_config, calculate, config);
err = calculate_program_full_path(pre_config, core_config, calculate, config);
if (_Py_INIT_FAILED(err)) {
return err;
}
Expand Down Expand Up @@ -993,7 +995,9 @@ calculate_path_impl(const _PyCoreConfig *core_config,


_PyInitError
_PyPathConfig_Calculate_impl(_PyPathConfig *config, const _PyCoreConfig *core_config)
_PyPathConfig_Calculate_impl(_PyPathConfig *path_config,
const _PyPreConfig *pre_config,
const _PyCoreConfig *core_config)
{
PyCalculatePath calculate;
memset(&calculate, 0, sizeof(calculate));
Expand All @@ -1003,7 +1007,8 @@ _PyPathConfig_Calculate_impl(_PyPathConfig *config, const _PyCoreConfig *core_co
goto done;
}

err = calculate_path_impl(core_config, &calculate, config);
err = calculate_path_impl(pre_config, core_config,
&calculate, path_config);
if (_Py_INIT_FAILED(err)) {
goto done;
}
Expand Down
Loading