From 5802bd63e3e0b3f1aad2ae34bd9900e813b46f88 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Sat, 13 Oct 2018 16:33:02 -0700 Subject: [PATCH 1/3] bpo-34725: Adds Py_SetProgramFullPath so embedders may override sys.executable --- Doc/c-api/init.rst | 36 +++++++++--- Doc/whatsnew/3.8.rst | 2 + Include/internal/pystate.h | 3 +- Include/pylifecycle.h | 16 +++--- .../2018-10-13-16-30-54.bpo-34725.j52rIS.rst | 1 + PC/getpathp.c | 4 ++ Python/coreconfig.c | 24 ++++++++ Python/pathconfig.c | 57 +++++++++++-------- 8 files changed, 103 insertions(+), 40 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2018-10-13-16-30-54.bpo-34725.j52rIS.rst diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 6b5290a7ebbb00..74f81bcf1b9e0a 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -28,6 +28,7 @@ The following functions can be safely called before Python is initialized: * :c:func:`PyMem_SetupDebugHooks` * :c:func:`PyObject_SetArenaAllocator` * :c:func:`Py_SetPath` + * :c:func:`Py_SetProgramFullPath` * :c:func:`Py_SetProgramName` * :c:func:`Py_SetPythonHome` * :c:func:`Py_SetStandardStreamEncoding` @@ -225,6 +226,7 @@ Initializing and finalizing the interpreter .. c:function:: void Py_Initialize() .. index:: + single: Py_SetProgramFullPath() single: Py_SetProgramName() single: PyEval_InitThreads() single: modules (in module sys) @@ -355,9 +357,8 @@ Process-wide parameters This is used by :c:func:`Py_GetPath` and some other functions below to find the Python run-time libraries relative to the interpreter executable. The default value is ``'python'``. The argument should point to a - zero-terminated wide character string in static storage whose contents will not - change for the duration of the program's execution. No code in the Python - interpreter will change the contents of this storage. + zero-terminated wide character string. After this function returns, the string + may be modified or freed. Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a :c:type:`wchar_*` string. @@ -420,6 +421,26 @@ Process-wide parameters platform. +.. c:function:: void Py_SetProgramFullPath(const wchar_t *full_path) + + .. index:: + single: Py_Initialize() + single: main() + single: Py_GetPath() + + This function should be called before :c:func:`Py_Initialize` is called for + the first time, if it is called at all. It tells the interpreter the full + path to itself for cases where this cannot be inferred correctly. When this + function has been called, ``sys.executable`` will return it directly instead + of calculating the path another way. + After this function returns, the string may be modified or freed. + + Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a + :c:type:`wchar_*` string. + + .. versionadded:: 3.8 + + .. c:function:: wchar_t* Py_GetProgramFullPath() .. index:: @@ -427,10 +448,11 @@ Process-wide parameters single: executable (in module sys) Return the full program name of the Python executable; this is computed as a - side-effect of deriving the default module search path from the program name - (set by :c:func:`Py_SetProgramName` above). The returned string points into - static storage; the caller should not modify its value. The value is available - to Python code as ``sys.executable``. + side-effect of deriving the default module search path from the program name + (set by :c:func:`Py_SetProgramName` above), or set directly by + :c:func:`Py_SetProgramFullPath`. The returned string points into static + storage; the caller should not modify its value. The value is available to + Python code as ``sys.executable``. .. c:function:: wchar_t* Py_GetPath() diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index bd3283caadb8b3..949173f8d3605a 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -251,6 +251,8 @@ Build and C API Changes (Contributed by Antoine Pitrou in :issue:`32430`.) +* ``sys.executable`` can now be directly overridden when embedding by calling + :c:func:`Py_SetProgramFullPath` before initializing Python. Deprecated ========== diff --git a/Include/internal/pystate.h b/Include/internal/pystate.h index c93dda28954ae8..5d26c3ad4c3e4d 100644 --- a/Include/internal/pystate.h +++ b/Include/internal/pystate.h @@ -41,10 +41,9 @@ typedef struct _PyPathConfig { /* Full path to the Python program */ wchar_t *program_full_path; wchar_t *prefix; + wchar_t *exec_prefix; #ifdef MS_WINDOWS wchar_t *dll_path; -#else - wchar_t *exec_prefix; #endif /* Set by Py_SetPath(), or computed by _PyPathConfig_Init() */ wchar_t *module_search_path; diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h index 04e672e96e17cb..46a11eaec66af7 100644 --- a/Include/pylifecycle.h +++ b/Include/pylifecycle.h @@ -7,12 +7,6 @@ extern "C" { #endif -PyAPI_FUNC(void) Py_SetProgramName(const wchar_t *); -PyAPI_FUNC(wchar_t *) Py_GetProgramName(void); - -PyAPI_FUNC(void) Py_SetPythonHome(const wchar_t *); -PyAPI_FUNC(wchar_t *) Py_GetPythonHome(void); - #ifndef Py_LIMITED_API /* Only used by applications that embed the interpreter and need to * override the standard encoding determination mechanism @@ -86,8 +80,16 @@ PyAPI_FUNC(int) Py_Main(int argc, wchar_t **argv); PyAPI_FUNC(int) _Py_UnixMain(int argc, char **argv); #endif -/* In getpath.c */ +/* In pathconfig.c */ +PyAPI_FUNC(void) Py_SetProgramName(const wchar_t *); +PyAPI_FUNC(wchar_t *) Py_GetProgramName(void); + +PyAPI_FUNC(void) Py_SetPythonHome(const wchar_t *); +PyAPI_FUNC(wchar_t *) Py_GetPythonHome(void); + +PyAPI_FUNC(void) Py_SetProgramFullPath(const wchar_t *); PyAPI_FUNC(wchar_t *) Py_GetProgramFullPath(void); + PyAPI_FUNC(wchar_t *) Py_GetPrefix(void); PyAPI_FUNC(wchar_t *) Py_GetExecPrefix(void); PyAPI_FUNC(wchar_t *) Py_GetPath(void); diff --git a/Misc/NEWS.d/next/C API/2018-10-13-16-30-54.bpo-34725.j52rIS.rst b/Misc/NEWS.d/next/C API/2018-10-13-16-30-54.bpo-34725.j52rIS.rst new file mode 100644 index 00000000000000..b5bc1bf0c72325 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2018-10-13-16-30-54.bpo-34725.j52rIS.rst @@ -0,0 +1 @@ +Adds _Py_SetProgramFullPath so embedders may override sys.executable diff --git a/PC/getpathp.c b/PC/getpathp.c index ada02899966a53..1af7d1adf4e8a9 100644 --- a/PC/getpathp.c +++ b/PC/getpathp.c @@ -982,6 +982,10 @@ calculate_path_impl(const _PyCoreConfig *core_config, if (config->prefix == NULL) { return _Py_INIT_NO_MEMORY(); } + config->exec_prefix = _PyMem_RawWcsdup(prefix); + if (config->exec_prefix == NULL) { + return _Py_INIT_NO_MEMORY(); + } return _Py_INIT_OK(); } diff --git a/Python/coreconfig.c b/Python/coreconfig.c index fae32e533aa9cd..4d8c121935fa96 100644 --- a/Python/coreconfig.c +++ b/Python/coreconfig.c @@ -585,6 +585,23 @@ config_init_program_name(_PyCoreConfig *config) return _Py_INIT_OK(); } +static _PyInitError +config_init_executable(_PyCoreConfig *config) +{ + assert(config->executable == NULL); + + /* If Py_SetProgramFullPath() was called, use its value */ + const wchar_t *program_full_path = _Py_path_config.program_full_path; + if (program_full_path != NULL) { + config->executable = _PyMem_RawWcsdup(program_full_path); + if (config->executable == NULL) { + return _Py_INIT_NO_MEMORY(); + } + return _Py_INIT_OK(); + } + + return _Py_INIT_OK(); +} static const wchar_t* config_get_xoption(const _PyCoreConfig *config, wchar_t *name) @@ -1291,6 +1308,13 @@ _PyCoreConfig_Read(_PyCoreConfig *config) } } + if (config->executable == NULL) { + err = config_init_executable(config); + if (_Py_INIT_FAILED(err)) { + return err; + } + } + if (config->utf8_mode < 0 || config->coerce_c_locale < 0) { config_init_locale(config); } diff --git a/Python/pathconfig.c b/Python/pathconfig.c index 4e0830f4cf9afb..6b14531e9d9c3e 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -46,10 +46,9 @@ _PyPathConfig_Clear(_PyPathConfig *config) CLEAR(config->prefix); CLEAR(config->program_full_path); + CLEAR(config->exec_prefix); #ifdef MS_WINDOWS CLEAR(config->dll_path); -#else - CLEAR(config->exec_prefix); #endif CLEAR(config->module_search_path); CLEAR(config->home); @@ -71,8 +70,8 @@ _PyPathConfig_Calculate(_PyPathConfig *path_config, PyMemAllocatorEx old_alloc; _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - /* Calculate program_full_path, prefix, exec_prefix (Unix) - or dll_path (Windows), and module_search_path */ + /* Calculate program_full_path, prefix, exec_prefix, + dll_path (Windows), and module_search_path */ err = _PyPathConfig_Calculate_impl(&new_config, core_config); if (_Py_INIT_FAILED(err)) { goto err; @@ -123,10 +122,9 @@ _PyPathConfig_SetGlobal(const _PyPathConfig *config) COPY_ATTR(program_full_path); COPY_ATTR(prefix); + COPY_ATTR(exec_prefix); #ifdef MS_WINDOWS COPY_ATTR(dll_path); -#else - COPY_ATTR(exec_prefix); #endif COPY_ATTR(module_search_path); COPY_ATTR(program_name); @@ -205,12 +203,11 @@ _PyCoreConfig_SetPathConfig(const _PyCoreConfig *core_config) if (copy_wstr(&path_config.prefix, core_config->prefix) < 0) { goto no_memory; } -#ifdef MS_WINDOWS - if (copy_wstr(&path_config.dll_path, core_config->dll_path) < 0) { + if (copy_wstr(&path_config.exec_prefix, core_config->exec_prefix) < 0) { goto no_memory; } -#else - if (copy_wstr(&path_config.exec_prefix, core_config->exec_prefix) < 0) { +#ifdef MS_WINDOWS + if (copy_wstr(&path_config.dll_path, core_config->dll_path) < 0) { goto no_memory; } #endif @@ -314,12 +311,8 @@ _PyCoreConfig_CalculatePathConfig(_PyCoreConfig *config) } if (config->exec_prefix == NULL) { -#ifdef MS_WINDOWS - wchar_t *exec_prefix = path_config.prefix; -#else - wchar_t *exec_prefix = path_config.exec_prefix; -#endif - if (copy_wstr(&config->exec_prefix, exec_prefix) < 0) { + if (copy_wstr(&config->exec_prefix, + path_config.exec_prefix) < 0) { goto no_memory; } } @@ -376,7 +369,8 @@ _PyCoreConfig_InitPathConfig(_PyCoreConfig *config) } if (config->base_exec_prefix == NULL) { - if (copy_wstr(&config->base_exec_prefix, config->exec_prefix) < 0) { + if (copy_wstr(&config->base_exec_prefix, + config->exec_prefix) < 0) { return _Py_INIT_NO_MEMORY(); } } @@ -432,12 +426,11 @@ Py_SetPath(const wchar_t *path) int alloc_error = (new_config.program_full_path == NULL); new_config.prefix = _PyMem_RawWcsdup(L""); alloc_error |= (new_config.prefix == NULL); + new_config.exec_prefix = _PyMem_RawWcsdup(L""); + alloc_error |= (new_config.exec_prefix == NULL); #ifdef MS_WINDOWS new_config.dll_path = _PyMem_RawWcsdup(L""); alloc_error |= (new_config.dll_path == NULL); -#else - new_config.exec_prefix = _PyMem_RawWcsdup(L""); - alloc_error |= (new_config.exec_prefix == NULL); #endif new_config.module_search_path = _PyMem_RawWcsdup(path); alloc_error |= (new_config.module_search_path == NULL); @@ -500,6 +493,26 @@ Py_SetProgramName(const wchar_t *program_name) } } +void +Py_SetProgramFullPath(const wchar_t *program_full_path) +{ + if (program_full_path == NULL || program_full_path[0] == L'\0') { + return; + } + + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + + PyMem_RawFree(_Py_path_config.program_full_path); + _Py_path_config.program_full_path = _PyMem_RawWcsdup(program_full_path); + + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + + if (_Py_path_config.program_full_path == NULL) { + Py_FatalError("Py_SetProgramFullPath() failed: out of memory"); + } +} + wchar_t * Py_GetPath(void) @@ -520,12 +533,8 @@ Py_GetPrefix(void) wchar_t * Py_GetExecPrefix(void) { -#ifdef MS_WINDOWS - return Py_GetPrefix(); -#else pathconfig_global_init(); return _Py_path_config.exec_prefix; -#endif } From aeafe22f0a35edceab4457fa7fbf9df48cc2fa53 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Mon, 22 Oct 2018 09:11:29 -0400 Subject: [PATCH 2/3] Make API internal --- Doc/c-api/init.rst | 36 +++++++----------------------------- Doc/whatsnew/3.8.rst | 2 -- Include/pylifecycle.h | 2 +- Python/pathconfig.c | 4 ++-- 4 files changed, 10 insertions(+), 34 deletions(-) diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 74f81bcf1b9e0a..6b5290a7ebbb00 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -28,7 +28,6 @@ The following functions can be safely called before Python is initialized: * :c:func:`PyMem_SetupDebugHooks` * :c:func:`PyObject_SetArenaAllocator` * :c:func:`Py_SetPath` - * :c:func:`Py_SetProgramFullPath` * :c:func:`Py_SetProgramName` * :c:func:`Py_SetPythonHome` * :c:func:`Py_SetStandardStreamEncoding` @@ -226,7 +225,6 @@ Initializing and finalizing the interpreter .. c:function:: void Py_Initialize() .. index:: - single: Py_SetProgramFullPath() single: Py_SetProgramName() single: PyEval_InitThreads() single: modules (in module sys) @@ -357,8 +355,9 @@ Process-wide parameters This is used by :c:func:`Py_GetPath` and some other functions below to find the Python run-time libraries relative to the interpreter executable. The default value is ``'python'``. The argument should point to a - zero-terminated wide character string. After this function returns, the string - may be modified or freed. + zero-terminated wide character string in static storage whose contents will not + change for the duration of the program's execution. No code in the Python + interpreter will change the contents of this storage. Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a :c:type:`wchar_*` string. @@ -421,26 +420,6 @@ Process-wide parameters platform. -.. c:function:: void Py_SetProgramFullPath(const wchar_t *full_path) - - .. index:: - single: Py_Initialize() - single: main() - single: Py_GetPath() - - This function should be called before :c:func:`Py_Initialize` is called for - the first time, if it is called at all. It tells the interpreter the full - path to itself for cases where this cannot be inferred correctly. When this - function has been called, ``sys.executable`` will return it directly instead - of calculating the path another way. - After this function returns, the string may be modified or freed. - - Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a - :c:type:`wchar_*` string. - - .. versionadded:: 3.8 - - .. c:function:: wchar_t* Py_GetProgramFullPath() .. index:: @@ -448,11 +427,10 @@ Process-wide parameters single: executable (in module sys) Return the full program name of the Python executable; this is computed as a - side-effect of deriving the default module search path from the program name - (set by :c:func:`Py_SetProgramName` above), or set directly by - :c:func:`Py_SetProgramFullPath`. The returned string points into static - storage; the caller should not modify its value. The value is available to - Python code as ``sys.executable``. + side-effect of deriving the default module search path from the program name + (set by :c:func:`Py_SetProgramName` above). The returned string points into + static storage; the caller should not modify its value. The value is available + to Python code as ``sys.executable``. .. c:function:: wchar_t* Py_GetPath() diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index 949173f8d3605a..bd3283caadb8b3 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -251,8 +251,6 @@ Build and C API Changes (Contributed by Antoine Pitrou in :issue:`32430`.) -* ``sys.executable`` can now be directly overridden when embedding by calling - :c:func:`Py_SetProgramFullPath` before initializing Python. Deprecated ========== diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h index 46a11eaec66af7..67b74f9d62c917 100644 --- a/Include/pylifecycle.h +++ b/Include/pylifecycle.h @@ -87,7 +87,7 @@ PyAPI_FUNC(wchar_t *) Py_GetProgramName(void); PyAPI_FUNC(void) Py_SetPythonHome(const wchar_t *); PyAPI_FUNC(wchar_t *) Py_GetPythonHome(void); -PyAPI_FUNC(void) Py_SetProgramFullPath(const wchar_t *); +PyAPI_FUNC(void) _Py_SetProgramFullPath(const wchar_t *); PyAPI_FUNC(wchar_t *) Py_GetProgramFullPath(void); PyAPI_FUNC(wchar_t *) Py_GetPrefix(void); diff --git a/Python/pathconfig.c b/Python/pathconfig.c index 6b14531e9d9c3e..b61d5520bc83c5 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -494,7 +494,7 @@ Py_SetProgramName(const wchar_t *program_name) } void -Py_SetProgramFullPath(const wchar_t *program_full_path) +_Py_SetProgramFullPath(const wchar_t *program_full_path) { if (program_full_path == NULL || program_full_path[0] == L'\0') { return; @@ -509,7 +509,7 @@ Py_SetProgramFullPath(const wchar_t *program_full_path) PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); if (_Py_path_config.program_full_path == NULL) { - Py_FatalError("Py_SetProgramFullPath() failed: out of memory"); + Py_FatalError("_Py_SetProgramFullPath() failed: out of memory"); } } From 2378e5d1fcc833f366e93bd455cd8248144082cb Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Mon, 22 Oct 2018 09:12:39 -0400 Subject: [PATCH 3/3] Remove _Py_SetProgramFullPath from limited API --- Include/pylifecycle.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h index 67b74f9d62c917..ae878c5c1e6796 100644 --- a/Include/pylifecycle.h +++ b/Include/pylifecycle.h @@ -87,7 +87,9 @@ PyAPI_FUNC(wchar_t *) Py_GetProgramName(void); PyAPI_FUNC(void) Py_SetPythonHome(const wchar_t *); PyAPI_FUNC(wchar_t *) Py_GetPythonHome(void); +#ifndef Py_LIMITED_API PyAPI_FUNC(void) _Py_SetProgramFullPath(const wchar_t *); +#endif PyAPI_FUNC(wchar_t *) Py_GetProgramFullPath(void); PyAPI_FUNC(wchar_t *) Py_GetPrefix(void);