diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index fd4221f0816d24b..06c4ca1619d7ce1 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -208,16 +208,16 @@ extern void _PyEval_DeactivateOpCache(void); /* --- _Py_EnterRecursiveCall() ----------------------------------------- */ -static inline int _Py_MakeRecCheck(PyThreadState *tstate) { +static inline int _Py_ReachedRecursionLimit(PyThreadState *tstate) { uintptr_t here_addr = _Py_get_machine_stack_pointer(); _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; - // Overflow if stack pointer is between soft limit and the base of the hardware stack. - // If it is below the hardware stack base, assume that we have the wrong stack limits, and do nothing. - // We could have the wrong stack limits because of limited platform support, or user-space threads. + // Possible overflow if stack pointer is beyond the soft limit. + // _Py_CheckRecursiveCall will check for corner cases and + // report an error if there is an overflow. #if _Py_STACK_GROWS_DOWN - return here_addr < _tstate->c_stack_soft_limit && here_addr >= _tstate->c_stack_soft_limit - 2 * _PyOS_STACK_MARGIN_BYTES; + return here_addr < _tstate->c_stack_soft_limit; #else - return here_addr > _tstate->c_stack_soft_limit && here_addr <= _tstate->c_stack_soft_limit + 2 * _PyOS_STACK_MARGIN_BYTES; + return here_addr > _tstate->c_stack_soft_limit; #endif } @@ -232,7 +232,7 @@ PyAPI_FUNC(int) _Py_CheckRecursiveCallPy( static inline int _Py_EnterRecursiveCallTstate(PyThreadState *tstate, const char *where) { - return (_Py_MakeRecCheck(tstate) && _Py_CheckRecursiveCall(tstate, where)); + return (_Py_ReachedRecursionLimit(tstate) && _Py_CheckRecursiveCall(tstate, where)); } static inline int _Py_EnterRecursiveCall(const char *where) { @@ -246,8 +246,6 @@ static inline void _Py_LeaveRecursiveCallTstate(PyThreadState *tstate) { PyAPI_FUNC(void) _Py_InitializeRecursionLimits(PyThreadState *tstate); -PyAPI_FUNC(int) _Py_ReachedRecursionLimit(PyThreadState *tstate); - // Export for test_peg_generator PyAPI_FUNC(int) _Py_ReachedRecursionLimitWithMargin( PyThreadState *tstate, diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index c9e918bceda9fc7..ca6819d2cd44730 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -316,15 +316,20 @@ static uintptr_t return_pointer_as_int(char* p) { static inline uintptr_t _Py_get_machine_stack_pointer(void) { -#if _Py__has_builtin(__builtin_frame_address) || defined(__GNUC__) - return (uintptr_t)__builtin_frame_address(0); -#elif defined(_MSC_VER) - return (uintptr_t)_AddressOfReturnAddress(); + uintptr_t result; +#if defined(_M_ARM64) + result = __getReg(31); +#elif defined(_M_X64) || defined(_M_IX86) + result = (uintptr_t)_AddressOfReturnAddress(); +#elif defined(__aarch64__) + __asm__ ("mov %0, sp" : "=r" (result)); +#elif defined(__x86_64__) + __asm__("{movq %%rsp, %0" : "=r" (result)); #else char here; - /* Avoid compiler warning about returning stack address */ - return return_pointer_as_int(&here); + result = (uintptr_t)&here; #endif + return result; } static inline intptr_t diff --git a/Include/internal/pycore_pythonrun.h b/Include/internal/pycore_pythonrun.h index 2a544edc431e6b1..66dd7cd843b04fc 100644 --- a/Include/internal/pycore_pythonrun.h +++ b/Include/internal/pycore_pythonrun.h @@ -46,7 +46,8 @@ extern PyObject * _Py_CompileStringObjectWithModule( * stack consumption of PyEval_EvalDefault */ #if (defined(Py_DEBUG) \ || defined(_Py_ADDRESS_SANITIZER) \ - || defined(_Py_THREAD_SANITIZER)) + || defined(_Py_THREAD_SANITIZER)) \ + || defined(_Py_UNDEFINED_BEHAVIOR_SANITIZER) # define _PyOS_LOG2_STACK_MARGIN 12 #else # define _PyOS_LOG2_STACK_MARGIN 11 diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 62804e2fa2d68e9..1b4ad5a22ee271c 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -324,16 +324,6 @@ def requires(resource, msg=None): if resource == 'gui' and not _is_gui_available(): raise ResourceDenied(_is_gui_available.reason) -def _get_kernel_version(sysname="Linux"): - import platform - if platform.system() != sysname: - return None - version_txt = platform.release().split('-', 1)[0] - try: - return tuple(map(int, version_txt.split('.'))) - except ValueError: - return None - def _requires_unix_version(sysname, min_version): """Decorator raising SkipTest if the OS is `sysname` and the version is less than `min_version`. diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index c5ced3cc6134b96..2d1533c46b98f33 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -74,6 +74,27 @@ def debug_build(program): return name.casefold().endswith("_d".casefold()) +def getpath_which(program_name): + if sys.platform != 'cygwin': + return shutil.which(program_name) + + # shutil.which() checks for os.access(fn, os.F_OK | os.X_OK), whereas + # getpath.isxfile() doesn't. The difference matters on Cygwin. + import stat + def isxfile(fn): + try: + st = os.stat(fn) + except OSError: + return False + return stat.S_ISREG(st.st_mode) + + for p in os.environ['PATH'].split(':'): + p = os.path.join(p, program_name) + if isxfile(p): + return p + return None + + def remove_python_envvars(): env = dict(os.environ) # Remove PYTHON* environment variables to get deterministic environment @@ -92,6 +113,8 @@ def setUp(self): exename += ext exepath = builddir else: + if sys.platform == 'cygwin': + exename += '.exe' exepath = os.path.join(builddir, 'Programs') self.test_exe = exe = os.path.join(exepath, exename) if not os.path.exists(exe): @@ -328,6 +351,8 @@ def test_pre_initialization_api(self): expected_path = self.test_exe else: expected_path = os.path.join(os.getcwd(), "_testembed") + if sys.platform == 'cygwin': + expected_path += '.exe' expected_output = f"sys.executable: {expected_path}\n" self.assertIn(expected_output, out) self.assertEqual(err, '') @@ -872,12 +897,16 @@ def get_expected_config(self, expected_preconfig, expected, default_executable = os.path.abspath(expected['program_name']) else: default_executable = os.path.join(os.getcwd(), '_testembed') + if sys.platform == 'cygwin': + default_executable += '.exe' if expected['executable'] is self.GET_DEFAULT_CONFIG: expected['executable'] = default_executable if expected['base_executable'] is self.GET_DEFAULT_CONFIG: expected['base_executable'] = default_executable if expected['program_name'] is self.GET_DEFAULT_CONFIG: expected['program_name'] = './_testembed' + if sys.platform == 'cygwin': + expected['program_name'] += '.exe' config = configs['config'] for key, value in expected.items(): @@ -1370,7 +1399,7 @@ def default_program_name(self, config): if MACOS: executable = self.test_exe else: - executable = shutil.which(program_name) or '' + executable = getpath_which(program_name) or '' config.update({ 'program_name': program_name, 'base_executable': executable, @@ -1469,6 +1498,13 @@ def tmpdir_with_python(self, subdir=None): shutil.copystat(self.test_exe, exec_copy) self.test_exe = exec_copy + if sys.platform == "cygwin": + # Copy libpython DLL + exe_path = os.path.dirname(sys.executable) + libpython_dll = sysconfig.get_config_var('DLLLIBRARY') + shutil.copy2(os.path.join(exe_path, libpython_dll), + os.path.join(tmpdir, libpython_dll)) + yield tmpdir def test_init_setpythonhome(self): diff --git a/Lib/test/test_pyexpat.py b/Lib/test/test_pyexpat.py index 10dca684accee3c..cddf52d569bcd31 100644 --- a/Lib/test/test_pyexpat.py +++ b/Lib/test/test_pyexpat.py @@ -861,7 +861,7 @@ def test_trigger_leak(self): def test_deeply_nested_content_model(self): # This should raise a RecursionError and not crash. # See https://github.com/python/cpython/issues/145986. - N = 500_000 + N = 800_000 data = ( b'= (6, 16) and kernel_version < (6, 18): - # See https://github.com/python/cpython/issues/139310. - self.skipTest("upstream Linux kernel issue") - key = bytes.fromhex('c939cc13397c1d37de6ae0e1cb7c423c') iv = bytes.fromhex('b3d8cc017cbb89b39e0f67e2') plain = bytes.fromhex('c3b3c41f113a31b73d9a5cd432103069') diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 9d2960664abfad5..1ff5d7cf0c51dd8 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -500,6 +500,7 @@ def test_executable_symlinks(self): # gh-124651: test quoted strings @unittest.skipIf(os.name == 'nt', 'contains invalid characters on Windows') + @unittest.skipIf(sys.platform == 'cygwin', 'fail to locate cygpython DLL') def test_special_chars_bash(self): """ Test that the template strings are quoted properly (bash) @@ -714,6 +715,12 @@ def test_zippath_from_non_installed_posix(self): os.mkdir(bindir) python_exe = os.path.basename(sys.executable) shutil.copy2(sys.executable, os.path.join(bindir, python_exe)) + if sys.platform == 'cygwin': + # Copy libpython DLL + exe_path = os.path.dirname(sys.executable) + libpython_dll = sysconfig.get_config_var('DLLLIBRARY') + shutil.copy2(os.path.join(exe_path, libpython_dll), + os.path.join(bindir, libpython_dll)) libdir = os.path.join(non_installed_dir, platlibdir, self.lib[1]) os.makedirs(libdir) landmark = os.path.join(libdir, "os.py") diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index 0653a43a8b17766..bd2762d55ef6961 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -319,6 +319,14 @@ def setup_python(self, context): if not os.path.islink(path): os.chmod(path, 0o755) + if not self.symlinks and sys.platform == 'cygwin': + # Copy libpython DLL + libpython_dll = sysconfig.get_config_var('DLLLIBRARY') + if not os.path.exists(os.path.join(binpath, libpython_dll)): + exe_path = os.path.dirname(sys.executable) + shutil.copy(os.path.join(exe_path, libpython_dll), + os.path.join(binpath, libpython_dll)) + else: def setup_python(self, context): """ diff --git a/Makefile.pre.in b/Makefile.pre.in index 242aca0dc05eec2..2b34b009fd745a0 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -3139,7 +3139,7 @@ Python/asm_trampoline_universal2.o: $(srcdir)/Python/asm_trampoline_aarch64.S $( Python/emscripten_trampoline_inner.wasm: $(srcdir)/Python/emscripten_trampoline_inner.c # emcc has a path that ends with emsdk/upstream/emscripten/emcc, we're looking for emsdk/upstream/bin/clang. - $$(dirname $$(dirname $(CC)))/bin/clang -o $@ $< -mgc -O2 -Wl,--no-entry -Wl,--import-table -Wl,--import-memory -target wasm32-unknown-unknown -nostdlib + $$(em-config LLVM_ROOT)/clang -o $@ $< -mgc -O2 -Wl,--no-entry -Wl,--import-table -Wl,--import-memory -target wasm32-unknown-unknown -nostdlib Python/emscripten_trampoline_wasm.c: Python/emscripten_trampoline_inner.wasm $(PYTHON_FOR_REGEN) $(srcdir)/Platforms/emscripten/prepare_external_wasm.py $< $@ getWasmTrampolineModule diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 278984ddb17c1a2..7246cc06ffff036 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -41,8 +41,13 @@ char **main_argv; #define PROGRAM "test_embed" /* Use path starting with "./" avoids a search along the PATH */ -#define PROGRAM_NAME L"./_testembed" -#define PROGRAM_NAME_UTF8 "./_testembed" +#ifdef __CYGWIN__ +# define PROGRAM_NAME L"./_testembed.exe" +# define PROGRAM_NAME_UTF8 "./_testembed.exe" +#else +# define PROGRAM_NAME L"./_testembed" +# define PROGRAM_NAME_UTF8 "./_testembed" +#endif #define INIT_LOOPS 4 diff --git a/Python/ceval.c b/Python/ceval.c index 5661200e74d0a55..5dbb116ded165ec 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -49,20 +49,6 @@ _Py_ReachedRecursionLimitWithMargin(PyThreadState *tstate, int margin_count) #endif } -void -_Py_EnterRecursiveCallUnchecked(PyThreadState *tstate) -{ - uintptr_t here_addr = _Py_get_machine_stack_pointer(); - _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; -#if _Py_STACK_GROWS_DOWN - if (here_addr < _tstate->c_stack_hard_limit) { -#else - if (here_addr > _tstate->c_stack_hard_limit) { -#endif - Py_FatalError("Unchecked stack overflow."); - } -} - #if defined(__s390x__) # define Py_C_STACK_SIZE 320000 #elif defined(_WIN32) @@ -278,7 +264,7 @@ PyUnstable_ThreadState_ResetStackProtection(PyThreadState *tstate) /* The function _Py_EnterRecursiveCallTstate() only calls _Py_CheckRecursiveCall() - if the stack pointer is between the stack base and c_stack_hard_limit. */ + if the stack pointer is beyond c_stack_soft_limit. */ int _Py_CheckRecursiveCall(PyThreadState *tstate, const char *where) { @@ -287,16 +273,21 @@ _Py_CheckRecursiveCall(PyThreadState *tstate, const char *where) assert(_tstate->c_stack_soft_limit != 0); assert(_tstate->c_stack_hard_limit != 0); #if _Py_STACK_GROWS_DOWN - assert(here_addr >= _tstate->c_stack_hard_limit - _PyOS_STACK_MARGIN_BYTES); if (here_addr < _tstate->c_stack_hard_limit) { - /* Overflowing while handling an overflow. Give up. */ + if (here_addr < _tstate->c_stack_hard_limit - _PyOS_STACK_MARGIN_BYTES) { + // Far out of bounds -- Assume stack switching has occurred + return 0; + } int kbytes_used = (int)(_tstate->c_stack_top - here_addr)/1024; #else - assert(here_addr <= _tstate->c_stack_hard_limit + _PyOS_STACK_MARGIN_BYTES); if (here_addr > _tstate->c_stack_hard_limit) { - /* Overflowing while handling an overflow. Give up. */ + if (here_addr > _tstate->c_stack_hard_limit + _PyOS_STACK_MARGIN_BYTES) { + // Far out of bounds -- Assume stack switching has occurred + return 0; + } int kbytes_used = (int)(here_addr - _tstate->c_stack_top)/1024; #endif + /* Too much stack used to safely raise an exception. Give up. */ char buffer[80]; snprintf(buffer, 80, "Unrecoverable stack overflow (used %d kB)%s", kbytes_used, where); Py_FatalError(buffer); @@ -1146,19 +1137,6 @@ _PyEval_GetIter(_PyStackRef iterable, _PyStackRef *index_or_null, int yield_from return PyStackRef_FromPyObjectSteal(iter_o); } -Py_NO_INLINE int -_Py_ReachedRecursionLimit(PyThreadState *tstate) { - uintptr_t here_addr = _Py_get_machine_stack_pointer(); - _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; - assert(_tstate->c_stack_hard_limit != 0); -#if _Py_STACK_GROWS_DOWN - return here_addr <= _tstate->c_stack_soft_limit; -#else - return here_addr >= _tstate->c_stack_soft_limit; -#endif -} - - #if (defined(__GNUC__) && __GNUC__ >= 10 && !defined(__clang__)) && defined(__x86_64__) /* * gh-129987: The SLP autovectorizer can cause poor code generation for @@ -1169,7 +1147,7 @@ _Py_ReachedRecursionLimit(PyThreadState *tstate) { * (prior to GCC 9, 40% performance drop), so we have to selectively disable * it. */ -#define DONT_SLP_VECTORIZE __attribute__((optimize ("no-tree-slp-vectorize"))) +#define DONT_SLP_VECTORIZE __attribute__((optimize ("no-tree-slp-vectorize", "no-omit-frame-pointer"))) #else #define DONT_SLP_VECTORIZE #endif diff --git a/Python/jit.c b/Python/jit.c index 67dd88f510040eb..ade8b8d36e789d4 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -33,7 +33,7 @@ #include "pycore_jit.h" -// Memory management stuff: //////////////////////////////////////////////////// +// Memory management stuff: /////////////////////////////////////////////////// #ifndef MS_WINDOWS #include