degrade gracefully

Author: maurycy

Include/internal/pycore_global_objects_fini_generated.h

@@ -54,6 +54,9 @@ struct _Py_global_strings {
         STRUCT_FOR_STR(native, "<native>")
         STRUCT_FOR_STR(str_replace_inf, "1e309")
         STRUCT_FOR_STR(type_params, ".type_params")
+        STRUCT_FOR_STR(unknown_file, "<unknown file>")
+        STRUCT_FOR_STR(unknown_function, "<unknown function>")
+        STRUCT_FOR_STR(unreadable_frame, "<unreadable frame>")
         STRUCT_FOR_STR(utf_8, "utf-8")
     } literals;
 

Include/internal/pycore_global_strings.h

@@ -3854,5 +3854,155 @@ def test_get_stats_disabled_raises(self):
             client_socket.sendall(b"done")
 
 
+@requires_remote_subprocess_debugging()
+@skip_if_not_supported
+@unittest.skipIf(
+    sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED,
+    "Test only runs on Linux with process_vm_readv support",
+)
+class TestMetadataDegradation(RemoteInspectionTestBase):
+    """Tests for graceful degradation of oversized code-object metadata."""
+
+    @contextmanager
+    def _running_target(self, script_body):
+        """Run a target script (socket handshake prepended), yield (process, socket)."""
+        port = find_unused_port()
+        script = (
+            textwrap.dedent(
+                f"""\
+                import time, socket
+                sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+                sock.connect(('localhost', {port}))
+                """
+            )
+            + textwrap.dedent(script_body)
+        )
+
+        with os_helper.temp_dir() as work_dir:
+            script_dir = os.path.join(work_dir, "script_pkg")
+            os.mkdir(script_dir)
+
+            server_socket = _create_server_socket(port)
+            script_name = _make_test_script(script_dir, "script", script)
+            client_socket = None
+
+            try:
+                with _managed_subprocess([sys.executable, script_name]) as p:
+                    client_socket, _ = server_socket.accept()
+                    server_socket.close()
+                    server_socket = None
+                    yield p, client_socket
+            finally:
+                _cleanup_sockets(client_socket, server_socket)
+
+    def _sample_until_frame(self, pid, predicate):
+        """Sample until a frame matching predicate appears; return that frame."""
+        unwinder = RemoteUnwinder(pid, all_threads=True)
+        traces = _get_stack_trace_with_retry(
+            unwinder,
+            condition=lambda t: self._find_frame_in_trace(t, predicate)
+            is not None,
+        )
+        return self._find_frame_in_trace(traces, predicate)
+
+    def test_long_qualname_truncated_not_dropped(self):
+        """A qualname longer than 1024 chars is truncated with a marker
+        instead of failing the whole sample."""
+        script_body = """\
+            src = ("def " + "f" * 1100 + "():\\n"
+                   "    sock.sendall(b'ready')\\n"
+                   "    time.sleep(10_000)\\n")
+            ns = {"sock": sock, "time": time}
+            exec(src, ns)
+            ns["f" * 1100]()
+            """
+        with self._running_target(script_body) as (p, client_socket):
+            _wait_for_signal(client_socket, b"ready")
+            frame = self._sample_until_frame(
+                p.pid, lambda f: f.funcname.startswith("fff")
+            )
+            self.assertEqual(
+                frame.funcname, "f" * 1024 + "(len=1100)"
+            )
+
+    def test_long_filename_truncated(self):
+        """A filename longer than 1024 chars is truncated with a marker
+        instead of failing the whole sample."""
+        script_body = """\
+            src = ("def g():\\n"
+                   "    sock.sendall(b'ready')\\n"
+                   "    time.sleep(10_000)\\n")
+            ns = {"sock": sock, "time": time}
+            exec(compile(src, "x" * 1500 + ".py", "exec"), ns)
+            ns["g"]()
+            """
+        with self._running_target(script_body) as (p, client_socket):
+            _wait_for_signal(client_socket, b"ready")
+            frame = self._sample_until_frame(
+                p.pid, lambda f: f.funcname == "g"
+            )
+            self.assertEqual(
+                frame.filename, "x" * 1024 + "(len=1503)"
+            )
+
+    def test_large_linetable_keeps_line_numbers(self):
+        """A linetable larger than the old 4096-byte cap keeps line numbers."""
+        script_body = """\
+            body = "\\n".join("    x%d = %d" % (i, i) for i in range(4000))
+            src = ("def big():\\n" + body + "\\n"
+                   "    sock.sendall(b'ready')\\n"
+                   "    time.sleep(10_000)\\n")
+            ns = {"sock": sock, "time": time}
+            exec(src, ns)
+            sock.sendall(b"lt:%d\\n" % len(ns["big"].__code__.co_linetable))
+            ns["big"]()
+            """
+        with self._running_target(script_body) as (p, client_socket):
+            buffer = _wait_for_signal(client_socket, [b"lt:", b"ready"])
+            linetable_size = int(
+                buffer.partition(b"lt:")[2].partition(b"\n")[0]
+            )
+            self.assertGreater(linetable_size, 4096)
+
+            frame = self._sample_until_frame(
+                p.pid, lambda f: f.funcname == "big"
+            )
+            self.assertIsNotNone(
+                frame.location, "line info degraded for large linetable"
+            )
+            self.assertGreater(frame.location.lineno, 4000)
+
+    @unittest.skipIf(
+        sys.platform == "win32",
+        "Process death maps to ProcessLookupError only on POSIX platforms",
+    )
+    def test_dead_process_raises_not_degrades(self):
+        """Death of the target raises ProcessLookupError instead of
+        degrading to synthetic frames."""
+        script_body = """\
+            sock.sendall(b"ready")
+            time.sleep(10_000)
+            """
+        with self._running_target(script_body) as (p, client_socket):
+            _wait_for_signal(client_socket, b"ready")
+            unwinder = RemoteUnwinder(p.pid, all_threads=True)
+            _get_stack_trace_with_retry(unwinder)
+
+            p.kill()
+            p.wait()
+
+            for _ in busy_retry(SHORT_TIMEOUT, error=False):
+                try:
+                    unwinder.get_stack_trace()
+                except ProcessLookupError:
+                    break
+                except RuntimeError:
+                    continue
+            else:
+                self.fail(
+                    "ProcessLookupError never raised for dead process"
+                )
+
+
 if __name__ == "__main__":
     unittest.main()

Include/internal/pycore_runtime_init_generated.h

@@ -628,6 +628,26 @@ def test_same_line_different_columns(self):
         collector, count = self.roundtrip(samples)
         self.assertEqual(count, 3)
 
+    def test_synthetic_frames_roundtrip(self):
+        """Degraded/sentinel frames (location=None) survive the binary format."""
+        frames = [
+            FrameInfo(("~", None, name, None))
+            for name in (
+                "<GC>",
+                "<native>",
+                "<unknown function>",
+                "<unknown file>",
+                "<unreadable frame>",
+            )
+        ]
+        frames.append(FrameInfo(("app.py", None, "<unknown function>", None)))
+        frames.append(FrameInfo(("<unknown file>", None, "real_func", None)))
+        samples = [[make_interpreter(0, [make_thread(1, frames)])]]
+
+        collector, count = self.roundtrip(samples)
+        self.assertEqual(count, 1)
+        self.assert_samples_equal(samples, collector)
+
 
 class TestBinaryEdgeCases(BinaryFormatTestBase):
     """Tests for edge cases in binary format."""

Include/internal/pycore_unicodeobject_generated.h

@@ -144,6 +144,7 @@ typedef enum _WIN32_THREADSTATE {
 #define SIZEOF_INTERPRETER_STATE sizeof(PyInterpreterState)
 
 /* Maximum sizes for validation to prevent buffer overflows from corrupted data */
+#define MAX_REMOTE_READ ((Py_ssize_t)(256 * 1024))  /* 256 KB max for variable-length object reads */
 #define MAX_STACK_CHUNK_SIZE (16 * 1024 * 1024)  /* 16 MB max for stack chunks */
 #define MAX_LONG_DIGITS 64  /* Allows values up to ~2^1920 */
 #define MAX_SET_TABLE_SIZE (1 << 20)  /* 1 million entries max for set iteration */
@@ -180,7 +181,7 @@ typedef enum _WIN32_THREADSTATE {
 #define set_exception_cause(unwinder, exc_type, message)                              \
     do {                                                                              \
         assert(PyErr_Occurred() && "function returned -1 without setting exception"); \
-        if (unwinder->debug && !_Py_RemoteDebug_HasPermissionError()) {               \
+        if (unwinder->debug && !_Py_RemoteDebug_IsFatalReadError()) {                 \
             _set_debug_exception_cause(exc_type, message);                            \
         }                                                                             \
     } while (0)

Lib/test/test_external_inspection.py

@@ -338,11 +338,16 @@ parse_code_object(RemoteUnwinderObject *unwinder,
                   PyObject **result,
                   const CodeObjectContext *ctx)
 {
+    _Py_DECLARE_STR(unknown_function, "<unknown function>");
+    _Py_DECLARE_STR(unknown_file, "<unknown file>");
+    _Py_DECLARE_STR(unreadable_frame, "<unreadable frame>");
+
     void *key = (void *)ctx->code_addr;
     CachedCodeMetadata *meta = NULL;
     PyObject *func = NULL;
     PyObject *file = NULL;
     PyObject *linetable = NULL;
+    int code_metadata_incomplete = 0;
 
 #ifdef Py_GIL_DISABLED
     // In free threading builds, code object addresses might have the low bit set
@@ -366,30 +371,59 @@ parse_code_object(RemoteUnwinderObject *unwinder,
         if (_Py_RemoteDebug_PagedReadRemoteMemory(
                 &unwinder->handle, real_address, SIZEOF_CODE_OBJ, code_object) < 0)
         {
-            set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read code object");
-            goto error;
+            if (_Py_RemoteDebug_IsFatalReadError()) {
+                set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read code object");
+                goto error;
+            }
+            PyErr_Clear();
+            PyObject *tuple = make_frame_info(
+                unwinder, _Py_LATIN1_CHR('~'), Py_None,
+                &_Py_STR(unreadable_frame), Py_None);
+            if (tuple == NULL) {
+                goto error;
+            }
+            *result = tuple;
+            return 0;
         }
 
         func = read_py_str(unwinder,
             GET_MEMBER(uintptr_t, code_object, unwinder->debug_offsets.code_object.qualname), 1024);
         if (!func) {
-            set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read function name from code object");
-            goto error;
+            if (_Py_RemoteDebug_IsFatalReadError()) {
+                set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read function name from code object");
+                goto error;
+            }
+            PyErr_Clear();
+            func = Py_NewRef(&_Py_STR(unknown_function));
+            code_metadata_incomplete = 1;
         }
 
         file = read_py_str(unwinder,
             GET_MEMBER(uintptr_t, code_object, unwinder->debug_offsets.code_object.filename), 1024);
         if (!file) {
-            set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read filename from code object");
-            goto error;
+            if (_Py_RemoteDebug_IsFatalReadError()) {
+                set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read filename from code object");
+                goto error;
+            }
+            PyErr_Clear();
+            file = Py_NewRef(&_Py_STR(unknown_file));
+            code_metadata_incomplete = 1;
+        }
+
+        if (code_metadata_incomplete) {
+            goto degraded;
         }
 
         linetable = read_py_bytes(unwinder,
             GET_MEMBER(uintptr_t, code_object, unwinder->debug_offsets.code_object.linetable),
             MAX_LINETABLE_SIZE);
         if (!linetable) {
-            set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read linetable from code object");
-            goto error;
+            if (_Py_RemoteDebug_IsFatalReadError()) {
+                set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read linetable from code object");
+                goto error;
+            }
+            PyErr_Clear();
+            goto degraded;
         }
 
         meta = PyMem_RawMalloc(sizeof(CachedCodeMetadata));
@@ -542,6 +576,18 @@ parse_code_object(RemoteUnwinderObject *unwinder,
     *result = tuple;
     return 0;
 
+degraded: {
+    PyObject *degraded_tuple = make_frame_info(unwinder, file, Py_None,
+                                               func, Py_None);
+    Py_CLEAR(func);
+    Py_CLEAR(file);
+    if (!degraded_tuple) {
+        return -1;
+    }
+    *result = degraded_tuple;
+    return 0;
+}
+
 error:
     Py_XDECREF(func);
     Py_XDECREF(file);