From 206fedfd9a226ec4661ea73fdd85908b1864a2f5 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 24 Jul 2026 23:46:43 +0000 Subject: [PATCH] Playground: cover the leaked-fc leak on failed restore MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two paths could leave a firecracker process alive after a query- triggered restore failed, holding fc-tap- and breaking the next /query with "Open tap device failed: … Resource busy (os error 16). Invalid TUN/TAP Backend provided by fc-tap-N": 1. vm_manager._restore_snapshot: _boot's own except handler already reaps on _configure_boot failure, but if the guest agent never answers /health (_wait_for_agent) or the daemon never reports /ready (_wait_for_daemon_ready), we fall through with vm.pid still set and no cleanup. Wrap both waits and shut down the fc process + teardown_tap on failure. 2. main._dispatch_query: even with the vm_manager fix, an out-of-band raise (e.g. an aiohttp ServerDisconnectedError before we ever entered _restore_snapshot's try) can still leak. Kick the VM before the second attempt so ensure_tap on attempt-2 always sees a clean slot. --- playground/server/main.py | 10 ++++++++++ playground/server/vm_manager.py | 25 ++++++++++++++++++++----- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/playground/server/main.py b/playground/server/main.py index ec1301e47d..405fe6d503 100644 --- a/playground/server/main.py +++ b/playground/server/main.py @@ -436,6 +436,16 @@ async def _dispatch_query(self, system_name: str, sql: bytes if attempt == 1: self.sink.write_event(system=system_name, kind="ensure-failed", detail=f"attempt {attempt}: {e!r}") + # Belt-and-suspenders: even though vm_manager's + # _restore_snapshot + _boot except handlers reap + # their own fc process on failure, corners we don't + # yet cover (e.g. a network-level ServerDisconnect + # while waiting for the guest agent) can still + # leave an fc process alive holding fc-tap-. + # Force a clean teardown before the second attempt + # so ensure_tap can recreate the TAP. + with contextlib.suppress(Exception): + await self.vmm.kick(system_name, "ensure-failed-retry") await asyncio.sleep(0.5) continue raise diff --git a/playground/server/vm_manager.py b/playground/server/vm_manager.py index 600731225e..40f39cb0e5 100644 --- a/playground/server/vm_manager.py +++ b/playground/server/vm_manager.py @@ -660,11 +660,26 @@ async def _restore_snapshot(self, vm: VM) -> None: if vm.system.name in DATALAKE_FILTERED: await net.enable_filtered_internet(vm.slot) await self._boot(vm, restore_snapshot=True) - await self._wait_for_agent(vm, timeout=60) - # Block here until the system's daemon reports ready, so the - # first user query doesn't time out mid-startup. Big upper bound - # for slow JVMs (Doris/Druid/Trino). - await self._wait_for_daemon_ready(vm, timeout=600) + try: + await self._wait_for_agent(vm, timeout=60) + # Block here until the system's daemon reports ready, so the + # first user query doesn't time out mid-startup. Big upper + # bound for slow JVMs (Doris/Druid/Trino). + await self._wait_for_daemon_ready(vm, timeout=600) + except Exception: + # _boot's own except handler covers _configure_boot failure, + # but if the guest agent never comes up or the daemon never + # reports ready, the fc process is left alive holding + # fc-tap-. The next /query then hits + # Open tap device failed: … Resource busy (os error 16) + # even though vm.state was left at "snapshotted". Reap the + # process + delete the TAP so ensure_tap can recreate a + # fresh one on the next attempt. + with contextlib.suppress(Exception): + await self._shutdown(vm) + with contextlib.suppress(Exception): + await net.teardown_tap(vm.slot) + raise vm.state = "ready" vm.ready_since = time.time() # Baseline the firecracker's current jiffy counter so the