diff --git a/playground/server/main.py b/playground/server/main.py index ec1301e47d..405fe6d503 100644 --- a/playground/server/main.py +++ b/playground/server/main.py @@ -436,6 +436,16 @@ async def _dispatch_query(self, system_name: str, sql: bytes if attempt == 1: self.sink.write_event(system=system_name, kind="ensure-failed", detail=f"attempt {attempt}: {e!r}") + # Belt-and-suspenders: even though vm_manager's + # _restore_snapshot + _boot except handlers reap + # their own fc process on failure, corners we don't + # yet cover (e.g. a network-level ServerDisconnect + # while waiting for the guest agent) can still + # leave an fc process alive holding fc-tap-. + # Force a clean teardown before the second attempt + # so ensure_tap can recreate the TAP. + with contextlib.suppress(Exception): + await self.vmm.kick(system_name, "ensure-failed-retry") await asyncio.sleep(0.5) continue raise diff --git a/playground/server/vm_manager.py b/playground/server/vm_manager.py index 600731225e..40f39cb0e5 100644 --- a/playground/server/vm_manager.py +++ b/playground/server/vm_manager.py @@ -660,11 +660,26 @@ async def _restore_snapshot(self, vm: VM) -> None: if vm.system.name in DATALAKE_FILTERED: await net.enable_filtered_internet(vm.slot) await self._boot(vm, restore_snapshot=True) - await self._wait_for_agent(vm, timeout=60) - # Block here until the system's daemon reports ready, so the - # first user query doesn't time out mid-startup. Big upper bound - # for slow JVMs (Doris/Druid/Trino). - await self._wait_for_daemon_ready(vm, timeout=600) + try: + await self._wait_for_agent(vm, timeout=60) + # Block here until the system's daemon reports ready, so the + # first user query doesn't time out mid-startup. Big upper + # bound for slow JVMs (Doris/Druid/Trino). + await self._wait_for_daemon_ready(vm, timeout=600) + except Exception: + # _boot's own except handler covers _configure_boot failure, + # but if the guest agent never comes up or the daemon never + # reports ready, the fc process is left alive holding + # fc-tap-. The next /query then hits + # Open tap device failed: … Resource busy (os error 16) + # even though vm.state was left at "snapshotted". Reap the + # process + delete the TAP so ensure_tap can recreate a + # fresh one on the next attempt. + with contextlib.suppress(Exception): + await self._shutdown(vm) + with contextlib.suppress(Exception): + await net.teardown_tap(vm.slot) + raise vm.state = "ready" vm.ready_since = time.time() # Baseline the firecracker's current jiffy counter so the