Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions playground/server/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -436,6 +436,16 @@ async def _dispatch_query(self, system_name: str, sql: bytes
if attempt == 1:
self.sink.write_event(system=system_name, kind="ensure-failed",
detail=f"attempt {attempt}: {e!r}")
# Belt-and-suspenders: even though vm_manager's
# _restore_snapshot + _boot except handlers reap
# their own fc process on failure, corners we don't
# yet cover (e.g. a network-level ServerDisconnect
# while waiting for the guest agent) can still
# leave an fc process alive holding fc-tap-<slot>.
# Force a clean teardown before the second attempt
# so ensure_tap can recreate the TAP.
with contextlib.suppress(Exception):
await self.vmm.kick(system_name, "ensure-failed-retry")
await asyncio.sleep(0.5)
continue
raise
Expand Down
25 changes: 20 additions & 5 deletions playground/server/vm_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -660,11 +660,26 @@ async def _restore_snapshot(self, vm: VM) -> None:
if vm.system.name in DATALAKE_FILTERED:
await net.enable_filtered_internet(vm.slot)
await self._boot(vm, restore_snapshot=True)
await self._wait_for_agent(vm, timeout=60)
# Block here until the system's daemon reports ready, so the
# first user query doesn't time out mid-startup. Big upper bound
# for slow JVMs (Doris/Druid/Trino).
await self._wait_for_daemon_ready(vm, timeout=600)
try:
await self._wait_for_agent(vm, timeout=60)
# Block here until the system's daemon reports ready, so the
# first user query doesn't time out mid-startup. Big upper
# bound for slow JVMs (Doris/Druid/Trino).
await self._wait_for_daemon_ready(vm, timeout=600)
except Exception:
# _boot's own except handler covers _configure_boot failure,
# but if the guest agent never comes up or the daemon never
# reports ready, the fc process is left alive holding
# fc-tap-<slot>. The next /query then hits
# Open tap device failed: … Resource busy (os error 16)
# even though vm.state was left at "snapshotted". Reap the
# process + delete the TAP so ensure_tap can recreate a
# fresh one on the next attempt.
with contextlib.suppress(Exception):
await self._shutdown(vm)
with contextlib.suppress(Exception):
await net.teardown_tap(vm.slot)
raise
vm.state = "ready"
vm.ready_since = time.time()
# Baseline the firecracker's current jiffy counter so the
Expand Down
Loading