From 02bab02fa93a9e5693c00193605ec3c4597e4360 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Wed, 12 Aug 2026 13:49:46 +0200 Subject: [PATCH] fix: bind /dev/fuse into the payload container when available --contain gives the payload a minimal /dev which does not include /dev/fuse. Apptainer needs it to mount an image with fuse-overlayfs, so payloads which start a container of their own (e.g. lb-run for an application needing an older OS) fail on any host where the kernel overlay is unavailable: FATAL: container creation failed: image driver mount failure: image driver fuse-overlayfs instance exited with error: fuse-overlayfs exited: fuse: device /dev/fuse not found. Kernel module not loaded? This is silent and site dependent: where the overlay module is loaded apptainer uses the kernel overlay and never needs fuse, so the same payload succeeds. Of 140 test jobs at one site, 54 landed on nodes with the module loaded and worked, while the other 86 failed this way. /dev/fuse is also what squashfuse needs to mount SIF images without privileges, so this affects payloads using SIF regardless of whether the kernel overlay is available. The bind is conditional because apptainer fails if a bind source does not exist, mirroring the existing handling of /cvmfs. --- .../Resources/Computing/SingularityComputingElement.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/DIRAC/Resources/Computing/SingularityComputingElement.py b/src/DIRAC/Resources/Computing/SingularityComputingElement.py index 5f019588ff1..b40e7112638 100644 --- a/src/DIRAC/Resources/Computing/SingularityComputingElement.py +++ b/src/DIRAC/Resources/Computing/SingularityComputingElement.py @@ -346,6 +346,11 @@ def submitJob(self, executableFile, proxy=None, **kwargs): # Now prepare start singularity # Mount /cvmfs in if it exists on the host withCVMFS = os.path.isdir("/cvmfs") + # Payloads may need to start a container of their own (e.g. lb-run for an + # application which needs an older OS). As --contain gives them a minimal + # /dev, apptainer is left without /dev/fuse and so cannot fall back to + # fuse-overlayfs on hosts where the kernel overlay is unavailable. + withFuse = os.path.exists("/dev/fuse") innerCmd = os.path.join(self.__innerdir, "dirac_container.sh") outerCmd = ["apptainer", "exec"] outerCmd.extend(["--contain"]) # use minimal /dev and empty other directories (e.g. /tmp and $HOME) @@ -356,6 +361,8 @@ def submitJob(self, executableFile, proxy=None, **kwargs): outerCmd.append("--userns") if withCVMFS: outerCmd.extend(["--bind", "/cvmfs"]) + if withFuse: + outerCmd.extend(["--bind", "/dev/fuse"]) if not self.__installDIRACInContainer: outerCmd.extend(["--bind", "{0}:{0}:ro".format(self.__findInstallBaseDir())])