From 7d31f7f512d7d32c368768598e3b5f930eb6029d Mon Sep 17 00:00:00 2001 From: MickLesk <47820557+MickLesk@users.noreply.github.com> Date: Fri, 28 Aug 2026 10:18:44 +0200 Subject: [PATCH] Pass through every selected GPU, not just the first one Reported against unsloth on a host with an Intel iGPU and an NVIDIA card: the userspace driver went in, the script said "NVIDIA GPU configured", and torch.cuda.is_available() was False. /dev/nvidia* was never in the container config. Removing the iGPU and rerunning fixed it, which is the tell: the fault only exists when two vendors are present. Three things, each of which alone was enough to produce that. prompt_select refused to ask when stdin was not a terminal -- but the read it guards takes /dev/tty, not stdin. So `curl ... | bash`, which the docs themselves recommend for run.sh, silently took the default without ever printing the menu. It now asks whenever a terminal is reachable. Both backends then picked exactly one GPU type, defaulting to the first, and the collection order is INTEL, AMD, NVIDIA. The default on a mixed host was therefore always the iGPU. Device nodes are disjoint and an iGPU beside a discrete card is the normal case, so the choice is now multi-select and defaults to all of them. Nothing valid typed passes everything through rather than nothing: a spare device node is harmless, a missing one is this bug. Both branches used to start their dev counter at zero, which was fine while only one could run and would have overwritten entries now that several can. Finally, hwaccel.func reported success on libcuda.so.1 alone. lspci inside a container sees every host GPU whether or not it was passed through, so it will install the stack for a card the container cannot reach. It now also requires at least one /dev/nvidia* node and, when there is none, says that instead of claiming success. --- core/core.func | 8 +- incus/backend.func | 173 +++++++++++++++++++++++--------------- lib/hwaccel.func | 22 ++++- pve/backend.func | 201 +++++++++++++++++++++++++-------------------- 4 files changed, 245 insertions(+), 159 deletions(-) diff --git a/core/core.func b/core/core.func index 17a1b31..c4d6169 100644 --- a/core/core.func +++ b/core/core.func @@ -1856,8 +1856,12 @@ prompt_select() { return 0 fi - # Check if running in a TTY - if [[ ! -t 0 ]]; then + # The read below takes /dev/tty, not stdin, so a piped or exhausted stdin says + # nothing about whether the operator can answer. Testing -t 0 here meant that + # `curl ... | bash` silently took the default without ever printing the menu -- + # which on a mixed-GPU host handed the container the iGPU and never said so. + # Ask whenever a terminal is reachable at all. + if [[ ! -r /dev/tty ]]; then echo "${options[$((default - 1))]}" return 0 fi diff --git a/incus/backend.func b/incus/backend.func index 80d6a11..bbea9f5 100644 --- a/incus/backend.func +++ b/incus/backend.func @@ -654,80 +654,117 @@ incus_configure_device_passthrough() { fi if [[ "$gpu_count" -gt 0 ]]; then - selected_gpu="${available_gpus[0]}" + # Several types can be attached together, and picking exactly one used to + # default to the first -- always INTEL, by collection order. On a host with + # an iGPU beside a discrete card that silently attached the iGPU only. + local -a selected_types=("${available_gpus[@]}") if [[ "$gpu_count" -gt 1 ]]; then - if declare -f prompt_select >/dev/null 2>&1; then - selected_gpu=$(prompt_select "Which GPU type to passthrough?" 1 60 "${available_gpus[@]}") + echo -e "\n${INFO} Multiple GPU types detected:" + local gi + for gi in "${!available_gpus[@]}"; do + echo " $((gi + 1))) ${available_gpus[$gi]}" + done + echo " A) Pass through ALL of them" + + local type_selection="" + read -r -t 60 -p "Select GPU type(s) (1-${gpu_count}, comma-separated, A=all) [timeout 60s, default=all]: " type_selection = 1 && gnum <= gpu_count)); then + selected_types+=("${available_gpus[$((gnum - 1))]}") + fi + done + # A spare device is harmless; a missing one is the bug being fixed. + if [[ ${#selected_types[@]} -eq 0 ]]; then + msg_warn "No valid selection - passing through all detected GPUs" + selected_types=("${available_gpus[@]}") + fi fi fi - selected_gpu="${selected_gpu^^}" - - local sel added=0 failed=0 idx=0 - case "$selected_gpu" in - INTEL) - for sel in "${INTEL_GPUS[@]}"; do - if _incus_add_gpu_physical "gpu${idx}" "$sel"; then - added=$((added + 1)) - else - failed=$((failed + 1)) - msg_warn "Could not add Intel GPU (${sel})" + + # idx is shared across types so the device names stay unique. + local sel added failed idx=0 + local -a configured_types=() + for selected_gpu in "${selected_types[@]}"; do + added=0 + failed=0 + case "$selected_gpu" in + INTEL) + for sel in "${INTEL_GPUS[@]}"; do + if _incus_add_gpu_physical "gpu${idx}" "$sel"; then + added=$((added + 1)) + else + failed=$((failed + 1)) + msg_warn "Could not add Intel GPU (${sel})" + fi + idx=$((idx + 1)) + done + # Vendor fallback if PCI/id adds failed entirely + if [[ "$added" -eq 0 ]]; then + _incus_add_gpu_physical "gpu0" "vendorid=8086" && added=1 || true fi - idx=$((idx + 1)) - done - # Vendor fallback if PCI/id adds failed entirely - if [[ "$added" -eq 0 ]]; then - _incus_add_gpu_physical "gpu0" "vendorid=8086" && added=1 || true - fi - if [[ "$added" -gt 0 ]]; then - export GPU_TYPE="INTEL" - msg_ok "Intel GPU passthrough configured (${added})" - fi - ;; - AMD) - for sel in "${AMD_GPUS[@]}"; do - if _incus_add_gpu_physical "gpu${idx}" "$sel"; then - added=$((added + 1)) - else - failed=$((failed + 1)) - msg_warn "Could not add AMD GPU (${sel})" + if [[ "$added" -gt 0 ]]; then + configured_types+=("INTEL") + msg_ok "Intel GPU passthrough configured (${added})" fi - idx=$((idx + 1)) - done - if [[ "$added" -eq 0 ]]; then - _incus_add_gpu_physical "gpu0" "vendorid=1002" && added=1 || true - fi - if [[ "$added" -gt 0 ]]; then - export GPU_TYPE="AMD" - msg_ok "AMD GPU passthrough configured (${added})" - fi - ;; - NVIDIA) - for sel in "${NVIDIA_GPUS[@]}"; do - if _incus_add_gpu_physical "gpu${idx}" "$sel"; then - added=$((added + 1)) - else - failed=$((failed + 1)) - msg_warn "Could not add NVIDIA GPU (${sel})" + ;; + AMD) + for sel in "${AMD_GPUS[@]}"; do + if _incus_add_gpu_physical "gpu${idx}" "$sel"; then + added=$((added + 1)) + else + failed=$((failed + 1)) + msg_warn "Could not add AMD GPU (${sel})" + fi + idx=$((idx + 1)) + done + if [[ "$added" -eq 0 ]]; then + _incus_add_gpu_physical "gpu0" "vendorid=1002" && added=1 || true fi - idx=$((idx + 1)) - done - if [[ "$added" -eq 0 ]]; then - _incus_add_gpu_physical "gpu0" "vendorid=10de" && added=1 || true - fi - # Optional NVIDIA char nodes (driver interface) - local ndev nidx=0 - for ndev in "${NVIDIA_CHAR_DEVS[@]}"; do - local nname - nname=$(basename "$ndev" | tr -c 'A-Za-z0-9' '_') - incus config device add "${CT_NAME}" "nvidia_${nname}_${nidx}" unix-char "source=${ndev}" "path=${ndev}" >>"${LOGFILE:-$INCUS_BUILD_LOG}" 2>&1 || true - nidx=$((nidx + 1)) - done - if [[ "$added" -gt 0 || "$nidx" -gt 0 ]]; then - export GPU_TYPE="NVIDIA" - msg_ok "NVIDIA GPU passthrough configured (gpu=${added}, char=${nidx})" - fi - ;; - esac + if [[ "$added" -gt 0 ]]; then + configured_types+=("AMD") + msg_ok "AMD GPU passthrough configured (${added})" + fi + ;; + NVIDIA) + for sel in "${NVIDIA_GPUS[@]}"; do + if _incus_add_gpu_physical "gpu${idx}" "$sel"; then + added=$((added + 1)) + else + failed=$((failed + 1)) + msg_warn "Could not add NVIDIA GPU (${sel})" + fi + idx=$((idx + 1)) + done + if [[ "$added" -eq 0 ]]; then + _incus_add_gpu_physical "gpu0" "vendorid=10de" && added=1 || true + fi + # Optional NVIDIA char nodes (driver interface) + local ndev nidx=0 + for ndev in "${NVIDIA_CHAR_DEVS[@]}"; do + local nname + nname=$(basename "$ndev" | tr -c 'A-Za-z0-9' '_') + incus config device add "${CT_NAME}" "nvidia_${nname}_${nidx}" unix-char "source=${ndev}" "path=${ndev}" >>"${LOGFILE:-$INCUS_BUILD_LOG}" 2>&1 || true + nidx=$((nidx + 1)) + done + if [[ "$added" -gt 0 || "$nidx" -gt 0 ]]; then + configured_types+=("NVIDIA") + msg_ok "NVIDIA GPU passthrough configured (gpu=${added}, char=${nidx})" + fi + ;; + esac + done + + # Space separated, so a reader can test for one type without caring how + # many others came with it. + export GPU_TYPE="${configured_types[*]}" if [[ -z "${GPU_TYPE:-}" ]]; then msg_warn "GPU passthrough requested but no device could be attached (check permissions / incus info --resources)" diff --git a/lib/hwaccel.func b/lib/hwaccel.func index 76fa7da..025dde4 100644 --- a/lib/hwaccel.func +++ b/lib/hwaccel.func @@ -867,8 +867,26 @@ NVIDIA_PIN # ldconfig rather than dpkg: the library can arrive by bind mount too, and what # matters is whether a CUDA program can link it. Not a non-zero return, since # acceleration is optional and install scripts run under an ERR trap. - if ldconfig -p 2>/dev/null | grep -q 'libcuda\.so\.1'; then - msg_ok "NVIDIA GPU configured (host driver ${nvidia_host_version})" + # Libraries alone are not enough, and this is the half that used to be missed: + # lspci sees every GPU on the host whether or not it was passed through, so + # this function will happily install the userspace stack for a card the + # container cannot reach. That reported success while torch.cuda.is_available() + # stayed False, and the real fault -- no device nodes -- went unmentioned. + local -a nvidia_nodes=() + local _nvdev + for _nvdev in /dev/nvidia*; do + [[ -c "$_nvdev" ]] && nvidia_nodes+=("$_nvdev") + done + + local have_libcuda=0 + ldconfig -p 2>/dev/null | grep -q 'libcuda\.so\.1' && have_libcuda=1 + + if [[ $have_libcuda -eq 1 && ${#nvidia_nodes[@]} -gt 0 ]]; then + msg_ok "NVIDIA GPU configured (host driver ${nvidia_host_version}, ${#nvidia_nodes[@]} device nodes)" + elif [[ ${#nvidia_nodes[@]} -eq 0 ]]; then + msg_error "NVIDIA GPU not usable: no /dev/nvidia* device nodes in this container" + msg_warn "The userspace driver is in place, but the devices were never passed through." + msg_warn "On the host, check the container config lists /dev/nvidia0, /dev/nvidiactl and /dev/nvidia-uvm." else msg_error "NVIDIA GPU not usable: libcuda.so.1 is not available in this container" msg_warn "The host driver is ${nvidia_host_version}. CUDA workloads will fail until a matching userspace library is present." diff --git a/pve/backend.func b/pve/backend.func index 4ce87b8..59cd62f 100644 --- a/pve/backend.func +++ b/pve/backend.func @@ -576,111 +576,136 @@ EOF return 0 fi - local selected_gpu="" + # Several types can be passed through at once: the device nodes are disjoint, + # and an iGPU next to a discrete card is the normal case rather than an + # either/or. This used to pick exactly one, defaulting to the first entry -- + # always INTEL, since that is the order they are collected in. On a host with + # an Intel iGPU and an NVIDIA card that silently passed through the iGPU and + # left /dev/nvidia* out, so CUDA never saw a device. + local -a selected_types=() if [[ $gpu_count -eq 1 ]]; then - selected_gpu="${available_gpus[0]}" - msg_ok "Automatically configuring ${selected_gpu} GPU passthrough" + selected_types=("${available_gpus[0]}") + msg_ok "Automatically configuring ${selected_types[0]} GPU passthrough" else echo -e "\n${INFO} Multiple GPU types detected:" - for gpu in "${available_gpus[@]}"; do - echo " - $gpu" + local i + for i in "${!available_gpus[@]}"; do + echo " $((i + 1))) ${available_gpus[$i]}" done - selected_gpu=$(prompt_select "Which GPU type to passthrough?" 1 60 "${available_gpus[@]}") - selected_gpu="${selected_gpu^^}" + echo " A) Pass through ALL of them" - # Validate selection - local valid=0 - for gpu in "${available_gpus[@]}"; do - [[ "$selected_gpu" == "$gpu" ]] && valid=1 - done + local type_selection="" + read -r -t 60 -p "Select GPU type(s) (1-${gpu_count}, comma-separated, A=all) [timeout 60s, default=all]: " type_selection = 1 && num <= gpu_count)); then + selected_types+=("${available_gpus[$((num - 1))]}") + fi + done + # Nothing usable typed. Passing everything through beats passing nothing: + # a spare device node is harmless, a missing one is the bug being fixed. + if [[ ${#selected_types[@]} -eq 0 ]]; then + msg_warn "No valid selection - passing through all detected GPUs" + selected_types=("${available_gpus[@]}") + fi fi fi - # Apply passthrough configuration based on selection - local dev_idx=0 - - case "$selected_gpu" in - INTEL | AMD) - local devices=() - [[ "$selected_gpu" == "INTEL" ]] && devices=("${INTEL_DEVICES[@]}") - [[ "$selected_gpu" == "AMD" ]] && devices=("${AMD_DEVICES[@]}") - - # Use pct set to add devices with proper dev0/dev1 format - # GIDs will be detected and set after container starts - local dev_index=0 - for dev in "${devices[@]}"; do - # Add to config using pct set (will be visible in GUI) - echo "dev${dev_index}: ${dev},gid=44" >>"$LXC_CONFIG" - dev_index=$((dev_index + 1)) - done - - export GPU_TYPE="$selected_gpu" - msg_ok "${selected_gpu} GPU passthrough configured (${#devices[@]} devices)" - ;; + # One counter across every selected type: each branch used to start at dev0, + # which was harmless while only one branch could run and would overwrite + # entries now that several can. + local dev_index=0 + local -a configured_types=() + + local selected_gpu + for selected_gpu in "${selected_types[@]}"; do + case "$selected_gpu" in + INTEL | AMD) + local devices=() + [[ "$selected_gpu" == "INTEL" ]] && devices=("${INTEL_DEVICES[@]}") + [[ "$selected_gpu" == "AMD" ]] && devices=("${AMD_DEVICES[@]}") + + # dev0/dev1 format so the entries show up in the GUI. + # GIDs are detected and corrected once the container has started. + for dev in "${devices[@]}"; do + echo "dev${dev_index}: ${dev},gid=44" >>"$LXC_CONFIG" + dev_index=$((dev_index + 1)) + done - NVIDIA) - if [[ ${#NVIDIA_DEVICES[@]} -eq 0 ]]; then - msg_warn "No NVIDIA devices available for passthrough" - return 0 - fi + configured_types+=("$selected_gpu") + msg_ok "${selected_gpu} GPU passthrough configured (${#devices[@]} devices)" + ;; - # Separate per-card device nodes (/dev/nvidia0, /dev/nvidia1, ...) from - # the shared control nodes (nvidiactl, nvidia-uvm*, nvidia-caps/*, - # nvidia-modeset), so a host with several NVIDIA cards can pass through - # only some of them instead of always all-or-nothing. - local -a nvidia_cards=() nvidia_shared=() - for dev in "${NVIDIA_DEVICES[@]}"; do - if [[ "$dev" =~ ^/dev/nvidia[0-9]+$ ]]; then - nvidia_cards+=("$dev") - else - nvidia_shared+=("$dev") + NVIDIA) + if [[ ${#NVIDIA_DEVICES[@]} -eq 0 ]]; then + msg_warn "No NVIDIA devices available for passthrough" + continue fi - done - local -a selected_cards=("${nvidia_cards[@]}") - if [[ ${#nvidia_cards[@]} -gt 1 ]]; then - echo -e "\n${INFO} Multiple NVIDIA GPUs detected:" - local i - for i in "${!nvidia_cards[@]}"; do - echo " $((i + 1))) ${nvidia_cards[$i]}" + # Separate per-card device nodes (/dev/nvidia0, /dev/nvidia1, ...) from + # the shared control nodes (nvidiactl, nvidia-uvm*, nvidia-caps/*, + # nvidia-modeset), so a host with several NVIDIA cards can pass through + # only some of them instead of always all-or-nothing. + local -a nvidia_cards=() nvidia_shared=() + for dev in "${NVIDIA_DEVICES[@]}"; do + if [[ "$dev" =~ ^/dev/nvidia[0-9]+$ ]]; then + nvidia_cards+=("$dev") + else + nvidia_shared+=("$dev") + fi done - echo " A) Configure ALL GPUs" - local gpu_selection="" - read -r -t 60 -p "Select GPU(s) to pass through (1-${#nvidia_cards[@]}, A=all) [timeout 60s, default=all]: " gpu_selection = 1 && num <= ${#nvidia_cards[@]})); then - selected_cards+=("${nvidia_cards[$((num - 1))]}") - fi + + local -a selected_cards=("${nvidia_cards[@]}") + if [[ ${#nvidia_cards[@]} -gt 1 ]]; then + echo -e "\n${INFO} Multiple NVIDIA GPUs detected:" + local i + for i in "${!nvidia_cards[@]}"; do + echo " $((i + 1))) ${nvidia_cards[$i]}" done - [[ ${#selected_cards[@]} -eq 0 ]] && selected_cards=("${nvidia_cards[@]}") + echo " A) Configure ALL GPUs" + local gpu_selection="" + read -r -t 60 -p "Select GPU(s) to pass through (1-${#nvidia_cards[@]}, A=all) [timeout 60s, default=all]: " gpu_selection = 1 && num <= ${#nvidia_cards[@]})); then + selected_cards+=("${nvidia_cards[$((num - 1))]}") + fi + done + [[ ${#selected_cards[@]} -eq 0 ]] && selected_cards=("${nvidia_cards[@]}") + fi fi - fi - # Use pct set for NVIDIA devices - local -a devices_to_add=("${selected_cards[@]}" "${nvidia_shared[@]}") - local dev_index=0 - for dev in "${devices_to_add[@]}"; do - echo "dev${dev_index}: ${dev},gid=44" >>"$LXC_CONFIG" - dev_index=$((dev_index + 1)) - done + local -a devices_to_add=("${selected_cards[@]}" "${nvidia_shared[@]}") + for dev in "${devices_to_add[@]}"; do + echo "dev${dev_index}: ${dev},gid=44" >>"$LXC_CONFIG" + dev_index=$((dev_index + 1)) + done - export GPU_TYPE="NVIDIA" - msg_ok "NVIDIA GPU passthrough configured (${#selected_cards[@]}/${#nvidia_cards[@]} GPU(s), ${#devices_to_add[@]} devices) - install drivers in container if needed" - ;; - esac + configured_types+=("NVIDIA") + msg_ok "NVIDIA GPU passthrough configured (${#selected_cards[@]}/${#nvidia_cards[@]} GPU(s), ${#devices_to_add[@]} devices) - install drivers in container if needed" + ;; + esac + done + + # Space separated, so the readers below can test for one type without + # caring how many others came with it. + export GPU_TYPE="${configured_types[*]}" } # Additional device passthrough @@ -705,7 +730,9 @@ EOF configure_gpu_passthrough configure_additional_devices - if [[ "${GPU_TYPE:-}" == "AMD" ]]; then + # GPU_TYPE can now name several types at once, so this asks whether AMD is + # among them rather than whether it is the only one. + if [[ " ${GPU_TYPE:-} " == *" AMD "* ]]; then local rocm_extra=4 local new_disk_size=$((PCT_DISK_SIZE + rocm_extra)) if pct resize "$CTID" rootfs "${new_disk_size}G" >/dev/null 2>&1; then