diff --git a/core/core.func b/core/core.func index 17a1b31..c4d6169 100644 --- a/core/core.func +++ b/core/core.func @@ -1856,8 +1856,12 @@ prompt_select() { return 0 fi - # Check if running in a TTY - if [[ ! -t 0 ]]; then + # The read below takes /dev/tty, not stdin, so a piped or exhausted stdin says + # nothing about whether the operator can answer. Testing -t 0 here meant that + # `curl ... | bash` silently took the default without ever printing the menu -- + # which on a mixed-GPU host handed the container the iGPU and never said so. + # Ask whenever a terminal is reachable at all. + if [[ ! -r /dev/tty ]]; then echo "${options[$((default - 1))]}" return 0 fi diff --git a/incus/backend.func b/incus/backend.func index 80d6a11..bbea9f5 100644 --- a/incus/backend.func +++ b/incus/backend.func @@ -654,80 +654,117 @@ incus_configure_device_passthrough() { fi if [[ "$gpu_count" -gt 0 ]]; then - selected_gpu="${available_gpus[0]}" + # Several types can be attached together, and picking exactly one used to + # default to the first -- always INTEL, by collection order. On a host with + # an iGPU beside a discrete card that silently attached the iGPU only. + local -a selected_types=("${available_gpus[@]}") if [[ "$gpu_count" -gt 1 ]]; then - if declare -f prompt_select >/dev/null 2>&1; then - selected_gpu=$(prompt_select "Which GPU type to passthrough?" 1 60 "${available_gpus[@]}") + echo -e "\n${INFO} Multiple GPU types detected:" + local gi + for gi in "${!available_gpus[@]}"; do + echo " $((gi + 1))) ${available_gpus[$gi]}" + done + echo " A) Pass through ALL of them" + + local type_selection="" + read -r -t 60 -p "Select GPU type(s) (1-${gpu_count}, comma-separated, A=all) [timeout 60s, default=all]: " type_selection = 1 && gnum <= gpu_count)); then + selected_types+=("${available_gpus[$((gnum - 1))]}") + fi + done + # A spare device is harmless; a missing one is the bug being fixed. + if [[ ${#selected_types[@]} -eq 0 ]]; then + msg_warn "No valid selection - passing through all detected GPUs" + selected_types=("${available_gpus[@]}") + fi fi fi - selected_gpu="${selected_gpu^^}" - - local sel added=0 failed=0 idx=0 - case "$selected_gpu" in - INTEL) - for sel in "${INTEL_GPUS[@]}"; do - if _incus_add_gpu_physical "gpu${idx}" "$sel"; then - added=$((added + 1)) - else - failed=$((failed + 1)) - msg_warn "Could not add Intel GPU (${sel})" + + # idx is shared across types so the device names stay unique. + local sel added failed idx=0 + local -a configured_types=() + for selected_gpu in "${selected_types[@]}"; do + added=0 + failed=0 + case "$selected_gpu" in + INTEL) + for sel in "${INTEL_GPUS[@]}"; do + if _incus_add_gpu_physical "gpu${idx}" "$sel"; then + added=$((added + 1)) + else + failed=$((failed + 1)) + msg_warn "Could not add Intel GPU (${sel})" + fi + idx=$((idx + 1)) + done + # Vendor fallback if PCI/id adds failed entirely + if [[ "$added" -eq 0 ]]; then + _incus_add_gpu_physical "gpu0" "vendorid=8086" && added=1 || true fi - idx=$((idx + 1)) - done - # Vendor fallback if PCI/id adds failed entirely - if [[ "$added" -eq 0 ]]; then - _incus_add_gpu_physical "gpu0" "vendorid=8086" && added=1 || true - fi - if [[ "$added" -gt 0 ]]; then - export GPU_TYPE="INTEL" - msg_ok "Intel GPU passthrough configured (${added})" - fi - ;; - AMD) - for sel in "${AMD_GPUS[@]}"; do - if _incus_add_gpu_physical "gpu${idx}" "$sel"; then - added=$((added + 1)) - else - failed=$((failed + 1)) - msg_warn "Could not add AMD GPU (${sel})" + if [[ "$added" -gt 0 ]]; then + configured_types+=("INTEL") + msg_ok "Intel GPU passthrough configured (${added})" fi - idx=$((idx + 1)) - done - if [[ "$added" -eq 0 ]]; then - _incus_add_gpu_physical "gpu0" "vendorid=1002" && added=1 || true - fi - if [[ "$added" -gt 0 ]]; then - export GPU_TYPE="AMD" - msg_ok "AMD GPU passthrough configured (${added})" - fi - ;; - NVIDIA) - for sel in "${NVIDIA_GPUS[@]}"; do - if _incus_add_gpu_physical "gpu${idx}" "$sel"; then - added=$((added + 1)) - else - failed=$((failed + 1)) - msg_warn "Could not add NVIDIA GPU (${sel})" + ;; + AMD) + for sel in "${AMD_GPUS[@]}"; do + if _incus_add_gpu_physical "gpu${idx}" "$sel"; then + added=$((added + 1)) + else + failed=$((failed + 1)) + msg_warn "Could not add AMD GPU (${sel})" + fi + idx=$((idx + 1)) + done + if [[ "$added" -eq 0 ]]; then + _incus_add_gpu_physical "gpu0" "vendorid=1002" && added=1 || true fi - idx=$((idx + 1)) - done - if [[ "$added" -eq 0 ]]; then - _incus_add_gpu_physical "gpu0" "vendorid=10de" && added=1 || true - fi - # Optional NVIDIA char nodes (driver interface) - local ndev nidx=0 - for ndev in "${NVIDIA_CHAR_DEVS[@]}"; do - local nname - nname=$(basename "$ndev" | tr -c 'A-Za-z0-9' '_') - incus config device add "${CT_NAME}" "nvidia_${nname}_${nidx}" unix-char "source=${ndev}" "path=${ndev}" >>"${LOGFILE:-$INCUS_BUILD_LOG}" 2>&1 || true - nidx=$((nidx + 1)) - done - if [[ "$added" -gt 0 || "$nidx" -gt 0 ]]; then - export GPU_TYPE="NVIDIA" - msg_ok "NVIDIA GPU passthrough configured (gpu=${added}, char=${nidx})" - fi - ;; - esac + if [[ "$added" -gt 0 ]]; then + configured_types+=("AMD") + msg_ok "AMD GPU passthrough configured (${added})" + fi + ;; + NVIDIA) + for sel in "${NVIDIA_GPUS[@]}"; do + if _incus_add_gpu_physical "gpu${idx}" "$sel"; then + added=$((added + 1)) + else + failed=$((failed + 1)) + msg_warn "Could not add NVIDIA GPU (${sel})" + fi + idx=$((idx + 1)) + done + if [[ "$added" -eq 0 ]]; then + _incus_add_gpu_physical "gpu0" "vendorid=10de" && added=1 || true + fi + # Optional NVIDIA char nodes (driver interface) + local ndev nidx=0 + for ndev in "${NVIDIA_CHAR_DEVS[@]}"; do + local nname + nname=$(basename "$ndev" | tr -c 'A-Za-z0-9' '_') + incus config device add "${CT_NAME}" "nvidia_${nname}_${nidx}" unix-char "source=${ndev}" "path=${ndev}" >>"${LOGFILE:-$INCUS_BUILD_LOG}" 2>&1 || true + nidx=$((nidx + 1)) + done + if [[ "$added" -gt 0 || "$nidx" -gt 0 ]]; then + configured_types+=("NVIDIA") + msg_ok "NVIDIA GPU passthrough configured (gpu=${added}, char=${nidx})" + fi + ;; + esac + done + + # Space separated, so a reader can test for one type without caring how + # many others came with it. + export GPU_TYPE="${configured_types[*]}" if [[ -z "${GPU_TYPE:-}" ]]; then msg_warn "GPU passthrough requested but no device could be attached (check permissions / incus info --resources)" diff --git a/lib/hwaccel.func b/lib/hwaccel.func index 76fa7da..025dde4 100644 --- a/lib/hwaccel.func +++ b/lib/hwaccel.func @@ -867,8 +867,26 @@ NVIDIA_PIN # ldconfig rather than dpkg: the library can arrive by bind mount too, and what # matters is whether a CUDA program can link it. Not a non-zero return, since # acceleration is optional and install scripts run under an ERR trap. - if ldconfig -p 2>/dev/null | grep -q 'libcuda\.so\.1'; then - msg_ok "NVIDIA GPU configured (host driver ${nvidia_host_version})" + # Libraries alone are not enough, and this is the half that used to be missed: + # lspci sees every GPU on the host whether or not it was passed through, so + # this function will happily install the userspace stack for a card the + # container cannot reach. That reported success while torch.cuda.is_available() + # stayed False, and the real fault -- no device nodes -- went unmentioned. + local -a nvidia_nodes=() + local _nvdev + for _nvdev in /dev/nvidia*; do + [[ -c "$_nvdev" ]] && nvidia_nodes+=("$_nvdev") + done + + local have_libcuda=0 + ldconfig -p 2>/dev/null | grep -q 'libcuda\.so\.1' && have_libcuda=1 + + if [[ $have_libcuda -eq 1 && ${#nvidia_nodes[@]} -gt 0 ]]; then + msg_ok "NVIDIA GPU configured (host driver ${nvidia_host_version}, ${#nvidia_nodes[@]} device nodes)" + elif [[ ${#nvidia_nodes[@]} -eq 0 ]]; then + msg_error "NVIDIA GPU not usable: no /dev/nvidia* device nodes in this container" + msg_warn "The userspace driver is in place, but the devices were never passed through." + msg_warn "On the host, check the container config lists /dev/nvidia0, /dev/nvidiactl and /dev/nvidia-uvm." else msg_error "NVIDIA GPU not usable: libcuda.so.1 is not available in this container" msg_warn "The host driver is ${nvidia_host_version}. CUDA workloads will fail until a matching userspace library is present." diff --git a/pve/backend.func b/pve/backend.func index 4ce87b8..59cd62f 100644 --- a/pve/backend.func +++ b/pve/backend.func @@ -576,111 +576,136 @@ EOF return 0 fi - local selected_gpu="" + # Several types can be passed through at once: the device nodes are disjoint, + # and an iGPU next to a discrete card is the normal case rather than an + # either/or. This used to pick exactly one, defaulting to the first entry -- + # always INTEL, since that is the order they are collected in. On a host with + # an Intel iGPU and an NVIDIA card that silently passed through the iGPU and + # left /dev/nvidia* out, so CUDA never saw a device. + local -a selected_types=() if [[ $gpu_count -eq 1 ]]; then - selected_gpu="${available_gpus[0]}" - msg_ok "Automatically configuring ${selected_gpu} GPU passthrough" + selected_types=("${available_gpus[0]}") + msg_ok "Automatically configuring ${selected_types[0]} GPU passthrough" else echo -e "\n${INFO} Multiple GPU types detected:" - for gpu in "${available_gpus[@]}"; do - echo " - $gpu" + local i + for i in "${!available_gpus[@]}"; do + echo " $((i + 1))) ${available_gpus[$i]}" done - selected_gpu=$(prompt_select "Which GPU type to passthrough?" 1 60 "${available_gpus[@]}") - selected_gpu="${selected_gpu^^}" + echo " A) Pass through ALL of them" - # Validate selection - local valid=0 - for gpu in "${available_gpus[@]}"; do - [[ "$selected_gpu" == "$gpu" ]] && valid=1 - done + local type_selection="" + read -r -t 60 -p "Select GPU type(s) (1-${gpu_count}, comma-separated, A=all) [timeout 60s, default=all]: " type_selection = 1 && num <= gpu_count)); then + selected_types+=("${available_gpus[$((num - 1))]}") + fi + done + # Nothing usable typed. Passing everything through beats passing nothing: + # a spare device node is harmless, a missing one is the bug being fixed. + if [[ ${#selected_types[@]} -eq 0 ]]; then + msg_warn "No valid selection - passing through all detected GPUs" + selected_types=("${available_gpus[@]}") + fi fi fi - # Apply passthrough configuration based on selection - local dev_idx=0 - - case "$selected_gpu" in - INTEL | AMD) - local devices=() - [[ "$selected_gpu" == "INTEL" ]] && devices=("${INTEL_DEVICES[@]}") - [[ "$selected_gpu" == "AMD" ]] && devices=("${AMD_DEVICES[@]}") - - # Use pct set to add devices with proper dev0/dev1 format - # GIDs will be detected and set after container starts - local dev_index=0 - for dev in "${devices[@]}"; do - # Add to config using pct set (will be visible in GUI) - echo "dev${dev_index}: ${dev},gid=44" >>"$LXC_CONFIG" - dev_index=$((dev_index + 1)) - done - - export GPU_TYPE="$selected_gpu" - msg_ok "${selected_gpu} GPU passthrough configured (${#devices[@]} devices)" - ;; + # One counter across every selected type: each branch used to start at dev0, + # which was harmless while only one branch could run and would overwrite + # entries now that several can. + local dev_index=0 + local -a configured_types=() + + local selected_gpu + for selected_gpu in "${selected_types[@]}"; do + case "$selected_gpu" in + INTEL | AMD) + local devices=() + [[ "$selected_gpu" == "INTEL" ]] && devices=("${INTEL_DEVICES[@]}") + [[ "$selected_gpu" == "AMD" ]] && devices=("${AMD_DEVICES[@]}") + + # dev0/dev1 format so the entries show up in the GUI. + # GIDs are detected and corrected once the container has started. + for dev in "${devices[@]}"; do + echo "dev${dev_index}: ${dev},gid=44" >>"$LXC_CONFIG" + dev_index=$((dev_index + 1)) + done - NVIDIA) - if [[ ${#NVIDIA_DEVICES[@]} -eq 0 ]]; then - msg_warn "No NVIDIA devices available for passthrough" - return 0 - fi + configured_types+=("$selected_gpu") + msg_ok "${selected_gpu} GPU passthrough configured (${#devices[@]} devices)" + ;; - # Separate per-card device nodes (/dev/nvidia0, /dev/nvidia1, ...) from - # the shared control nodes (nvidiactl, nvidia-uvm*, nvidia-caps/*, - # nvidia-modeset), so a host with several NVIDIA cards can pass through - # only some of them instead of always all-or-nothing. - local -a nvidia_cards=() nvidia_shared=() - for dev in "${NVIDIA_DEVICES[@]}"; do - if [[ "$dev" =~ ^/dev/nvidia[0-9]+$ ]]; then - nvidia_cards+=("$dev") - else - nvidia_shared+=("$dev") + NVIDIA) + if [[ ${#NVIDIA_DEVICES[@]} -eq 0 ]]; then + msg_warn "No NVIDIA devices available for passthrough" + continue fi - done - local -a selected_cards=("${nvidia_cards[@]}") - if [[ ${#nvidia_cards[@]} -gt 1 ]]; then - echo -e "\n${INFO} Multiple NVIDIA GPUs detected:" - local i - for i in "${!nvidia_cards[@]}"; do - echo " $((i + 1))) ${nvidia_cards[$i]}" + # Separate per-card device nodes (/dev/nvidia0, /dev/nvidia1, ...) from + # the shared control nodes (nvidiactl, nvidia-uvm*, nvidia-caps/*, + # nvidia-modeset), so a host with several NVIDIA cards can pass through + # only some of them instead of always all-or-nothing. + local -a nvidia_cards=() nvidia_shared=() + for dev in "${NVIDIA_DEVICES[@]}"; do + if [[ "$dev" =~ ^/dev/nvidia[0-9]+$ ]]; then + nvidia_cards+=("$dev") + else + nvidia_shared+=("$dev") + fi done - echo " A) Configure ALL GPUs" - local gpu_selection="" - read -r -t 60 -p "Select GPU(s) to pass through (1-${#nvidia_cards[@]}, A=all) [timeout 60s, default=all]: " gpu_selection = 1 && num <= ${#nvidia_cards[@]})); then - selected_cards+=("${nvidia_cards[$((num - 1))]}") - fi + + local -a selected_cards=("${nvidia_cards[@]}") + if [[ ${#nvidia_cards[@]} -gt 1 ]]; then + echo -e "\n${INFO} Multiple NVIDIA GPUs detected:" + local i + for i in "${!nvidia_cards[@]}"; do + echo " $((i + 1))) ${nvidia_cards[$i]}" done - [[ ${#selected_cards[@]} -eq 0 ]] && selected_cards=("${nvidia_cards[@]}") + echo " A) Configure ALL GPUs" + local gpu_selection="" + read -r -t 60 -p "Select GPU(s) to pass through (1-${#nvidia_cards[@]}, A=all) [timeout 60s, default=all]: " gpu_selection = 1 && num <= ${#nvidia_cards[@]})); then + selected_cards+=("${nvidia_cards[$((num - 1))]}") + fi + done + [[ ${#selected_cards[@]} -eq 0 ]] && selected_cards=("${nvidia_cards[@]}") + fi fi - fi - # Use pct set for NVIDIA devices - local -a devices_to_add=("${selected_cards[@]}" "${nvidia_shared[@]}") - local dev_index=0 - for dev in "${devices_to_add[@]}"; do - echo "dev${dev_index}: ${dev},gid=44" >>"$LXC_CONFIG" - dev_index=$((dev_index + 1)) - done + local -a devices_to_add=("${selected_cards[@]}" "${nvidia_shared[@]}") + for dev in "${devices_to_add[@]}"; do + echo "dev${dev_index}: ${dev},gid=44" >>"$LXC_CONFIG" + dev_index=$((dev_index + 1)) + done - export GPU_TYPE="NVIDIA" - msg_ok "NVIDIA GPU passthrough configured (${#selected_cards[@]}/${#nvidia_cards[@]} GPU(s), ${#devices_to_add[@]} devices) - install drivers in container if needed" - ;; - esac + configured_types+=("NVIDIA") + msg_ok "NVIDIA GPU passthrough configured (${#selected_cards[@]}/${#nvidia_cards[@]} GPU(s), ${#devices_to_add[@]} devices) - install drivers in container if needed" + ;; + esac + done + + # Space separated, so the readers below can test for one type without + # caring how many others came with it. + export GPU_TYPE="${configured_types[*]}" } # Additional device passthrough @@ -705,7 +730,9 @@ EOF configure_gpu_passthrough configure_additional_devices - if [[ "${GPU_TYPE:-}" == "AMD" ]]; then + # GPU_TYPE can now name several types at once, so this asks whether AMD is + # among them rather than whether it is the only one. + if [[ " ${GPU_TYPE:-} " == *" AMD "* ]]; then local rocm_extra=4 local new_disk_size=$((PCT_DISK_SIZE + rocm_extra)) if pct resize "$CTID" rootfs "${new_disk_size}G" >/dev/null 2>&1; then