From ccfb9eab8c9d0d68f5fba68de69a58a492a72624 Mon Sep 17 00:00:00 2001 From: Mikhail Kot Date: Mon, 24 Aug 2026 13:10:25 +0100 Subject: [PATCH] Windows support for duckdb Signed-off-by: Mikhail Kot --- .github/workflows/ci.yml | 3 ++- scripts/duckdb-r2-resolve.sh | 17 +++++++++++++---- vortex-duckdb/Cargo.toml | 4 +++- vortex-duckdb/build.rs | 25 ++++++++++++++++++++----- vortex-duckdb/cpp/include/vector.h | 2 +- vortex-duckdb/cpp/vector.cpp | 6 +++++- vortex-duckdb/src/table_function.rs | 26 +++++++++++++++----------- 7 files changed, 59 insertions(+), 24 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d5e642417c3..74040e7e79b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -448,11 +448,12 @@ jobs: - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" + - name: Rust Tests (Windows) run: | cargo nextest run --cargo-profile ci --locked --workspace --all-features --no-fail-fast ` --exclude vortex-bench ` - --exclude vortex-python --exclude vortex-duckdb ` + --exclude vortex-python ` --exclude vortex-fuzz --exclude vortex-cuda --exclude vortex-cuda-ffi ` --exclude vortex-nvcomp --exclude vortex-cub --exclude vortex-test-e2e-cuda --exclude vortex-python-cuda ` --exclude duckdb-bench ` diff --git a/scripts/duckdb-r2-resolve.sh b/scripts/duckdb-r2-resolve.sh index 221c1f2bc8d..4b9764e65f4 100755 --- a/scripts/duckdb-r2-resolve.sh +++ b/scripts/duckdb-r2-resolve.sh @@ -30,11 +30,18 @@ echo "DuckDB $version release=$release" entries=$(mktemp) trap 'rm -f "$entries"' EXIT -for archive in \ - libduckdb-linux-amd64.zip \ - libduckdb-linux-arm64.zip \ - libduckdb-osx-universal.zip; do +archives="libduckdb-linux-amd64.zip +libduckdb-linux-arm64.zip +libduckdb-osx-universal.zip" +# Windows archives are only mirrored from DuckDB releases as for now +if [ "$release" = "true" ]; then + archives="$archives +libduckdb-windows-amd64.zip +libduckdb-windows-arm64.zip" +fi + +for archive in $archives; do url="${PUBLIC_BASE_URL}/${ref_dir}/${archive}" code=$(curl -o /dev/null -s -w '%{http_code}' --head "$url" || echo 000) if [ "$code" = "200" ]; then @@ -47,6 +54,8 @@ for archive in \ *linux-amd64*) runner=ubuntu-latest; os=linux; arch=amd64 ;; *linux-arm64*) runner=ubuntu-24.04-arm; os=linux; arch=arm64 ;; *osx-universal*) runner=macos-14; os=osx; arch=universal ;; + *windows-amd64*) runner=ubuntu-latest; os=windows; arch=amd64 ;; + *windows-arm64*) runner=ubuntu-latest; os=windows; arch=arm64 ;; esac jq -nc \ --arg archive "$archive" \ diff --git a/vortex-duckdb/Cargo.toml b/vortex-duckdb/Cargo.toml index fd611af030d..76b0b27ea1a 100644 --- a/vortex-duckdb/Cargo.toml +++ b/vortex-duckdb/Cargo.toml @@ -23,10 +23,12 @@ name = "vortex_duckdb" path = "src/lib.rs" crate-type = ["rlib"] +[target.'cfg(unix)'.dependencies] +custom-labels = { workspace = true } + [dependencies] async-fs = { workspace = true } bitvec = { workspace = true } -custom-labels = { workspace = true } futures = { workspace = true } itertools = { workspace = true } kanal = { workspace = true } diff --git a/vortex-duckdb/build.rs b/vortex-duckdb/build.rs index 825772b819e..7ac01c9265a 100644 --- a/vortex-duckdb/build.rs +++ b/vortex-duckdb/build.rs @@ -8,7 +8,10 @@ use std::env; use std::fs; use std::io; +#[cfg(unix)] use std::os::unix::fs::symlink; +#[cfg(windows)] +use std::os::windows::fs::symlink_dir as symlink; use std::path::Path; use std::path::PathBuf; use std::process::Command; @@ -403,6 +406,8 @@ fn download_prebuilt(version: &DuckDBVersion, library_dir: &Path, target: &str) "aarch64-apple-darwin" | "x86_64-apple-darwin" => ("osx", "universal"), "x86_64-unknown-linux-gnu" => ("linux", "amd64"), "aarch64-unknown-linux-gnu" => ("linux", "arm64"), + "x86_64-pc-windows-msvc" => ("windows", "amd64"), + "aarch64-pc-windows-msvc" => ("windows", "arm64"), _ => { println!("cargo:error=Unsupported target {target}"); exit(1); @@ -570,6 +575,7 @@ fn bindgen_c2rust(crate_dir: &Path, duckdb_include_dir: &Path) { /// Generate libvortex_duckdb.* fn compile_cpp(duckdb_include_dir: &Path) { let mut build = cc::Build::new(); + let has_debuginfo = env::var("DEBUG") .map(|v| !matches!(v.as_str(), "false" | "0" | "none" | "")) .unwrap_or(false); @@ -578,9 +584,19 @@ fn compile_cpp(duckdb_include_dir: &Path) { } else { build.define("NDEBUG", None); } + + if build.get_compiler().is_like_msvc() { + build.flag("/W4").include(duckdb_include_dir); + } else { + build + .flags(["-Wall", "-Wextra", "-Wpedantic", "-Werror"]) + // We don't want compiler warnings inside duckdb headers, pass as flags + .flag("-isystem") + .flag(duckdb_include_dir); + } + build .std("c++20") - .flags(["-Wall", "-Wextra", "-Wpedantic", "-Werror"]) .cpp(true) // Duckdb 1.5.5 uses C++11. spatial_overrides.o uses // duckdb::ScalarFunctionCatalogEntry::Name which is constexpr but not @@ -588,9 +604,6 @@ fn compile_cpp(duckdb_include_dir: &Path) { // emits this symbol with STB_GNU_UNIQUE and this conflicts on link stage // in duckdb-vortex where libvortex_duckdb.a is linked statically .flag_if_supported("-fno-gnu-unique") - // We don't want compiler warnings inside duckdb headers, pass as flags - .flag("-isystem") - .flag(duckdb_include_dir) .include("include") .include("cpp/include") .files(SOURCE_FILES) @@ -755,7 +768,9 @@ fn main() { // Set rpath for binaries built directly from this crate. This is not // inherited by downstream crates. - println!("cargo:rustc-link-arg=-Wl,-rpath,{library_dir_str}"); + if !cfg!(windows) { + println!("cargo:rustc-link-arg=-Wl,-rpath,{library_dir_str}"); + } // Export the library path for downstream crates via the `links` manifest key. // Downstream crates can access this via `env::var("DEP_DUCKDB_LIB_DIR")` in their build.rs diff --git a/vortex-duckdb/cpp/include/vector.h b/vortex-duckdb/cpp/include/vector.h index 983a7ecc479..40189f9341d 100644 --- a/vortex-duckdb/cpp/include/vector.h +++ b/vortex-duckdb/cpp/include/vector.h @@ -32,7 +32,7 @@ void duckdb_vx_vector_set_data_ptr(duckdb_vector ffi_vector, void *ptr); // Converts a duckdb flat vector into a Sequence vector. void duckdb_vx_sequence_vector(duckdb_vector c_vector, int64_t start, int64_t step, idx_t capacity); -void duckdb_vector_flatten(duckdb_vector vector, unsigned long len); +void duckdb_vector_flatten(duckdb_vector vector, idx_t len); duckdb_value duckdb_vx_vector_get_value(duckdb_vector ffi_vector, idx_t index); diff --git a/vortex-duckdb/cpp/vector.cpp b/vortex-duckdb/cpp/vector.cpp index 47e0492d094..9f985af9eae 100644 --- a/vortex-duckdb/cpp/vector.cpp +++ b/vortex-duckdb/cpp/vector.cpp @@ -113,7 +113,7 @@ extern "C" duckdb_value duckdb_vx_vector_get_value(duckdb_vector ffi_vector, idx return reinterpret_cast(value.release()); } -void duckdb_vector_flatten(duckdb_vector vector, unsigned long len) { +void duckdb_vector_flatten(duckdb_vector vector, idx_t len) { auto dvector = reinterpret_cast(vector); dvector->Flatten(len); } @@ -131,7 +131,11 @@ void duckdb_vx_vector_set_all_valid(duckdb_vector ffi_vector) { case FSST_VECTOR: return FSSTVector::Validity(vector).Reset(); default: +#if defined(_MSC_VER) && !defined(__clang__) + __assume(false); +#else __builtin_unreachable(); +#endif } } diff --git a/vortex-duckdb/src/table_function.rs b/vortex-duckdb/src/table_function.rs index d7d45ef8798..daeb69d3eb6 100644 --- a/vortex-duckdb/src/table_function.rs +++ b/vortex-duckdb/src/table_function.rs @@ -8,6 +8,7 @@ use std::sync::atomic::AtomicBool; use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; +#[cfg(unix)] use custom_labels::CURRENT_LABELSET; use futures::future::BoxFuture; use itertools::Itertools; @@ -32,6 +33,7 @@ use vortex::error::vortex_bail; use vortex::expr::BoundExpression; use vortex::expr::Expression; use vortex::extension::uuid::Uuid; +#[cfg(unix)] use vortex::metrics::tracing::get_global_labels; use vortex::scalar::Scalar; use vortex::scalar_fn::fns::binary::Binary; @@ -338,19 +340,21 @@ fn build_partials( } pub fn init_local(bind_data: &BindState, global: &GlobalState) -> LocalState { - unsafe { - use custom_labels::sys; - - if sys::current().is_null() { - let ls = sys::new(0); - sys::replace(ls); - }; - } + #[cfg(unix)] + { + unsafe { + use custom_labels::sys; + if sys::current().is_null() { + let ls = sys::new(0); + sys::replace(ls); + }; + } - let global_labels = get_global_labels(); + let global_labels = get_global_labels(); - for (key, value) in global_labels { - CURRENT_LABELSET.set(key, value); + for (key, value) in global_labels { + CURRENT_LABELSET.set(key, value); + } } let partials = build_partials(&global.aggregates, &bind_data.columns, &bind_data.dtype)