feat(client-v2): decode QBit in the Native format

CHANGELOG.md

@@ -23,8 +23,13 @@
   binary type encoding is read back to the concrete `QBit(...)` type). In the
   JDBC driver (`jdbc-v2`) `QBit` maps to `java.sql.Types.ARRAY` and is returned as a `java.sql.Array` from
   `getObject`/`getArray`. Previously `QBit` was an unimplemented type constant and reading or writing such a column
-  failed. Reading `QBit` through the `Native` output format is not supported — the server transmits it there using a
-  different internal layout — and fails fast with a clear error; use a `RowBinary` format instead.
+  failed. A plain top-level `QBit` column with a `Float32`, `Float64`, or `BFloat16` element type is also read through
+  the `Native` output format: there the server transmits it using its internal bit-plane-transposed
+  `Tuple(FixedString(...))` layout, and the client reverses that transposition to reconstruct the same
+  `float[]`/`double[]` vector as `RowBinary`. A `QBit` that is strided (`QBit(element_type, dimension, stride)`),
+  wrapped in `Nullable`/`LowCardinality`, nested inside another type (e.g. `Array`/`Tuple`/`Map(String, QBit(...))`),
+  or carrying any other element type is not yet decoded over `Native` and fails fast with a clear error directing you
+  to a `RowBinary` format such as `RowBinaryWithNamesAndTypes`.
   (https://github.com/ClickHouse/clickhouse-java/issues/2610)
 - **[client-v2, jdbc-v2]** Added TLS cipher suite selection. `Client.Builder.setSSLCipherSuites(String...)` (client-v2)
   and the comma-separated `ssl_cipher_suites` connection property (client-v2 and jdbc-v2) restrict the cipher suites

client-v2/src/main/java/com/clickhouse/client/api/data_formats/NativeFormatReader.java

@@ -91,23 +91,23 @@ private boolean readBlock() throws IOException {
             names.add(column.getColumnName());
             types.add(column.getDataType().name());
 
-            if (containsQBit(column)) {
-                // QBit is transmitted in the Native format using its internal bit-transposed
-                // Tuple(FixedString(...)) layout, which is NOT the Array(element_type)-like
-                // representation used in RowBinary (the only representation this reader decodes for
-                // QBit). Reading it through the columnar/per-row paths below would misread those bytes
-                // and desynchronize the block, corrupting the columns that follow. Fail loudly instead
-                // of silently decoding garbage. This also covers a QBit nested inside another type
-                // (e.g. Map(String, QBit(...))). QBit can be read through a RowBinary format.
+            List<Object> values;
+            if (isNativeDecodableQBit(column)) {
+                // Decode the Native bit-plane layout (docs/qbit-encoding.md).
+                values = binaryStreamReader.readQBitNative(column, nRows);
+            } else if (containsQBit(column)) {
+                // Any other QBit shape (non-float element, strided, Nullable/LowCardinality, or nested)
+                // uses a Native layout this reader does not decode; fail loudly rather than misread the
+                // block and desynchronize the columns that follow (docs/qbit-encoding.md).
                 throw new ClientException("Reading column '" + column.getColumnName() + "' ("
-                        + column.getOriginalTypeName() + ") from the Native format is not supported "
-                        + "because it contains a QBit type: QBit is serialized in the Native format "
-                        + "using an internal layout this reader does not decode. Use a RowBinary format "
-                        + "(e.g. RowBinaryWithNamesAndTypes) to read QBit values");
-            }
-
-            List<Object> values = new ArrayList<>(nRows);
-            if (column.isArray()) {
+                        + column.getOriginalTypeName() + ") from the Native format is not supported: "
+                        + "this reader decodes only a plain top-level QBit column with a Float32, Float64 "
+                        + "or BFloat16 element type. A QBit whose element type is none of those, or that is "
+                        + "strided, wrapped in Nullable/LowCardinality, or nested inside another type "
+                        + "(e.g. Array/Tuple/Map), is not decoded. Use a RowBinary format "
+                        + "(e.g. RowBinaryWithNamesAndTypes) to read such QBit values");
+            } else if (column.isArray()) {
+                values = new ArrayList<>(nRows);
                 int[] sizes = new int[nRows];
                 for (int j = 0; j < nRows; j++) {
                     sizes[j] = Math.toIntExact(binaryStreamReader.readLongLE());
@@ -116,6 +116,7 @@ private boolean readBlock() throws IOException {
                     values.add(binaryStreamReader.readArrayItem(column.getNestedColumns().get(0), sizes[0]));
                 }
             } else {
+                values = new ArrayList<>(nRows);
                 for (int j = 0; j < nRows; j++) {
                     Object value = binaryStreamReader.readValue(column);
                     values.add(value);
@@ -132,12 +133,37 @@ private boolean readBlock() throws IOException {
     }
 
     /**
-     * Returns {@code true} if {@code column} is a {@code QBit} or contains a {@code QBit} anywhere in
-     * its nested type tree (e.g. {@code Array(QBit(...))}, {@code Tuple(..., QBit(...))},
-     * {@code Map(String, QBit(...))}). {@code QBit} uses a different, internal wire layout in the
-     * Native format than in RowBinary, so this reader cannot decode it and rejects such columns
-     * up-front rather than misreading the block. {@code Nullable}/{@code LowCardinality} wrappers are
-     * flags on the column, so a wrapped {@code QBit} still reports {@code dataType == QBit} here.
+     * Returns {@code true} for a plain top-level {@code QBit(Float32|Float64|BFloat16, dimension)} the
+     * reader can decode from the Native bit-plane layout — not {@code Nullable}/{@code LowCardinality},
+     * not strided, not nested. Other QBit shapes are caught by {@link #containsQBit} and rejected in
+     * {@link #readBlock} (see {@code docs/qbit-encoding.md}).
+     */
+    private static boolean isNativeDecodableQBit(ClickHouseColumn column) {
+        if (column.getDataType() != ClickHouseDataType.QBit
+                || column.isNullable() || column.isLowCardinality()
+                || column.getNestedColumns().isEmpty()) {
+            return false;
+        }
+        // A strided QBit has a third parameter (stride) and a plane count this decoder does not handle.
+        if (column.getParameters().size() > 2) {
+            return false;
+        }
+        switch (column.getNestedColumns().get(0).getDataType()) {
+            case Float32:
+            case Float64:
+            case BFloat16:
+                return true;
+            default:
+                return false;
+        }
+    }
+
+    /**
+     * Returns {@code true} if {@code column} is or contains a {@code QBit} anywhere in its nested type
+     * tree (e.g. {@code Array}/{@code Tuple}/{@code Map(String, QBit(...))}). Used by {@link #readBlock}
+     * to reject the QBit shapes {@link #isNativeDecodableQBit} does not decode. {@code Nullable}/
+     * {@code LowCardinality} are column flags, so a wrapped {@code QBit} still reports
+     * {@code dataType == QBit} here.
      */
     private static boolean containsQBit(ClickHouseColumn column) {
         if (column.getDataType() == ClickHouseDataType.QBit) {

client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java

@@ -662,6 +662,87 @@
         return convertArray(array, typeHint == null ? arrayDefaultTypeHint : typeHint);
     }
 
+    /**
+     * Reads a plain, non-strided {@code QBit(Float32|Float64|BFloat16, dimension)} column from the
+     * Native format, reversing the server's bit-plane transpose into one vector per row. Values match
+     * the RowBinary path ({@link #readQBit}), so a {@code QBit} round-trips equally through either
+     * format. See {@code docs/qbit-encoding.md} for the wire layout and the transpose math.
+     *
+     * @param column QBit column information (element type in the nested column, dimension in precision)
+     * @param nRows  number of rows in the current block
+     * @return one materialized vector per row, in row order
+     * @throws IOException when an IO error occurs
+     */
+    public List<Object> readQBitNative(ClickHouseColumn column, int nRows) throws IOException {
+        ClickHouseDataType elementType = column.getNestedColumns().get(0).getDataType();
+        final boolean isDouble = elementType == ClickHouseDataType.Float64;
+        final boolean isBFloat16 = elementType == ClickHouseDataType.BFloat16;
+        if (!isDouble && !isBFloat16 && elementType != ClickHouseDataType.Float32) {
+            throw new ClientException("QBit Native decoding supports only Float32, Float64 and BFloat16 "
+                    + "element types, got: " + elementType);
+        }
+
+        final int elementBits = elementType.getByteLength() * 8; // 16 (BFloat16), 32 (Float32), 64 (Float64)
+        final int dimension = column.getPrecision();
+        final int bytesPerPlane = (dimension + 7) / 8;
+        final int totalBits = bytesPerPlane * 8;
+
+        // Planes are column-major: each holds nRows * bytesPerPlane bytes (docs/qbit-encoding.md).
+        // Size it in a long and guard against int overflow, since both factors come off the wire.
+        final long planeBytesLong = (long) nRows * bytesPerPlane;
+        if (planeBytesLong > Integer.MAX_VALUE) {
+            throw new ClientException("QBit Native block too large to decode: " + nRows + " rows x "
+                    + bytesPerPlane + " bytes per bit plane exceeds the maximum array size ("
+                    + Integer.MAX_VALUE + ")");
+        }
+        final int planeBytes = (int) planeBytesLong;
+        byte[][] planes = new byte[elementBits][];
+        for (int p = 0; p < elementBits; p++) {
+            planes[p] = readNBytes(input, planeBytes);
+        }
+
+        // Per-element byte offset and bit mask within a plane row (docs/qbit-encoding.md).
+        int[] elementByte = new int[dimension];
+        int[] elementMask = new int[dimension];
+        for (int j = 0; j < dimension; j++) {
+            int bitIndex = (totalBits - 1) - (j ^ 7);
+            elementByte[j] = bitIndex >> 3;
+            elementMask[j] = 1 << (bitIndex & 7);
+        }
+
+        List<Object> values = new ArrayList<>(nRows);
+        for (int r = 0; r < nRows; r++) {
+            final int rowBase = r * bytesPerPlane;
+            long[] bits = new long[dimension];
+            for (int p = 0; p < elementBits; p++) {
+                final byte[] plane = planes[p];
+                final long planeBit = 1L << (elementBits - 1 - p);
+                for (int j = 0; j < dimension; j++) {
+                    if ((plane[rowBase + elementByte[j]] & elementMask[j]) != 0) {
+                        bits[j] |= planeBit;
+                    }
+                }
+            }
+
+            ArrayValue vector;
+            if (isDouble) {
+                vector = new ArrayValue(double.class, dimension);
+                for (int j = 0; j < dimension; j++) {
+                    vector.set(j, Double.longBitsToDouble(bits[j]));
+                }
+            } else {
+                vector = new ArrayValue(float.class, dimension);
+                for (int j = 0; j < dimension; j++) {
+                    // BFloat16 carries the high 16 bits of a Float32; widen it the same way readBFloat16LE does.
+                    int intBits = isBFloat16 ? ((int) bits[j] << 16) : (int) bits[j];
+                    vector.set(j, Float.intBitsToFloat(intBits));
+                }
+            }
+            values.add(convertArray(vector, arrayDefaultTypeHint));
+        }
+        return values;
+    }
+
     /**
      * Reads a array into an ArrayValue object.
      * @param column - column information

client-v2/src/test/java/com/clickhouse/client/api/data_formats/NativeFormatReaderQBitTest.java

@@ -0,0 +1,99 @@
+package com.clickhouse.client.api.data_formats;
+
+import com.clickhouse.client.api.ClientException;
+import com.clickhouse.client.api.data_formats.internal.BinaryStreamReader;
+import com.clickhouse.client.api.query.QuerySettings;
+import com.clickhouse.data.format.BinaryStreamUtils;
+
+import org.testng.Assert;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+
+/**
+ * QBit handling in {@link NativeFormatReader}, driven by Native-format blocks captured from ClickHouse
+ * 26.5.1 and by minimal hand-built block headers. These run without a live server, so the block-level
+ * routing (decode a plain float QBit, reject every other shape) stays covered on the coverage build,
+ * whose older server skips the QBit integration tests.
+ */
+public class NativeFormatReaderQBitTest {
+
+    // Full single-row Native blocks (SELECT CAST(<vec> AS QBit(<type>)) FORMAT Native), one per decodable
+    // element type, with the vector each encodes.
+    @DataProvider(name = "decodableQBitBlocks")
+    public static Object[][] decodableQBitBlocks() {
+        return new Object[][] {
+                {"Float32",
+                        "010103766563105142697428466c6f617433322c2033290206010101010101010404000000000000000000000000000000000000000000",
+                        false, new float[] {1f, -2f, 3.5f}, null},
+                {"Float64",
+                        "010103766563105142697428466c6f617436342c20332902060101010101010101010104040000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+                        true, null, new double[] {1d, -2d, 3.5d}},
+                {"BFloat16",
+                        "01010376656311514269742842466c6f617431362c20382900fe01010101e1995500000000000000",
+                        false, new float[] {1f, 2f, 4f, 8f, 16f, 32f, 64f, 128f}, null},
+        };
+    }
+
+    @Test(dataProvider = "decodableQBitBlocks")
+    public void testDecodesPlainQBitColumn(String label, String blockHex, boolean isDouble,
+            float[] expectedFloat, double[] expectedDouble) {
+        NativeFormatReader reader = nativeReader(fromHex(blockHex));
+
+        Assert.assertNotNull(reader.next(), label);
+        if (isDouble) {
+            Assert.assertEquals(reader.getDoubleArray("vec"), expectedDouble, label);
+        } else {
+            Assert.assertEquals(reader.getFloatArray("vec"), expectedFloat, label);
+        }
+        Assert.assertNull(reader.next(), label);
+    }
+
+    // QBit shapes the Native reader cannot decode: nested inside a container, strided, wrapped in Nullable,
+    // or a non-float element type. Each must be rejected up front instead of misreading the block.
+    @DataProvider(name = "undecodableQBitTypes")
+    public static Object[][] undecodableQBitTypes() {
+        return new Object[][] {
+                {"m", "Map(String, QBit(Float32, 3))"},
+                {"vec", "QBit(Float32, 4, 2)"},
+                {"vec", "Nullable(QBit(Float32, 3))"},
+                {"vec", "QBit(Int8, 3)"},
+        };
+    }
+
+    @Test(dataProvider = "undecodableQBitTypes")
+    public void testRejectsUndecodableQBitColumn(String colName, String colType) throws IOException {
+        // Only the block header (column name + type) is needed: readBlock rejects the column before reading
+        // any payload.
+        byte[] block = nativeBlockHeader(colName, colType);
+        ClientException ex = Assert.expectThrows(ClientException.class, () -> nativeReader(block));
+        Assert.assertTrue(ex.getMessage().contains("QBit") && ex.getMessage().contains("Native"),
+                colType + ": " + ex.getMessage());
+    }
+
+    private static NativeFormatReader nativeReader(byte[] block) {
+        return new NativeFormatReader(new ByteArrayInputStream(block),
+                new QuerySettings().setUseTimeZone("UTC"),
+                new BinaryStreamReader.CachingByteBufferAllocator());
+    }
+
+    private static byte[] nativeBlockHeader(String columnName, String columnType) throws IOException {
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+        BinaryStreamUtils.writeVarInt(out, 1); // one column
+        BinaryStreamUtils.writeVarInt(out, 1); // one row
+        BinaryStreamUtils.writeString(out, columnName);
+        BinaryStreamUtils.writeString(out, columnType);
+        return out.toByteArray();
+    }
+
+    private static byte[] fromHex(String hex) {
+        byte[] out = new byte[hex.length() / 2];
+        for (int i = 0; i < out.length; i++) {
+            out[i] = (byte) Integer.parseInt(hex.substring(2 * i, 2 * i + 2), 16);
+        }
+        return out;
+    }
+}