feat(kernel): consolidated DAIS gap-closure + PuPr feature set

KERNEL_REV

@@ -1 +1 @@
-9b90406
+4d301455f7e70de9cb747e0f1143b8a3df8c5b48

auth/oauth/m2m/m2m.go

@@ -36,6 +36,15 @@ type authClient struct {
 	mx           sync.Mutex
 }
 
+// M2MCredentials exposes the raw client-credentials so the SEA-via-kernel backend
+// can drive the kernel's own M2M flow, keeping cfg.Authenticator the single source
+// of truth for auth mode. It structurally satisfies the M2MCredentialsProvider
+// interface the kernel backend asserts (defined in internal/backend/kernel, so the
+// secret-reading capability is not part of the driver's public API).
+func (c *authClient) M2MCredentials() (clientID, clientSecret string) {
+	return c.clientID, c.clientSecret
+}
+
 // Auth will start the OAuth Authorization Flow to authenticate the cli client
 // using the users credentials in the browser. Compatible with SSO.
 func (c *authClient) Authenticate(r *http.Request) error {

auth/oauth/u2m/authenticator.go

@@ -78,6 +78,13 @@ type u2mAuthenticator struct {
 	mx          sync.Mutex
 }
 
+// U2MClientID exposes the cloud-inferred OAuth client id so the SEA-via-kernel
+// backend uses the same client id for the kernel's browser/PKCE flow that the
+// Thrift path would, keeping cfg.Authenticator the single source of truth for auth
+// mode. It structurally satisfies the U2MCredentialsProvider interface the kernel
+// backend asserts (defined in internal/backend/kernel, off the public API).
+func (c *u2mAuthenticator) U2MClientID() string { return c.clientID }
+
 // Auth will start the OAuth Authorization Flow to authenticate the cli client
 // using the users credentials in the browser. Compatible with SSO.
 func (c *u2mAuthenticator) Authenticate(r *http.Request) error {

doc.go

@@ -200,37 +200,55 @@ path databricks_sql_kernel, which would match nothing):
 	# kernel logs plus its HTTP stack (hyper/reqwest):
 	DBSQL_KERNEL_DEBUG=1 RUST_LOG=debug ./your_app 2>&1
 
-The kernel backend currently supports PAT authentication; reading scalar, nested,
-and complex-typed results (CloudFetch is handled transparently); context
-cancellation during execute (a cancelled ctx fires a real server-side cancel; on
-the read path cancellation is honored at result-batch boundaries, not mid-fetch);
-and the TLS, proxy, and session-conf (query tags, statement timeout, time zone)
-connection options. OAuth (M2M/U2M), initial catalog/schema,
-WithEnableMetricViewMetadata, a non-default WithPort, and a non-https protocol
-are not yet supported and return a clear error at connect time rather than being
-silently ignored (the kernel backend connects over https:443 and has no port or
-scheme setter); likewise WithTimeout (a server query timeout the kernel C ABI
-can't set) and WithRetries used to disable retries (the kernel retries
-internally). Bound query parameters and staging operations
-(PUT/GET/REMOVE on a Unity Catalog volume, which need a local file transfer this
-backend cannot perform) are likewise not yet supported and return a clear error
-at execute time (they are per-statement, not connect-time). None of these is
-silently ignored. (Metadata is issued as ordinary SQL —
-SHOW/DESCRIBE/information_schema — and runs on this backend like any other
-query.)
+The kernel backend currently supports PAT and OAuth (M2M via WithClientCredentials,
+and U2M via the authType=oauthU2M DSN param — the interactive browser/PKCE flow is
+owned by the kernel) authentication; reading scalar, nested, and complex-typed
+results (CloudFetch is handled transparently); context cancellation during execute
+(a cancelled ctx fires a real server-side cancel; on the read path cancellation is
+honored at result-batch boundaries, not mid-fetch); the initial namespace
+(WithInitialNamespace catalog/schema, applied post-connect via USE CATALOG / USE
+SCHEMA since the kernel C ABI has no namespace setter); metric-view metadata
+(WithEnableMetricViewMetadata, sent as the same server session conf the Thrift
+backend uses); and the TLS, proxy, and session-conf (query tags, statement timeout,
+time zone) connection options. WithTimeout (a server query timeout the kernel C ABI
+can't set) and WithRetries used to disable retries (the kernel retries internally)
+return a clear error at connect time rather than being silently ignored; token-
+provider, external/static, and federated authenticators are likewise not supported
+and rejected loudly. Staging operations (PUT/GET/REMOVE on a Unity Catalog volume,
+which need a local file transfer this backend cannot perform) are not yet supported
+and return a clear error at execute time (they are per-statement, not connect-time).
+Bound query parameters (positional and named) are supported — bound over the C ABI
+to match the Thrift path. None of these is silently ignored. (Metadata is issued as
+ordinary SQL — SHOW/DESCRIBE/information_schema — and runs on this backend like any
+other query.)
+
+OAuth U2M is interactive. On a cache miss (no valid cached refresh token) opening a
+connection launches the system browser and blocks until the user completes login or
+the kernel's built-in callback timeout (~120s) expires — and because the kernel C
+ABI cannot interrupt session open mid-call, a deadline on the connection context is
+not honored during that window (nor can the timeout be shortened; the C ABI exposes
+no override). A cached, still-valid token opens without a browser. Use PAT or OAuth
+M2M for headless / deadline-bound connects.
 
 WithMaxRows and retry tuning (WithRetries with a positive limit) are accepted but
 not applied on the kernel path: the kernel manages result fetching and retries
 internally, below the C ABI, with no user-facing knob.
 
-Two further kernel-backend caveats. The INTERVAL types (year-month / day-time)
-listed in the type table below are not yet handled by the kernel scanner and
-return a scan error; use the default (Thrift) backend for interval columns. And
-the kernel backend does not yet surface a per-statement server query id on the
-success path, so a QueryIdCallback (see below) fires with "" and no
-EXECUTE_STATEMENT telemetry is emitted for kernel queries. (On a query failure the
-server query id IS available: the returned error is a DBExecutionError whose
-QueryId() carries it — see the Errors section.)
+Features that live above the backend seam are inherited unchanged: the
+database/sql connection pool (each connection wraps one kernel session),
+per-connection telemetry (CREATE_SESSION, EXECUTE_STATEMENT, and DELETE_SESSION
+are recorded for kernel connections just like Thrift), and the telemetry
+exporter's circuit breaker are all backend-agnostic. Result types are rendered to
+match the Thrift backend byte-for-byte: scalars, DECIMAL (exact string),
+TIMESTAMP / TIMESTAMP_NTZ (both shifted into the session time zone, as Thrift
+does), INTERVAL year-month and day-time, nested Array/Map/Struct and VARIANT (as
+JSON), and GEOMETRY (WKT). The per-statement server query id is surfaced on the
+success path, so a QueryIdCallback (see below) fires with the real id and
+EXECUTE_STATEMENT telemetry carries it.
+
+One kernel-backend limitation remains on the read path: context cancellation is
+honored at result-batch boundaries, not mid-fetch (an in-flight CloudFetch batch
+runs to completion before the cancel takes effect).
 
 # Programmatically Retrieving Connection and Query Id
 

internal/arrowscan/arrowscan.go

@@ -37,10 +37,12 @@ import (
 // matching the Thrift path — a float64 would lose precision beyond ~17 digits;
 // see databricks-sql-go#274). Nested types (List/Map/Struct, and VARIANT which
 // arrives nested) render to a JSON string byte-identical to the Thrift path;
-// GEOMETRY arrives as a WKB/WKT string and is handled by the string arm. NULLs
-// map to nil. A genuinely unhandled type (e.g. interval/duration) returns an
-// error rather than a silently wrong value. loc renders DATE / TIMESTAMP in the
-// session time zone (nil = UTC, arrow's ToTime default).
+// GEOMETRY arrives as a WKB/WKT string and is handled by the string arm. INTERVAL
+// day-time/year-month arrive as native arrow duration/month-interval and format to
+// the same string the Thrift path receives pre-formatted from the server. NULLs
+// map to nil. A genuinely unhandled type returns an error rather than a silently
+// wrong value. loc renders DATE / TIMESTAMP in the session time zone (nil = UTC,
+// arrow's ToTime default).
 func ScanCell(col arrow.Array, row int, loc *time.Location) (driver.Value, error) {
 	return ScanCellCached(col, row, loc, nil)
 }
@@ -175,16 +177,105 @@ func ScanCellCached(col arrow.Array, row int, loc *time.Location, keys *StructKe
 	case *array.Decimal128:
 		dt := col.DataType().(*arrow.Decimal128Type)
 		return decimalfmt.ExactString(c.Value(row), dt.Scale), nil
+	case *array.Duration:
+		// INTERVAL DAY TO SECOND arrives as an arrow duration. The kernel returns the
+		// native arrow value, so we format it Go-side to the same "D HH:MM:SS.nnnnnnnnn"
+		// string the Thrift path gets pre-formatted from the server (its native-interval
+		// config is off in prod, so it never scans a duration array — hence there is no
+		// shared renderer to reuse, and this stays kernel-side).
+		dt := col.DataType().(*arrow.DurationType)
+		return formatDayTimeInterval(int64(c.Value(row)), dt.Unit), nil
+	case *array.MonthInterval:
+		// INTERVAL YEAR TO MONTH arrives as a month count; Thrift's server string is
+		// "years-months".
+		return formatYearMonthInterval(int32(c.Value(row))), nil
 	case *array.List, *array.LargeList, *array.FixedSizeList, *array.Map, *array.Struct:
 		// Nested types (and VARIANT, which arrives as a nested value) render to a
 		// JSON string matching the Thrift path.
 		return renderJSONString(col, row, loc, keys)
 	default:
-		return nil, fmt.Errorf("scanning arrow type %s is not supported "+
-			"(intervals are not yet handled)", col.DataType())
+		return nil, fmt.Errorf("scanning arrow type %s is not supported", col.DataType())
 	}
 }
 
+// formatDayTimeInterval renders an arrow duration (in the given time unit) as the
+// Thrift path's "D HH:MM:SS.nnnnnnnnn" — days, then zero-padded hours:minutes:seconds
+// with 9 fractional digits, negated with a leading '-'.
+func formatDayTimeInterval(v int64, unit arrow.TimeUnit) string {
+	neg := v < 0
+	// Derive every component from the SIGNED value (Go integer division/modulo
+	// truncate toward zero, so each component simply carries v's sign), then take
+	// the magnitude of each bounded component when formatting. We deliberately do
+	// NOT negate the full magnitude up front (`v = -v`): at math.MinInt64 that
+	// wraps back to a negative value, so the components would come out negative
+	// *and* a '-' would be prepended — a doubly-negated garbage string — and
+	// math.MinInt64 μs is a representable Spark day-time bound.
+	//
+	// We also must NOT scale the full magnitude up to nanoseconds first: Spark
+	// day-time intervals run up to ~Long.MaxValue microseconds (~292 years), so
+	// v*1e3 (or *1e6/*1e9) would overflow int64 and silently produce a wrong
+	// string. Deriving seconds by dividing keeps every intermediate in range; only
+	// the bounded sub-second remainder is scaled up.
+	var secs, frac int64
+	switch unit {
+	case arrow.Second:
+		secs = v
+	case arrow.Millisecond:
+		secs = v / 1e3
+		frac = (v % 1e3) * 1e6
+	case arrow.Microsecond:
+		secs = v / 1e6
+		frac = (v % 1e6) * 1e3
+	default: // Nanosecond
+		secs = v / 1e9
+		frac = v % 1e9
+	}
+	days := secs / 86400
+	rem := secs % 86400
+	h := rem / 3600
+	rem %= 3600
+	m := rem / 60
+	s := rem % 60
+	sign := ""
+	if neg {
+		sign = "-"
+	}
+	// Every component is bounded well within int64 (days ≤ ~1.07e8, h < 24, m/s <
+	// 60, frac < 1e9), so abs64 here can never hit the math.MinInt64 abs overflow.
+	return fmt.Sprintf("%s%d %02d:%02d:%02d.%09d", sign, abs64(days), abs64(h), abs64(m), abs64(s), abs64(frac))
+}
+
+// formatYearMonthInterval renders a month count as the Thrift path's "years-months",
+// negated with a leading '-'.
+func formatYearMonthInterval(months int32) string {
+	neg := months < 0
+	// Widen to int64 BEFORE negating: negating math.MinInt32 as an int32 overflows
+	// (wraps back negative → doubly-negated garbage), but math.MinInt32 fits in
+	// int64 where the negation is exact. math.MinInt32 months is a representable
+	// Spark year-month bound.
+	m := int64(months)
+	if neg {
+		m = -m
+	}
+	y := m / 12
+	mo := m % 12
+	sign := ""
+	if neg {
+		sign = "-"
+	}
+	return fmt.Sprintf("%s%d-%d", sign, y, mo)
+}
+
+// abs64 returns the absolute value of x. It is only ever called on components
+// already bounded well within int64 (see formatDayTimeInterval), so the classic
+// abs(math.MinInt64) overflow can't arise; the trivial form is intentional.
+func abs64(x int64) int64 {
+	if x < 0 {
+		return -x
+	}
+	return x
+}
+
 // inLocation renders t in loc, matching the Thrift path's .In(location); a nil
 // loc leaves the value in UTC (arrow's ToTime default).
 func inLocation(t time.Time, loc *time.Location) time.Time {

internal/arrowscan/arrowscan_test.go

@@ -101,19 +101,101 @@ func TestScanCellScalars(t *testing.T) {
 	})
 
 	t.Run("unsupported_type_errors", func(t *testing.T) {
-		// A duration (INTERVAL) is not yet handled: must error, not return a
-		// wrong value.
-		b := array.NewDurationBuilder(pool, &arrow.DurationType{Unit: arrow.Microsecond})
+		// An unhandled arrow type must error, not return a silently wrong value.
+		// Duration/MonthInterval are now handled (see TestScanCellInterval); use a
+		// type with no scan arm.
+		b := array.NewTime32Builder(pool, &arrow.Time32Type{Unit: arrow.Second})
 		defer b.Release()
 		b.Append(1000)
 		arr := b.NewArray()
 		defer arr.Release()
 		if _, err := ScanCell(arr, 0, nil); err == nil {
-			t.Error("scanning a Duration should return an unsupported-type error")
+			t.Error("scanning an unhandled arrow type should return an error")
 		}
 	})
 }
 
+// INTERVAL day-time (arrow duration) and year-month (arrow month-interval) arrive
+// as native arrow values on the kernel path and must format to the exact string the
+// Thrift path receives pre-formatted from the server: "D HH:MM:SS.nnnnnnnnn" and
+// "years-months", with negatives signed. (These formatters were validated live
+// kernel==Thrift in the PuPr POC; this is the regression guard.)
+func TestScanCellInterval(t *testing.T) {
+	pool := memory.NewGoAllocator()
+
+	dayTime := []struct {
+		name string
+		unit arrow.TimeUnit
+		v    int64
+		want string
+	}{
+		{"one_day_us", arrow.Microsecond, 86400 * 1_000_000, "1 00:00:00.000000000"},
+		{"day_to_sec_us", arrow.Microsecond, 90061_500000, "1 01:01:01.500000000"},
+		{"seconds_unit", arrow.Second, 3661, "0 01:01:01.000000000"},
+		{"negative_us", arrow.Microsecond, -90061_500000, "-1 01:01:01.500000000"},
+		// A large microsecond magnitude (~106.75M days, near Long.MaxValue μs) must
+		// NOT overflow int64 while scaling to nanoseconds — regression guard for the
+		// prior multiply-first bug that produced a wrong/negative string here.
+		{"large_us_no_overflow", arrow.Microsecond, 9223372036854775807, "106751991 04:00:54.775807000"},
+		// math.MinInt64 μs is a representable negative bound. Negating the full
+		// magnitude (`v = -v`) wraps it back negative, doubly-negating into garbage;
+		// deriving components from the signed value renders it correctly. Its
+		// magnitude is exactly one μs past MaxInt64, so the last fractional digit is
+		// 8 where the MaxInt64 case above is 7.
+		{"min_int64_us", arrow.Microsecond, -9223372036854775808, "-106751991 04:00:54.775808000"},
+		// Same MinInt64 wrap-on-negate hazard at the nanosecond unit (no scaling
+		// involved — this isolates the negation bug from the multiply-overflow one).
+		{"min_int64_ns", arrow.Nanosecond, -9223372036854775808, "-106751 23:47:16.854775808"},
+	}
+	for _, tc := range dayTime {
+		t.Run("daytime_"+tc.name, func(t *testing.T) {
+			b := array.NewDurationBuilder(pool, &arrow.DurationType{Unit: tc.unit})
+			defer b.Release()
+			b.Append(arrow.Duration(tc.v))
+			arr := b.NewArray()
+			defer arr.Release()
+			v, err := ScanCell(arr, 0, nil)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if v.(string) != tc.want {
+				t.Errorf("got %q, want %q", v, tc.want)
+			}
+		})
+	}
+
+	yearMonth := []struct {
+		name   string
+		months int32
+		want   string
+	}{
+		{"two_years", 24, "2-0"},
+		{"year_and_month", 13, "1-1"},
+		{"months_only", 5, "0-5"},
+		{"negative", -13, "-1-1"},
+		// math.MinInt32 months is a representable negative bound. Negating it as an
+		// int32 (`months = -months`) overflows and wraps back negative, doubly-
+		// negating into garbage; widening to int64 before negating renders it right.
+		{"min_int32", -2147483648, "-178956970-8"},
+	}
+	for _, tc := range yearMonth {
+		t.Run("yearmonth_"+tc.name, func(t *testing.T) {
+			b := array.NewMonthIntervalBuilder(pool)
+			defer b.Release()
+			b.Append(arrow.MonthInterval(tc.months))
+			arr := b.NewArray()
+			defer arr.Release()
+			v, err := ScanCell(arr, 0, nil)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if v.(string) != tc.want {
+				t.Errorf("got %q, want %q", v, tc.want)
+			}
+		})
+	}
+}
+
 // ScanCell renders DATE / TIMESTAMP in the requested location, matching the
 // Thrift path's .In(location); a nil location leaves the value in UTC.
 func TestScanCellTimestampLocation(t *testing.T) {