From 8f71431790012ee51411f48fd5073ed586604ba6 Mon Sep 17 00:00:00 2001 From: Ben Knight Date: Thu, 16 Jul 2026 13:57:45 +0000 Subject: [PATCH] fix: don't aggregate columns for clustered columnstore indexes in describe_indexes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A clustered columnstore index (type 5) stores the whole table and has no key columns, so sys.index_columns lists every column for it. describe_indexes was aggregating all of them into the `columns` field, which: - is meaningless: reconciliation (index_config_changes) matches a CCI by name and type only and never compares its columns, and - was the underlying cause of the STRING_AGG 8000-byte overflow in #735 — the nvarchar(max) cast merged there stops the *error* but still builds a ~25 KB column list on every reconcile of a wide table. Guard the key-column aggregation with `i.[type] <> 5` so a CCI reports no columns. The nvarchar(max) cast is kept as defense for wide *nonclustered* columnstore indexes (type 6), whose columns are genuine user-chosen identity. Extends the wide-columnstore functional test to assert describe_indexes reports no columns for the CCI. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 1 + .../sqlserver/macros/adapters/indexes.sql | 11 ++++- .../adapter/mssql/test_index_config.py | 42 +++++++++++++++++++ 3 files changed, 53 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 22650cf3..dee394c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ - Fix `columnstore IF EXISTS` guard to check `object_id('schema.table')` correctly. - Gate the `optimize_for_sequential_key` and `resumable` index options on the detected engine version: they require SQL Server 2019+, so on 2017/2016 the index is now built without them (with a warning) instead of failing with "is not a recognized CREATE INDEX option". - Cast the aggregated column names in `sqlserver__describe_indexes` to `nvarchar(max)` so index reconciliation no longer fails with `STRING_AGG` error 9829 on wide tables (e.g. a clustered columnstore index spanning enough columns to exceed the 8000-byte cap). [#735](https://github.com/dbt-msft/dbt-sqlserver/issues/735) +- Stop `sqlserver__describe_indexes` from aggregating column names for clustered columnstore indexes (type 5). A CCI stores the whole table and has no key columns; its column list is not part of the index's identity and reconciliation matches it by name/type only, so listing every table column was both meaningless and the underlying cause of the `STRING_AGG` overflow above. [#735](https://github.com/dbt-msft/dbt-sqlserver/issues/735) - Escape single quotes in `query_tag` before building `OPTION (LABEL)` clause. - Map Python `float` to SQL Server `float`, not `bigint`. - Set default port to `1433` (instead of Postgres `5432`) in `dbt init` profile template. diff --git a/dbt/include/sqlserver/macros/adapters/indexes.sql b/dbt/include/sqlserver/macros/adapters/indexes.sql index 27e80255..7ce2be3d 100644 --- a/dbt/include/sqlserver/macros/adapters/indexes.sql +++ b/dbt/include/sqlserver/macros/adapters/indexes.sql @@ -378,13 +378,22 @@ from sys.indexes i {{ information_schema_hints() }} outer apply ( /* STRING_AGG ... WITHIN GROUP requires SQL Server 2017+, the floor - of this adapter's CI matrix */ + of this adapter's CI matrix. + A clustered columnstore index (type 5) has no key columns: it stores + the whole table, so sys.index_columns lists EVERY column for it. Those + columns are not part of the index's identity — reconciliation matches + the CCI by name/type and never compares its columns (see + index_config_changes) — and aggregating them all is what overflowed + STRING_AGG on wide tables (issue #735). Skip them: report no columns + for a CCI. The nvarchar(max) cast still guards wide *nonclustered* + columnstore indexes (type 6), whose columns ARE user-chosen identity. */ select string_agg(cast(col.[name] as nvarchar(max)), ', ') within group (order by ic.key_ordinal) as cols from sys.index_columns ic {{ information_schema_hints() }} inner join sys.columns col {{ information_schema_hints() }} on col.object_id = ic.object_id and col.column_id = ic.column_id where ic.object_id = i.object_id and ic.index_id = i.index_id and ic.is_included_column = 0 + and i.[type] <> 5 ) key_cols outer apply ( select string_agg(cast(col.[name] as nvarchar(max)), ', ') as cols diff --git a/tests/functional/adapter/mssql/test_index_config.py b/tests/functional/adapter/mssql/test_index_config.py index 60b382bf..1eadab76 100644 --- a/tests/functional/adapter/mssql/test_index_config.py +++ b/tests/functional/adapter/mssql/test_index_config.py @@ -703,6 +703,25 @@ def test_invalid_index_configs(self, project): {_wide_columnstore_columns} """ +# Calls the adapter's describe_indexes macro on a relation and logs, for the +# clustered columnstore index (type 5), how many column names it reports. The +# CCI stores the whole table but has no key columns, so describe_indexes must +# report none for it — both because those columns are not part of the index's +# identity and to avoid re-aggregating every column name on each reconcile. +VALIDATE_CCI_DESCRIBE_MACRO = """ +{% macro validate_cci_describe_columns(schema, identifier) -%} + {% set relation = api.Relation.create( + database=target.database, schema=schema, identifier=identifier, type='table' + ) %} + {% set described = sqlserver__describe_indexes(relation) %} + {% for row in described.rows %} + {% if row['type'] == 'clustered columnstore' %} + {{ log("cci_columns_len: " ~ (row['columns'] | string | length)) }} + {% endif %} + {% endfor %} +{% endmacro %} +""" + def get_index_rows(project, unique_schema, table_name): sql = indexes_def.format(schema_name=unique_schema, table_name=table_name) @@ -781,6 +800,10 @@ class TestSQLServerWideColumnstoreReconcile: def models(self): return {"wide_columnstore.sql": models__wide_columnstore_sql} + @pytest.fixture(scope="class") + def macros(self): + return {"validate_cci_describe_columns.sql": VALIDATE_CCI_DESCRIBE_MACRO} + def test_wide_columnstore_reconcile_does_not_overflow(self, project, unique_schema): # First run creates the wide table plus its clustered columnstore index. run_dbt(["run", "--models", "wide_columnstore"]) @@ -805,6 +828,25 @@ def test_wide_columnstore_reconcile_does_not_overflow(self, project, unique_sche )[0] assert cci_count == 1 + def test_cci_described_with_no_columns(self, project, unique_schema): + # Build the wide table + its clustered columnstore index. + run_dbt(["run", "--models", "wide_columnstore"]) + + # describe_indexes must report NO columns for the CCI: they are the whole + # table, are not part of the index's identity, and aggregating them all + # is what overflowed STRING_AGG on wide tables (issue #735). + kwargs = {"schema": unique_schema, "identifier": "wide_columnstore"} + _, log_output = run_dbt_and_capture( + [ + "--debug", + "run-operation", + "validate_cci_describe_columns", + "--args", + str(kwargs), + ] + ) + assert "cci_columns_len: 0" in log_output + class TestSQLServerDropUnmanagedIndexes: @pytest.fixture(scope="class")