From 6151a3535ebf000b1760af83cb588e73f2c69049 Mon Sep 17 00:00:00 2001 From: Markus Eriksson Date: Thu, 20 Aug 2026 23:06:32 +0200 Subject: [PATCH 01/11] Add plan for ClickHouse grammar gaps Identifies two statement shapes the parser currently rejects that were each validated against a full ClickHouse server (26.7.4.58): CREATE OR REPLACE MATERIALIZED VIEW and the TTL GROUP BY ... SET action. Co-authored-by: CommandCodeBot --- PLAN.md | 96 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 PLAN.md diff --git a/PLAN.md b/PLAN.md new file mode 100644 index 0000000..f449bdd --- /dev/null +++ b/PLAN.md @@ -0,0 +1,96 @@ +# Plan: Close ClickHouse parser grammar gaps + +Status: **implemented and verified.** Both gaps below are fixed on this +branch; see each gap's “Implemented” section for the change and the +“Verification” section at the end for the results. + +## Why + +The goal is a pure-Go, dialect-faithful parser that fully covers the +statement shapes below. The gaps were found by running the parser against a +corpus of real migration statements; each item is a statement shape this +parser rejected, and fixing them removes the need for any keyword-prefix +fallback when classifying statements. + +## Validation method + +Each candidate statement was checked with the full ClickHouse server via +Docker (`clickhouse/clickhouse-server:latest`, version 26.7.4.58), not +`clickhouse local`: `clickhouse local` uses a restricted parser and wrongly +rejects statements the real engine accepts (it rejected +`CREATE OR REPLACE MATERIALIZED VIEW`, which the full server parses). +Candidates were run as real DDL against the server (with prerequisite +tables created), so a pass means the engine actually accepts the syntax. + +## Gap 1 — `CREATE OR REPLACE MATERIALIZED VIEW` + +**Statement:** `CREATE OR REPLACE MATERIALIZED VIEW mv TO dest AS SELECT * FROM src` + +**Validated:** accepted by ClickHouse server 26.7.4.58 (DDL executed +successfully). + +**Implemented:** +- `parser/parser_table.go` — `parseDDL` guard now accepts `MATERIALIZED` + after `CREATE OR REPLACE` (error message updated to + `TEMPORARY|TABLE|VIEW|FUNCTION|DICTIONARY|MATERIALIZED`). +- `parser/ast.go` — `CreateMaterializedView` gained an `OrReplace` field + (mirroring `CreateView.OrReplace`). +- `parser/parser_view.go` — `parseCreateMaterializedView` accepts the + `orReplace` flag and sets it on the node. +- `parser/format.go` — `FormatSQL` renders `CREATE OR REPLACE MATERIALIZED + VIEW` when the flag is set. +- Existing materialized-view fixtures' goldens regenerated with the new + field. + +## Gap 2 — TTL with `GROUP BY` action (TTL delete-by-group) + +**Statement:** +`CREATE TABLE t (id UInt64, created DateTime, x UInt64) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY id SET x = sum(x)` + +**Validated:** accepted by ClickHouse server 26.7.4.58. Engine constraints +confirmed: the `SET` assignment must contain an aggregate function +(`SET x = 1` fails with `BAD_TTL_EXPRESSION`), `SET` is optional after +`GROUP BY` (`GROUP BY id` alone creates the table), and `GROUP BY` keys must +be a prefix of the primary key. + +**Implemented:** +- `parser/ast.go` — new nodes `TTLPolicyGroupBy` (expr list + `Set` + assignments) and `TTLPolicySetExpr` on `TTLPolicyRule`, with + `Pos`/`End`/`Accept`. +- `parser/parser_table.go` — `tryParseTTLPolicy` handles a leading `GROUP + BY` without an action keyword; new `parseTTLPolicyGroupBy` parses the key + list (comma-separated, kept in a `ColumnExprList`) and the optional + `SET = [, ...]` assignments. +- `parser/parser_common.go` — `parseIdentOrKeyword` helper for assignment + target names (which may be keywords). +- `parser/format.go` — `FormatSQL` for `TTLPolicyGroupBy`/`TTLPolicySetExpr` + (one `SET`, comma-joined assignments). +- `parser/ast_visitor.go` / `parser/walk.go` — visitor methods and walk + cases for both new node types. + +## Test coverage + +Fixtures added under `parser/testdata/ddl/` with regenerated goldens in +`output/`, `format/`, and `format/beautify/`: + +- `create_or_replace_materialized_view.sql` +- `create_table_ttl_group_by.sql` — three engine-validated variants: single + `SET`, `SET`-less `GROUP BY id`, and multi-key/multi-`SET` + (`GROUP BY id, created SET x = sum(x), total = count()`). + +## Verification + +- `make test` — full suite passes + (`ok github.com/AfterShip/clickhouse-sql-parser/parser`, 58.5% coverage). +- `make update_test` — goldens regenerated and committed to the working + tree; existing TTL/MV goldens updated only by the new fields. +- `gofmt -l` — clean. (`golangci-lint` is not installed in this + environment, so `make lint` cannot run; gofmt and a `Walk` smoke check + were used instead.) +- `Walk` smoke test on the new TTL AST reaches all new nodes + (`TTLPolicyRule: 1, TTLPolicyGroupBy: 1, TTLPolicySetExpr: 2`). +- CLI round-trip: both statements parse and re-format byte-identically. +- Engine parity: every accepted form above (single/multi key, with/without + `SET`, single/multi assignment) was executed against + `clickhouse/clickhouse-server:latest` and matches the parser's accepted + input. \ No newline at end of file From 4f9c4962a67295ed01c07f80ce3810a3642d5046 Mon Sep 17 00:00:00 2001 From: Markus Eriksson Date: Fri, 21 Aug 2026 09:05:26 +0200 Subject: [PATCH 02/11] Add support for CREATE OR REPLACE MATERIALIZED VIEW and TTL GROUP BY Allow MATERIALIZED after CREATE OR REPLACE (guard + CreateMaterializedView.OrReplace), and parse the TTL GROUP BY [SET = ] action into new TTLPolicyGroupBy/TTLPolicySetExpr nodes. Both forms validated against a full ClickHouse server (26.7.4.58) before implementation. Co-authored-by: CommandCodeBot --- parser/ast.go | 67 ++ parser/ast_visitor.go | 16 + parser/format.go | 28 +- parser/parser_common.go | 11 + parser/parser_table.go | 63 +- parser/parser_view.go | 6 +- .../create_or_replace_materialized_view.sql | 1 + .../ddl/create_table_ttl_group_by.sql | 5 + .../create_or_replace_materialized_view.sql | 11 + .../beautify/create_table_ttl_group_by.sql | 39 + .../create_or_replace_materialized_view.sql | 5 + .../ddl/format/create_table_ttl_group_by.sql | 11 + .../ddl/output/bug_001.sql.golden.json | 1 + ...te_materialized_view_basic.sql.golden.json | 1 + ..._view_rmv_depends_on_multi.sql.golden.json | 1 + ...ew_rmv_engine_with_columns.sql.golden.json | 1 + ...iew_with_comment_before_as.sql.golden.json | 1 + ...rialized_view_with_definer.sql.golden.json | 1 + ...ew_with_empty_table_schema.sql.golden.json | 1 + ...materialized_view_with_gcs.sql.golden.json | 1 + ...rialized_view_with_refresh.sql.golden.json | 1 + .../create_mv_with_not_op.sql.golden.json | 1 + .../create_mv_with_order_by.sql.golden.json | 2 + ..._replace_materialized_view.sql.golden.json | 104 +++ .../create_table_ttl_group_by.sql.golden.json | 831 ++++++++++++++++++ ...eate_table_with_ttl_policy.sql.golden.json | 18 +- ..._table_modify_ttl_multiple.sql.golden.json | 3 +- parser/walk.go | 19 + 28 files changed, 1237 insertions(+), 13 deletions(-) create mode 100644 parser/testdata/ddl/create_or_replace_materialized_view.sql create mode 100644 parser/testdata/ddl/create_table_ttl_group_by.sql create mode 100644 parser/testdata/ddl/format/beautify/create_or_replace_materialized_view.sql create mode 100644 parser/testdata/ddl/format/beautify/create_table_ttl_group_by.sql create mode 100644 parser/testdata/ddl/format/create_or_replace_materialized_view.sql create mode 100644 parser/testdata/ddl/format/create_table_ttl_group_by.sql create mode 100644 parser/testdata/ddl/output/create_or_replace_materialized_view.sql.golden.json create mode 100644 parser/testdata/ddl/output/create_table_ttl_group_by.sql.golden.json diff --git a/parser/ast.go b/parser/ast.go index 3566641..2a94f6a 100644 --- a/parser/ast.go +++ b/parser/ast.go @@ -1406,6 +1406,7 @@ func (c *CreateTable) Accept(visitor ASTVisitor) error { type CreateMaterializedView struct { CreatePos Pos // position of CREATE|ATTACH keyword StatementEnd Pos + OrReplace bool Name *TableIdentifier IfNotExists bool OnCluster *ClusterClause @@ -2476,6 +2477,64 @@ type TTLPolicyRule struct { ToVolume *StringLiteral ToDisk *StringLiteral Action *TTLPolicyRuleAction + GroupBy *TTLPolicyGroupBy +} + +type TTLPolicyGroupBy struct { + GroupByPos Pos + GroupByEnd Pos + Expr Expr + Set []*TTLPolicySetExpr +} + +type TTLPolicySetExpr struct { + SetPos Pos + Name *Ident + Expr Expr +} + +func (t *TTLPolicyGroupBy) Pos() Pos { + return t.GroupByPos +} + +func (t *TTLPolicyGroupBy) End() Pos { + return t.GroupByEnd +} + +func (t *TTLPolicyGroupBy) Accept(visitor ASTVisitor) error { + visitor.Enter(t) + defer visitor.Leave(t) + if t.Expr != nil { + if err := t.Expr.Accept(visitor); err != nil { + return err + } + } + for _, set := range t.Set { + if err := set.Accept(visitor); err != nil { + return err + } + } + return visitor.VisitTTLPolicyGroupBy(t) +} + +func (t *TTLPolicySetExpr) Pos() Pos { + return t.SetPos +} + +func (t *TTLPolicySetExpr) End() Pos { + return t.Expr.End() +} + +func (t *TTLPolicySetExpr) Accept(visitor ASTVisitor) error { + visitor.Enter(t) + defer visitor.Leave(t) + if err := t.Name.Accept(visitor); err != nil { + return err + } + if err := t.Expr.Accept(visitor); err != nil { + return err + } + return visitor.VisitTTLPolicySetExpr(t) } func (t *TTLPolicyRule) Pos() Pos { @@ -2483,6 +2542,9 @@ func (t *TTLPolicyRule) Pos() Pos { } func (t *TTLPolicyRule) End() Pos { + if t.GroupBy != nil { + return t.GroupBy.End() + } if t.Action != nil { return t.Action.End() } @@ -2510,6 +2572,11 @@ func (t *TTLPolicyRule) Accept(visitor ASTVisitor) error { return err } } + if t.GroupBy != nil { + if err := t.GroupBy.Accept(visitor); err != nil { + return err + } + } return visitor.VisitTTLPolicyRule(t) } diff --git a/parser/ast_visitor.go b/parser/ast_visitor.go index ab27d96..a4878c7 100644 --- a/parser/ast_visitor.go +++ b/parser/ast_visitor.go @@ -76,6 +76,8 @@ type ASTVisitor interface { VisitTTLPolicy(expr *TTLPolicy) error VisitTTLPolicyRule(expr *TTLPolicyRule) error VisitTTLPolicyItemAction(expr *TTLPolicyRuleAction) error + VisitTTLPolicyGroupBy(expr *TTLPolicyGroupBy) error + VisitTTLPolicySetExpr(expr *TTLPolicySetExpr) error VisitRefreshExpr(expr *RefreshExpr) error VisitOrderByExpr(expr *OrderExpr) error VisitOrderByListExpr(expr *OrderByClause) error @@ -740,6 +742,20 @@ func (v *DefaultASTVisitor) VisitTTLPolicyItemAction(expr *TTLPolicyRuleAction) return nil } +func (v *DefaultASTVisitor) VisitTTLPolicyGroupBy(expr *TTLPolicyGroupBy) error { + if v.Visit != nil { + return v.Visit(expr) + } + return nil +} + +func (v *DefaultASTVisitor) VisitTTLPolicySetExpr(expr *TTLPolicySetExpr) error { + if v.Visit != nil { + return v.Visit(expr) + } + return nil +} + func (v *DefaultASTVisitor) VisitRefreshExpr(expr *RefreshExpr) error { if v.Visit != nil { return v.Visit(expr) diff --git a/parser/format.go b/parser/format.go index 1f55d2d..7465b52 100644 --- a/parser/format.go +++ b/parser/format.go @@ -886,7 +886,11 @@ func (c *CreateLiveView) FormatSQL(formatter *Formatter) { } func (c *CreateMaterializedView) FormatSQL(formatter *Formatter) { - formatter.WriteString("CREATE MATERIALIZED VIEW ") + formatter.WriteString("CREATE") + if c.OrReplace { + formatter.WriteString(" OR REPLACE") + } + formatter.WriteString(" MATERIALIZED VIEW ") if c.IfNotExists { formatter.WriteString("IF NOT EXISTS ") } @@ -2596,9 +2600,31 @@ func (t *TTLPolicyRule) FormatSQL(formatter *Formatter) { formatter.WriteExpr(t.ToDisk) } else if t.Action != nil { formatter.WriteExpr(t.Action) + } else if t.GroupBy != nil { + formatter.WriteExpr(t.GroupBy) + } +} + +func (t *TTLPolicyGroupBy) FormatSQL(formatter *Formatter) { + formatter.WriteString("GROUP BY ") + formatter.WriteExpr(t.Expr) + if len(t.Set) > 0 { + formatter.WriteString(" SET ") + for i, set := range t.Set { + if i > 0 { + formatter.WriteString(", ") + } + formatter.WriteExpr(set) + } } } +func (t *TTLPolicySetExpr) FormatSQL(formatter *Formatter) { + formatter.WriteExpr(t.Name) + formatter.WriteString(" = ") + formatter.WriteExpr(t.Expr) +} + func (t *TTLPolicyRuleAction) FormatSQL(formatter *Formatter) { formatter.WriteString(t.Action) if t.Codec != nil { diff --git a/parser/parser_common.go b/parser/parser_common.go index 7532601..a39da76 100644 --- a/parser/parser_common.go +++ b/parser/parser_common.go @@ -168,6 +168,17 @@ func (p *Parser) tryParseIdent() *Ident { } } +// parseIdentOrKeyword parses the current token as an identifier, accepting +// both plain identifiers and keyword tokens as the name. Use it only in +// positions where context has already proven the token is a name and not the +// start of a clause or expression. +func (p *Parser) parseIdentOrKeyword() (*Ident, error) { + if p.matchTokenKind(TokenKindIdent, TokenKindKeyword) { + return p.parseAnyKeyword() + } + return nil, fmt.Errorf("expected , but got %q", p.currentTokenKind()) +} + // parseAnyKeyword parses the current token as an identifier, accepting // any keyword token — reserved or not — as the name. Use it only in positions // where context has already proven the token is a name and not the start of a diff --git a/parser/parser_table.go b/parser/parser_table.go index 6525050..86eff5a 100644 --- a/parser/parser_table.go +++ b/parser/parser_table.go @@ -12,8 +12,8 @@ func (p *Parser) parseDDL(pos Pos) (DDL, error) { p.matchKeyword(KeywordAttach): _ = p.lexer.consumeToken() orReplace := p.tryConsumeKeywords(KeywordOr, KeywordReplace) - if orReplace && !p.matchOneOfKeywords(KeywordTemporary, KeywordTable, KeywordView, KeywordFunction, KeywordDictionary) { - return nil, fmt.Errorf("expected keyword: TEMPORARY|TABLE|VIEW|FUNCTION|DICTIONARY, but got %q", p.currentTokenString()) + if orReplace && !p.matchOneOfKeywords(KeywordTemporary, KeywordTable, KeywordView, KeywordFunction, KeywordDictionary, KeywordMaterialized) { + return nil, fmt.Errorf("expected keyword: TEMPORARY|TABLE|VIEW|FUNCTION|DICTIONARY|MATERIALIZED, but got %q", p.currentTokenString()) } switch { case p.matchKeyword(KeywordNamed): @@ -28,7 +28,7 @@ func (p *Parser) parseDDL(pos Pos) (DDL, error) { case p.matchKeyword(KeywordFunction): return p.parseCreateFunction(pos, orReplace) case p.matchKeyword(KeywordMaterialized): - return p.parseCreateMaterializedView(pos) + return p.parseCreateMaterializedView(pos, orReplace) case p.matchKeyword(KeywordLive): return p.parseCreateLiveView(pos) case p.matchKeyword(KeywordView): @@ -1242,6 +1242,12 @@ func (p *Parser) tryParseTTLPolicy(pos Pos) (*TTLPolicy, error) { } action.Codec = codec rule = &TTLPolicyRule{RulePos: pos, Action: action} + case p.matchKeyword(KeywordGroup): + groupBy, err := p.parseTTLPolicyGroupBy(pos) + if err != nil { + return nil, err + } + rule = &TTLPolicyRule{RulePos: pos, GroupBy: groupBy} default: return nil, nil // nolint } @@ -1261,6 +1267,57 @@ func (p *Parser) tryParseTTLPolicy(pos Pos) (*TTLPolicy, error) { return policy, nil } +// parseTTLPolicyGroupBy parses the TTL GROUP BY action: +// GROUP BY [SET = [, ...]] +func (p *Parser) parseTTLPolicyGroupBy(pos Pos) (*TTLPolicyGroupBy, error) { + if err := p.expectKeyword(KeywordGroup); err != nil { + return nil, err + } + if err := p.expectKeyword(KeywordBy); err != nil { + return nil, err + } + exprList := &ColumnExprList{ListPos: p.Pos()} + for { + expr, err := p.parseExpr(p.Pos()) + if err != nil { + return nil, err + } + exprList.Items = append(exprList.Items, expr) + exprList.ListEnd = expr.End() + if p.tryConsumeTokenKind(TokenKindComma) == nil { + break + } + } + groupBy := &TTLPolicyGroupBy{ + GroupByPos: pos, + GroupByEnd: exprList.End(), + Expr: exprList, + } + if !p.tryConsumeKeywords(KeywordSet) { + return groupBy, nil + } + for { + setPos := p.Pos() + name, err := p.parseIdentOrKeyword() + if err != nil { + return nil, err + } + if err := p.expectTokenKind(TokenKindSingleEQ); err != nil { + return nil, err + } + value, err := p.parseSubExpr(p.Pos(), precedenceIn) + if err != nil { + return nil, err + } + groupBy.Set = append(groupBy.Set, &TTLPolicySetExpr{SetPos: setPos, Name: name, Expr: value}) + groupBy.GroupByEnd = value.End() + if p.tryConsumeTokenKind(TokenKindComma) == nil { + break + } + } + return groupBy, nil +} + func (p *Parser) parseTTLExpr(pos Pos) (*TTLExpr, error) { columnExpr, err := p.parseExpr(pos) if err != nil { diff --git a/parser/parser_view.go b/parser/parser_view.go index 02e31bc..effbb9c 100644 --- a/parser/parser_view.go +++ b/parser/parser_view.go @@ -16,7 +16,9 @@ import "fmt" // [DEFINER = { user | CURRENT_USER }] [SQL SECURITY { DEFINER | NONE }] // AS SELECT ... // [COMMENT 'comment'] -func (p *Parser) parseCreateMaterializedView(pos Pos) (*CreateMaterializedView, error) { +// +//nolint:funlen +func (p *Parser) parseCreateMaterializedView(pos Pos, orReplace bool) (*CreateMaterializedView, error) { if err := p.expectKeyword(KeywordMaterialized); err != nil { return nil, err } @@ -24,7 +26,7 @@ func (p *Parser) parseCreateMaterializedView(pos Pos) (*CreateMaterializedView, return nil, err } - createMaterializedView := &CreateMaterializedView{CreatePos: pos} + createMaterializedView := &CreateMaterializedView{CreatePos: pos, OrReplace: orReplace} // parse IF NOT EXISTS clause if exists var err error diff --git a/parser/testdata/ddl/create_or_replace_materialized_view.sql b/parser/testdata/ddl/create_or_replace_materialized_view.sql new file mode 100644 index 0000000..a72697c --- /dev/null +++ b/parser/testdata/ddl/create_or_replace_materialized_view.sql @@ -0,0 +1 @@ +CREATE OR REPLACE MATERIALIZED VIEW mv TO dest AS SELECT * FROM src; \ No newline at end of file diff --git a/parser/testdata/ddl/create_table_ttl_group_by.sql b/parser/testdata/ddl/create_table_ttl_group_by.sql new file mode 100644 index 0000000..9a5d3d7 --- /dev/null +++ b/parser/testdata/ddl/create_table_ttl_group_by.sql @@ -0,0 +1,5 @@ +CREATE TABLE t (id UInt64, created DateTime, x UInt64) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY id SET x = sum(x); + +CREATE TABLE t (id UInt64, created DateTime, x UInt64) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY id; + +CREATE TABLE t (id UInt64, created DateTime, x UInt64, total UInt64) ENGINE = MergeTree() ORDER BY (id, created) TTL created + INTERVAL 1 DAY GROUP BY id, created SET x = sum(x), total = count(); \ No newline at end of file diff --git a/parser/testdata/ddl/format/beautify/create_or_replace_materialized_view.sql b/parser/testdata/ddl/format/beautify/create_or_replace_materialized_view.sql new file mode 100644 index 0000000..89c3c5a --- /dev/null +++ b/parser/testdata/ddl/format/beautify/create_or_replace_materialized_view.sql @@ -0,0 +1,11 @@ +-- Origin SQL: +CREATE OR REPLACE MATERIALIZED VIEW mv TO dest AS SELECT * FROM src; + +-- Beautify SQL: +CREATE OR REPLACE MATERIALIZED VIEW mv +TO dest +AS + SELECT + * + FROM + src; diff --git a/parser/testdata/ddl/format/beautify/create_table_ttl_group_by.sql b/parser/testdata/ddl/format/beautify/create_table_ttl_group_by.sql new file mode 100644 index 0000000..c2c11e1 --- /dev/null +++ b/parser/testdata/ddl/format/beautify/create_table_ttl_group_by.sql @@ -0,0 +1,39 @@ +-- Origin SQL: +CREATE TABLE t (id UInt64, created DateTime, x UInt64) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY id SET x = sum(x); + +CREATE TABLE t (id UInt64, created DateTime, x UInt64) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY id; + +CREATE TABLE t (id UInt64, created DateTime, x UInt64, total UInt64) ENGINE = MergeTree() ORDER BY (id, created) TTL created + INTERVAL 1 DAY GROUP BY id, created SET x = sum(x), total = count(); + +-- Beautify SQL: +CREATE TABLE t +( + id UInt64, + created DateTime, + x UInt64 +) +ENGINE = MergeTree() +ORDER BY + id +TTL created + INTERVAL 1 DAY GROUP BY id SET x = sum(x); +CREATE TABLE t +( + id UInt64, + created DateTime, + x UInt64 +) +ENGINE = MergeTree() +ORDER BY + id +TTL created + INTERVAL 1 DAY GROUP BY id; +CREATE TABLE t +( + id UInt64, + created DateTime, + x UInt64, + total UInt64 +) +ENGINE = MergeTree() +ORDER BY + (id, created) +TTL created + INTERVAL 1 DAY GROUP BY id, created SET x = sum(x), total = count(); diff --git a/parser/testdata/ddl/format/create_or_replace_materialized_view.sql b/parser/testdata/ddl/format/create_or_replace_materialized_view.sql new file mode 100644 index 0000000..360ff72 --- /dev/null +++ b/parser/testdata/ddl/format/create_or_replace_materialized_view.sql @@ -0,0 +1,5 @@ +-- Origin SQL: +CREATE OR REPLACE MATERIALIZED VIEW mv TO dest AS SELECT * FROM src; + +-- Format SQL: +CREATE OR REPLACE MATERIALIZED VIEW mv TO dest AS SELECT * FROM src; diff --git a/parser/testdata/ddl/format/create_table_ttl_group_by.sql b/parser/testdata/ddl/format/create_table_ttl_group_by.sql new file mode 100644 index 0000000..e355fca --- /dev/null +++ b/parser/testdata/ddl/format/create_table_ttl_group_by.sql @@ -0,0 +1,11 @@ +-- Origin SQL: +CREATE TABLE t (id UInt64, created DateTime, x UInt64) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY id SET x = sum(x); + +CREATE TABLE t (id UInt64, created DateTime, x UInt64) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY id; + +CREATE TABLE t (id UInt64, created DateTime, x UInt64, total UInt64) ENGINE = MergeTree() ORDER BY (id, created) TTL created + INTERVAL 1 DAY GROUP BY id, created SET x = sum(x), total = count(); + +-- Format SQL: +CREATE TABLE t (id UInt64, created DateTime, x UInt64) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY id SET x = sum(x); +CREATE TABLE t (id UInt64, created DateTime, x UInt64) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY id; +CREATE TABLE t (id UInt64, created DateTime, x UInt64, total UInt64) ENGINE = MergeTree() ORDER BY (id, created) TTL created + INTERVAL 1 DAY GROUP BY id, created SET x = sum(x), total = count(); diff --git a/parser/testdata/ddl/output/bug_001.sql.golden.json b/parser/testdata/ddl/output/bug_001.sql.golden.json index 9716734..dcc786c 100644 --- a/parser/testdata/ddl/output/bug_001.sql.golden.json +++ b/parser/testdata/ddl/output/bug_001.sql.golden.json @@ -2,6 +2,7 @@ { "CreatePos": 0, "StatementEnd": 635, + "OrReplace": false, "Name": { "Database": { "Name": "db", diff --git a/parser/testdata/ddl/output/create_materialized_view_basic.sql.golden.json b/parser/testdata/ddl/output/create_materialized_view_basic.sql.golden.json index 282afa8..fc6eda9 100644 --- a/parser/testdata/ddl/output/create_materialized_view_basic.sql.golden.json +++ b/parser/testdata/ddl/output/create_materialized_view_basic.sql.golden.json @@ -2,6 +2,7 @@ { "CreatePos": 0, "StatementEnd": 537, + "OrReplace": false, "Name": { "Database": { "Name": "infra_bm", diff --git a/parser/testdata/ddl/output/create_materialized_view_rmv_depends_on_multi.sql.golden.json b/parser/testdata/ddl/output/create_materialized_view_rmv_depends_on_multi.sql.golden.json index 229f78b..3c5a270 100644 --- a/parser/testdata/ddl/output/create_materialized_view_rmv_depends_on_multi.sql.golden.json +++ b/parser/testdata/ddl/output/create_materialized_view_rmv_depends_on_multi.sql.golden.json @@ -2,6 +2,7 @@ { "CreatePos": 0, "StatementEnd": 258, + "OrReplace": false, "Name": { "Database": { "Name": "db1", diff --git a/parser/testdata/ddl/output/create_materialized_view_rmv_engine_with_columns.sql.golden.json b/parser/testdata/ddl/output/create_materialized_view_rmv_engine_with_columns.sql.golden.json index ebf5e6f..81fceca 100644 --- a/parser/testdata/ddl/output/create_materialized_view_rmv_engine_with_columns.sql.golden.json +++ b/parser/testdata/ddl/output/create_materialized_view_rmv_engine_with_columns.sql.golden.json @@ -2,6 +2,7 @@ { "CreatePos": 0, "StatementEnd": 833, + "OrReplace": false, "Name": { "Database": { "Name": "db1", diff --git a/parser/testdata/ddl/output/create_materialized_view_with_comment_before_as.sql.golden.json b/parser/testdata/ddl/output/create_materialized_view_with_comment_before_as.sql.golden.json index 334a26b..f97db25 100644 --- a/parser/testdata/ddl/output/create_materialized_view_with_comment_before_as.sql.golden.json +++ b/parser/testdata/ddl/output/create_materialized_view_with_comment_before_as.sql.golden.json @@ -2,6 +2,7 @@ { "CreatePos": 0, "StatementEnd": 302, + "OrReplace": false, "Name": { "Database": { "Name": "db", diff --git a/parser/testdata/ddl/output/create_materialized_view_with_definer.sql.golden.json b/parser/testdata/ddl/output/create_materialized_view_with_definer.sql.golden.json index 7fee8ad..e5463ed 100644 --- a/parser/testdata/ddl/output/create_materialized_view_with_definer.sql.golden.json +++ b/parser/testdata/ddl/output/create_materialized_view_with_definer.sql.golden.json @@ -2,6 +2,7 @@ { "CreatePos": 0, "StatementEnd": 355, + "OrReplace": false, "Name": { "Database": null, "Table": { diff --git a/parser/testdata/ddl/output/create_materialized_view_with_empty_table_schema.sql.golden.json b/parser/testdata/ddl/output/create_materialized_view_with_empty_table_schema.sql.golden.json index 475f7e6..d01f81e 100644 --- a/parser/testdata/ddl/output/create_materialized_view_with_empty_table_schema.sql.golden.json +++ b/parser/testdata/ddl/output/create_materialized_view_with_empty_table_schema.sql.golden.json @@ -2,6 +2,7 @@ { "CreatePos": 0, "StatementEnd": 460, + "OrReplace": false, "Name": { "Database": { "Name": "test", diff --git a/parser/testdata/ddl/output/create_materialized_view_with_gcs.sql.golden.json b/parser/testdata/ddl/output/create_materialized_view_with_gcs.sql.golden.json index ed3d0de..ae8809c 100644 --- a/parser/testdata/ddl/output/create_materialized_view_with_gcs.sql.golden.json +++ b/parser/testdata/ddl/output/create_materialized_view_with_gcs.sql.golden.json @@ -2,6 +2,7 @@ { "CreatePos": 0, "StatementEnd": 206, + "OrReplace": false, "Name": { "Database": { "Name": "database_name", diff --git a/parser/testdata/ddl/output/create_materialized_view_with_refresh.sql.golden.json b/parser/testdata/ddl/output/create_materialized_view_with_refresh.sql.golden.json index 9a19fb6..40438a9 100644 --- a/parser/testdata/ddl/output/create_materialized_view_with_refresh.sql.golden.json +++ b/parser/testdata/ddl/output/create_materialized_view_with_refresh.sql.golden.json @@ -2,6 +2,7 @@ { "CreatePos": 0, "StatementEnd": 302, + "OrReplace": false, "Name": { "Database": null, "Table": { diff --git a/parser/testdata/ddl/output/create_mv_with_not_op.sql.golden.json b/parser/testdata/ddl/output/create_mv_with_not_op.sql.golden.json index dc3e3f4..1134a4d 100644 --- a/parser/testdata/ddl/output/create_mv_with_not_op.sql.golden.json +++ b/parser/testdata/ddl/output/create_mv_with_not_op.sql.golden.json @@ -2,6 +2,7 @@ { "CreatePos": 0, "StatementEnd": 559, + "OrReplace": false, "Name": { "Database": { "Name": "infra_bm", diff --git a/parser/testdata/ddl/output/create_mv_with_order_by.sql.golden.json b/parser/testdata/ddl/output/create_mv_with_order_by.sql.golden.json index fa0688d..dd2be4b 100644 --- a/parser/testdata/ddl/output/create_mv_with_order_by.sql.golden.json +++ b/parser/testdata/ddl/output/create_mv_with_order_by.sql.golden.json @@ -2,6 +2,7 @@ { "CreatePos": 0, "StatementEnd": 135, + "OrReplace": false, "Name": { "Database": null, "Table": { @@ -168,6 +169,7 @@ { "CreatePos": 138, "StatementEnd": 259, + "OrReplace": false, "Name": { "Database": null, "Table": { diff --git a/parser/testdata/ddl/output/create_or_replace_materialized_view.sql.golden.json b/parser/testdata/ddl/output/create_or_replace_materialized_view.sql.golden.json new file mode 100644 index 0000000..7fed6e5 --- /dev/null +++ b/parser/testdata/ddl/output/create_or_replace_materialized_view.sql.golden.json @@ -0,0 +1,104 @@ +[ + { + "CreatePos": 0, + "StatementEnd": 67, + "OrReplace": true, + "Name": { + "Database": null, + "Table": { + "Name": "mv", + "QuoteType": 1, + "NamePos": 36, + "NameEnd": 38 + } + }, + "IfNotExists": false, + "OnCluster": null, + "Refresh": null, + "RandomizeFor": null, + "DependsOn": null, + "Settings": null, + "HasAppend": false, + "Engine": null, + "TableSchema": null, + "HasEmpty": false, + "Destination": { + "ToPos": 39, + "TableIdentifier": { + "Database": null, + "Table": { + "Name": "dest", + "QuoteType": 1, + "NamePos": 42, + "NameEnd": 46 + } + }, + "TableSchema": null + }, + "SubQuery": { + "HasParen": false, + "Select": { + "SelectPos": 50, + "StatementEnd": 67, + "With": null, + "Top": null, + "HasDistinct": false, + "DistinctOn": null, + "SelectItems": [ + { + "Expr": { + "Name": "*", + "QuoteType": 0, + "NamePos": 57, + "NameEnd": 57 + }, + "Modifiers": [], + "Alias": null + } + ], + "From": { + "FromPos": 59, + "Expr": { + "Table": { + "TablePos": 64, + "TableEnd": 67, + "Alias": null, + "Expr": { + "Database": null, + "Table": { + "Name": "src", + "QuoteType": 1, + "NamePos": 64, + "NameEnd": 67 + } + }, + "HasFinal": false + }, + "StatementEnd": 67, + "SampleRatio": null, + "HasFinal": false + } + }, + "Window": null, + "Prewhere": null, + "Where": null, + "GroupBy": null, + "WithTotal": false, + "Having": null, + "OrderBy": null, + "LimitBy": null, + "Limit": null, + "Settings": null, + "Format": null, + "UnionAll": null, + "UnionDistinct": null, + "Except": null, + "Intersect": null + } + }, + "Populate": false, + "Comment": null, + "Definer": null, + "SQLSecurity": "" + } +] \ No newline at end of file diff --git a/parser/testdata/ddl/output/create_table_ttl_group_by.sql.golden.json b/parser/testdata/ddl/output/create_table_ttl_group_by.sql.golden.json new file mode 100644 index 0000000..64f5a69 --- /dev/null +++ b/parser/testdata/ddl/output/create_table_ttl_group_by.sql.golden.json @@ -0,0 +1,831 @@ +[ + { + "CreatePos": 0, + "StatementEnd": 116, + "OrReplace": false, + "Name": { + "Database": null, + "Table": { + "Name": "t", + "QuoteType": 1, + "NamePos": 13, + "NameEnd": 14 + } + }, + "IfNotExists": false, + "UUID": null, + "OnCluster": null, + "TableSchema": { + "SchemaPos": 15, + "SchemaEnd": 53, + "Columns": [ + { + "NamePos": 16, + "ColumnEnd": 25, + "Name": { + "Ident": { + "Name": "id", + "QuoteType": 1, + "NamePos": 16, + "NameEnd": 18 + }, + "DotIdent": null + }, + "Type": { + "Name": { + "Name": "UInt64", + "QuoteType": 1, + "NamePos": 19, + "NameEnd": 25 + } + }, + "NotNull": null, + "Nullable": null, + "DefaultExpr": null, + "MaterializedExpr": null, + "AliasExpr": null, + "Codec": null, + "TTL": null, + "Comment": null, + "CompressionCodec": null + }, + { + "NamePos": 27, + "ColumnEnd": 43, + "Name": { + "Ident": { + "Name": "created", + "QuoteType": 1, + "NamePos": 27, + "NameEnd": 34 + }, + "DotIdent": null + }, + "Type": { + "Name": { + "Name": "DateTime", + "QuoteType": 1, + "NamePos": 35, + "NameEnd": 43 + } + }, + "NotNull": null, + "Nullable": null, + "DefaultExpr": null, + "MaterializedExpr": null, + "AliasExpr": null, + "Codec": null, + "TTL": null, + "Comment": null, + "CompressionCodec": null + }, + { + "NamePos": 45, + "ColumnEnd": 53, + "Name": { + "Ident": { + "Name": "x", + "QuoteType": 1, + "NamePos": 45, + "NameEnd": 46 + }, + "DotIdent": null + }, + "Type": { + "Name": { + "Name": "UInt64", + "QuoteType": 1, + "NamePos": 47, + "NameEnd": 53 + } + }, + "NotNull": null, + "Nullable": null, + "DefaultExpr": null, + "MaterializedExpr": null, + "AliasExpr": null, + "Codec": null, + "TTL": null, + "Comment": null, + "CompressionCodec": null + } + ], + "AliasTable": null, + "TableFunction": null + }, + "Engine": { + "EnginePos": 55, + "EngineEnd": 116, + "Name": "MergeTree", + "Params": { + "LeftParenPos": 73, + "RightParenPos": 74, + "Items": { + "ListPos": 74, + "ListEnd": 74, + "HasDistinct": false, + "Items": [] + }, + "ColumnArgList": null + }, + "PrimaryKey": null, + "PartitionBy": null, + "SampleBy": null, + "TTL": { + "TTLPos": 88, + "ListEnd": 116, + "Items": [ + { + "TTLPos": 88, + "Expr": { + "LeftExpr": { + "Name": "created", + "QuoteType": 1, + "NamePos": 92, + "NameEnd": 99 + }, + "Operation": "+", + "RightExpr": { + "IntervalPos": 102, + "Expr": { + "NumPos": 111, + "NumEnd": 112, + "Literal": "1", + "Base": 10 + }, + "Unit": { + "Name": "DAY", + "QuoteType": 1, + "NamePos": 113, + "NameEnd": 116 + } + }, + "HasGlobal": false, + "HasNot": false + }, + "Policy": { + "Item": { + "RulePos": 117, + "ToVolume": null, + "ToDisk": null, + "Action": null, + "GroupBy": { + "GroupByPos": 117, + "GroupByEnd": 142, + "Expr": { + "ListPos": 126, + "ListEnd": 128, + "HasDistinct": false, + "Items": [ + { + "Name": "id", + "QuoteType": 1, + "NamePos": 126, + "NameEnd": 128 + } + ] + }, + "Set": [ + { + "SetPos": 133, + "Name": { + "Name": "x", + "QuoteType": 1, + "NamePos": 133, + "NameEnd": 134 + }, + "Expr": { + "Name": { + "Name": "sum", + "QuoteType": 1, + "NamePos": 137, + "NameEnd": 140 + }, + "Params": { + "LeftParenPos": 140, + "RightParenPos": 142, + "Items": { + "ListPos": 141, + "ListEnd": 142, + "HasDistinct": false, + "Items": [ + { + "Expr": { + "Name": "x", + "QuoteType": 1, + "NamePos": 141, + "NameEnd": 142 + }, + "Alias": null + } + ] + }, + "ColumnArgList": null + } + } + } + ] + } + }, + "Where": null, + "GroupBy": null + } + } + ] + }, + "Settings": null, + "OrderBy": { + "OrderPos": 76, + "ListEnd": 87, + "Items": [ + { + "OrderPos": 76, + "Expr": { + "Name": "id", + "QuoteType": 1, + "NamePos": 85, + "NameEnd": 87 + }, + "Alias": null, + "Direction": "", + "Fill": null + } + ], + "Interpolate": null + } + }, + "SubQuery": null, + "TableFunction": null, + "HasTemporary": false, + "Comment": null + }, + { + "CreatePos": 146, + "StatementEnd": 262, + "OrReplace": false, + "Name": { + "Database": null, + "Table": { + "Name": "t", + "QuoteType": 1, + "NamePos": 159, + "NameEnd": 160 + } + }, + "IfNotExists": false, + "UUID": null, + "OnCluster": null, + "TableSchema": { + "SchemaPos": 161, + "SchemaEnd": 199, + "Columns": [ + { + "NamePos": 162, + "ColumnEnd": 171, + "Name": { + "Ident": { + "Name": "id", + "QuoteType": 1, + "NamePos": 162, + "NameEnd": 164 + }, + "DotIdent": null + }, + "Type": { + "Name": { + "Name": "UInt64", + "QuoteType": 1, + "NamePos": 165, + "NameEnd": 171 + } + }, + "NotNull": null, + "Nullable": null, + "DefaultExpr": null, + "MaterializedExpr": null, + "AliasExpr": null, + "Codec": null, + "TTL": null, + "Comment": null, + "CompressionCodec": null + }, + { + "NamePos": 173, + "ColumnEnd": 189, + "Name": { + "Ident": { + "Name": "created", + "QuoteType": 1, + "NamePos": 173, + "NameEnd": 180 + }, + "DotIdent": null + }, + "Type": { + "Name": { + "Name": "DateTime", + "QuoteType": 1, + "NamePos": 181, + "NameEnd": 189 + } + }, + "NotNull": null, + "Nullable": null, + "DefaultExpr": null, + "MaterializedExpr": null, + "AliasExpr": null, + "Codec": null, + "TTL": null, + "Comment": null, + "CompressionCodec": null + }, + { + "NamePos": 191, + "ColumnEnd": 199, + "Name": { + "Ident": { + "Name": "x", + "QuoteType": 1, + "NamePos": 191, + "NameEnd": 192 + }, + "DotIdent": null + }, + "Type": { + "Name": { + "Name": "UInt64", + "QuoteType": 1, + "NamePos": 193, + "NameEnd": 199 + } + }, + "NotNull": null, + "Nullable": null, + "DefaultExpr": null, + "MaterializedExpr": null, + "AliasExpr": null, + "Codec": null, + "TTL": null, + "Comment": null, + "CompressionCodec": null + } + ], + "AliasTable": null, + "TableFunction": null + }, + "Engine": { + "EnginePos": 201, + "EngineEnd": 262, + "Name": "MergeTree", + "Params": { + "LeftParenPos": 219, + "RightParenPos": 220, + "Items": { + "ListPos": 220, + "ListEnd": 220, + "HasDistinct": false, + "Items": [] + }, + "ColumnArgList": null + }, + "PrimaryKey": null, + "PartitionBy": null, + "SampleBy": null, + "TTL": { + "TTLPos": 234, + "ListEnd": 262, + "Items": [ + { + "TTLPos": 234, + "Expr": { + "LeftExpr": { + "Name": "created", + "QuoteType": 1, + "NamePos": 238, + "NameEnd": 245 + }, + "Operation": "+", + "RightExpr": { + "IntervalPos": 248, + "Expr": { + "NumPos": 257, + "NumEnd": 258, + "Literal": "1", + "Base": 10 + }, + "Unit": { + "Name": "DAY", + "QuoteType": 1, + "NamePos": 259, + "NameEnd": 262 + } + }, + "HasGlobal": false, + "HasNot": false + }, + "Policy": { + "Item": { + "RulePos": 263, + "ToVolume": null, + "ToDisk": null, + "Action": null, + "GroupBy": { + "GroupByPos": 263, + "GroupByEnd": 274, + "Expr": { + "ListPos": 272, + "ListEnd": 274, + "HasDistinct": false, + "Items": [ + { + "Name": "id", + "QuoteType": 1, + "NamePos": 272, + "NameEnd": 274 + } + ] + }, + "Set": null + } + }, + "Where": null, + "GroupBy": null + } + } + ] + }, + "Settings": null, + "OrderBy": { + "OrderPos": 222, + "ListEnd": 233, + "Items": [ + { + "OrderPos": 222, + "Expr": { + "Name": "id", + "QuoteType": 1, + "NamePos": 231, + "NameEnd": 233 + }, + "Alias": null, + "Direction": "", + "Fill": null + } + ], + "Interpolate": null + } + }, + "SubQuery": null, + "TableFunction": null, + "HasTemporary": false, + "Comment": null + }, + { + "CreatePos": 277, + "StatementEnd": 418, + "OrReplace": false, + "Name": { + "Database": null, + "Table": { + "Name": "t", + "QuoteType": 1, + "NamePos": 290, + "NameEnd": 291 + } + }, + "IfNotExists": false, + "UUID": null, + "OnCluster": null, + "TableSchema": { + "SchemaPos": 292, + "SchemaEnd": 344, + "Columns": [ + { + "NamePos": 293, + "ColumnEnd": 302, + "Name": { + "Ident": { + "Name": "id", + "QuoteType": 1, + "NamePos": 293, + "NameEnd": 295 + }, + "DotIdent": null + }, + "Type": { + "Name": { + "Name": "UInt64", + "QuoteType": 1, + "NamePos": 296, + "NameEnd": 302 + } + }, + "NotNull": null, + "Nullable": null, + "DefaultExpr": null, + "MaterializedExpr": null, + "AliasExpr": null, + "Codec": null, + "TTL": null, + "Comment": null, + "CompressionCodec": null + }, + { + "NamePos": 304, + "ColumnEnd": 320, + "Name": { + "Ident": { + "Name": "created", + "QuoteType": 1, + "NamePos": 304, + "NameEnd": 311 + }, + "DotIdent": null + }, + "Type": { + "Name": { + "Name": "DateTime", + "QuoteType": 1, + "NamePos": 312, + "NameEnd": 320 + } + }, + "NotNull": null, + "Nullable": null, + "DefaultExpr": null, + "MaterializedExpr": null, + "AliasExpr": null, + "Codec": null, + "TTL": null, + "Comment": null, + "CompressionCodec": null + }, + { + "NamePos": 322, + "ColumnEnd": 330, + "Name": { + "Ident": { + "Name": "x", + "QuoteType": 1, + "NamePos": 322, + "NameEnd": 323 + }, + "DotIdent": null + }, + "Type": { + "Name": { + "Name": "UInt64", + "QuoteType": 1, + "NamePos": 324, + "NameEnd": 330 + } + }, + "NotNull": null, + "Nullable": null, + "DefaultExpr": null, + "MaterializedExpr": null, + "AliasExpr": null, + "Codec": null, + "TTL": null, + "Comment": null, + "CompressionCodec": null + }, + { + "NamePos": 332, + "ColumnEnd": 344, + "Name": { + "Ident": { + "Name": "total", + "QuoteType": 1, + "NamePos": 332, + "NameEnd": 337 + }, + "DotIdent": null + }, + "Type": { + "Name": { + "Name": "UInt64", + "QuoteType": 1, + "NamePos": 338, + "NameEnd": 344 + } + }, + "NotNull": null, + "Nullable": null, + "DefaultExpr": null, + "MaterializedExpr": null, + "AliasExpr": null, + "Codec": null, + "TTL": null, + "Comment": null, + "CompressionCodec": null + } + ], + "AliasTable": null, + "TableFunction": null + }, + "Engine": { + "EnginePos": 346, + "EngineEnd": 418, + "Name": "MergeTree", + "Params": { + "LeftParenPos": 364, + "RightParenPos": 365, + "Items": { + "ListPos": 365, + "ListEnd": 365, + "HasDistinct": false, + "Items": [] + }, + "ColumnArgList": null + }, + "PrimaryKey": null, + "PartitionBy": null, + "SampleBy": null, + "TTL": { + "TTLPos": 390, + "ListEnd": 418, + "Items": [ + { + "TTLPos": 390, + "Expr": { + "LeftExpr": { + "Name": "created", + "QuoteType": 1, + "NamePos": 394, + "NameEnd": 401 + }, + "Operation": "+", + "RightExpr": { + "IntervalPos": 404, + "Expr": { + "NumPos": 413, + "NumEnd": 414, + "Literal": "1", + "Base": 10 + }, + "Unit": { + "Name": "DAY", + "QuoteType": 1, + "NamePos": 415, + "NameEnd": 418 + } + }, + "HasGlobal": false, + "HasNot": false + }, + "Policy": { + "Item": { + "RulePos": 419, + "ToVolume": null, + "ToDisk": null, + "Action": null, + "GroupBy": { + "GroupByPos": 419, + "GroupByEnd": 470, + "Expr": { + "ListPos": 428, + "ListEnd": 439, + "HasDistinct": false, + "Items": [ + { + "Name": "id", + "QuoteType": 1, + "NamePos": 428, + "NameEnd": 430 + }, + { + "Name": "created", + "QuoteType": 1, + "NamePos": 432, + "NameEnd": 439 + } + ] + }, + "Set": [ + { + "SetPos": 444, + "Name": { + "Name": "x", + "QuoteType": 1, + "NamePos": 444, + "NameEnd": 445 + }, + "Expr": { + "Name": { + "Name": "sum", + "QuoteType": 1, + "NamePos": 448, + "NameEnd": 451 + }, + "Params": { + "LeftParenPos": 451, + "RightParenPos": 453, + "Items": { + "ListPos": 452, + "ListEnd": 453, + "HasDistinct": false, + "Items": [ + { + "Expr": { + "Name": "x", + "QuoteType": 1, + "NamePos": 452, + "NameEnd": 453 + }, + "Alias": null + } + ] + }, + "ColumnArgList": null + } + } + }, + { + "SetPos": 456, + "Name": { + "Name": "total", + "QuoteType": 1, + "NamePos": 456, + "NameEnd": 461 + }, + "Expr": { + "Name": { + "Name": "count", + "QuoteType": 1, + "NamePos": 464, + "NameEnd": 469 + }, + "Params": { + "LeftParenPos": 469, + "RightParenPos": 470, + "Items": { + "ListPos": 470, + "ListEnd": 470, + "HasDistinct": false, + "Items": [] + }, + "ColumnArgList": null + } + } + } + ] + } + }, + "Where": null, + "GroupBy": null + } + } + ] + }, + "Settings": null, + "OrderBy": { + "OrderPos": 367, + "ListEnd": 388, + "Items": [ + { + "OrderPos": 367, + "Expr": { + "LeftParenPos": 376, + "RightParenPos": 388, + "Items": { + "ListPos": 377, + "ListEnd": 388, + "HasDistinct": false, + "Items": [ + { + "Expr": { + "Name": "id", + "QuoteType": 1, + "NamePos": 377, + "NameEnd": 379 + }, + "Alias": null + }, + { + "Expr": { + "Name": "created", + "QuoteType": 1, + "NamePos": 381, + "NameEnd": 388 + }, + "Alias": null + } + ] + }, + "ColumnArgList": null + }, + "Alias": null, + "Direction": "", + "Fill": null + } + ], + "Interpolate": null + } + }, + "SubQuery": null, + "TableFunction": null, + "HasTemporary": false, + "Comment": null + } +] \ No newline at end of file diff --git a/parser/testdata/ddl/output/create_table_with_ttl_policy.sql.golden.json b/parser/testdata/ddl/output/create_table_with_ttl_policy.sql.golden.json index 39be9e7..8bfdf53 100644 --- a/parser/testdata/ddl/output/create_table_with_ttl_policy.sql.golden.json +++ b/parser/testdata/ddl/output/create_table_with_ttl_policy.sql.golden.json @@ -174,7 +174,8 @@ "ActionEnd": 137, "Action": "DELETE", "Codec": null - } + }, + "GroupBy": null }, "Where": null, "GroupBy": null @@ -217,7 +218,8 @@ "Literal": "aaa" }, "ToDisk": null, - "Action": null + "Action": null, + "GroupBy": null }, "Where": null, "GroupBy": null @@ -260,7 +262,8 @@ "LiteralEnd": 216, "Literal": "bbb" }, - "Action": null + "Action": null, + "GroupBy": null }, "Where": null, "GroupBy": null @@ -469,7 +472,8 @@ "ActionEnd": 371, "Action": "DELETE", "Codec": null - } + }, + "GroupBy": null }, "Where": { "WherePos": 372, @@ -754,7 +758,8 @@ "Base": 10 } } - } + }, + "GroupBy": null }, "Where": null, "GroupBy": null @@ -815,7 +820,8 @@ "Base": 10 } } - } + }, + "GroupBy": null }, "Where": null, "GroupBy": null diff --git a/parser/testdata/dml/output/alter_table_modify_ttl_multiple.sql.golden.json b/parser/testdata/dml/output/alter_table_modify_ttl_multiple.sql.golden.json index 2086e21..f0615e7 100644 --- a/parser/testdata/dml/output/alter_table_modify_ttl_multiple.sql.golden.json +++ b/parser/testdata/dml/output/alter_table_modify_ttl_multiple.sql.golden.json @@ -104,7 +104,8 @@ "LiteralEnd": 125, "Literal": "gcs" }, - "Action": null + "Action": null, + "GroupBy": null }, "Where": null, "GroupBy": null diff --git a/parser/walk.go b/parser/walk.go index 67ff901..9321d1e 100644 --- a/parser/walk.go +++ b/parser/walk.go @@ -1190,6 +1190,25 @@ func Walk(node Expr, fn WalkFunc) bool { if !Walk(n.Action, fn) { return false } + if !Walk(n.GroupBy, fn) { + return false + } + case *TTLPolicyGroupBy: + if !Walk(n.Expr, fn) { + return false + } + for _, set := range n.Set { + if !Walk(set, fn) { + return false + } + } + case *TTLPolicySetExpr: + if !Walk(n.Name, fn) { + return false + } + if !Walk(n.Expr, fn) { + return false + } case *TTLPolicyRuleAction: if !Walk(n.Codec, fn) { return false From f1db1958446f98d2aceeb3c55e92ed15f55c01a2 Mon Sep 17 00:00:00 2001 From: Markus Eriksson Date: Fri, 21 Aug 2026 09:07:40 +0200 Subject: [PATCH 03/11] Remove PLAN.md The plan is complete; the two grammar gaps it tracked are implemented. Co-authored-by: CommandCodeBot --- PLAN.md | 96 --------------------------------------------------------- 1 file changed, 96 deletions(-) delete mode 100644 PLAN.md diff --git a/PLAN.md b/PLAN.md deleted file mode 100644 index f449bdd..0000000 --- a/PLAN.md +++ /dev/null @@ -1,96 +0,0 @@ -# Plan: Close ClickHouse parser grammar gaps - -Status: **implemented and verified.** Both gaps below are fixed on this -branch; see each gap's “Implemented” section for the change and the -“Verification” section at the end for the results. - -## Why - -The goal is a pure-Go, dialect-faithful parser that fully covers the -statement shapes below. The gaps were found by running the parser against a -corpus of real migration statements; each item is a statement shape this -parser rejected, and fixing them removes the need for any keyword-prefix -fallback when classifying statements. - -## Validation method - -Each candidate statement was checked with the full ClickHouse server via -Docker (`clickhouse/clickhouse-server:latest`, version 26.7.4.58), not -`clickhouse local`: `clickhouse local` uses a restricted parser and wrongly -rejects statements the real engine accepts (it rejected -`CREATE OR REPLACE MATERIALIZED VIEW`, which the full server parses). -Candidates were run as real DDL against the server (with prerequisite -tables created), so a pass means the engine actually accepts the syntax. - -## Gap 1 — `CREATE OR REPLACE MATERIALIZED VIEW` - -**Statement:** `CREATE OR REPLACE MATERIALIZED VIEW mv TO dest AS SELECT * FROM src` - -**Validated:** accepted by ClickHouse server 26.7.4.58 (DDL executed -successfully). - -**Implemented:** -- `parser/parser_table.go` — `parseDDL` guard now accepts `MATERIALIZED` - after `CREATE OR REPLACE` (error message updated to - `TEMPORARY|TABLE|VIEW|FUNCTION|DICTIONARY|MATERIALIZED`). -- `parser/ast.go` — `CreateMaterializedView` gained an `OrReplace` field - (mirroring `CreateView.OrReplace`). -- `parser/parser_view.go` — `parseCreateMaterializedView` accepts the - `orReplace` flag and sets it on the node. -- `parser/format.go` — `FormatSQL` renders `CREATE OR REPLACE MATERIALIZED - VIEW` when the flag is set. -- Existing materialized-view fixtures' goldens regenerated with the new - field. - -## Gap 2 — TTL with `GROUP BY` action (TTL delete-by-group) - -**Statement:** -`CREATE TABLE t (id UInt64, created DateTime, x UInt64) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY id SET x = sum(x)` - -**Validated:** accepted by ClickHouse server 26.7.4.58. Engine constraints -confirmed: the `SET` assignment must contain an aggregate function -(`SET x = 1` fails with `BAD_TTL_EXPRESSION`), `SET` is optional after -`GROUP BY` (`GROUP BY id` alone creates the table), and `GROUP BY` keys must -be a prefix of the primary key. - -**Implemented:** -- `parser/ast.go` — new nodes `TTLPolicyGroupBy` (expr list + `Set` - assignments) and `TTLPolicySetExpr` on `TTLPolicyRule`, with - `Pos`/`End`/`Accept`. -- `parser/parser_table.go` — `tryParseTTLPolicy` handles a leading `GROUP - BY` without an action keyword; new `parseTTLPolicyGroupBy` parses the key - list (comma-separated, kept in a `ColumnExprList`) and the optional - `SET = [, ...]` assignments. -- `parser/parser_common.go` — `parseIdentOrKeyword` helper for assignment - target names (which may be keywords). -- `parser/format.go` — `FormatSQL` for `TTLPolicyGroupBy`/`TTLPolicySetExpr` - (one `SET`, comma-joined assignments). -- `parser/ast_visitor.go` / `parser/walk.go` — visitor methods and walk - cases for both new node types. - -## Test coverage - -Fixtures added under `parser/testdata/ddl/` with regenerated goldens in -`output/`, `format/`, and `format/beautify/`: - -- `create_or_replace_materialized_view.sql` -- `create_table_ttl_group_by.sql` — three engine-validated variants: single - `SET`, `SET`-less `GROUP BY id`, and multi-key/multi-`SET` - (`GROUP BY id, created SET x = sum(x), total = count()`). - -## Verification - -- `make test` — full suite passes - (`ok github.com/AfterShip/clickhouse-sql-parser/parser`, 58.5% coverage). -- `make update_test` — goldens regenerated and committed to the working - tree; existing TTL/MV goldens updated only by the new fields. -- `gofmt -l` — clean. (`golangci-lint` is not installed in this - environment, so `make lint` cannot run; gofmt and a `Walk` smoke check - were used instead.) -- `Walk` smoke test on the new TTL AST reaches all new nodes - (`TTLPolicyRule: 1, TTLPolicyGroupBy: 1, TTLPolicySetExpr: 2`). -- CLI round-trip: both statements parse and re-format byte-identically. -- Engine parity: every accepted form above (single/multi key, with/without - `SET`, single/multi assignment) was executed against - `clickhouse/clickhouse-server:latest` and matches the parser's accepted - input. \ No newline at end of file From 9dec20eb7ddc9ec8afad267f6a8d1971c1bb5ada Mon Sep 17 00:00:00 2001 From: Markus Eriksson Date: Fri, 21 Aug 2026 09:23:41 +0200 Subject: [PATCH 04/11] Reuse existing AST nodes for TTL GROUP BY action Model the TTL GROUP BY [SET = ] action with the existing GroupByClause and UpdateAssignment nodes instead of dedicated TTLPolicyGroupBy/TTLPolicySetExpr types, and drop the now-redundant TTLPolicy.GroupBy field. Net removal of ~130 lines. Co-authored-by: CommandCodeBot --- parser/ast.go | 86 +----- parser/ast_visitor.go | 16 -- parser/format.go | 15 - parser/parser_common.go | 11 - parser/parser_table.go | 74 +---- .../beautify/create_table_ttl_group_by.sql | 9 +- .../create_table_ttl_group_by.sql.golden.json | 256 ++++++++++-------- ...eate_table_with_ttl_policy.sql.golden.json | 36 +-- ..._table_modify_ttl_multiple.sql.golden.json | 6 +- parser/walk.go | 14 - 10 files changed, 198 insertions(+), 325 deletions(-) diff --git a/parser/ast.go b/parser/ast.go index 2a94f6a..dc64d2a 100644 --- a/parser/ast.go +++ b/parser/ast.go @@ -2477,64 +2477,8 @@ type TTLPolicyRule struct { ToVolume *StringLiteral ToDisk *StringLiteral Action *TTLPolicyRuleAction - GroupBy *TTLPolicyGroupBy -} - -type TTLPolicyGroupBy struct { - GroupByPos Pos - GroupByEnd Pos - Expr Expr - Set []*TTLPolicySetExpr -} - -type TTLPolicySetExpr struct { - SetPos Pos - Name *Ident - Expr Expr -} - -func (t *TTLPolicyGroupBy) Pos() Pos { - return t.GroupByPos -} - -func (t *TTLPolicyGroupBy) End() Pos { - return t.GroupByEnd -} - -func (t *TTLPolicyGroupBy) Accept(visitor ASTVisitor) error { - visitor.Enter(t) - defer visitor.Leave(t) - if t.Expr != nil { - if err := t.Expr.Accept(visitor); err != nil { - return err - } - } - for _, set := range t.Set { - if err := set.Accept(visitor); err != nil { - return err - } - } - return visitor.VisitTTLPolicyGroupBy(t) -} - -func (t *TTLPolicySetExpr) Pos() Pos { - return t.SetPos -} - -func (t *TTLPolicySetExpr) End() Pos { - return t.Expr.End() -} - -func (t *TTLPolicySetExpr) Accept(visitor ASTVisitor) error { - visitor.Enter(t) - defer visitor.Leave(t) - if err := t.Name.Accept(visitor); err != nil { - return err - } - if err := t.Expr.Accept(visitor); err != nil { - return err - } - return visitor.VisitTTLPolicySetExpr(t) + GroupBy *GroupByClause + Set []*UpdateAssignment } func (t *TTLPolicyRule) Pos() Pos { @@ -2542,6 +2486,9 @@ func (t *TTLPolicyRule) Pos() Pos { } func (t *TTLPolicyRule) End() Pos { + if len(t.Set) > 0 { + return t.Set[len(t.Set)-1].End() + } if t.GroupBy != nil { return t.GroupBy.End() } @@ -2577,29 +2524,27 @@ func (t *TTLPolicyRule) Accept(visitor ASTVisitor) error { return err } } + for _, set := range t.Set { + if err := set.Accept(visitor); err != nil { + return err + } + } return visitor.VisitTTLPolicyRule(t) } type TTLPolicy struct { - Item *TTLPolicyRule - Where *WhereClause - GroupBy *GroupByClause + Item *TTLPolicyRule + Where *WhereClause } func (t *TTLPolicy) Pos() Pos { if t.Item != nil { return t.Item.Pos() } - if t.Where != nil { - return t.Where.Pos() - } - return t.GroupBy.Pos() + return t.Where.Pos() } func (t *TTLPolicy) End() Pos { - if t.GroupBy != nil { - return t.GroupBy.End() - } if t.Where != nil { return t.Where.End() } @@ -2619,11 +2564,6 @@ func (t *TTLPolicy) Accept(visitor ASTVisitor) error { return err } } - if t.GroupBy != nil { - if err := t.GroupBy.Accept(visitor); err != nil { - return err - } - } return visitor.VisitTTLPolicy(t) } diff --git a/parser/ast_visitor.go b/parser/ast_visitor.go index a4878c7..ab27d96 100644 --- a/parser/ast_visitor.go +++ b/parser/ast_visitor.go @@ -76,8 +76,6 @@ type ASTVisitor interface { VisitTTLPolicy(expr *TTLPolicy) error VisitTTLPolicyRule(expr *TTLPolicyRule) error VisitTTLPolicyItemAction(expr *TTLPolicyRuleAction) error - VisitTTLPolicyGroupBy(expr *TTLPolicyGroupBy) error - VisitTTLPolicySetExpr(expr *TTLPolicySetExpr) error VisitRefreshExpr(expr *RefreshExpr) error VisitOrderByExpr(expr *OrderExpr) error VisitOrderByListExpr(expr *OrderByClause) error @@ -742,20 +740,6 @@ func (v *DefaultASTVisitor) VisitTTLPolicyItemAction(expr *TTLPolicyRuleAction) return nil } -func (v *DefaultASTVisitor) VisitTTLPolicyGroupBy(expr *TTLPolicyGroupBy) error { - if v.Visit != nil { - return v.Visit(expr) - } - return nil -} - -func (v *DefaultASTVisitor) VisitTTLPolicySetExpr(expr *TTLPolicySetExpr) error { - if v.Visit != nil { - return v.Visit(expr) - } - return nil -} - func (v *DefaultASTVisitor) VisitRefreshExpr(expr *RefreshExpr) error { if v.Visit != nil { return v.Visit(expr) diff --git a/parser/format.go b/parser/format.go index 7465b52..1e2067e 100644 --- a/parser/format.go +++ b/parser/format.go @@ -2585,10 +2585,6 @@ func (t *TTLPolicy) FormatSQL(formatter *Formatter) { formatter.WriteByte(whitespace) formatter.WriteExpr(t.Where) } - if t.GroupBy != nil { - formatter.WriteByte(whitespace) - formatter.WriteExpr(t.GroupBy) - } } func (t *TTLPolicyRule) FormatSQL(formatter *Formatter) { @@ -2603,11 +2599,6 @@ func (t *TTLPolicyRule) FormatSQL(formatter *Formatter) { } else if t.GroupBy != nil { formatter.WriteExpr(t.GroupBy) } -} - -func (t *TTLPolicyGroupBy) FormatSQL(formatter *Formatter) { - formatter.WriteString("GROUP BY ") - formatter.WriteExpr(t.Expr) if len(t.Set) > 0 { formatter.WriteString(" SET ") for i, set := range t.Set { @@ -2619,12 +2610,6 @@ func (t *TTLPolicyGroupBy) FormatSQL(formatter *Formatter) { } } -func (t *TTLPolicySetExpr) FormatSQL(formatter *Formatter) { - formatter.WriteExpr(t.Name) - formatter.WriteString(" = ") - formatter.WriteExpr(t.Expr) -} - func (t *TTLPolicyRuleAction) FormatSQL(formatter *Formatter) { formatter.WriteString(t.Action) if t.Codec != nil { diff --git a/parser/parser_common.go b/parser/parser_common.go index a39da76..7532601 100644 --- a/parser/parser_common.go +++ b/parser/parser_common.go @@ -168,17 +168,6 @@ func (p *Parser) tryParseIdent() *Ident { } } -// parseIdentOrKeyword parses the current token as an identifier, accepting -// both plain identifiers and keyword tokens as the name. Use it only in -// positions where context has already proven the token is a name and not the -// start of a clause or expression. -func (p *Parser) parseIdentOrKeyword() (*Ident, error) { - if p.matchTokenKind(TokenKindIdent, TokenKindKeyword) { - return p.parseAnyKeyword() - } - return nil, fmt.Errorf("expected , but got %q", p.currentTokenKind()) -} - // parseAnyKeyword parses the current token as an identifier, accepting // any keyword token — reserved or not — as the name. Use it only in positions // where context has already proven the token is a name and not the start of a diff --git a/parser/parser_table.go b/parser/parser_table.go index 86eff5a..50b4fe6 100644 --- a/parser/parser_table.go +++ b/parser/parser_table.go @@ -1243,11 +1243,24 @@ func (p *Parser) tryParseTTLPolicy(pos Pos) (*TTLPolicy, error) { action.Codec = codec rule = &TTLPolicyRule{RulePos: pos, Action: action} case p.matchKeyword(KeywordGroup): - groupBy, err := p.parseTTLPolicyGroupBy(pos) + rule = &TTLPolicyRule{RulePos: pos} + groupBy, err := p.parseGroupByClause(pos) if err != nil { return nil, err } - rule = &TTLPolicyRule{RulePos: pos, GroupBy: groupBy} + rule.GroupBy = groupBy + if p.tryConsumeKeywords(KeywordSet) { + for { + set, err := p.parseUpdateAssignment(p.Pos()) + if err != nil { + return nil, err + } + rule.Set = append(rule.Set, set) + if p.tryConsumeTokenKind(TokenKindComma) == nil { + break + } + } + } default: return nil, nil // nolint } @@ -1258,66 +1271,9 @@ func (p *Parser) tryParseTTLPolicy(pos Pos) (*TTLPolicy, error) { return nil, err } policy.Where = where - - groupBy, err := p.tryParseGroupByClause(p.Pos()) - if err != nil { - return nil, err - } - policy.GroupBy = groupBy return policy, nil } -// parseTTLPolicyGroupBy parses the TTL GROUP BY action: -// GROUP BY [SET = [, ...]] -func (p *Parser) parseTTLPolicyGroupBy(pos Pos) (*TTLPolicyGroupBy, error) { - if err := p.expectKeyword(KeywordGroup); err != nil { - return nil, err - } - if err := p.expectKeyword(KeywordBy); err != nil { - return nil, err - } - exprList := &ColumnExprList{ListPos: p.Pos()} - for { - expr, err := p.parseExpr(p.Pos()) - if err != nil { - return nil, err - } - exprList.Items = append(exprList.Items, expr) - exprList.ListEnd = expr.End() - if p.tryConsumeTokenKind(TokenKindComma) == nil { - break - } - } - groupBy := &TTLPolicyGroupBy{ - GroupByPos: pos, - GroupByEnd: exprList.End(), - Expr: exprList, - } - if !p.tryConsumeKeywords(KeywordSet) { - return groupBy, nil - } - for { - setPos := p.Pos() - name, err := p.parseIdentOrKeyword() - if err != nil { - return nil, err - } - if err := p.expectTokenKind(TokenKindSingleEQ); err != nil { - return nil, err - } - value, err := p.parseSubExpr(p.Pos(), precedenceIn) - if err != nil { - return nil, err - } - groupBy.Set = append(groupBy.Set, &TTLPolicySetExpr{SetPos: setPos, Name: name, Expr: value}) - groupBy.GroupByEnd = value.End() - if p.tryConsumeTokenKind(TokenKindComma) == nil { - break - } - } - return groupBy, nil -} - func (p *Parser) parseTTLExpr(pos Pos) (*TTLExpr, error) { columnExpr, err := p.parseExpr(pos) if err != nil { diff --git a/parser/testdata/ddl/format/beautify/create_table_ttl_group_by.sql b/parser/testdata/ddl/format/beautify/create_table_ttl_group_by.sql index c2c11e1..10421f7 100644 --- a/parser/testdata/ddl/format/beautify/create_table_ttl_group_by.sql +++ b/parser/testdata/ddl/format/beautify/create_table_ttl_group_by.sql @@ -15,7 +15,8 @@ CREATE TABLE t ENGINE = MergeTree() ORDER BY id -TTL created + INTERVAL 1 DAY GROUP BY id SET x = sum(x); +TTL created + INTERVAL 1 DAY GROUP BY + id SET x = sum(x); CREATE TABLE t ( id UInt64, @@ -25,7 +26,8 @@ CREATE TABLE t ENGINE = MergeTree() ORDER BY id -TTL created + INTERVAL 1 DAY GROUP BY id; +TTL created + INTERVAL 1 DAY GROUP BY + id; CREATE TABLE t ( id UInt64, @@ -36,4 +38,5 @@ CREATE TABLE t ENGINE = MergeTree() ORDER BY (id, created) -TTL created + INTERVAL 1 DAY GROUP BY id, created SET x = sum(x), total = count(); +TTL created + INTERVAL 1 DAY GROUP BY + id, created SET x = sum(x), total = count(); diff --git a/parser/testdata/ddl/output/create_table_ttl_group_by.sql.golden.json b/parser/testdata/ddl/output/create_table_ttl_group_by.sql.golden.json index 64f5a69..993edb3 100644 --- a/parser/testdata/ddl/output/create_table_ttl_group_by.sql.golden.json +++ b/parser/testdata/ddl/output/create_table_ttl_group_by.sql.golden.json @@ -171,64 +171,73 @@ "Action": null, "GroupBy": { "GroupByPos": 117, - "GroupByEnd": 142, + "GroupByEnd": 128, + "AggregateType": "", "Expr": { "ListPos": 126, "ListEnd": 128, "HasDistinct": false, "Items": [ { - "Name": "id", - "QuoteType": 1, - "NamePos": 126, - "NameEnd": 128 + "Expr": { + "Name": "id", + "QuoteType": 1, + "NamePos": 126, + "NameEnd": 128 + }, + "Alias": null } ] }, - "Set": [ - { - "SetPos": 133, - "Name": { + "WithCube": false, + "WithRollup": false, + "WithTotals": false + }, + "Set": [ + { + "AssignmentPos": 133, + "Column": { + "Ident": { "Name": "x", "QuoteType": 1, "NamePos": 133, "NameEnd": 134 }, - "Expr": { - "Name": { - "Name": "sum", - "QuoteType": 1, - "NamePos": 137, - "NameEnd": 140 + "DotIdent": null + }, + "Expr": { + "Name": { + "Name": "sum", + "QuoteType": 1, + "NamePos": 137, + "NameEnd": 140 + }, + "Params": { + "LeftParenPos": 140, + "RightParenPos": 142, + "Items": { + "ListPos": 141, + "ListEnd": 142, + "HasDistinct": false, + "Items": [ + { + "Expr": { + "Name": "x", + "QuoteType": 1, + "NamePos": 141, + "NameEnd": 142 + }, + "Alias": null + } + ] }, - "Params": { - "LeftParenPos": 140, - "RightParenPos": 142, - "Items": { - "ListPos": 141, - "ListEnd": 142, - "HasDistinct": false, - "Items": [ - { - "Expr": { - "Name": "x", - "QuoteType": 1, - "NamePos": 141, - "NameEnd": 142 - }, - "Alias": null - } - ] - }, - "ColumnArgList": null - } + "ColumnArgList": null } } - ] - } + } + ] }, - "Where": null, - "GroupBy": null + "Where": null } } ] @@ -432,24 +441,30 @@ "GroupBy": { "GroupByPos": 263, "GroupByEnd": 274, + "AggregateType": "", "Expr": { "ListPos": 272, "ListEnd": 274, "HasDistinct": false, "Items": [ { - "Name": "id", - "QuoteType": 1, - "NamePos": 272, - "NameEnd": 274 + "Expr": { + "Name": "id", + "QuoteType": 1, + "NamePos": 272, + "NameEnd": 274 + }, + "Alias": null } ] }, - "Set": null - } + "WithCube": false, + "WithRollup": false, + "WithTotals": false + }, + "Set": null }, - "Where": null, - "GroupBy": null + "Where": null } } ] @@ -682,98 +697,113 @@ "Action": null, "GroupBy": { "GroupByPos": 419, - "GroupByEnd": 470, + "GroupByEnd": 439, + "AggregateType": "", "Expr": { "ListPos": 428, "ListEnd": 439, "HasDistinct": false, "Items": [ { - "Name": "id", - "QuoteType": 1, - "NamePos": 428, - "NameEnd": 430 + "Expr": { + "Name": "id", + "QuoteType": 1, + "NamePos": 428, + "NameEnd": 430 + }, + "Alias": null }, { - "Name": "created", - "QuoteType": 1, - "NamePos": 432, - "NameEnd": 439 + "Expr": { + "Name": "created", + "QuoteType": 1, + "NamePos": 432, + "NameEnd": 439 + }, + "Alias": null } ] }, - "Set": [ - { - "SetPos": 444, - "Name": { + "WithCube": false, + "WithRollup": false, + "WithTotals": false + }, + "Set": [ + { + "AssignmentPos": 444, + "Column": { + "Ident": { "Name": "x", "QuoteType": 1, "NamePos": 444, "NameEnd": 445 }, - "Expr": { - "Name": { - "Name": "sum", - "QuoteType": 1, - "NamePos": 448, - "NameEnd": 451 - }, - "Params": { - "LeftParenPos": 451, - "RightParenPos": 453, - "Items": { - "ListPos": 452, - "ListEnd": 453, - "HasDistinct": false, - "Items": [ - { - "Expr": { - "Name": "x", - "QuoteType": 1, - "NamePos": 452, - "NameEnd": 453 - }, - "Alias": null - } - ] - }, - "ColumnArgList": null - } - } + "DotIdent": null }, - { - "SetPos": 456, + "Expr": { "Name": { + "Name": "sum", + "QuoteType": 1, + "NamePos": 448, + "NameEnd": 451 + }, + "Params": { + "LeftParenPos": 451, + "RightParenPos": 453, + "Items": { + "ListPos": 452, + "ListEnd": 453, + "HasDistinct": false, + "Items": [ + { + "Expr": { + "Name": "x", + "QuoteType": 1, + "NamePos": 452, + "NameEnd": 453 + }, + "Alias": null + } + ] + }, + "ColumnArgList": null + } + } + }, + { + "AssignmentPos": 456, + "Column": { + "Ident": { "Name": "total", "QuoteType": 1, "NamePos": 456, "NameEnd": 461 }, - "Expr": { - "Name": { - "Name": "count", - "QuoteType": 1, - "NamePos": 464, - "NameEnd": 469 + "DotIdent": null + }, + "Expr": { + "Name": { + "Name": "count", + "QuoteType": 1, + "NamePos": 464, + "NameEnd": 469 + }, + "Params": { + "LeftParenPos": 469, + "RightParenPos": 470, + "Items": { + "ListPos": 470, + "ListEnd": 470, + "HasDistinct": false, + "Items": [] }, - "Params": { - "LeftParenPos": 469, - "RightParenPos": 470, - "Items": { - "ListPos": 470, - "ListEnd": 470, - "HasDistinct": false, - "Items": [] - }, - "ColumnArgList": null - } + "ColumnArgList": null } } - ] - } + } + ] }, - "Where": null, - "GroupBy": null + "Where": null } } ] diff --git a/parser/testdata/ddl/output/create_table_with_ttl_policy.sql.golden.json b/parser/testdata/ddl/output/create_table_with_ttl_policy.sql.golden.json index 8bfdf53..5b8f57f 100644 --- a/parser/testdata/ddl/output/create_table_with_ttl_policy.sql.golden.json +++ b/parser/testdata/ddl/output/create_table_with_ttl_policy.sql.golden.json @@ -175,10 +175,10 @@ "Action": "DELETE", "Codec": null }, - "GroupBy": null + "GroupBy": null, + "Set": null }, - "Where": null, - "GroupBy": null + "Where": null } }, { @@ -219,10 +219,10 @@ }, "ToDisk": null, "Action": null, - "GroupBy": null + "GroupBy": null, + "Set": null }, - "Where": null, - "GroupBy": null + "Where": null } }, { @@ -263,10 +263,10 @@ "Literal": "bbb" }, "Action": null, - "GroupBy": null + "GroupBy": null, + "Set": null }, - "Where": null, - "GroupBy": null + "Where": null } } ] @@ -473,7 +473,8 @@ "Action": "DELETE", "Codec": null }, - "GroupBy": null + "GroupBy": null, + "Set": null }, "Where": { "WherePos": 372, @@ -517,8 +518,7 @@ "HasGlobal": false, "HasNot": false } - }, - "GroupBy": null + } } } ] @@ -759,10 +759,10 @@ } } }, - "GroupBy": null + "GroupBy": null, + "Set": null }, - "Where": null, - "GroupBy": null + "Where": null } }, { @@ -821,10 +821,10 @@ } } }, - "GroupBy": null + "GroupBy": null, + "Set": null }, - "Where": null, - "GroupBy": null + "Where": null } } ] diff --git a/parser/testdata/dml/output/alter_table_modify_ttl_multiple.sql.golden.json b/parser/testdata/dml/output/alter_table_modify_ttl_multiple.sql.golden.json index f0615e7..66bed1e 100644 --- a/parser/testdata/dml/output/alter_table_modify_ttl_multiple.sql.golden.json +++ b/parser/testdata/dml/output/alter_table_modify_ttl_multiple.sql.golden.json @@ -105,10 +105,10 @@ "Literal": "gcs" }, "Action": null, - "GroupBy": null + "GroupBy": null, + "Set": null }, - "Where": null, - "GroupBy": null + "Where": null } }, { diff --git a/parser/walk.go b/parser/walk.go index 9321d1e..5cbcd7c 100644 --- a/parser/walk.go +++ b/parser/walk.go @@ -1177,9 +1177,6 @@ func Walk(node Expr, fn WalkFunc) bool { if !Walk(n.Where, fn) { return false } - if !Walk(n.GroupBy, fn) { - return false - } case *TTLPolicyRule: if !Walk(n.ToVolume, fn) { return false @@ -1193,22 +1190,11 @@ func Walk(node Expr, fn WalkFunc) bool { if !Walk(n.GroupBy, fn) { return false } - case *TTLPolicyGroupBy: - if !Walk(n.Expr, fn) { - return false - } for _, set := range n.Set { if !Walk(set, fn) { return false } } - case *TTLPolicySetExpr: - if !Walk(n.Name, fn) { - return false - } - if !Walk(n.Expr, fn) { - return false - } case *TTLPolicyRuleAction: if !Walk(n.Codec, fn) { return false From 16d38192748bdf2845e6c72373d74822d1a84f34 Mon Sep 17 00:00:00 2001 From: Markus Eriksson Date: Fri, 21 Aug 2026 09:44:49 +0200 Subject: [PATCH 05/11] Reject query-level GROUP BY forms in TTL GROUP BY A TTL GROUP BY action only accepts a plain expression list; the query-level forms (ALL, CUBE/ROLLUP/GROUPING SETS, and WITH CUBE/ROLLUP/TOTALS) are rejected by ClickHouse inside a TTL. Co-authored-by: CommandCodeBot --- parser/parser_table.go | 6 ++++++ parser/parser_test.go | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/parser/parser_table.go b/parser/parser_table.go index 50b4fe6..cb07d71 100644 --- a/parser/parser_table.go +++ b/parser/parser_table.go @@ -1248,6 +1248,12 @@ func (p *Parser) tryParseTTLPolicy(pos Pos) (*TTLPolicy, error) { if err != nil { return nil, err } + // The TTL GROUP BY action only accepts a plain expression list; + // the query-level forms (ALL, CUBE/ROLLUP/GROUPING SETS, WITH + // CUBE/ROLLUP/TOTALS) are rejected by ClickHouse in a TTL. + if groupBy.AggregateType != "" || groupBy.WithCube || groupBy.WithRollup || groupBy.WithTotals { + return nil, fmt.Errorf("unexpected token: %q, expected expression list in TTL GROUP BY", p.currentTokenString()) + } rule.GroupBy = groupBy if p.tryConsumeKeywords(KeywordSet) { for { diff --git a/parser/parser_test.go b/parser/parser_test.go index 142c4e8..8bb30d9 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -192,6 +192,13 @@ func TestParser_InvalidSyntax(t *testing.T) { "CREATE TABLE t (x String DEFAULT CAST(a +, 'String'))", "CREATE TABLE t (x String MATERIALIZED a +)", "CREATE TABLE t (x String ALIAS a +)", + // A TTL GROUP BY action only accepts a plain expression list; the + // query-level GROUP BY forms are rejected by ClickHouse in a TTL. + "CREATE TABLE t (id UInt64, created DateTime) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY id WITH TOTALS", + "CREATE TABLE t (id UInt64, created DateTime) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY id WITH ROLLUP", + "CREATE TABLE t (id UInt64, created DateTime) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY id WITH CUBE", + "CREATE TABLE t (id UInt64, created DateTime) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY ALL", + "CREATE TABLE t (id UInt64, created DateTime) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY GROUPING SETS (id)", // Invalid ARRAY JOIN types (only ARRAY JOIN, LEFT ARRAY JOIN, and INNER ARRAY JOIN are valid) "SELECT * FROM t RIGHT ARRAY JOIN arr AS a", // RIGHT ARRAY JOIN not supported "SELECT * FROM t FULL ARRAY JOIN arr AS a", // FULL ARRAY JOIN not supported From 83181bcdba40c824183776126a757a18049c845b Mon Sep 17 00:00:00 2001 From: Markus Eriksson Date: Fri, 21 Aug 2026 10:26:56 +0200 Subject: [PATCH 06/11] Parse TTL GROUP BY keys as a plain expression list ClickHouse rejects query-level GROUP BY modifiers (ALL, CUBE/ROLLUP/ GROUPING SETS, WITH CUBE/ROLLUP/TOTALS) inside a TTL GROUP BY, but ALL and CUBE/ROLLUP(...) read there as ordinary key expressions and can be valid (e.g. a column named all). Parse the keys directly as a plain list so the clause is built from the expected shape and future query-level GROUP BY sugar stays out of the TTL grammar. Co-authored-by: CommandCodeBot --- parser/parser_table.go | 29 +++++++++++++++++++++-------- parser/parser_test.go | 5 +++-- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/parser/parser_table.go b/parser/parser_table.go index cb07d71..302655f 100644 --- a/parser/parser_table.go +++ b/parser/parser_table.go @@ -1243,18 +1243,31 @@ func (p *Parser) tryParseTTLPolicy(pos Pos) (*TTLPolicy, error) { action.Codec = codec rule = &TTLPolicyRule{RulePos: pos, Action: action} case p.matchKeyword(KeywordGroup): - rule = &TTLPolicyRule{RulePos: pos} - groupBy, err := p.parseGroupByClause(pos) + // A TTL GROUP BY action takes only a plain expression list. + // ClickHouse rejects the query-level forms (ALL, + // CUBE/ROLLUP/GROUPING SETS, WITH CUBE/ROLLUP/TOTALS) inside a + // TTL, so the keys are parsed directly as a list instead of + // reusing parseGroupByClause and then discarding those forms: + // building the clause from the expected shape keeps any future + // query-level GROUP BY sugar out of the TTL grammar as well. + if err := p.expectKeyword(KeywordGroup); err != nil { + return nil, err + } + if err := p.expectKeyword(KeywordBy); err != nil { + return nil, err + } + keys, err := p.parseColumnExprList(p.Pos()) if err != nil { return nil, err } - // The TTL GROUP BY action only accepts a plain expression list; - // the query-level forms (ALL, CUBE/ROLLUP/GROUPING SETS, WITH - // CUBE/ROLLUP/TOTALS) are rejected by ClickHouse in a TTL. - if groupBy.AggregateType != "" || groupBy.WithCube || groupBy.WithRollup || groupBy.WithTotals { - return nil, fmt.Errorf("unexpected token: %q, expected expression list in TTL GROUP BY", p.currentTokenString()) + rule = &TTLPolicyRule{ + RulePos: pos, + GroupBy: &GroupByClause{ + GroupByPos: pos, + GroupByEnd: keys.End(), + Expr: keys, + }, } - rule.GroupBy = groupBy if p.tryConsumeKeywords(KeywordSet) { for { set, err := p.parseUpdateAssignment(p.Pos()) diff --git a/parser/parser_test.go b/parser/parser_test.go index 8bb30d9..ba7fa4f 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -193,11 +193,12 @@ func TestParser_InvalidSyntax(t *testing.T) { "CREATE TABLE t (x String MATERIALIZED a +)", "CREATE TABLE t (x String ALIAS a +)", // A TTL GROUP BY action only accepts a plain expression list; the - // query-level GROUP BY forms are rejected by ClickHouse in a TTL. + // query-level modifiers are syntax errors for ClickHouse in a TTL. + // (GROUP BY ALL and CUBE/ROLLUP(...) read as ordinary key + // expressions there and are only rejected semantically.) "CREATE TABLE t (id UInt64, created DateTime) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY id WITH TOTALS", "CREATE TABLE t (id UInt64, created DateTime) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY id WITH ROLLUP", "CREATE TABLE t (id UInt64, created DateTime) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY id WITH CUBE", - "CREATE TABLE t (id UInt64, created DateTime) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY ALL", "CREATE TABLE t (id UInt64, created DateTime) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY GROUPING SETS (id)", // Invalid ARRAY JOIN types (only ARRAY JOIN, LEFT ARRAY JOIN, and INNER ARRAY JOIN are valid) "SELECT * FROM t RIGHT ARRAY JOIN arr AS a", // RIGHT ARRAY JOIN not supported From 39740e0784218ebca0002b6830795fd11a6c8d4d Mon Sep 17 00:00:00 2001 From: Markus Eriksson Date: Fri, 21 Aug 2026 11:13:14 +0200 Subject: [PATCH 07/11] Fix TTL GROUP BY SET parsing, spans, and bare keyword keys - Do not consume the comma separating TTL expressions as a SET assignment separator: probe for the next assignment and roll back when the comma starts the next TTL rule. - Include the policy span in TTLExpr.End() so StatementEnd/ListEnd cover GROUP BY/SET actions. - Accept bare keyword TTL GROUP BY keys (e.g. GROUP BY ALL SET ...) by falling back to a keyword identifier when an expression cannot start. Co-authored-by: CommandCodeBot --- parser/ast.go | 3 + parser/parser_table.go | 50 +- .../ddl/create_table_ttl_group_by.sql | 6 +- .../beautify/create_table_ttl_group_by.sql | 26 + .../ddl/format/create_table_ttl_group_by.sql | 6 + .../create_table_ttl_group_by.sql.golden.json | 639 +++++++++++++++++- ...eate_table_with_ttl_policy.sql.golden.json | 14 +- 7 files changed, 692 insertions(+), 52 deletions(-) diff --git a/parser/ast.go b/parser/ast.go index dc64d2a..40492bb 100644 --- a/parser/ast.go +++ b/parser/ast.go @@ -2578,6 +2578,9 @@ func (t *TTLExpr) Pos() Pos { } func (t *TTLExpr) End() Pos { + if t.Policy != nil { + return t.Policy.End() + } return t.Expr.End() } diff --git a/parser/parser_table.go b/parser/parser_table.go index 302655f..f2e2fa7 100644 --- a/parser/parser_table.go +++ b/parser/parser_table.go @@ -1256,9 +1256,33 @@ func (p *Parser) tryParseTTLPolicy(pos Pos) (*TTLPolicy, error) { if err := p.expectKeyword(KeywordBy); err != nil { return nil, err } - keys, err := p.parseColumnExprList(p.Pos()) - if err != nil { - return nil, err + keys := &ColumnExprList{ListPos: p.Pos()} + for { + var key Expr + var err error + if p.matchTokenKind(TokenKindKeyword) { + // Bare keywords (e.g. ALL) are valid TTL GROUP BY keys + // even when followed by SET or a closing engine clause; + // parseColumnExpr only reads a keyword as an identifier + // for a narrower lookahead, so fall back to it when an + // expression cannot start here. + savedState := p.lexer.saveState() + key, err = p.parseExpr(p.Pos()) + if err != nil { + p.lexer.restoreState(savedState) + key, err = p.parseAnyKeyword() + } + } else { + key, err = p.parseExpr(p.Pos()) + } + if err != nil { + return nil, err + } + keys.Items = append(keys.Items, key) + keys.ListEnd = key.End() + if p.tryConsumeTokenKind(TokenKindComma) == nil { + break + } } rule = &TTLPolicyRule{ RulePos: pos, @@ -1269,15 +1293,27 @@ func (p *Parser) tryParseTTLPolicy(pos Pos) (*TTLPolicy, error) { }, } if p.tryConsumeKeywords(KeywordSet) { + set, err := p.parseUpdateAssignment(p.Pos()) + if err != nil { + return nil, err + } + rule.Set = append(rule.Set, set) for { + // A comma either continues the SET assignment list or + // starts the next TTL expression of a multi-value TTL + // clause; consume it and probe for another assignment, + // rolling both back when none follows so parseTTLClause + // can treat the comma as a rule separator. + savedState := p.lexer.saveState() + if p.tryConsumeTokenKind(TokenKindComma) == nil { + break + } set, err := p.parseUpdateAssignment(p.Pos()) if err != nil { - return nil, err - } - rule.Set = append(rule.Set, set) - if p.tryConsumeTokenKind(TokenKindComma) == nil { + p.lexer.restoreState(savedState) break } + rule.Set = append(rule.Set, set) } } default: diff --git a/parser/testdata/ddl/create_table_ttl_group_by.sql b/parser/testdata/ddl/create_table_ttl_group_by.sql index 9a5d3d7..387ced0 100644 --- a/parser/testdata/ddl/create_table_ttl_group_by.sql +++ b/parser/testdata/ddl/create_table_ttl_group_by.sql @@ -2,4 +2,8 @@ CREATE TABLE t (id UInt64, created DateTime, x UInt64) ENGINE = MergeTree() ORDE CREATE TABLE t (id UInt64, created DateTime, x UInt64) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY id; -CREATE TABLE t (id UInt64, created DateTime, x UInt64, total UInt64) ENGINE = MergeTree() ORDER BY (id, created) TTL created + INTERVAL 1 DAY GROUP BY id, created SET x = sum(x), total = count(); \ No newline at end of file +CREATE TABLE t (id UInt64, created DateTime, x UInt64, total UInt64) ENGINE = MergeTree() ORDER BY (id, created) TTL created + INTERVAL 1 DAY GROUP BY id, created SET x = sum(x), total = count(); + +CREATE TABLE t (id UInt64, created DateTime, x UInt64) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY id SET x = sum(x), created + INTERVAL 2 DAY DELETE; + +CREATE TABLE t (id UInt64, created DateTime, x UInt64) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY ALL SET x = sum(x); \ No newline at end of file diff --git a/parser/testdata/ddl/format/beautify/create_table_ttl_group_by.sql b/parser/testdata/ddl/format/beautify/create_table_ttl_group_by.sql index 10421f7..36612a1 100644 --- a/parser/testdata/ddl/format/beautify/create_table_ttl_group_by.sql +++ b/parser/testdata/ddl/format/beautify/create_table_ttl_group_by.sql @@ -5,6 +5,10 @@ CREATE TABLE t (id UInt64, created DateTime, x UInt64) ENGINE = MergeTree() ORDE CREATE TABLE t (id UInt64, created DateTime, x UInt64, total UInt64) ENGINE = MergeTree() ORDER BY (id, created) TTL created + INTERVAL 1 DAY GROUP BY id, created SET x = sum(x), total = count(); +CREATE TABLE t (id UInt64, created DateTime, x UInt64) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY id SET x = sum(x), created + INTERVAL 2 DAY DELETE; + +CREATE TABLE t (id UInt64, created DateTime, x UInt64) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY ALL SET x = sum(x); + -- Beautify SQL: CREATE TABLE t ( @@ -40,3 +44,25 @@ ORDER BY (id, created) TTL created + INTERVAL 1 DAY GROUP BY id, created SET x = sum(x), total = count(); +CREATE TABLE t +( + id UInt64, + created DateTime, + x UInt64 +) +ENGINE = MergeTree() +ORDER BY + id +TTL created + INTERVAL 1 DAY GROUP BY + id SET x = sum(x), created + INTERVAL 2 DAY DELETE; +CREATE TABLE t +( + id UInt64, + created DateTime, + x UInt64 +) +ENGINE = MergeTree() +ORDER BY + id +TTL created + INTERVAL 1 DAY GROUP BY + ALL SET x = sum(x); diff --git a/parser/testdata/ddl/format/create_table_ttl_group_by.sql b/parser/testdata/ddl/format/create_table_ttl_group_by.sql index e355fca..3dd1762 100644 --- a/parser/testdata/ddl/format/create_table_ttl_group_by.sql +++ b/parser/testdata/ddl/format/create_table_ttl_group_by.sql @@ -5,7 +5,13 @@ CREATE TABLE t (id UInt64, created DateTime, x UInt64) ENGINE = MergeTree() ORDE CREATE TABLE t (id UInt64, created DateTime, x UInt64, total UInt64) ENGINE = MergeTree() ORDER BY (id, created) TTL created + INTERVAL 1 DAY GROUP BY id, created SET x = sum(x), total = count(); +CREATE TABLE t (id UInt64, created DateTime, x UInt64) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY id SET x = sum(x), created + INTERVAL 2 DAY DELETE; + +CREATE TABLE t (id UInt64, created DateTime, x UInt64) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY ALL SET x = sum(x); + -- Format SQL: CREATE TABLE t (id UInt64, created DateTime, x UInt64) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY id SET x = sum(x); CREATE TABLE t (id UInt64, created DateTime, x UInt64) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY id; CREATE TABLE t (id UInt64, created DateTime, x UInt64, total UInt64) ENGINE = MergeTree() ORDER BY (id, created) TTL created + INTERVAL 1 DAY GROUP BY id, created SET x = sum(x), total = count(); +CREATE TABLE t (id UInt64, created DateTime, x UInt64) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY id SET x = sum(x), created + INTERVAL 2 DAY DELETE; +CREATE TABLE t (id UInt64, created DateTime, x UInt64) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY ALL SET x = sum(x); diff --git a/parser/testdata/ddl/output/create_table_ttl_group_by.sql.golden.json b/parser/testdata/ddl/output/create_table_ttl_group_by.sql.golden.json index 993edb3..ec90a80 100644 --- a/parser/testdata/ddl/output/create_table_ttl_group_by.sql.golden.json +++ b/parser/testdata/ddl/output/create_table_ttl_group_by.sql.golden.json @@ -1,7 +1,7 @@ [ { "CreatePos": 0, - "StatementEnd": 116, + "StatementEnd": 142, "OrReplace": false, "Name": { "Database": null, @@ -115,7 +115,7 @@ }, "Engine": { "EnginePos": 55, - "EngineEnd": 116, + "EngineEnd": 142, "Name": "MergeTree", "Params": { "LeftParenPos": 73, @@ -133,7 +133,7 @@ "SampleBy": null, "TTL": { "TTLPos": 88, - "ListEnd": 116, + "ListEnd": 142, "Items": [ { "TTLPos": 88, @@ -179,13 +179,10 @@ "HasDistinct": false, "Items": [ { - "Expr": { - "Name": "id", - "QuoteType": 1, - "NamePos": 126, - "NameEnd": 128 - }, - "Alias": null + "Name": "id", + "QuoteType": 1, + "NamePos": 126, + "NameEnd": 128 } ] }, @@ -270,7 +267,7 @@ }, { "CreatePos": 146, - "StatementEnd": 262, + "StatementEnd": 274, "OrReplace": false, "Name": { "Database": null, @@ -384,7 +381,7 @@ }, "Engine": { "EnginePos": 201, - "EngineEnd": 262, + "EngineEnd": 274, "Name": "MergeTree", "Params": { "LeftParenPos": 219, @@ -402,7 +399,7 @@ "SampleBy": null, "TTL": { "TTLPos": 234, - "ListEnd": 262, + "ListEnd": 274, "Items": [ { "TTLPos": 234, @@ -448,13 +445,10 @@ "HasDistinct": false, "Items": [ { - "Expr": { - "Name": "id", - "QuoteType": 1, - "NamePos": 272, - "NameEnd": 274 - }, - "Alias": null + "Name": "id", + "QuoteType": 1, + "NamePos": 272, + "NameEnd": 274 } ] }, @@ -497,7 +491,7 @@ }, { "CreatePos": 277, - "StatementEnd": 418, + "StatementEnd": 470, "OrReplace": false, "Name": { "Database": null, @@ -641,7 +635,7 @@ }, "Engine": { "EnginePos": 346, - "EngineEnd": 418, + "EngineEnd": 470, "Name": "MergeTree", "Params": { "LeftParenPos": 364, @@ -659,7 +653,7 @@ "SampleBy": null, "TTL": { "TTLPos": 390, - "ListEnd": 418, + "ListEnd": 470, "Items": [ { "TTLPos": 390, @@ -705,22 +699,16 @@ "HasDistinct": false, "Items": [ { - "Expr": { - "Name": "id", - "QuoteType": 1, - "NamePos": 428, - "NameEnd": 430 - }, - "Alias": null + "Name": "id", + "QuoteType": 1, + "NamePos": 428, + "NameEnd": 430 }, { - "Expr": { - "Name": "created", - "QuoteType": 1, - "NamePos": 432, - "NameEnd": 439 - }, - "Alias": null + "Name": "created", + "QuoteType": 1, + "NamePos": 432, + "NameEnd": 439 } ] }, @@ -857,5 +845,582 @@ "TableFunction": null, "HasTemporary": false, "Comment": null + }, + { + "CreatePos": 474, + "StatementEnd": 650, + "OrReplace": false, + "Name": { + "Database": null, + "Table": { + "Name": "t", + "QuoteType": 1, + "NamePos": 487, + "NameEnd": 488 + } + }, + "IfNotExists": false, + "UUID": null, + "OnCluster": null, + "TableSchema": { + "SchemaPos": 489, + "SchemaEnd": 527, + "Columns": [ + { + "NamePos": 490, + "ColumnEnd": 499, + "Name": { + "Ident": { + "Name": "id", + "QuoteType": 1, + "NamePos": 490, + "NameEnd": 492 + }, + "DotIdent": null + }, + "Type": { + "Name": { + "Name": "UInt64", + "QuoteType": 1, + "NamePos": 493, + "NameEnd": 499 + } + }, + "NotNull": null, + "Nullable": null, + "DefaultExpr": null, + "MaterializedExpr": null, + "AliasExpr": null, + "Codec": null, + "TTL": null, + "Comment": null, + "CompressionCodec": null + }, + { + "NamePos": 501, + "ColumnEnd": 517, + "Name": { + "Ident": { + "Name": "created", + "QuoteType": 1, + "NamePos": 501, + "NameEnd": 508 + }, + "DotIdent": null + }, + "Type": { + "Name": { + "Name": "DateTime", + "QuoteType": 1, + "NamePos": 509, + "NameEnd": 517 + } + }, + "NotNull": null, + "Nullable": null, + "DefaultExpr": null, + "MaterializedExpr": null, + "AliasExpr": null, + "Codec": null, + "TTL": null, + "Comment": null, + "CompressionCodec": null + }, + { + "NamePos": 519, + "ColumnEnd": 527, + "Name": { + "Ident": { + "Name": "x", + "QuoteType": 1, + "NamePos": 519, + "NameEnd": 520 + }, + "DotIdent": null + }, + "Type": { + "Name": { + "Name": "UInt64", + "QuoteType": 1, + "NamePos": 521, + "NameEnd": 527 + } + }, + "NotNull": null, + "Nullable": null, + "DefaultExpr": null, + "MaterializedExpr": null, + "AliasExpr": null, + "Codec": null, + "TTL": null, + "Comment": null, + "CompressionCodec": null + } + ], + "AliasTable": null, + "TableFunction": null + }, + "Engine": { + "EnginePos": 529, + "EngineEnd": 650, + "Name": "MergeTree", + "Params": { + "LeftParenPos": 547, + "RightParenPos": 548, + "Items": { + "ListPos": 548, + "ListEnd": 548, + "HasDistinct": false, + "Items": [] + }, + "ColumnArgList": null + }, + "PrimaryKey": null, + "PartitionBy": null, + "SampleBy": null, + "TTL": { + "TTLPos": 562, + "ListEnd": 650, + "Items": [ + { + "TTLPos": 562, + "Expr": { + "LeftExpr": { + "Name": "created", + "QuoteType": 1, + "NamePos": 566, + "NameEnd": 573 + }, + "Operation": "+", + "RightExpr": { + "IntervalPos": 576, + "Expr": { + "NumPos": 585, + "NumEnd": 586, + "Literal": "1", + "Base": 10 + }, + "Unit": { + "Name": "DAY", + "QuoteType": 1, + "NamePos": 587, + "NameEnd": 590 + } + }, + "HasGlobal": false, + "HasNot": false + }, + "Policy": { + "Item": { + "RulePos": 591, + "ToVolume": null, + "ToDisk": null, + "Action": null, + "GroupBy": { + "GroupByPos": 591, + "GroupByEnd": 602, + "AggregateType": "", + "Expr": { + "ListPos": 600, + "ListEnd": 602, + "HasDistinct": false, + "Items": [ + { + "Name": "id", + "QuoteType": 1, + "NamePos": 600, + "NameEnd": 602 + } + ] + }, + "WithCube": false, + "WithRollup": false, + "WithTotals": false + }, + "Set": [ + { + "AssignmentPos": 607, + "Column": { + "Ident": { + "Name": "x", + "QuoteType": 1, + "NamePos": 607, + "NameEnd": 608 + }, + "DotIdent": null + }, + "Expr": { + "Name": { + "Name": "sum", + "QuoteType": 1, + "NamePos": 611, + "NameEnd": 614 + }, + "Params": { + "LeftParenPos": 614, + "RightParenPos": 616, + "Items": { + "ListPos": 615, + "ListEnd": 616, + "HasDistinct": false, + "Items": [ + { + "Expr": { + "Name": "x", + "QuoteType": 1, + "NamePos": 615, + "NameEnd": 616 + }, + "Alias": null + } + ] + }, + "ColumnArgList": null + } + } + } + ] + }, + "Where": null + } + }, + { + "TTLPos": 562, + "Expr": { + "LeftExpr": { + "Name": "created", + "QuoteType": 1, + "NamePos": 619, + "NameEnd": 626 + }, + "Operation": "+", + "RightExpr": { + "IntervalPos": 629, + "Expr": { + "NumPos": 638, + "NumEnd": 639, + "Literal": "2", + "Base": 10 + }, + "Unit": { + "Name": "DAY", + "QuoteType": 1, + "NamePos": 640, + "NameEnd": 643 + } + }, + "HasGlobal": false, + "HasNot": false + }, + "Policy": { + "Item": { + "RulePos": 644, + "ToVolume": null, + "ToDisk": null, + "Action": { + "ActionPos": 644, + "ActionEnd": 650, + "Action": "DELETE", + "Codec": null + }, + "GroupBy": null, + "Set": null + }, + "Where": null + } + } + ] + }, + "Settings": null, + "OrderBy": { + "OrderPos": 550, + "ListEnd": 561, + "Items": [ + { + "OrderPos": 550, + "Expr": { + "Name": "id", + "QuoteType": 1, + "NamePos": 559, + "NameEnd": 561 + }, + "Alias": null, + "Direction": "", + "Fill": null + } + ], + "Interpolate": null + } + }, + "SubQuery": null, + "TableFunction": null, + "HasTemporary": false, + "Comment": null + }, + { + "CreatePos": 653, + "StatementEnd": 796, + "OrReplace": false, + "Name": { + "Database": null, + "Table": { + "Name": "t", + "QuoteType": 1, + "NamePos": 666, + "NameEnd": 667 + } + }, + "IfNotExists": false, + "UUID": null, + "OnCluster": null, + "TableSchema": { + "SchemaPos": 668, + "SchemaEnd": 706, + "Columns": [ + { + "NamePos": 669, + "ColumnEnd": 678, + "Name": { + "Ident": { + "Name": "id", + "QuoteType": 1, + "NamePos": 669, + "NameEnd": 671 + }, + "DotIdent": null + }, + "Type": { + "Name": { + "Name": "UInt64", + "QuoteType": 1, + "NamePos": 672, + "NameEnd": 678 + } + }, + "NotNull": null, + "Nullable": null, + "DefaultExpr": null, + "MaterializedExpr": null, + "AliasExpr": null, + "Codec": null, + "TTL": null, + "Comment": null, + "CompressionCodec": null + }, + { + "NamePos": 680, + "ColumnEnd": 696, + "Name": { + "Ident": { + "Name": "created", + "QuoteType": 1, + "NamePos": 680, + "NameEnd": 687 + }, + "DotIdent": null + }, + "Type": { + "Name": { + "Name": "DateTime", + "QuoteType": 1, + "NamePos": 688, + "NameEnd": 696 + } + }, + "NotNull": null, + "Nullable": null, + "DefaultExpr": null, + "MaterializedExpr": null, + "AliasExpr": null, + "Codec": null, + "TTL": null, + "Comment": null, + "CompressionCodec": null + }, + { + "NamePos": 698, + "ColumnEnd": 706, + "Name": { + "Ident": { + "Name": "x", + "QuoteType": 1, + "NamePos": 698, + "NameEnd": 699 + }, + "DotIdent": null + }, + "Type": { + "Name": { + "Name": "UInt64", + "QuoteType": 1, + "NamePos": 700, + "NameEnd": 706 + } + }, + "NotNull": null, + "Nullable": null, + "DefaultExpr": null, + "MaterializedExpr": null, + "AliasExpr": null, + "Codec": null, + "TTL": null, + "Comment": null, + "CompressionCodec": null + } + ], + "AliasTable": null, + "TableFunction": null + }, + "Engine": { + "EnginePos": 708, + "EngineEnd": 796, + "Name": "MergeTree", + "Params": { + "LeftParenPos": 726, + "RightParenPos": 727, + "Items": { + "ListPos": 727, + "ListEnd": 727, + "HasDistinct": false, + "Items": [] + }, + "ColumnArgList": null + }, + "PrimaryKey": null, + "PartitionBy": null, + "SampleBy": null, + "TTL": { + "TTLPos": 741, + "ListEnd": 796, + "Items": [ + { + "TTLPos": 741, + "Expr": { + "LeftExpr": { + "Name": "created", + "QuoteType": 1, + "NamePos": 745, + "NameEnd": 752 + }, + "Operation": "+", + "RightExpr": { + "IntervalPos": 755, + "Expr": { + "NumPos": 764, + "NumEnd": 765, + "Literal": "1", + "Base": 10 + }, + "Unit": { + "Name": "DAY", + "QuoteType": 1, + "NamePos": 766, + "NameEnd": 769 + } + }, + "HasGlobal": false, + "HasNot": false + }, + "Policy": { + "Item": { + "RulePos": 770, + "ToVolume": null, + "ToDisk": null, + "Action": null, + "GroupBy": { + "GroupByPos": 770, + "GroupByEnd": 782, + "AggregateType": "", + "Expr": { + "ListPos": 779, + "ListEnd": 782, + "HasDistinct": false, + "Items": [ + { + "Name": "ALL", + "QuoteType": 1, + "NamePos": 779, + "NameEnd": 782 + } + ] + }, + "WithCube": false, + "WithRollup": false, + "WithTotals": false + }, + "Set": [ + { + "AssignmentPos": 787, + "Column": { + "Ident": { + "Name": "x", + "QuoteType": 1, + "NamePos": 787, + "NameEnd": 788 + }, + "DotIdent": null + }, + "Expr": { + "Name": { + "Name": "sum", + "QuoteType": 1, + "NamePos": 791, + "NameEnd": 794 + }, + "Params": { + "LeftParenPos": 794, + "RightParenPos": 796, + "Items": { + "ListPos": 795, + "ListEnd": 796, + "HasDistinct": false, + "Items": [ + { + "Expr": { + "Name": "x", + "QuoteType": 1, + "NamePos": 795, + "NameEnd": 796 + }, + "Alias": null + } + ] + }, + "ColumnArgList": null + } + } + } + ] + }, + "Where": null + } + } + ] + }, + "Settings": null, + "OrderBy": { + "OrderPos": 729, + "ListEnd": 740, + "Items": [ + { + "OrderPos": 729, + "Expr": { + "Name": "id", + "QuoteType": 1, + "NamePos": 738, + "NameEnd": 740 + }, + "Alias": null, + "Direction": "", + "Fill": null + } + ], + "Interpolate": null + } + }, + "SubQuery": null, + "TableFunction": null, + "HasTemporary": false, + "Comment": null } ] \ No newline at end of file diff --git a/parser/testdata/ddl/output/create_table_with_ttl_policy.sql.golden.json b/parser/testdata/ddl/output/create_table_with_ttl_policy.sql.golden.json index 5b8f57f..3ab8cc6 100644 --- a/parser/testdata/ddl/output/create_table_with_ttl_policy.sql.golden.json +++ b/parser/testdata/ddl/output/create_table_with_ttl_policy.sql.golden.json @@ -1,7 +1,7 @@ [ { "CreatePos": 0, - "StatementEnd": 203, + "StatementEnd": 216, "OrReplace": false, "Name": { "Database": null, @@ -85,7 +85,7 @@ }, "Engine": { "EnginePos": 51, - "EngineEnd": 203, + "EngineEnd": 216, "Name": "MergeTree", "Params": null, "PrimaryKey": null, @@ -134,7 +134,7 @@ "SampleBy": null, "TTL": { "TTLPos": 106, - "ListEnd": 203, + "ListEnd": 216, "Items": [ { "TTLPos": 106, @@ -299,7 +299,7 @@ }, { "CreatePos": 221, - "StatementEnd": 364, + "StatementEnd": 396, "OrReplace": false, "Name": { "Database": null, @@ -383,7 +383,7 @@ }, "Engine": { "EnginePos": 285, - "EngineEnd": 364, + "EngineEnd": 396, "Name": "MergeTree", "Params": null, "PrimaryKey": null, @@ -432,7 +432,7 @@ "SampleBy": null, "TTL": { "TTLPos": 340, - "ListEnd": 364, + "ListEnd": 396, "Items": [ { "TTLPos": 340, @@ -701,7 +701,7 @@ "SampleBy": null, "TTL": { "TTLPos": 542, - "ListEnd": 614, + "ListEnd": 642, "Items": [ { "TTLPos": 542, From 3033f4639ca1be28e2690a5b867e3a4717d9b649 Mon Sep 17 00:00:00 2001 From: Markus Eriksson Date: Fri, 21 Aug 2026 11:34:07 +0200 Subject: [PATCH 08/11] Read TTL SET assignments with full expression precedence A TTL SET assignment has no trailing clause to bound the right-hand side, unlike ALTER TABLE UPDATE's IN PARTITION, so parse it with full expression precedence (e.g. SET x = max(y) > 0). Also restrict OR REPLACE MATERIALIZED VIEW to the CREATE path: ClickHouse rejects an ATTACH OR REPLACE combination. Co-authored-by: CommandCodeBot --- parser/parser_table.go | 41 ++- parser/parser_test.go | 3 + .../ddl/create_table_ttl_group_by.sql | 4 +- .../beautify/create_table_ttl_group_by.sql | 14 + .../ddl/format/create_table_ttl_group_by.sql | 3 + .../create_table_ttl_group_by.sql.golden.json | 307 ++++++++++++++++++ 6 files changed, 367 insertions(+), 5 deletions(-) diff --git a/parser/parser_table.go b/parser/parser_table.go index f2e2fa7..0de4c5f 100644 --- a/parser/parser_table.go +++ b/parser/parser_table.go @@ -10,10 +10,19 @@ func (p *Parser) parseDDL(pos Pos) (DDL, error) { switch { case p.matchKeyword(KeywordCreate), p.matchKeyword(KeywordAttach): + isAttach := p.matchKeyword(KeywordAttach) _ = p.lexer.consumeToken() orReplace := p.tryConsumeKeywords(KeywordOr, KeywordReplace) - if orReplace && !p.matchOneOfKeywords(KeywordTemporary, KeywordTable, KeywordView, KeywordFunction, KeywordDictionary, KeywordMaterialized) { - return nil, fmt.Errorf("expected keyword: TEMPORARY|TABLE|VIEW|FUNCTION|DICTIONARY|MATERIALIZED, but got %q", p.currentTokenString()) + if orReplace { + // MATERIALIZED VIEW accepts OR REPLACE only under CREATE; + // ClickHouse rejects an ATTACH OR REPLACE combination. + if isAttach { + if !p.matchOneOfKeywords(KeywordTemporary, KeywordTable, KeywordView, KeywordFunction, KeywordDictionary) { + return nil, fmt.Errorf("expected keyword: TEMPORARY|TABLE|VIEW|FUNCTION|DICTIONARY, but got %q", p.currentTokenString()) + } + } else if !p.matchOneOfKeywords(KeywordTemporary, KeywordTable, KeywordView, KeywordFunction, KeywordDictionary, KeywordMaterialized) { + return nil, fmt.Errorf("expected keyword: TEMPORARY|TABLE|VIEW|FUNCTION|DICTIONARY|MATERIALIZED, but got %q", p.currentTokenString()) + } } switch { case p.matchKeyword(KeywordNamed): @@ -1293,7 +1302,7 @@ func (p *Parser) tryParseTTLPolicy(pos Pos) (*TTLPolicy, error) { }, } if p.tryConsumeKeywords(KeywordSet) { - set, err := p.parseUpdateAssignment(p.Pos()) + set, err := p.parseTTLPolicySet(p.Pos()) if err != nil { return nil, err } @@ -1308,7 +1317,7 @@ func (p *Parser) tryParseTTLPolicy(pos Pos) (*TTLPolicy, error) { if p.tryConsumeTokenKind(TokenKindComma) == nil { break } - set, err := p.parseUpdateAssignment(p.Pos()) + set, err := p.parseTTLPolicySet(p.Pos()) if err != nil { p.lexer.restoreState(savedState) break @@ -1329,6 +1338,30 @@ func (p *Parser) tryParseTTLPolicy(pos Pos) (*TTLPolicy, error) { return policy, nil } +// parseTTLPolicySet parses one TTL SET assignment: = . +// Unlike parseUpdateAssignment, the right-hand side is read with full +// expression precedence: a TTL SET has no trailing clause like the IN +// PARTITION that bounds an ALTER TABLE UPDATE assignment, so expressions +// such as `SET x = max(y) > 0` are valid here. +func (p *Parser) parseTTLPolicySet(pos Pos) (*UpdateAssignment, error) { + column, err := p.ParseNestedIdentifier(p.Pos()) + if err != nil { + return nil, err + } + if err := p.expectTokenKind(TokenKindSingleEQ); err != nil { + return nil, err + } + expr, err := p.parseExpr(p.Pos()) + if err != nil { + return nil, err + } + return &UpdateAssignment{ + AssignmentPos: pos, + Column: column, + Expr: expr, + }, nil +} + func (p *Parser) parseTTLExpr(pos Pos) (*TTLExpr, error) { columnExpr, err := p.parseExpr(pos) if err != nil { diff --git a/parser/parser_test.go b/parser/parser_test.go index ba7fa4f..806e2eb 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -200,6 +200,9 @@ func TestParser_InvalidSyntax(t *testing.T) { "CREATE TABLE t (id UInt64, created DateTime) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY id WITH ROLLUP", "CREATE TABLE t (id UInt64, created DateTime) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY id WITH CUBE", "CREATE TABLE t (id UInt64, created DateTime) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY GROUPING SETS (id)", + // OR REPLACE is only accepted for MATERIALIZED VIEW under CREATE, + // never under ATTACH + "ATTACH OR REPLACE MATERIALIZED VIEW mv TO dest AS SELECT * FROM src", // Invalid ARRAY JOIN types (only ARRAY JOIN, LEFT ARRAY JOIN, and INNER ARRAY JOIN are valid) "SELECT * FROM t RIGHT ARRAY JOIN arr AS a", // RIGHT ARRAY JOIN not supported "SELECT * FROM t FULL ARRAY JOIN arr AS a", // FULL ARRAY JOIN not supported diff --git a/parser/testdata/ddl/create_table_ttl_group_by.sql b/parser/testdata/ddl/create_table_ttl_group_by.sql index 387ced0..8e64f25 100644 --- a/parser/testdata/ddl/create_table_ttl_group_by.sql +++ b/parser/testdata/ddl/create_table_ttl_group_by.sql @@ -6,4 +6,6 @@ CREATE TABLE t (id UInt64, created DateTime, x UInt64, total UInt64) ENGINE = Me CREATE TABLE t (id UInt64, created DateTime, x UInt64) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY id SET x = sum(x), created + INTERVAL 2 DAY DELETE; -CREATE TABLE t (id UInt64, created DateTime, x UInt64) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY ALL SET x = sum(x); \ No newline at end of file +CREATE TABLE t (id UInt64, created DateTime, x UInt64) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY ALL SET x = sum(x); + +CREATE TABLE t (id UInt64, created DateTime, x UInt64, y UInt64) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY id SET x = max(y) > 0; \ No newline at end of file diff --git a/parser/testdata/ddl/format/beautify/create_table_ttl_group_by.sql b/parser/testdata/ddl/format/beautify/create_table_ttl_group_by.sql index 36612a1..631bc9b 100644 --- a/parser/testdata/ddl/format/beautify/create_table_ttl_group_by.sql +++ b/parser/testdata/ddl/format/beautify/create_table_ttl_group_by.sql @@ -9,6 +9,8 @@ CREATE TABLE t (id UInt64, created DateTime, x UInt64) ENGINE = MergeTree() ORDE CREATE TABLE t (id UInt64, created DateTime, x UInt64) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY ALL SET x = sum(x); +CREATE TABLE t (id UInt64, created DateTime, x UInt64, y UInt64) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY id SET x = max(y) > 0; + -- Beautify SQL: CREATE TABLE t ( @@ -66,3 +68,15 @@ ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY ALL SET x = sum(x); +CREATE TABLE t +( + id UInt64, + created DateTime, + x UInt64, + y UInt64 +) +ENGINE = MergeTree() +ORDER BY + id +TTL created + INTERVAL 1 DAY GROUP BY + id SET x = max(y) > 0; diff --git a/parser/testdata/ddl/format/create_table_ttl_group_by.sql b/parser/testdata/ddl/format/create_table_ttl_group_by.sql index 3dd1762..c24e683 100644 --- a/parser/testdata/ddl/format/create_table_ttl_group_by.sql +++ b/parser/testdata/ddl/format/create_table_ttl_group_by.sql @@ -9,9 +9,12 @@ CREATE TABLE t (id UInt64, created DateTime, x UInt64) ENGINE = MergeTree() ORDE CREATE TABLE t (id UInt64, created DateTime, x UInt64) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY ALL SET x = sum(x); +CREATE TABLE t (id UInt64, created DateTime, x UInt64, y UInt64) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY id SET x = max(y) > 0; + -- Format SQL: CREATE TABLE t (id UInt64, created DateTime, x UInt64) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY id SET x = sum(x); CREATE TABLE t (id UInt64, created DateTime, x UInt64) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY id; CREATE TABLE t (id UInt64, created DateTime, x UInt64, total UInt64) ENGINE = MergeTree() ORDER BY (id, created) TTL created + INTERVAL 1 DAY GROUP BY id, created SET x = sum(x), total = count(); CREATE TABLE t (id UInt64, created DateTime, x UInt64) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY id SET x = sum(x), created + INTERVAL 2 DAY DELETE; CREATE TABLE t (id UInt64, created DateTime, x UInt64) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY ALL SET x = sum(x); +CREATE TABLE t (id UInt64, created DateTime, x UInt64, y UInt64) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY id SET x = max(y) > 0; diff --git a/parser/testdata/ddl/output/create_table_ttl_group_by.sql.golden.json b/parser/testdata/ddl/output/create_table_ttl_group_by.sql.golden.json index ec90a80..4264fcc 100644 --- a/parser/testdata/ddl/output/create_table_ttl_group_by.sql.golden.json +++ b/parser/testdata/ddl/output/create_table_ttl_group_by.sql.golden.json @@ -1422,5 +1422,312 @@ "TableFunction": null, "HasTemporary": false, "Comment": null + }, + { + "CreatePos": 800, + "StatementEnd": 957, + "OrReplace": false, + "Name": { + "Database": null, + "Table": { + "Name": "t", + "QuoteType": 1, + "NamePos": 813, + "NameEnd": 814 + } + }, + "IfNotExists": false, + "UUID": null, + "OnCluster": null, + "TableSchema": { + "SchemaPos": 815, + "SchemaEnd": 863, + "Columns": [ + { + "NamePos": 816, + "ColumnEnd": 825, + "Name": { + "Ident": { + "Name": "id", + "QuoteType": 1, + "NamePos": 816, + "NameEnd": 818 + }, + "DotIdent": null + }, + "Type": { + "Name": { + "Name": "UInt64", + "QuoteType": 1, + "NamePos": 819, + "NameEnd": 825 + } + }, + "NotNull": null, + "Nullable": null, + "DefaultExpr": null, + "MaterializedExpr": null, + "AliasExpr": null, + "Codec": null, + "TTL": null, + "Comment": null, + "CompressionCodec": null + }, + { + "NamePos": 827, + "ColumnEnd": 843, + "Name": { + "Ident": { + "Name": "created", + "QuoteType": 1, + "NamePos": 827, + "NameEnd": 834 + }, + "DotIdent": null + }, + "Type": { + "Name": { + "Name": "DateTime", + "QuoteType": 1, + "NamePos": 835, + "NameEnd": 843 + } + }, + "NotNull": null, + "Nullable": null, + "DefaultExpr": null, + "MaterializedExpr": null, + "AliasExpr": null, + "Codec": null, + "TTL": null, + "Comment": null, + "CompressionCodec": null + }, + { + "NamePos": 845, + "ColumnEnd": 853, + "Name": { + "Ident": { + "Name": "x", + "QuoteType": 1, + "NamePos": 845, + "NameEnd": 846 + }, + "DotIdent": null + }, + "Type": { + "Name": { + "Name": "UInt64", + "QuoteType": 1, + "NamePos": 847, + "NameEnd": 853 + } + }, + "NotNull": null, + "Nullable": null, + "DefaultExpr": null, + "MaterializedExpr": null, + "AliasExpr": null, + "Codec": null, + "TTL": null, + "Comment": null, + "CompressionCodec": null + }, + { + "NamePos": 855, + "ColumnEnd": 863, + "Name": { + "Ident": { + "Name": "y", + "QuoteType": 1, + "NamePos": 855, + "NameEnd": 856 + }, + "DotIdent": null + }, + "Type": { + "Name": { + "Name": "UInt64", + "QuoteType": 1, + "NamePos": 857, + "NameEnd": 863 + } + }, + "NotNull": null, + "Nullable": null, + "DefaultExpr": null, + "MaterializedExpr": null, + "AliasExpr": null, + "Codec": null, + "TTL": null, + "Comment": null, + "CompressionCodec": null + } + ], + "AliasTable": null, + "TableFunction": null + }, + "Engine": { + "EnginePos": 865, + "EngineEnd": 957, + "Name": "MergeTree", + "Params": { + "LeftParenPos": 883, + "RightParenPos": 884, + "Items": { + "ListPos": 884, + "ListEnd": 884, + "HasDistinct": false, + "Items": [] + }, + "ColumnArgList": null + }, + "PrimaryKey": null, + "PartitionBy": null, + "SampleBy": null, + "TTL": { + "TTLPos": 898, + "ListEnd": 957, + "Items": [ + { + "TTLPos": 898, + "Expr": { + "LeftExpr": { + "Name": "created", + "QuoteType": 1, + "NamePos": 902, + "NameEnd": 909 + }, + "Operation": "+", + "RightExpr": { + "IntervalPos": 912, + "Expr": { + "NumPos": 921, + "NumEnd": 922, + "Literal": "1", + "Base": 10 + }, + "Unit": { + "Name": "DAY", + "QuoteType": 1, + "NamePos": 923, + "NameEnd": 926 + } + }, + "HasGlobal": false, + "HasNot": false + }, + "Policy": { + "Item": { + "RulePos": 927, + "ToVolume": null, + "ToDisk": null, + "Action": null, + "GroupBy": { + "GroupByPos": 927, + "GroupByEnd": 938, + "AggregateType": "", + "Expr": { + "ListPos": 936, + "ListEnd": 938, + "HasDistinct": false, + "Items": [ + { + "Name": "id", + "QuoteType": 1, + "NamePos": 936, + "NameEnd": 938 + } + ] + }, + "WithCube": false, + "WithRollup": false, + "WithTotals": false + }, + "Set": [ + { + "AssignmentPos": 943, + "Column": { + "Ident": { + "Name": "x", + "QuoteType": 1, + "NamePos": 943, + "NameEnd": 944 + }, + "DotIdent": null + }, + "Expr": { + "LeftExpr": { + "Name": { + "Name": "max", + "QuoteType": 1, + "NamePos": 947, + "NameEnd": 950 + }, + "Params": { + "LeftParenPos": 950, + "RightParenPos": 952, + "Items": { + "ListPos": 951, + "ListEnd": 952, + "HasDistinct": false, + "Items": [ + { + "Expr": { + "Name": "y", + "QuoteType": 1, + "NamePos": 951, + "NameEnd": 952 + }, + "Alias": null + } + ] + }, + "ColumnArgList": null + } + }, + "Operation": "\u003e", + "RightExpr": { + "NumPos": 956, + "NumEnd": 957, + "Literal": "0", + "Base": 10 + }, + "HasGlobal": false, + "HasNot": false + } + } + ] + }, + "Where": null + } + } + ] + }, + "Settings": null, + "OrderBy": { + "OrderPos": 886, + "ListEnd": 897, + "Items": [ + { + "OrderPos": 886, + "Expr": { + "Name": "id", + "QuoteType": 1, + "NamePos": 895, + "NameEnd": 897 + }, + "Alias": null, + "Direction": "", + "Fill": null + } + ], + "Interpolate": null + } + }, + "SubQuery": null, + "TableFunction": null, + "HasTemporary": false, + "Comment": null } ] \ No newline at end of file From 46c2a9c2c10c8a08cea1a7bb970fc6e7464f4268 Mon Sep 17 00:00:00 2001 From: Markus Eriksson Date: Fri, 21 Aug 2026 11:46:55 +0200 Subject: [PATCH 09/11] Extract TTL GROUP BY action parsing into its own function Keep tryParseTTLPolicy's switch declarative: the GROUP BY case now delegates to parseTTLPolicyGroupBy, which owns the key list, the SET assignment probe, and rule construction. Behavior-neutral. Co-authored-by: CommandCodeBot --- parser/parser_table.go | 151 ++++++++++++++++++++++------------------- 1 file changed, 81 insertions(+), 70 deletions(-) diff --git a/parser/parser_table.go b/parser/parser_table.go index 0de4c5f..481f85b 100644 --- a/parser/parser_table.go +++ b/parser/parser_table.go @@ -1252,90 +1252,101 @@ func (p *Parser) tryParseTTLPolicy(pos Pos) (*TTLPolicy, error) { action.Codec = codec rule = &TTLPolicyRule{RulePos: pos, Action: action} case p.matchKeyword(KeywordGroup): - // A TTL GROUP BY action takes only a plain expression list. - // ClickHouse rejects the query-level forms (ALL, - // CUBE/ROLLUP/GROUPING SETS, WITH CUBE/ROLLUP/TOTALS) inside a - // TTL, so the keys are parsed directly as a list instead of - // reusing parseGroupByClause and then discarding those forms: - // building the clause from the expected shape keeps any future - // query-level GROUP BY sugar out of the TTL grammar as well. - if err := p.expectKeyword(KeywordGroup); err != nil { + groupBy, err := p.parseTTLPolicyGroupBy(pos) + if err != nil { return nil, err } - if err := p.expectKeyword(KeywordBy); err != nil { + rule = groupBy + default: + return nil, nil // nolint + } + policy := &TTLPolicy{Item: rule} + + where, err := p.tryParseWhereClause(p.Pos()) + if err != nil { + return nil, err + } + policy.Where = where + return policy, nil +} + +// parseTTLPolicyGroupBy parses the TTL GROUP BY action: a plain key +// expression list, optionally followed by SET = assignments. +// +// ClickHouse rejects the query-level GROUP BY forms (ALL, +// CUBE/ROLLUP/GROUPING SETS, WITH CUBE/ROLLUP/TOTALS) inside a TTL, so the +// keys are parsed directly as a list instead of reusing parseGroupByClause +// and then discarding those forms: building the clause from the expected +// shape keeps any future query-level GROUP BY sugar out of the TTL grammar +// as well. +func (p *Parser) parseTTLPolicyGroupBy(pos Pos) (*TTLPolicyRule, error) { + if err := p.expectKeyword(KeywordGroup); err != nil { + return nil, err + } + if err := p.expectKeyword(KeywordBy); err != nil { + return nil, err + } + keys := &ColumnExprList{ListPos: p.Pos()} + for { + var key Expr + var err error + if p.matchTokenKind(TokenKindKeyword) { + // Bare keywords (e.g. ALL) are valid TTL GROUP BY keys even + // when followed by SET or a closing engine clause; + // parseColumnExpr only reads a keyword as an identifier for a + // narrower lookahead, so fall back to it when an expression + // cannot start here. + savedState := p.lexer.saveState() + key, err = p.parseExpr(p.Pos()) + if err != nil { + p.lexer.restoreState(savedState) + key, err = p.parseAnyKeyword() + } + } else { + key, err = p.parseExpr(p.Pos()) + } + if err != nil { + return nil, err + } + keys.Items = append(keys.Items, key) + keys.ListEnd = key.End() + if p.tryConsumeTokenKind(TokenKindComma) == nil { + break + } + } + rule := &TTLPolicyRule{ + RulePos: pos, + GroupBy: &GroupByClause{ + GroupByPos: pos, + GroupByEnd: keys.End(), + Expr: keys, + }, + } + if p.tryConsumeKeywords(KeywordSet) { + set, err := p.parseTTLPolicySet(p.Pos()) + if err != nil { return nil, err } - keys := &ColumnExprList{ListPos: p.Pos()} + rule.Set = append(rule.Set, set) for { - var key Expr - var err error - if p.matchTokenKind(TokenKindKeyword) { - // Bare keywords (e.g. ALL) are valid TTL GROUP BY keys - // even when followed by SET or a closing engine clause; - // parseColumnExpr only reads a keyword as an identifier - // for a narrower lookahead, so fall back to it when an - // expression cannot start here. - savedState := p.lexer.saveState() - key, err = p.parseExpr(p.Pos()) - if err != nil { - p.lexer.restoreState(savedState) - key, err = p.parseAnyKeyword() - } - } else { - key, err = p.parseExpr(p.Pos()) - } - if err != nil { - return nil, err - } - keys.Items = append(keys.Items, key) - keys.ListEnd = key.End() + // A comma either continues the SET assignment list or starts + // the next TTL expression of a multi-value TTL clause; consume + // it and probe for another assignment, rolling both back when + // none follows so parseTTLClause can treat the comma as a rule + // separator. + savedState := p.lexer.saveState() if p.tryConsumeTokenKind(TokenKindComma) == nil { break } - } - rule = &TTLPolicyRule{ - RulePos: pos, - GroupBy: &GroupByClause{ - GroupByPos: pos, - GroupByEnd: keys.End(), - Expr: keys, - }, - } - if p.tryConsumeKeywords(KeywordSet) { set, err := p.parseTTLPolicySet(p.Pos()) if err != nil { - return nil, err + p.lexer.restoreState(savedState) + break } rule.Set = append(rule.Set, set) - for { - // A comma either continues the SET assignment list or - // starts the next TTL expression of a multi-value TTL - // clause; consume it and probe for another assignment, - // rolling both back when none follows so parseTTLClause - // can treat the comma as a rule separator. - savedState := p.lexer.saveState() - if p.tryConsumeTokenKind(TokenKindComma) == nil { - break - } - set, err := p.parseTTLPolicySet(p.Pos()) - if err != nil { - p.lexer.restoreState(savedState) - break - } - rule.Set = append(rule.Set, set) - } } - default: - return nil, nil // nolint } - policy := &TTLPolicy{Item: rule} - - where, err := p.tryParseWhereClause(p.Pos()) - if err != nil { - return nil, err - } - policy.Where = where - return policy, nil + return rule, nil } // parseTTLPolicySet parses one TTL SET assignment: = . From 4498afdad0189a84f40ce1e3ba44f8c7b220a2e6 Mon Sep 17 00:00:00 2001 From: Markus Eriksson Date: Fri, 21 Aug 2026 12:39:07 +0200 Subject: [PATCH 10/11] Lock in greedy TTL GROUP BY key list with a regression test A comma after a TTL GROUP BY key continues the key list, and the engine rejects a trailing TTL action after a comma-separated key (unlike after a complete SET assignment, where the comma starts the next TTL rule). Add the engine-rejected combined form to TestParser_InvalidSyntax so the behavior stays locked in. Co-authored-by: CommandCodeBot --- parser/parser_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/parser/parser_test.go b/parser/parser_test.go index 806e2eb..39ab64d 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -203,6 +203,12 @@ func TestParser_InvalidSyntax(t *testing.T) { // OR REPLACE is only accepted for MATERIALIZED VIEW under CREATE, // never under ATTACH "ATTACH OR REPLACE MATERIALIZED VIEW mv TO dest AS SELECT * FROM src", + // A TTL GROUP BY key list is greedy: after the keys the comma + // continues the list, so a trailing TTL action after a + // comma-separated key is rejected by ClickHouse (unlike after a + // complete SET assignment, where the comma starts the next TTL + // rule). + "CREATE TABLE t (id UInt64, created DateTime) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY id, created + INTERVAL 2 DAY DELETE", // Invalid ARRAY JOIN types (only ARRAY JOIN, LEFT ARRAY JOIN, and INNER ARRAY JOIN are valid) "SELECT * FROM t RIGHT ARRAY JOIN arr AS a", // RIGHT ARRAY JOIN not supported "SELECT * FROM t FULL ARRAY JOIN arr AS a", // FULL ARRAY JOIN not supported From fa6ab50a4c26b8deb5e00c789838222949fda8f7 Mon Sep 17 00:00:00 2001 From: Markus Eriksson Date: Fri, 21 Aug 2026 13:05:22 +0200 Subject: [PATCH 11/11] Restrict TTL WHERE clause to the DELETE action ClickHouse rejects a WHERE clause after RECOMPRESS, TO DISK/VOLUME, and GROUP BY; it belongs only to DELETE. Parse it only in the DELETE branch so the other actions leave WHERE unconsumed for the statement parser, and lock the rejected forms in with regression tests. Co-authored-by: CommandCodeBot --- parser/parser_table.go | 20 ++++++++++++-------- parser/parser_test.go | 6 ++++++ 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/parser/parser_table.go b/parser/parser_table.go index 481f85b..7e07434 100644 --- a/parser/parser_table.go +++ b/parser/parser_table.go @@ -1220,6 +1220,7 @@ func (p *Parser) parseTTLClause(pos Pos, allowMultiValues bool) ([]*TTLExpr, err func (p *Parser) tryParseTTLPolicy(pos Pos) (*TTLPolicy, error) { var rule *TTLPolicyRule + var where *WhereClause switch { case p.tryConsumeKeywords(KeywordTo): if p.tryConsumeKeywords(KeywordDisk) { @@ -1238,6 +1239,7 @@ func (p *Parser) tryParseTTLPolicy(pos Pos) (*TTLPolicy, error) { return nil, fmt.Errorf("unexpected token: %q, expected DISK or VOLUME", p.currentTokenKind()) } case p.matchKeyword(KeywordDelete), p.matchKeyword(KeywordRecompress): + isDelete := p.matchKeyword(KeywordDelete) token := p.current() _ = p.lexer.consumeToken() action := &TTLPolicyRuleAction{ @@ -1251,6 +1253,15 @@ func (p *Parser) tryParseTTLPolicy(pos Pos) (*TTLPolicy, error) { } action.Codec = codec rule = &TTLPolicyRule{RulePos: pos, Action: action} + // A TTL WHERE clause belongs only to the DELETE action; ClickHouse + // rejects it after RECOMPRESS, TO DISK/VOLUME, and GROUP BY, so it + // is left unconsumed for the statement parser in those cases. + if isDelete { + where, err = p.tryParseWhereClause(p.Pos()) + if err != nil { + return nil, err + } + } case p.matchKeyword(KeywordGroup): groupBy, err := p.parseTTLPolicyGroupBy(pos) if err != nil { @@ -1260,14 +1271,7 @@ func (p *Parser) tryParseTTLPolicy(pos Pos) (*TTLPolicy, error) { default: return nil, nil // nolint } - policy := &TTLPolicy{Item: rule} - - where, err := p.tryParseWhereClause(p.Pos()) - if err != nil { - return nil, err - } - policy.Where = where - return policy, nil + return &TTLPolicy{Item: rule, Where: where}, nil } // parseTTLPolicyGroupBy parses the TTL GROUP BY action: a plain key diff --git a/parser/parser_test.go b/parser/parser_test.go index 39ab64d..b84f210 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -209,6 +209,12 @@ func TestParser_InvalidSyntax(t *testing.T) { // complete SET assignment, where the comma starts the next TTL // rule). "CREATE TABLE t (id UInt64, created DateTime) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY id, created + INTERVAL 2 DAY DELETE", + // A TTL WHERE clause belongs only to the DELETE action; ClickHouse + // rejects it after GROUP BY, RECOMPRESS, and TO DISK/VOLUME. + "CREATE TABLE t (id UInt64, created DateTime, x UInt64) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY id SET x = sum(x) WHERE id > 0", + "CREATE TABLE t (id UInt64, created DateTime) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY GROUP BY id WHERE id > 0", + "CREATE TABLE t (id UInt64, created DateTime) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY RECOMPRESS CODEC(ZSTD(1)) WHERE id > 0", + "CREATE TABLE t (id UInt64, created DateTime) ENGINE = MergeTree() ORDER BY id TTL created + INTERVAL 1 DAY TO VOLUME 'v1' WHERE id > 0", // Invalid ARRAY JOIN types (only ARRAY JOIN, LEFT ARRAY JOIN, and INNER ARRAY JOIN are valid) "SELECT * FROM t RIGHT ARRAY JOIN arr AS a", // RIGHT ARRAY JOIN not supported "SELECT * FROM t FULL ARRAY JOIN arr AS a", // FULL ARRAY JOIN not supported