diff --git a/build.sbt b/build.sbt index 807c6178..22c7824b 100644 --- a/build.sbt +++ b/build.sbt @@ -20,7 +20,7 @@ ThisBuild / organization := "app.softnetwork" name := "softclient4es" -ThisBuild / version := "0.20.1" +ThisBuild / version := "0.20.2-SNAPSHOT" ThisBuild / scalaVersion := scala213 diff --git a/core/src/main/scala/app/softnetwork/elastic/client/repl/ReplCompleter.scala b/core/src/main/scala/app/softnetwork/elastic/client/repl/ReplCompleter.scala index bf8462a2..c1ecdc96 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/repl/ReplCompleter.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/repl/ReplCompleter.scala @@ -23,112 +23,28 @@ import java.util class ReplCompleter extends Completer { - private val simpleKeywords = Set( - // DQL - "SELECT", - "FROM", - "WHERE", - "GROUP", - "ORDER", - "HAVING", - "LIMIT", - "OFFSET", - "JOIN", - "LEFT", - "RIGHT", - "INNER", - "OUTER", - "UNION", - "INTERSECT", - "EXCEPT", + // Single source of truth for SQL keywords (#161): parser-derived registry + REPL extras. + private val simpleKeywords: Set[String] = ReplKeywords.all - // DML - "INSERT", - "INTO", - "VALUES", - "UPDATE", - "SET", - "DELETE", - "COPY", - "BULK", - - // DDL - "CREATE", - "ALTER", - "DROP", - "TRUNCATE", - "RENAME", - "TABLE", - "INDEX", - "VIEW", - "PIPELINE", - "WATCHER", - "ENRICH", - "POLICY", - "TRANSFORM", - - // Functions - "COUNT", - "SUM", - "AVG", - "MIN", - "MAX", - "DISTINCT", - "CAST", - "COALESCE", - "NULLIF", - "CASE", - "WHEN", - "THEN", - "ELSE", - "END", - - // Operators - "AND", - "OR", - "NOT", - "IN", - "LIKE", - "BETWEEN", - "IS", - "NULL", - "TRUE", - "FALSE", - - // Enrich/Watcher - "TYPE", - "MATCH", - "GEO_MATCH", - "RANGE", - "ON", - "EXECUTE", - "SCHEDULE", - "EVERY", - "CONDITION", - "ACTION", - "WEBHOOK", - "LOG", - - // Meta - "SHOW", - "DESCRIBE", - "EXPLAIN", - "BY", - "ALL", - "AS" - ) - - // Context-aware compound keywords + // Context-aware compound keywords (continuations offered after the trigger word). + // Hand-curated UX map; the underlying registry phrases (ORDER BY, PARTITION BY, + // UNION ALL, NULLS FIRST/LAST, ...) are pinned in SQLKeywords.compoundPhrases by + // ReplKeywordsSpec. GEO -> MATCH is a REPL-only convenience (extraWords). private val compoundKeywords = Map( - "GROUP" -> List("BY"), - "ORDER" -> List("BY"), - "LEFT" -> List("JOIN", "OUTER JOIN"), - "RIGHT" -> List("JOIN", "OUTER JOIN"), - "INNER" -> List("JOIN"), - "OUTER" -> List("JOIN"), - "UNION" -> List("ALL"), - "ENRICH" -> List("POLICY"), - "GEO" -> List("MATCH") + "GROUP" -> List("BY"), + "ORDER" -> List("BY"), + "PARTITION" -> List("BY"), + "LEFT" -> List("JOIN", "OUTER JOIN"), + "RIGHT" -> List("JOIN", "OUTER JOIN"), + "FULL" -> List("JOIN", "OUTER JOIN"), + "CROSS" -> List("JOIN"), + "INNER" -> List("JOIN"), + "OUTER" -> List("JOIN"), + "UNION" -> List("ALL"), + "NULLS" -> List("FIRST", "LAST"), + "IS" -> List("NULL", "NOT NULL"), + "ENRICH" -> List("POLICY"), + "GEO" -> List("MATCH") ) private val metaCommands = Set( diff --git a/core/src/main/scala/app/softnetwork/elastic/client/repl/ReplHighlighter.scala b/core/src/main/scala/app/softnetwork/elastic/client/repl/ReplHighlighter.scala index ccbf9758..2def9fab 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/repl/ReplHighlighter.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/repl/ReplHighlighter.scala @@ -21,36 +21,8 @@ import org.jline.utils.{AttributedString, AttributedStringBuilder, AttributedSty class ReplHighlighter extends Highlighter { - private val keywords = Set( - "SELECT", - "FROM", - "WHERE", - "INSERT", - "UPDATE", - "DELETE", - "CREATE", - "ALTER", - "DROP", - "TABLE", - "INDEX", - "VIEW", - "AND", - "OR", - "NOT", - "IN", - "LIKE", - "BETWEEN", - "JOIN", - "LEFT", - "RIGHT", - "INNER", - "OUTER", - "ENRICH", - "POLICY", - "WATCHER", - "TRANSFORM", - "PIPELINE" - ) + // Single source of truth for SQL keywords (#161): parser-derived registry + REPL extras. + private val keywords: Set[String] = ReplKeywords.all override def highlight(reader: LineReader, buffer: String): AttributedString = { val builder = new AttributedStringBuilder() diff --git a/core/src/main/scala/app/softnetwork/elastic/client/repl/ReplKeywords.scala b/core/src/main/scala/app/softnetwork/elastic/client/repl/ReplKeywords.scala new file mode 100644 index 00000000..e4dc4090 --- /dev/null +++ b/core/src/main/scala/app/softnetwork/elastic/client/repl/ReplKeywords.scala @@ -0,0 +1,42 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client.repl + +import app.softnetwork.elastic.sql.SQLKeywords + +/** The single keyword set both REPL components (highlighter + completer) consume (#161). + * + * `sqlWords` is the parser-derived truth (SQLKeywords registry, sql module). `extraWords` are + * REPL-only entries the parser does NOT accept today (legacy/roadmap completer entries kept for + * continuity - PD-2). A word the parser actually accepts must live in SQLKeywords, never here + * (guarded by ReplKeywordsSpec). + */ +object ReplKeywords { + + /** Parser-backed keywords (single uppercase words). */ + val sqlWords: Set[String] = SQLKeywords.highlightedWords + + /** REPL-only entries — NOT parser keywords at this baseline: + * INTERSECT/EXPLAIN/BULK/CONDITION/ACTION/TRANSFORM were advertised by the pre-#161 + * completer/highlighter; GEO is the compound trigger for "GEO MATCH". + */ + val extraWords: Set[String] = + Set("ACTION", "BULK", "CONDITION", "EXPLAIN", "GEO", "INTERSECT", "TRANSFORM") + + /** Everything the REPL highlights and completes. */ + val all: Set[String] = sqlWords ++ extraWords +} diff --git a/core/src/test/scala/app/softnetwork/elastic/client/repl/ReplKeywordsSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/repl/ReplKeywordsSpec.scala new file mode 100644 index 00000000..8a6781fd --- /dev/null +++ b/core/src/test/scala/app/softnetwork/elastic/client/repl/ReplKeywordsSpec.scala @@ -0,0 +1,251 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client.repl + +import app.softnetwork.elastic.sql.SQLKeywords +import org.jline.reader.{Candidate, ParsedLine} +import org.jline.utils.AttributedStyle +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import java.util +import scala.jdk.CollectionConverters._ + +class ReplKeywordsSpec extends AnyFlatSpec with Matchers { + + private val keywordStyle = AttributedStyle.DEFAULT.foreground(AttributedStyle.BLUE).bold() + private val stringStyle = AttributedStyle.DEFAULT.foreground(AttributedStyle.GREEN) + private val numberStyle = AttributedStyle.DEFAULT.foreground(AttributedStyle.YELLOW) + + private def styleOf(buffer: String, token: String): AttributedStyle = { + // ReplHighlighter.highlight ignores the reader parameter — null is safe here. + val highlighted = new ReplHighlighter().highlight(null, buffer) + val idx = buffer.indexOf(token) + idx should be >= 0 + highlighted.styleAt(idx) + } + + /** The 54 keywords issue #161 reported as drifted, split to single words ("AS IS" -> AS, IS; + * "OUTER JOIN" -> OUTER, JOIN) — pinned verbatim. + */ + private val issue161Words: Set[String] = Set( + "ACTION", + "ALL", + "AS", + "AVG", + "BULK", + "BY", + "CASE", + "CAST", + "COALESCE", + "CONDITION", + "COPY", + "COUNT", + "DESCRIBE", + "DISTINCT", + "ELSE", + "END", + "EVERY", + "EXCEPT", + "EXECUTE", + "EXPLAIN", + "FALSE", + "GEO", + "GEO_MATCH", + "GROUP", + "HAVING", + "INTERSECT", + "INTO", + "IS", + "JOIN", + "LIMIT", + "LOG", + "MATCH", + "MAX", + "MIN", + "NULL", + "NULLIF", + "OFFSET", + "ON", + "ORDER", + "OUTER", + "RANGE", + "RENAME", + "SCHEDULE", + "SET", + "SHOW", + "SUM", + "THEN", + "TRUE", + "TRUNCATE", + "TYPE", + "UNION", + "VALUES", + "WEBHOOK", + "WHEN" + ) + + /** The pre-#161 hardcoded highlighter set — regression pin (AC 4). */ + private val legacyHighlighterWords: Set[String] = Set( + "SELECT", + "FROM", + "WHERE", + "INSERT", + "UPDATE", + "DELETE", + "CREATE", + "ALTER", + "DROP", + "TABLE", + "INDEX", + "VIEW", + "AND", + "OR", + "NOT", + "IN", + "LIKE", + "BETWEEN", + "JOIN", + "LEFT", + "RIGHT", + "INNER", + "OUTER", + "ENRICH", + "POLICY", + "WATCHER", + "TRANSFORM", + "PIPELINE" + ) + + // --- anti-drift (AC 3) -------------------------------------------------------------- + + "ReplKeywords" should "be a superset of the parser keyword registry" in { + val missing = SQLKeywords.highlightedWords.diff(ReplKeywords.all) + withClue(s"parser keywords invisible to the REPL: $missing\n") { missing shouldBe empty } + } + + it should "keep REPL-only extras disjoint from parser keywords" in { + val overlap = ReplKeywords.extraWords.intersect(SQLKeywords.highlightedWords) + withClue( + s"these words are parser keywords and must move out of extraWords: $overlap\n" + ) { overlap shouldBe empty } + } + + it should "cover all 54 keywords reported by issue #161" in { + val missing = issue161Words.diff(ReplKeywords.all) + withClue(s"issue #161 keywords still missing: $missing\n") { missing shouldBe empty } + } + + it should "cover the full legacy highlighter set (no regression)" in { + val missing = legacyHighlighterWords.diff(ReplKeywords.all) + withClue(s"legacy highlighter keywords lost: $missing\n") { missing shouldBe empty } + } + + it should "back every parser-derived compound continuation with the registry" in { + SQLKeywords.compoundPhrases should contain allOf ( + "ORDER BY", + "GROUP BY", + "PARTITION BY", + "UNION ALL", + "NULLS FIRST", + "NULLS LAST" + ) + } + + // --- highlighter behaviour (AC 1, 2, 4) ---------------------------------------------- + + "ReplHighlighter" should "colourise LIMIT like SELECT and FROM" in { + val buffer = "SELECT * FROM emp LIMIT 100" + styleOf(buffer, "SELECT") shouldBe keywordStyle + styleOf(buffer, "FROM") shouldBe keywordStyle + styleOf(buffer, "LIMIT") shouldBe keywordStyle + } + + it should "colourise ORDER BY, GROUP BY, HAVING, OFFSET, DISTINCT and UNION" in { + val buffer = + "SELECT DISTINCT dept FROM emp GROUP BY dept HAVING COUNT(*) > 1 " + + "ORDER BY dept OFFSET 5 UNION ALL SELECT dept FROM emp2" + Seq("DISTINCT", "GROUP", "BY", "HAVING", "ORDER", "OFFSET", "UNION", "ALL", "COUNT") + .foreach(word => styleOf(buffer, word) shouldBe keywordStyle) + } + + it should "colourise lowercase keywords identically" in { + val buffer = "select * from emp limit 100" + styleOf(buffer, "select") shouldBe keywordStyle + styleOf(buffer, "limit") shouldBe keywordStyle + } + + it should "keep number highlighting and string-literal handling unchanged (no regression)" in { + val buffer = "SELECT * FROM emp WHERE name = 'John' LIMIT 100" + styleOf(buffer, "100") shouldBe numberStyle + // Baseline behaviour pin: split("\\b") fragments quoted literals (" = '", "John", "' ") + // so the green '[^']*' branch never sees a whole 'John' token — the inner word renders + // as a plain identifier, NOT as a keyword. Pin exactly that (see Dev Notes gotcha 12). + styleOf(buffer, "John") shouldBe AttributedStyle.DEFAULT + // stringStyle is referenced so the green branch constant stays pinned at compile time + stringStyle shouldBe AttributedStyle.DEFAULT.foreground(AttributedStyle.GREEN) + } + + it should "not colourise plain identifiers or 1-letter aliases" in { + val buffer = "SELECT e.salary FROM emp e" + styleOf(buffer, "salary") shouldBe AttributedStyle.DEFAULT + styleOf(buffer, "emp") shouldBe AttributedStyle.DEFAULT + // "e" must NOT be highlighted even though EValue.sql == "E" + styleOf(buffer, "e.") shouldBe AttributedStyle.DEFAULT + } + + // --- completer behaviour (AC 6) ------------------------------------------------------ + + private def parsedLine(fullLine: String, currentWord: String): ParsedLine = + new ParsedLine { + override def word(): String = currentWord + override def wordCursor(): Int = currentWord.length + override def wordIndex(): Int = fullLine.trim.split("\\s+").length - 1 + override def words(): util.List[String] = fullLine.trim.split("\\s+").toList.asJava + override def line(): String = fullLine + override def cursor(): Int = fullLine.length + } + + private def completionsFor(fullLine: String, currentWord: String): Seq[String] = { + val candidates = new util.ArrayList[Candidate]() + new ReplCompleter().complete(null, parsedLine(fullLine, currentWord), candidates) + candidates.asScala.map(_.value).toSeq + } + + "ReplCompleter" should "offer LIMIT from the registry" in { + completionsFor("SELECT * FROM emp LIM", "LIM") should contain("LIMIT") + } + + it should "offer registry-only newcomers such as UNNEST and OVER" in { + completionsFor("SELECT * FROM UNN", "UNN") should contain("UNNEST") + completionsFor("SELECT RANK() OV", "OV") should contain("OVER") + } + + it should "still offer legacy extras (INTERSECT) and compound continuations" in { + completionsFor("SELECT 1 INTERS", "INTERS") should contain("INTERSECT") + // NOTE: the baseline compound path derives previousWord = words(length - 2) from the + // TRIMMED buffer, so it only triggers once the continuation is being typed + // ("ORDER B" -> previous "ORDER"), not on "ORDER " with an empty current word. + // Pin the working form — do not "fix" the empty-word case in this story. + completionsFor("SELECT * FROM emp ORDER B", "B") should contain("BY") + completionsFor("SELECT x FROM t WHERE x IS N", "N") should contain allOf ("NULL", "NOT NULL") + } + + it should "keep meta command completion working" in { + completionsFor(".he", ".he") should contain(".help") + } +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLKeywords.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/SQLKeywords.scala new file mode 100644 index 00000000..7b8ab1d9 --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/SQLKeywords.scala @@ -0,0 +1,534 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.sql + +import app.softnetwork.elastic.sql.function.aggregate.{ + ARRAY_AGG, + AVG, + COUNT, + DENSE_RANK, + FIRST_VALUE, + LAST_VALUE, + MAX, + MIN, + OVER, + PARTITION_BY, + PERCENTILE_CONT, + PERCENTILE_DISC, + RANK, + ROW_NUMBER, + STDDEV, + STDDEV_POP, + STDDEV_SAMP, + SUM, + VARIANCE, + VAR_POP, + VAR_SAMP +} +import app.softnetwork.elastic.sql.function.cond.{ + Case, + Coalesce, + ELSE, + END, + Greatest, + IsNotNull, + IsNull, + Least, + NullIf, + THEN, + WHEN +} +import app.softnetwork.elastic.sql.function.convert.{Cast, Convert, TryCast} +import app.softnetwork.elastic.sql.function.geo.{Distance, Point} +import app.softnetwork.elastic.sql.function.math.{ + Abs, + Acos, + Asin, + Atan, + Atan2, + Ceil, + Cos, + Degrees, + Exp, + Floor, + Log, + Log10, + Pow, + Radians, + Round, + Sign, + Sin, + Sqrt, + Tan +} +import app.softnetwork.elastic.sql.function.string.{ + Concat, + For, + LeftOp, + Length, + Lower, + Ltrim, + Position, + RegexpLike, + Replace, + Reverse, + RightOp, + Rtrim, + Substring, + Trim, + Upper +} +import app.softnetwork.elastic.sql.function.time.{ + CurrentDate, + CurrentTime, + CurrentTimestamp, + DateAdd, + DateDiff, + DateFormat, + DateParse, + DateSub, + DateTimeAdd, + DateTimeFormat, + DateTimeParse, + DateTimeSub, + DateTrunc, + Extract, + LastDayOfMonth, + Now, + Today +} +import app.softnetwork.elastic.sql.operator.{ + AGAINST, + AND, + BETWEEN, + Child, + IN, + IS_NOT_NULL, + IS_NULL, + LIKE, + MATCH, + NOT, + Nested, + OR, + Parent, + RLIKE, + UNION +} +import app.softnetwork.elastic.sql.query.{ + Asc, + CrossJoin, + Desc, + Except, + From, + FullJoin, + GroupBy, + Having, + InnerJoin, + Join, + LeftJoin, + Limit, + NullsFirst, + NullsLast, + Offset, + On, + OrderBy, + RightJoin, + Select, + Unnest, + Where +} +import app.softnetwork.elastic.sql.time.{Interval, IsoField, TimeField} + +/** Single source of truth for the SQL keywords the parser understands (#161). + * + * The parser has four keyword surfaces; this object curates them into one place: + * 1. `TokenRegex` objects -> `clauseTokens` / `functionTokens` / `literalTokens` + * 1. `Parser.keyword("…")` literals -> `statementWords` + * 1. type / literal parser regexes -> `typeWords` / `literalWords` + * 1. `Parser.reservedKeywords` -> read-only, tied in by `SQLKeywordsSpec` + * + * Anti-drift: `SQLKeywordsSpec` (sql) scans the parser sources; `ReplKeywordsSpec` (core) asserts + * the REPL highlighter/completer consume this registry. When you add a SQL keyword (a new + * `TokenRegex` object or a new `keyword("…")` literal), add it here or those tests fail. + * + * Deliberately NOT listed: pure-symbol tokens (`=`, `::`, `||`, `?`, arithmetic operators) and geo + * distance units (`km`, `m`, `cm`, `mm`, `mi`, `yd`, `ft`, `in`, `nmi`) — unit codes are + * 1–3-letter strings that collide with short identifiers and must not be colourised or completed. + * (The word `IN` still highlights — it arrives via the `operator.IN` token; only the unit *tokens* + * are unlisted.) + * + * Known scan limitation: literal-value tokens declared `extends Value[...] with TokenRegex` (Null, + * PiValue, RandomValue, EValue, ParamValue, IdValue, IngestTimestampValue) do not match the + * Expr-scan pattern in `SQLKeywordsSpec` — a NEW token of that shape must be added to + * `literalTokens` by hand. + */ +object SQLKeywords { + + /** Clause, join, operator and CASE syntax keywords (word-bearing TokenRegex objects). */ + val clauseTokens: List[TokenRegex] = List( + Select, + Distinct, + From, + Where, + GroupBy, + Having, + OrderBy, + Asc, + Desc, + NullsFirst, + NullsLast, + Limit, + Offset, + Alias, + Except, + UNION, + InnerJoin, + LeftJoin, + RightJoin, + FullJoin, + CrossJoin, + Join, + On, + Unnest, + AND, + OR, + NOT, + IN, + LIKE, + RLIKE, + BETWEEN, + IS_NULL, + IS_NOT_NULL, + MATCH, + AGAINST, + Nested, + Child, + Parent, + Case, + WHEN, + THEN, + ELSE, + END, + OVER, + PARTITION_BY, + Interval, + For + ) + + /** Function-name keywords (aggregate, window, conditional, conversion, math, string, temporal, + * geo — all are TokenRegex via the `Function`-derived traits). + */ + val functionTokens: List[TokenRegex] = List( + COUNT, + MIN, + MAX, + AVG, + SUM, + STDDEV, + STDDEV_POP, + STDDEV_SAMP, + VARIANCE, + VAR_POP, + VAR_SAMP, + PERCENTILE_CONT, + PERCENTILE_DISC, + FIRST_VALUE, + LAST_VALUE, + ARRAY_AGG, + ROW_NUMBER, + RANK, + DENSE_RANK, + Coalesce, + IsNull, + IsNotNull, + NullIf, + Greatest, + Least, + Cast, + TryCast, + Convert, + Abs, + Ceil, + Floor, + Round, + Exp, + Log, + Log10, + Pow, + Sqrt, + Sign, + Sin, + Asin, + Cos, + Acos, + Tan, + Atan, + Atan2, + Degrees, + Radians, + Concat, + Lower, + Upper, + Trim, + Ltrim, + Rtrim, + Substring, + LeftOp, + RightOp, + Length, + Replace, + Reverse, + Position, + RegexpLike, + CurrentDate, + CurrentTime, + CurrentTimestamp, + Now, + Today, + DateTrunc, + Extract, + LastDayOfMonth, + DateDiff, + DateAdd, + DateSub, + DateParse, + DateFormat, + DateTimeAdd, + DateTimeSub, + DateTimeParse, + DateTimeFormat, + Point, + Distance, + TimeField.YEAR, + TimeField.MONTH_OF_YEAR, + TimeField.DAY_OF_MONTH, + TimeField.DAY_OF_WEEK, + TimeField.DAY_OF_YEAR, + TimeField.HOUR_OF_DAY, + TimeField.MINUTE_OF_HOUR, + TimeField.SECOND_OF_MINUTE, + TimeField.NANO_OF_SECOND, + TimeField.MICRO_OF_SECOND, + TimeField.MILLI_OF_SECOND, + TimeField.EPOCH_DAY, + TimeField.OFFSET_SECONDS, + IsoField.QUARTER_OF_YEAR, + IsoField.WEEK_OF_WEEK_BASED_YEAR + ) + + /** Word-like literal value tokens (TokenRegex). `EValue` ("E") is listed for completeness but is + * dropped from `highlightedWords` by the length->=2 rule. + */ + val literalTokens: List[TokenRegex] = List(Null, PiValue, RandomValue, EValue) + + /** Boolean literals are parsed by `TypeParser.boolean` (type/package.scala:59-60), not via + * TokenRegex. + */ + val literalWords: Set[String] = Set("TRUE", "FALSE") + + /** Statement-level keywords matched by `Parser.keyword("…")` (Parser.scala:1118) across + * Parser.scala / DmlParser.scala. Curated copy — `SQLKeywordsSpec` scans the sources and fails + * if a `keyword("…")` literal is missing here. + */ + val statementWords: Set[String] = Set( + "ADD", + "ALIAS", + "ALTER", + "ALWAYS", + "AS", + "AT", + "BY", + "CLUSTER", + "COLUMN", + "COMMENT", + "CONFLICT", + "COPY", + "CREATE", + "DATA", + "DAY", + "DEFAULT", + "DELETE", + "DELTA_LAKE", + "DESC", + "DESCRIBE", + "DO", + "DROP", + "END", + "ENRICH", + "EVERY", + "EXECUTE", + "EXISTS", + "FIELD", + "FIELDS", + "FILE_FORMAT", + "FOREACH", + "FROM", + "GEO_MATCH", + "HOUR", + "IF", + "INDEX", + "INPUT", + "INPUTS", + "INSERT", + "INTO", + "JSON", + "JSON_ARRAY", + "KEY", + "LANG", + "LICENSE", + "LIKE", + "LIMIT", + "LOG", + "MAPPING", + "MATCH", + "MATERIALIZED", + "MINUTE", + "MONTH", + "NAME", + "NEVER", + "NOT", + "NOTHING", + "NOW", + "NULL", + "ON", + "OPTION", + "OPTIONS", + "OR", + "PARAMS", + "PARQUET", + "PARTITION", + "PIPELINE", + "PIPELINES", + "POLICIES", + "POLICY", + "PRIMARY", + "PROCESSOR", + "PROCESSORS", + "RANGE", + "REFRESH", + "RENAME", + "REPLACE", + "RETURNS", + "SCHEDULE", + "SCRIPT", + "SECOND", + "SET", + "SETTING", + "SHOW", + "STATUS", + "TABLE", + "TABLES", + "TO", + "TRUE", + "TRUNCATE", + "TYPE", + "UPDATE", + "USING", + "VALUES", + "VIEW", + "VIEWS", + "WATCHER", + "WATCHERS", + "WEBHOOK", + "WHEN", + "WITH", + "WITHIN", + "YEAR" + ) + + /** SQL type names accepted by `TypeParser` (parser/type/package.scala:91-164). */ + val typeWords: Set[String] = Set( + "ARRAY", + "BIGINT", + "BINARY", + "BOOLEAN", + "BYTE", + "CHAR", + "DATE", + "DATETIME", + "DOUBLE", + "FLOAT", + "GEOPOINT", + "GEO_POINT", + "INT", + "INTEGER", + "KEYWORD", + "LONG", + "REAL", + "SHORT", + "SMALLINT", + "STRING", + "STRUCT", + "TEXT", + "TIME", + "TIMESTAMP", + "TINYINT", + "VARBINARY", + "VARCHAR" + ) + + /** Plural time-unit forms. `TimeUnit`'s regex accepts an optional trailing `s` + * (`\\b(?i)$sql(s)?\\b`, time/package.scala:102), so `INTERVAL 2 DAYS` parses — the singular + * forms arrive via tokens/statementWords, the plurals are enumerated here (they are not + * derivable from any token's `words`). + */ + val timeUnitPluralWords: Set[String] = Set( + "YEARS", + "MONTHS", + "QUARTERS", + "WEEKS", + "DAYS", + "HOURS", + "MINUTES", + "SECONDS" + ) + + /** All word-bearing tokens, for test/introspection use. */ + val tokens: List[TokenRegex] = clauseTokens ++ functionTokens ++ literalTokens + + /** Normalize a token's `words` into single uppercase words: the literal regex separator `\s+` + * (e.g. `"LEFT\\s+OUTER"`, From.scala:45) and real whitespace (e.g. `"ORDER BY"`, `"NULLS + * FIRST"`) both split; non-word entries (`"\\:\\:"`, `"\\|\\|"`, `"?"`) are filtered out. + * Replace BEFORE uppercasing — uppercasing `"\\s+"` would corrupt the separator to `"\\S+"`. + */ + private[sql] def wordsOf(token: TokenRegex): List[String] = + token.words + .map(_.replaceAll("\\\\s\\+", " ")) + .flatMap(_.split("\\s+").toList) + .map(_.toUpperCase) + .filter(_.matches("[A-Z][A-Z0-9_]*")) + + /** Every single-word keyword known to the parser (uppercase). */ + lazy val allWords: Set[String] = + tokens.flatMap(wordsOf).toSet ++ statementWords ++ typeWords ++ literalWords ++ + timeUnitPluralWords + + /** Words REPL components colourise/complete: all parser keywords, minus 1-letter words (`E` would + * colourise every `e` table alias). + */ + lazy val highlightedWords: Set[String] = allWords.filter(_.length >= 2) + + /** Multi-word phrases (compound keywords) for completer use, e.g. "ORDER BY", "LEFT OUTER", "IS + * NOT NULL", "UNION ALL", "NULLS FIRST", "PARTITION BY". + */ + lazy val compoundPhrases: Set[String] = + tokens + .flatMap(_.words) + .map(_.replaceAll("\\\\s\\+", " ").trim.replaceAll("\\s+", " ").toUpperCase) + .filter(p => p.contains(" ") && p.split(" ").forall(_.matches("[A-Z][A-Z0-9_]*"))) + .toSet +} diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLKeywordsSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLKeywordsSpec.scala new file mode 100644 index 00000000..81c53ae1 --- /dev/null +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLKeywordsSpec.scala @@ -0,0 +1,217 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.sql + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import java.io.File +import scala.io.Source + +/** Anti-drift guard for #161: the registry must track every keyword surface of the parser. + * Source-scan tests locate the sql module sources relative to the working directory (sbt runs + * tests from the repo root; module-local runs are also handled) and are cancelled (not failed) + * when the source tree is absent, e.g. when tests run against a published jar. + */ +class SQLKeywordsSpec extends AnyFlatSpec with Matchers { + + // --- source location helpers ------------------------------------------------------- + + private val sourceRootCandidates = Seq( + new File("sql/src/main/scala/app/softnetwork/elastic/sql"), + new File("src/main/scala/app/softnetwork/elastic/sql") + ) + + private def sourceRoot: Option[File] = sourceRootCandidates.find(_.isDirectory) + + private def scalaFiles(dir: File): Seq[File] = { + val (dirs, files) = + Option(dir.listFiles).getOrElse(Array.empty[File]).toSeq.partition(_.isDirectory) + files.filter(_.getName.endsWith(".scala")) ++ dirs.flatMap(scalaFiles) + } + + private def read(f: File): String = { + val src = Source.fromFile(f, "UTF-8") + try src.mkString + finally src.close() + } + + // --- registry invariants ----------------------------------------------------------- + + "SQLKeywords" should "expose non-empty, normalized uppercase word sets" in { + SQLKeywords.allWords should not be empty + SQLKeywords.highlightedWords should not be empty + all(SQLKeywords.allWords) should fullyMatch regex "[A-Z][A-Z0-9_]*" + SQLKeywords.highlightedWords.foreach(_.length should be >= 2) + } + + it should "split multi-word tokens into their component words" in { + SQLKeywords.wordsOf(app.softnetwork.elastic.sql.query.OrderBy) shouldBe List("ORDER", "BY") + SQLKeywords.wordsOf(app.softnetwork.elastic.sql.query.GroupBy) shouldBe List("GROUP", "BY") + SQLKeywords.wordsOf(app.softnetwork.elastic.sql.operator.UNION) shouldBe List("UNION", "ALL") + // regex-alternate words: LEFT\s+OUTER | LEFT (From.scala:44-46) + SQLKeywords.wordsOf( + app.softnetwork.elastic.sql.query.LeftJoin + ) should contain allOf ("LEFT", "OUTER") + // lowercase sql forms are uppercased (Select.scala:118) + SQLKeywords.wordsOf(app.softnetwork.elastic.sql.query.Except) shouldBe List("EXCEPT") + } + + it should "expose compound phrases for completer use" in { + SQLKeywords.compoundPhrases should contain allOf ( + "ORDER BY", + "GROUP BY", + "PARTITION BY", + "UNION ALL", + "IS NULL", + "IS NOT NULL", + "NULLS FIRST", + "NULLS LAST", + "LEFT OUTER", + "RIGHT OUTER", + "FULL OUTER" + ) + } + + it should "cover the core clause keywords from issue #161" in { + SQLKeywords.highlightedWords should contain allOf ( + "LIMIT", + "ORDER", + "BY", + "GROUP", + "HAVING", + "OFFSET", + "AS", + "ON", + "DISTINCT", + "UNION", + "INTO", + "VALUES", + "SET" + ) + } + + it should "cover pluralised time units (TimeUnit's regex accepts a trailing s)" in { + SQLKeywords.highlightedWords should contain allOf ("YEARS", "DAYS", "HOURS", "MINUTES") + } + + it should "never colourise geo distance-unit codes or 1-letter words" in { + val banned = Set("KM", "CM", "MM", "MI", "YD", "FT", "NMI", "E", "M") + SQLKeywords.highlightedWords.intersect(banned) shouldBe empty + } + + // --- anti-drift: keyword("…") literals --------------------------------------------- + + it should "cover every keyword(\"…\") literal used by the parser (anti-drift)" in { + assume(sourceRoot.isDefined, "sql source tree not on disk - source-scan skipped") + // \s* tolerates scalafmt-wrapped calls — Parser.scala:832/850 really do contain + // `(keyword(` with the literal on the next line; without it a wrapped literal + // would silently escape the scan. + // digit-tolerant char class: a future keyword("LOG10")-style literal must not escape + val keywordLiteral = """keyword\(\s*"([A-Za-z_][A-Za-z0-9_]*)"\s*\)""".r + val found: Set[String] = + scalaFiles(sourceRoot.get) + .flatMap(f => keywordLiteral.findAllMatchIn(read(f)).map(_.group(1).toUpperCase)) + .toSet + found should not be empty + val missing = found.diff(SQLKeywords.allWords) + withClue( + s"parser keyword(...) literals missing from SQLKeywords (add to statementWords): $missing\n" + ) { missing shouldBe empty } + // reverse direction: statementWords is documented as an exact curated copy of the + // keyword("…") literals — a literal removed from the parser must be removed here too, + // or the registry (and the REPL highlighting derived from it) silently rots. + val stale = SQLKeywords.statementWords.diff(found) + withClue( + s"SQLKeywords.statementWords entries with no keyword(...) literal backing (remove them): $stale\n" + ) { stale shouldBe empty } + } + + // --- anti-drift: new Expr("...") keyword tokens ------------------------------------- + + it should "cover every word-like Expr token declared in the sql module (anti-drift)" in { + assume(sourceRoot.isDefined, "sql source tree not on disk - source-scan skipped") + // NOTE: literal-value tokens declared `extends Value[...] with TokenRegex` + // (Null, PiValue, RandomValue, EValue, ParamValue, IdValue, IngestTimestampValue) + // do NOT match this pattern (their `sql` is an override, not an Expr argument) — + // a NEW token of that shape must be added to SQLKeywords.literalTokens by hand. + // Tokens that are deliberately NOT keywords (see SQLKeywords scaladoc). + val excluded: Set[String] = Set( + // geo distance units (function/geo/package.scala:61-72) + "KM", + "M", + "CM", + "MM", + "MI", + "YD", + "FT", + "IN", + "NMI", + // parser delimiters (parser/Delimiter.scala:25-34) - lowercase case/when/then/end + // duplicates of the cond tokens; symbols are filtered by the word pattern anyway + "CASE", + "WHEN", + "THEN", + "END", + "E" + ) + // \s+/\s* tolerate scalafmt wraps around `extends Expr("…")`, mirroring the + // keyword("…") scan above (AD-7) — a wrapped declaration must not escape the scan. + val exprDecl = """case object \w+\s+extends\s+Expr\(\s*"([^"]+)"\s*\)""".r + val declaredWords: Set[String] = + scalaFiles(sourceRoot.get) + .flatMap(f => exprDecl.findAllMatchIn(read(f)).map(_.group(1))) + .map(_.replaceAll("\\\\s\\+", " ")) + .flatMap(_.split("\\s+")) + .map(_.toUpperCase) + .filter(_.matches("[A-Z][A-Z0-9_]*")) + .toSet + declaredWords should not be empty + val missing = declaredWords.diff(SQLKeywords.allWords ++ excluded) + withClue( + s"Expr(...) token words missing from SQLKeywords (add the token object to " + + s"clauseTokens/functionTokens/literalTokens, or to the exclusion list above " + + s"with a rationale): $missing\n" + ) { missing shouldBe empty } + } + + // --- anti-drift: Parser.reservedKeywords (read-only tie-in, AD-5) ------------------- + + it should "cover Parser.reservedKeywords (scraped - the val is private)" in { + assume(sourceRoot.isDefined, "sql source tree not on disk - source-scan skipped") + val parserFile = new File(sourceRoot.get, "parser/Parser.scala") + assume(parserFile.isFile, "parser/Parser.scala not found - source-scan skipped") + val text = read(parserFile) + val start = text.indexOf("reservedKeywords = Seq(") + start should be >= 0 + val block = text.substring(start, text.indexOf(")", start)) + val entry = """"([a-z_0-9]+)"""".r + val reserved: Set[String] = + block.linesIterator + .filterNot(_.trim.startsWith("//")) // skip commented-out entries + .flatMap(l => entry.findAllMatchIn(l).map(_.group(1).toUpperCase)) + .toSet + reserved should not be empty + // Reserved words with no parseable token/keyword backing at this baseline — + // reserved-only entries, pinned here; shrink this set, never grow it silently. + val reservedOnly = Set("FORMAT_DATE", "FORMAT_DATETIME", "CURRENT_DATETIME") + val missing = reserved.diff(SQLKeywords.allWords ++ reservedOnly) + withClue(s"Parser.reservedKeywords entries missing from SQLKeywords: $missing\n") { + missing shouldBe empty + } + } +}