From b6c4923c3c66920aad6717533ecbede0a183e4aa Mon Sep 17 00:00:00 2001 From: fc-anjos Date: Mon, 27 Jul 2026 11:29:46 -0300 Subject: [PATCH] feat(ffi): expose semantic analysis --- README.md | 29 ++++- docs/reference.md | 9 +- src/ffi.rs | 8 +- src/ffi/semantics.rs | 275 +++++++++++++++++++++++++++++++++++++++ src/lib.rs | 3 +- src/parser/mod.rs | 48 ++++++- src/parser/sql_parser.rs | 13 ++ tests/test_ffi.rs | 104 ++++++++++++++- 8 files changed, 479 insertions(+), 10 deletions(-) create mode 100644 src/ffi/semantics.rs diff --git a/README.md b/README.md index 40a179e..7fc07b2 100644 --- a/README.md +++ b/README.md @@ -358,9 +358,36 @@ const char *sqlglot_version(void); char *sqlglot_parse(const char *sql, const char *dialect); char *sqlglot_transpile(const char *sql, const char *from_dialect, const char *to_dialect); char *sqlglot_generate(const char *ast_json, const char *dialect); -void sqlglot_free(char *ptr); /* must be called on every non-NULL return */ +char *sqlglot_qualify_columns(const char *ast_json, const char *schema_json, const char *dialect); +char *sqlglot_build_scope(const char *ast_json); +char *sqlglot_lineage( + const char *column, + const char *ast_json, + const char *schema_json, + const char *config_json +); +void sqlglot_free(char *ptr); /* releases an owned API result; NULL-safe */ ``` +Schema-aware calls accept an ordered table-path transport: + +```json +{ + "tables": [ + { + "path": ["public", "cards"], + "columns": [ + {"name": "id", "data_type": "BIGINT"}, + {"name": "amount", "data_type": "DECIMAL(18, 4)"} + ] + } + ] +} +``` + +Identifier path components are explicit, column order is retained for wildcard +expansion, and `data_type` uses the parser's SQL data-type grammar. + ### C usage example ```c diff --git a/docs/reference.md b/docs/reference.md index aac077d..fb44be6 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -2217,13 +2217,16 @@ The C header is generated by `cbindgen` and placed at `target/ffi/include/sqlglo | `sqlglot_transpile` | `char *sqlglot_transpile(const char *sql, const char *from_dialect, const char *to_dialect)` | Transpile a single SQL statement between dialects. Returns `NULL` on error. | | `sqlglot_generate` | `char *sqlglot_generate(const char *ast_json, const char *dialect)` | Generate SQL from a JSON-serialised `Statement`. Returns `NULL` on error. | | `sqlglot_generate_pretty` | `char *sqlglot_generate_pretty(const char *ast_json, const char *dialect)` | Generate formatted SQL with indentation and newlines from a JSON-serialised `Statement`. Returns `NULL` on error. | +| `sqlglot_qualify_columns` | `char *sqlglot_qualify_columns(const char *ast_json, const char *schema_json, const char *dialect)` | Qualify columns and expand wildcards using an ordered table-path schema. Returns JSON or `NULL` on error. | +| `sqlglot_build_scope` | `char *sqlglot_build_scope(const char *ast_json)` | Build the semantic scope tree for a JSON-serialised `Statement`. Returns JSON or `NULL` on error. | +| `sqlglot_lineage` | `char *sqlglot_lineage(const char *column, const char *ast_json, const char *schema_json, const char *config_json)` | Build lineage for one requested output column using a schema and lineage configuration. Returns JSON or `NULL` on error. | | `sqlglot_version` | `const char *sqlglot_version(void)` | Return the library version as a static string. **Do not free.** | -| `sqlglot_free` | `void sqlglot_free(char *ptr)` | Free a string returned by any `sqlglot_*` function. `NULL`-safe. | +| `sqlglot_free` | `void sqlglot_free(char *ptr)` | Release an owned string returned by the API. `NULL`-safe. | ### Memory Management -- Every non-`NULL` `char *` returned by `sqlglot_parse`, `sqlglot_transpile`, `sqlglot_generate`, or `sqlglot_generate_pretty` **must** be freed by calling `sqlglot_free`. -- `sqlglot_version` returns a pointer to static memory — **do not** free it. +- The caller owns every non-`NULL` `char *` returned by the API and must release it with `sqlglot_free`. +- `sqlglot_version` returns a borrowed pointer to static storage and must not be freed. - Passing `NULL` to `sqlglot_free` is a safe no-op. ### Dialect Parameter diff --git a/src/ffi.rs b/src/ffi.rs index c87c294..7de70e3 100644 --- a/src/ffi.rs +++ b/src/ffi.rs @@ -5,8 +5,9 @@ //! //! # Memory management //! -//! Every `*mut c_char` returned by a function in this module **must** be freed -//! by calling [`sqlglot_free`]. Failing to do so will leak memory. +//! The caller owns every non-null `*mut c_char` returned by this API and must +//! release it with [`sqlglot_free`]. [`sqlglot_version`] returns a borrowed +//! pointer to static storage and must not be freed. use std::ffi::{CStr, CString}; use std::os::raw::c_char; @@ -14,6 +15,9 @@ use std::ptr; use crate::dialects::Dialect; +mod semantics; +pub use semantics::{sqlglot_build_scope, sqlglot_lineage, sqlglot_qualify_columns}; + // ── helpers ────────────────────────────────────────────────────────────── /// Convert a nullable C string to an `Option<&str>`. diff --git a/src/ffi/semantics.rs b/src/ffi/semantics.rs new file mode 100644 index 0000000..372d32a --- /dev/null +++ b/src/ffi/semantics.rs @@ -0,0 +1,275 @@ +//! C FFI bindings for schema-aware semantic APIs. + +use std::collections::{BTreeMap, HashMap}; +use std::os::raw::c_char; +use std::ptr; + +use serde::{Deserialize, Serialize}; + +use super::{cstr_to_option, resolve_dialect, to_c_string}; +use crate::ast::{Expr, Statement, TableRef}; +use crate::dialects::Dialect; +use crate::optimizer::lineage::{LineageConfig, LineageGraph, LineageNode, lineage}; +use crate::optimizer::qualify_columns::qualify_columns; +use crate::optimizer::scope_analysis::{ColumnRef, Scope, ScopeType, Source, build_scope}; +use crate::parser::parse_data_type; +use crate::schema::MappingSchema; + +#[derive(Serialize)] +struct ColumnRefView<'a> { + table: &'a Option, + name: &'a str, +} + +#[derive(Serialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +enum SourceView<'a> { + Table { table: &'a TableRef }, + Scope { scope: Box> }, +} + +#[derive(Serialize)] +struct ScopeView<'a> { + scope_type: &'static str, + sources: BTreeMap<&'a str, SourceView<'a>>, + columns: Vec>, + external_columns: Vec>, + derived_table_scopes: Vec>, + subquery_scopes: Vec>, + union_scopes: Vec>, + cte_scopes: Vec>, + selected_sources: BTreeMap<&'a str, SourceView<'a>>, + is_correlated: bool, +} + +#[derive(Serialize)] +struct LineageNodeView<'a> { + name: &'a str, + expression: &'a Option, + source_name: &'a Option, + source: &'a Option, + downstream: Vec>, + alias: &'a Option, + depth: usize, +} + +#[derive(Serialize)] +struct LineageGraphView<'a> { + node: LineageNodeView<'a>, + sql: &'a Option, + dialect: Dialect, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct LineageConfigInput { + dialect: Option, + trim_qualifiers: Option, + sources: Option>, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct MappingSchemaInput { + tables: Vec, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct TableInput { + path: Vec, + columns: Vec, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct ColumnInput { + name: String, + data_type: String, +} + +fn column_view(column: &ColumnRef) -> ColumnRefView<'_> { + ColumnRefView { + table: &column.table, + name: &column.name, + } +} + +fn source_view(source: &Source) -> SourceView<'_> { + match source { + Source::Table(table) => SourceView::Table { table }, + Source::Scope(scope) => SourceView::Scope { + scope: Box::new(scope_view(scope)), + }, + } +} + +fn sources_view(sources: &HashMap) -> BTreeMap<&str, SourceView<'_>> { + sources + .iter() + .map(|(name, source)| (name.as_str(), source_view(source))) + .collect() +} + +fn scope_view(scope: &Scope) -> ScopeView<'_> { + ScopeView { + scope_type: match scope.scope_type { + ScopeType::Root => "root", + ScopeType::Subquery => "subquery", + ScopeType::DerivedTable => "derived_table", + ScopeType::Cte => "cte", + ScopeType::Union => "union", + ScopeType::Udtf => "udtf", + }, + sources: sources_view(&scope.sources), + columns: scope.columns.iter().map(column_view).collect(), + external_columns: scope.external_columns.iter().map(column_view).collect(), + derived_table_scopes: scope.derived_table_scopes.iter().map(scope_view).collect(), + subquery_scopes: scope.subquery_scopes.iter().map(scope_view).collect(), + union_scopes: scope.union_scopes.iter().map(scope_view).collect(), + cte_scopes: scope.cte_scopes.iter().map(scope_view).collect(), + selected_sources: sources_view(&scope.selected_sources), + is_correlated: scope.is_correlated, + } +} + +fn lineage_node_view(node: &LineageNode) -> LineageNodeView<'_> { + LineageNodeView { + name: &node.name, + expression: &node.expression, + source_name: &node.source_name, + source: &node.source, + downstream: node.downstream.iter().map(lineage_node_view).collect(), + alias: &node.alias, + depth: node.depth, + } +} + +fn lineage_graph_view(graph: &LineageGraph) -> LineageGraphView<'_> { + LineageGraphView { + node: lineage_node_view(&graph.node), + sql: &graph.sql, + dialect: graph.dialect, + } +} + +fn parse_statement(json: &str) -> Option { + serde_json::from_str(json).ok() +} + +fn mapping_schema(json: &str, dialect: Dialect) -> Option { + let input: MappingSchemaInput = serde_json::from_str(json).ok()?; + let mut schema = MappingSchema::new(dialect); + + for table in input.tables { + let path: Vec<&str> = table.path.iter().map(String::as_str).collect(); + let typed_columns = table + .columns + .into_iter() + .map(|column| { + parse_data_type(&column.data_type, dialect) + .ok() + .map(|data_type| (column.name, data_type)) + }) + .collect::>>()?; + schema.replace_table(&path, typed_columns).ok()?; + } + + Some(schema) +} + +fn serialize(value: &T) -> *mut c_char { + serde_json::to_string(value) + .map(to_c_string) + .unwrap_or(ptr::null_mut()) +} + +/// Qualify a JSON-serialized SQLGlot statement using a JSON schema mapping. +/// +/// Returns a heap-allocated JSON statement, or `NULL` for invalid input. +/// +/// # Safety +/// +/// Each non-null argument must point to a readable NUL-terminated string that +/// remains valid for the duration of the call. Invalid UTF-8 or invalid input +/// returns `NULL`. Free a non-null return value with [`super::sqlglot_free`]. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn sqlglot_qualify_columns( + ast_json: *const c_char, + schema_json: *const c_char, + dialect: *const c_char, +) -> *mut c_char { + let Some(statement) = (unsafe { cstr_to_option(ast_json) }).and_then(parse_statement) else { + return ptr::null_mut(); + }; + let dialect = resolve_dialect(unsafe { cstr_to_option(dialect) }); + let Some(schema) = + (unsafe { cstr_to_option(schema_json) }).and_then(|json| mapping_schema(json, dialect)) + else { + return ptr::null_mut(); + }; + + serialize(&qualify_columns(statement, &schema)) +} + +/// Build a scope tree from a JSON-serialized SQLGlot statement. +/// +/// Returns a heap-allocated JSON scope, or `NULL` for invalid input. +/// +/// # Safety +/// +/// A non-null `ast_json` must point to a readable NUL-terminated string that +/// remains valid for the duration of the call. Invalid UTF-8 or invalid input +/// returns `NULL`. Free a non-null return value with [`super::sqlglot_free`]. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn sqlglot_build_scope(ast_json: *const c_char) -> *mut c_char { + let Some(statement) = (unsafe { cstr_to_option(ast_json) }).and_then(parse_statement) else { + return ptr::null_mut(); + }; + + serialize(&scope_view(&build_scope(&statement))) +} + +/// Build requested output-column lineage from JSON-serialized SQLGlot inputs. +/// +/// Returns a heap-allocated JSON lineage graph, or `NULL` for invalid input or +/// a lineage error. +/// +/// # Safety +/// +/// Each non-null argument must point to a readable NUL-terminated string that +/// remains valid for the duration of the call. Invalid UTF-8 or invalid input +/// returns `NULL`. Free a non-null return value with [`super::sqlglot_free`]. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn sqlglot_lineage( + column: *const c_char, + ast_json: *const c_char, + schema_json: *const c_char, + config_json: *const c_char, +) -> *mut c_char { + let Some(column) = (unsafe { cstr_to_option(column) }) else { + return ptr::null_mut(); + }; + let Some(statement) = (unsafe { cstr_to_option(ast_json) }).and_then(parse_statement) else { + return ptr::null_mut(); + }; + let Some(config) = (unsafe { cstr_to_option(config_json) }) + .and_then(|json| serde_json::from_str::(json).ok()) + else { + return ptr::null_mut(); + }; + let dialect = resolve_dialect(config.dialect.as_deref()); + let Some(schema) = + (unsafe { cstr_to_option(schema_json) }).and_then(|json| mapping_schema(json, dialect)) + else { + return ptr::null_mut(); + }; + let lineage_config = LineageConfig::new(dialect) + .with_sources(config.sources.unwrap_or_default()) + .with_trim_qualifiers(config.trim_qualifiers.unwrap_or(true)); + let Ok(graph) = lineage(column, &statement, &schema, &lineage_config) else { + return ptr::null_mut(); + }; + + serialize(&lineage_graph_view(&graph)) +} diff --git a/src/lib.rs b/src/lib.rs index 9a6d99d..b26fe13 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -119,8 +119,7 @@ pub use optimizer::lineage::{ }; pub use optimizer::pushdown_predicates::pushdown_predicates; pub use optimizer::scope_analysis::{Scope, ScopeType, build_scope, find_all_in_scope}; -pub use parser::parse; -pub use parser::{parse_statements_with_comments, parse_with_comments}; +pub use parser::{parse, parse_data_type, parse_statements_with_comments, parse_with_comments}; pub use planner::{Plan, Projection, Step, StepId, plan}; /// Validate that a transformed AST doesn't contain constructs unsupported by the target dialect. diff --git a/src/parser/mod.rs b/src/parser/mod.rs index c70fc56..a0ab056 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -2,7 +2,7 @@ mod sql_parser; pub use sql_parser::Parser; -use crate::ast::Statement; +use crate::ast::{DataType, Statement}; use crate::dialects::{Dialect, is_tsql_family}; use crate::errors::Result; @@ -17,6 +17,18 @@ pub fn parse(sql: &str, dialect: Dialect) -> Result { parser.parse_statement() } +/// Parse a standalone SQL data type using the same grammar as casts and +/// column definitions. +/// +/// # Errors +/// +/// Returns a [`SqlglotError`](crate::errors::SqlglotError) if the input is not +/// a valid data type or contains trailing tokens. +pub fn parse_data_type(sql: &str, dialect: Dialect) -> Result { + let mut parser = Parser::new_with_bracket_identifiers(sql, is_tsql_family(dialect))?; + parser.parse_data_type_expression() +} + /// Parse a SQL string into a [`Statement`] AST, preserving SQL comments. /// /// Comments are attached to the nearest AST node and survive through @@ -52,3 +64,37 @@ pub fn parse_statements_with_comments(sql: &str, dialect: Dialect) -> Result Result { + let data_type = self.parse_data_type()?; + if self.peek_type() != &TokenType::Eof { + return Err(SqlglotError::ParserError { + message: format!( + "Expected end of data type, got {:?}", + self.peek().token_type + ), + }); + } + Ok(data_type) + } + // ── Comment helpers ──────────────────────────────────────────── /// Consume any comment tokens at the current position, accumulating diff --git a/tests/test_ffi.rs b/tests/test_ffi.rs index 48e62a1..e3a4555 100644 --- a/tests/test_ffi.rs +++ b/tests/test_ffi.rs @@ -1,6 +1,17 @@ use std::ffi::{CStr, CString}; -use sqlglot_rust::ffi::{sqlglot_free, sqlglot_generate_pretty, sqlglot_parse}; +use sqlglot_rust::ffi::{ + sqlglot_build_scope, sqlglot_free, sqlglot_generate_pretty, sqlglot_lineage, sqlglot_parse, + sqlglot_qualify_columns, +}; + +fn read_json(pointer: *mut std::os::raw::c_char) -> serde_json::Value { + assert!(!pointer.is_null()); + let value = unsafe { CStr::from_ptr(pointer) }.to_str().unwrap(); + let parsed = serde_json::from_str(value).unwrap(); + unsafe { sqlglot_free(pointer) }; + parsed +} #[test] fn pretty_generation_is_available_through_the_ffi() { @@ -24,3 +35,94 @@ fn pretty_generation_is_available_through_the_ffi() { sqlglot_free(ast_json); } } + +#[test] +fn semantic_analysis_is_available_through_the_ffi() { + let sql = CString::new( + "WITH active_cards AS (SELECT id, project_id FROM cards) \ + SELECT projects.id FROM projects \ + JOIN active_cards ON active_cards.project_id = projects.id", + ) + .unwrap(); + let postgres = CString::new("postgres").unwrap(); + let schema = CString::new( + r#"{ + "tables": [ + { + "path": ["cards"], + "columns": [ + {"name": "id", "data_type": "BIGINT"}, + {"name": "project_id", "data_type": "DECIMAL(18, 4)"} + ] + }, + { + "path": ["projects"], + "columns": [ + {"name": "id", "data_type": "BIGINT"} + ] + } + ] + }"#, + ) + .unwrap(); + + let ast = unsafe { sqlglot_parse(sql.as_ptr(), postgres.as_ptr()) }; + let qualified = unsafe { sqlglot_qualify_columns(ast, schema.as_ptr(), postgres.as_ptr()) }; + let qualified_json = read_json(qualified); + assert_eq!( + qualified_json["Select"]["columns"][0]["Expr"]["expr"]["Column"]["table"], + "projects" + ); + + let qualified = CString::new(serde_json::to_string(&qualified_json).unwrap()).unwrap(); + let scope = read_json(unsafe { sqlglot_build_scope(qualified.as_ptr()) }); + assert_eq!(scope["sources"]["active_cards"]["kind"], "scope"); + assert!(scope["sources"]["active_cards"]["scope"].is_object()); + assert_eq!(scope["cte_scopes"][0]["scope_type"], "cte"); + + let column = CString::new("id").unwrap(); + let config = CString::new(r#"{"dialect":"postgres"}"#).unwrap(); + let graph = read_json(unsafe { + sqlglot_lineage( + column.as_ptr(), + qualified.as_ptr(), + schema.as_ptr(), + config.as_ptr(), + ) + }); + assert_eq!(graph["node"]["name"], "id"); + assert_eq!(graph["node"]["downstream"][0]["source_name"], "projects"); + + unsafe { sqlglot_free(ast) }; +} + +#[test] +fn schema_paths_do_not_split_dots_inside_identifiers() { + let sql = CString::new(r#"SELECT id FROM "events.v2""#).unwrap(); + let postgres = CString::new("postgres").unwrap(); + let schema = CString::new( + r#"{ + "tables": [ + { + "path": ["events.v2"], + "columns": [ + {"name": "id", "data_type": "TIMESTAMP(6) WITH TIME ZONE"} + ] + } + ] + }"#, + ) + .unwrap(); + + let ast = unsafe { sqlglot_parse(sql.as_ptr(), postgres.as_ptr()) }; + let qualified = unsafe { sqlglot_qualify_columns(ast, schema.as_ptr(), postgres.as_ptr()) }; + + assert!(!qualified.is_null()); + let qualified_json = read_json(qualified); + assert_eq!( + qualified_json["Select"]["columns"][0]["Expr"]["expr"]["Column"]["table"], + "events.v2" + ); + + unsafe { sqlglot_free(ast) }; +}