diff --git a/NAMESPACE b/NAMESPACE index 6b0b410..a0452c5 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -5,6 +5,7 @@ export(commons_server) export(commons_ui) export(context_layer) export(data_source) +export(data_source_options) export(list_tables) export(measure) export(semantic_layer) diff --git a/R/data-source-options.R b/R/data-source-options.R new file mode 100644 index 0000000..ce90261 --- /dev/null +++ b/R/data-source-options.R @@ -0,0 +1,128 @@ +#' Configure a database data source +#' +#' `data_source_options()` controls which objects a DBI-backed [data_source()] +#' exposes. Character values select literal table names. Use [DBI::Id()] to +#' preserve catalog and schema components or to select a name containing dots. +#' +#' @param include Exact tables to expose: a character vector, a [DBI::Id()], or +#' a list of `DBI::Id()` objects. An ID without a `table` component represents +#' a namespace; namespace expansion requires backend catalog support and is +#' not yet available. +#' @return A `commons_data_source_options` object for [data_source()]. +#' @export +data_source_options <- function(include = NULL) { + structure( + list(include = normalize_connection_includes(include)), + class = "commons_data_source_options" + ) +} + +check_data_source_options <- function( + options, + tables, + kind, + call = rlang::caller_env() +) { + if (is.null(options)) { + return(NULL) + } + if (!identical(kind, "connection")) { + cli::cli_abort( + "{.arg options} can only be used with a DBI connection.", + call = call + ) + } + if (!is.null(tables)) { + cli::cli_abort( + "Supply only one of {.arg tables} and {.arg options}.", + call = call + ) + } + if (!inherits(options, "commons_data_source_options")) { + cli::cli_abort( + "{.arg options} must come from {.fn data_source_options}.", + call = call + ) + } + options +} + +normalize_connection_includes <- function( + include, + call = rlang::caller_env() +) { + if (is.null(include)) { + return(NULL) + } + + entries <- if (inherits(include, "Id")) { + list(include) + } else if (is.character(include)) { + if (anyNA(include) || any(!nzchar(include))) { + cli::cli_abort( + "Character entries in {.arg include} must be non-empty table names.", + call = call + ) + } + lapply(include, function(table) DBI::Id(table = table)) + } else if (is.list(include)) { + include + } else { + cli::cli_abort( + paste0( + "{.arg include} must be a character vector, a {.cls DBI::Id}, ", + "or a list of {.cls DBI::Id} objects." + ), + call = call + ) + } + + if (length(entries) == 0) { + cli::cli_abort( + "{.arg include} must select at least one object.", + call = call + ) + } + if (!all(vapply(entries, inherits, logical(1), "Id"))) { + cli::cli_abort( + "Every entry in {.arg include} must be a {.cls DBI::Id}.", + call = call + ) + } + + for (id in entries) { + components <- id@name + if (any(is.na(components) | components == "")) { + cli::cli_abort( + "{.cls DBI::Id} entries in {.arg include} must have non-empty components.", + call = call + ) + } + } + + entries +} + +check_exact_connection_includes <- function( + include, + call = rlang::caller_env() +) { + prefixes <- vapply( + include, + function(id) is.na(id@name["table"]), + logical(1) + ) + if (any(prefixes)) { + cli::cli_abort( + c( + "Namespace selections in {.arg options} are not yet supported.", + i = paste0( + "Select exact tables by adding a {.arg table} component to each ", + "{.fn DBI::Id}; backend namespace expansion will be added separately." + ) + ), + call = call + ) + } + invisible(include) +} diff --git a/R/data-source.R b/R/data-source.R index b49a272..671c4ca 100644 --- a/R/data-source.R +++ b/R/data-source.R @@ -39,6 +39,9 @@ #' #' For a board, a named character vector of pins to read: the names become #' table names, and the values are pin names passed to [pins::pin_read()]. +#' @param options Database selection controls from [data_source_options()]. +#' This argument is supported only when `...` is a +#' DBI connection. Supply only one of `tables` and `options`. #' @param dictionary An optional path to a data dictionary describing the #' source's tables and columns, in the #' [data-dict.yaml](https://data-dict.tidyverse.org/) format. See the @@ -85,10 +88,11 @@ #' list_tables(src) #' #' @export -data_source <- function(..., tables = NULL, dictionary = NULL) { +data_source <- function(..., tables = NULL, dictionary = NULL, options = NULL) { dots <- rlang::list2(...) dictionary <- as_data_dictionary(dictionary) kind <- data_source_kind(dots) + options <- check_data_source_options(options, tables, kind) local_commons_span( "commons_data_source_create", @@ -96,7 +100,12 @@ data_source <- function(..., tables = NULL, dictionary = NULL) { ) if (kind == "connection") { - return(data_source_connection(dots[[1]], tables, dictionary = dictionary)) + return(data_source_connection( + dots[[1]], + tables, + dictionary = dictionary, + options = options + )) } if (kind == "board") { return(data_source_board(dots[[1]], tables, dictionary = dictionary)) @@ -139,18 +148,39 @@ data_source_connection <- function( con, tables, dictionary = NULL, + options = NULL, call = rlang::caller_env() ) { span <- local_commons_span("commons_data_source_list_tables") - if (is.null(tables)) { + include <- options$include %||% tables + if (is.null(include)) { listed <- DBI::dbListTables(con) commons_span_set_attribute(span, "commons.data_source.n_tables", length(listed)) - return(new_data_source(con, listed, owned = FALSE, dictionary = dictionary)) + return(new_data_source( + con, + listed, + owned = FALSE, + dictionary = dictionary, + options = options + )) } - table_registry <- normalize_table_registry(tables, call = call) - check_table_ids_exist(con, table_registry, call = call) + if (!is.null(options)) { + check_exact_connection_includes(include, call = call) + } + argument <- if (is.null(options)) "tables" else "include" + table_registry <- normalize_table_registry( + include, + call = call, + argument = argument + ) + check_table_ids_exist( + con, + table_registry, + call = call, + argument = argument + ) commons_span_set_attribute( span, "commons.data_source.n_tables", @@ -162,7 +192,8 @@ data_source_connection <- function( table_registry$labels, owned = FALSE, table_ids = table_registry$ids, - dictionary = dictionary + dictionary = dictionary, + options = options ) } @@ -241,7 +272,8 @@ new_data_source <- function( owned, table_ids = table_ids_from_labels(tables), dictionary = NULL, - pending = NULL + pending = NULL, + options = NULL ) { # Disconnect only the DuckDB connection we created; a user-supplied connection # has its own owner and lifetime. @@ -263,7 +295,8 @@ new_data_source <- function( table_ids = table_ids, handle = handle, dictionary = dictionary, - pending = pending + pending = pending, + options = options ), class = "commons_data_source" ) @@ -561,7 +594,11 @@ duckdb_connect <- function() { con } -normalize_table_registry <- function(tables, call = rlang::caller_env()) { +normalize_table_registry <- function( + tables, + call = rlang::caller_env(), + argument = "tables" +) { entries <- table_entries(tables, call = call) ids <- lapply(entries, table_entry_id, call = call) labels <- vapply(ids, table_id_label, character(1), call = call) @@ -569,7 +606,7 @@ normalize_table_registry <- function(tables, call = rlang::caller_env()) { duplicated_labels <- unique(labels[duplicated(labels)]) if (length(duplicated_labels)) { cli::cli_abort( - "{.arg tables} must not contain duplicate labels: {.val {duplicated_labels}}.", + "{.arg {argument}} must not contain duplicate labels: {.val {duplicated_labels}}.", call = call ) } @@ -661,7 +698,12 @@ table_ids_from_labels <- function(tables) { # startup against a remote warehouse. Probe every table in a single zero-row # query instead, and fall back to per-table checks only to name the missing # tables when that probe fails. -check_table_ids_exist <- function(con, table_registry, call = rlang::caller_env()) { +check_table_ids_exist <- function( + con, + table_registry, + call = rlang::caller_env(), + argument = "tables" +) { probes <- vapply( table_registry$ids, function(id) { @@ -689,14 +731,14 @@ check_table_ids_exist <- function(con, table_registry, call = rlang::caller_env( if (length(missing) == 0) { cli::cli_abort( - "Failed to verify {.arg tables} against the connection.", + "Failed to verify {.arg {argument}} against the connection.", parent = probe_error, call = call ) } cli::cli_abort( - "{.arg tables} names table{?s} not on the connection: {.val {missing}}.", + "{.arg {argument}} names table{?s} not on the connection: {.val {missing}}.", call = call ) } diff --git a/man/data_source.Rd b/man/data_source.Rd index 500ba9e..ce15354 100644 --- a/man/data_source.Rd +++ b/man/data_source.Rd @@ -4,7 +4,7 @@ \alias{data_source} \title{Create a data source} \usage{ -data_source(..., tables = NULL, dictionary = NULL) +data_source(..., tables = NULL, dictionary = NULL, options = NULL) } \arguments{ \item{...}{A single DBI connection, a single \code{pins} board, or named data @@ -27,6 +27,10 @@ table names, and the values are pin names passed to \code{\link[pins:pin_read]{p source's tables and columns, in the \href{https://data-dict.tidyverse.org/}{data-dict.yaml} format. See the \verb{Data dictionaries} section.} + +\item{options}{Database selection controls from \code{\link[=data_source_options]{data_source_options()}}. +This argument is supported only when \code{...} is a +DBI connection. Supply only one of \code{tables} and \code{options}.} } \value{ A \code{commons_data_source} object. diff --git a/man/data_source_options.Rd b/man/data_source_options.Rd new file mode 100644 index 0000000..f81c698 --- /dev/null +++ b/man/data_source_options.Rd @@ -0,0 +1,22 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/data-source-options.R +\name{data_source_options} +\alias{data_source_options} +\title{Configure a database data source} +\usage{ +data_source_options(include = NULL) +} +\arguments{ +\item{include}{Exact tables to expose: a character vector, a \code{\link[DBI:Id]{DBI::Id()}}, or +a list of \code{DBI::Id()} objects. An ID without a \code{table} component represents +a namespace; namespace expansion requires backend catalog support and is +not yet available.} +} +\value{ +A \code{commons_data_source_options} object for \code{\link[=data_source]{data_source()}}. +} +\description{ +\code{data_source_options()} controls which objects a DBI-backed \code{\link[=data_source]{data_source()}} +exposes. Character values select literal table names. Use \code{\link[DBI:Id]{DBI::Id()}} to +preserve catalog and schema components or to select a name containing dots. +} diff --git a/tests/testthat/test-data-source-options.R b/tests/testthat/test-data-source-options.R new file mode 100644 index 0000000..0579e2a --- /dev/null +++ b/tests/testthat/test-data-source-options.R @@ -0,0 +1,102 @@ +test_that("data_source_options normalizes exact selectors", { + options <- data_source_options(include = c("sales", "crm.sales")) + + expect_s3_class(options, "commons_data_source_options") + expect_identical( + lapply(options$include, function(id) id@name), + list(c(table = "sales"), c(table = "crm.sales")) + ) + qualified <- DBI::Id(catalog = "analytics", schema = "crm", table = "sales") + expect_identical(data_source_options(qualified)$include[[1]], qualified) +}) + +test_that("data_source_options validates its values", { + expect_error(data_source_options(include = ""), "non-empty table names") + expect_error( + data_source_options(include = NA_character_), + "non-empty table names" + ) + expect_error( + data_source_options(include = character()), + "at least one object" + ) + expect_error(data_source_options(include = 1), "must be a character vector") + expect_error( + data_source_options(include = list("sales")), + "Every entry.*DBI::Id" + ) + expect_error( + data_source_options(include = DBI::Id(schema = "")), + "non-empty components" + ) +}) + +test_that("options character selectors are literal table names", { + con <- DBI::dbConnect(duckdb::duckdb()) + withr::defer(DBI::dbDisconnect(con, shutdown = TRUE)) + DBI::dbExecute(con, 'CREATE TABLE "crm.sales" (order_id VARCHAR)') + DBI::dbExecute(con, 'INSERT INTO "crm.sales" VALUES (\'o01\')') + + src <- data_source( + con, + options = data_source_options(include = "crm.sales") + ) + + expect_equal(list_tables(src), "crm.sales") + expect_identical(src$table_ids[["crm.sales"]]@name, c(table = "crm.sales")) + expect_equal( + source_describe(src, "crm.sales", n_sample = 1)$sample$order_id, + "o01" + ) +}) + +test_that("options preserve and quote structured identifiers", { + con <- DBI::dbConnect(duckdb::duckdb()) + withr::defer(DBI::dbDisconnect(con, shutdown = TRUE)) + DBI::dbExecute(con, 'CREATE SCHEMA "crm raw"') + id <- DBI::Id(schema = "crm raw", table = "sales.daily") + quoted <- DBI::dbQuoteIdentifier(con, id) + DBI::dbExecute(con, paste("CREATE TABLE", quoted, "(order_id VARCHAR)")) + DBI::dbExecute(con, paste("INSERT INTO", quoted, "VALUES ('o01')")) + + src <- data_source(con, options = data_source_options(include = id)) + + expect_equal(list_tables(src), "crm raw.sales.daily") + expect_identical(src$table_ids[["crm raw.sales.daily"]], id) + expect_equal( + source_describe(src, "crm raw.sales.daily", n_sample = 1)$sample$order_id, + "o01" + ) +}) + +test_that("data_source options are confined to DBI connections", { + con <- DBI::dbConnect(duckdb::duckdb()) + withr::defer(DBI::dbDisconnect(con, shutdown = TRUE)) + DBI::dbWriteTable(con, "sales", data.frame(id = 1L)) + options <- data_source_options(include = "sales") + + expect_error( + data_source(con, tables = "sales", options = options), + "only one of `tables` and `options`" + ) + expect_error( + data_source(con, options = list(include = "sales")), + "must come from" + ) + expect_error( + data_source(sales = data.frame(id = 1L), options = options), + "only be used with a DBI connection" + ) +}) + +test_that("namespace options stop at the backend expansion boundary", { + options <- data_source_options(include = DBI::Id(schema = "crm")) + expect_identical(options$include[[1]], DBI::Id(schema = "crm")) + + con <- DBI::dbConnect(duckdb::duckdb()) + withr::defer(DBI::dbDisconnect(con, shutdown = TRUE)) + expect_error( + data_source(con, options = options), + "Namespace selections in `options` are not yet supported" + ) +})