diff --git a/.licenserc.yaml b/.licenserc.yaml index ea80177032..b25fbaceb0 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -31,6 +31,8 @@ header: - "**/DEPENDENCIES.*.tsv" # Generated content by cargo-public-api - "**/public-api.txt" + # Generated content by trybuild + - "**/*.stderr" # Release distributions - "dist/*" - "target" diff --git a/.typos.toml b/.typos.toml index e9fa0028f5..cde21f7091 100644 --- a/.typos.toml +++ b/.typos.toml @@ -21,6 +21,7 @@ extend-ignore-identifiers-re = ["^bimap$"] [default.extend-words] AGS = "AGS" ags = "ags" +ser = "ser" [files] extend-exclude = ["**/testdata", "CHANGELOG.md", "**/public-api.txt"] diff --git a/Cargo.lock b/Cargo.lock index 633534233e..8d939f2894 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4002,11 +4002,24 @@ dependencies = [ "mimalloc", "stacker", "tokio", - "toml", + "toml 0.8.23", "tracing", "tracing-subscriber", ] +[[package]] +name = "iceberg-property-macro" +version = "0.10.0" +dependencies = [ + "iceberg", + "proc-macro2", + "quote", + "serde", + "serde_json", + "syn", + "trybuild", +] + [[package]] name = "iceberg-sqllogictest" version = "0.10.0" @@ -4025,7 +4038,7 @@ dependencies = [ "serde", "sqllogictest", "tokio", - "toml", + "toml 0.8.23", "tracing", ] @@ -6922,6 +6935,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +dependencies = [ + "serde_core", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -7663,6 +7685,12 @@ version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" +[[package]] +name = "target-triple" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3a6bfce3d99adfa72d24750a61f782f3036a81e7f86d8841ee1326deaebd171" + [[package]] name = "tempfile" version = "3.27.0" @@ -7676,6 +7704,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + [[package]] name = "termtree" version = "0.5.1" @@ -7887,11 +7924,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" dependencies = [ "serde", - "serde_spanned", + "serde_spanned 0.6.9", "toml_datetime 0.6.11", "toml_edit 0.22.27", ] +[[package]] +name = "toml" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c96ecdfa941c8fc4fcaed14f99ada8ebed502eef533015095a07e3301d4c3c" +dependencies = [ + "indexmap 2.14.0", + "serde_core", + "serde_spanned 1.1.1", + "toml_datetime 1.1.1+spec-1.1.0", + "toml_parser", + "toml_writer", + "winnow 1.0.3", +] + [[package]] name = "toml_datetime" version = "0.6.11" @@ -7918,7 +7970,7 @@ checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ "indexmap 2.14.0", "serde", - "serde_spanned", + "serde_spanned 0.6.9", "toml_datetime 0.6.11", "toml_write", "winnow 0.7.15", @@ -7951,6 +8003,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" +[[package]] +name = "toml_writer" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" + [[package]] name = "tower" version = "0.5.3" @@ -8090,6 +8148,21 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "trybuild" +version = "1.0.120" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e605bf6b39357663d8ba4e984f8be8da8df6bb32e81031d6889024ea8fd68e4" +dependencies = [ + "glob", + "serde", + "serde_derive", + "serde_json", + "target-triple", + "termcolor", + "toml 1.1.3+spec-1.1.0", +] + [[package]] name = "twox-hash" version = "2.1.2" diff --git a/Cargo.toml b/Cargo.toml index a789ef1967..8e69861a35 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ members = [ "crates/iceberg", "crates/integration_tests", "crates/integrations/*", + "crates/property-macro", "crates/sqllogictest", "crates/storage/*", "crates/test_utils", @@ -125,7 +126,9 @@ ordered-float = "4" parquet = "58.4" pilota = "0.11.10" pretty_assertions = "1.4" +proc-macro2 = "1" pyo3 = "0.28" +quote = "1" rand = "0.9.3" regex = "1.11.3" reqwest = { version = "0.12.12", default-features = false, features = ["json"] } @@ -141,6 +144,7 @@ sqllogictest = "0.29" sqlx = { version = "0.8.1", default-features = false } stacker = "0.1.20" strum = "0.27.2" +syn = "2" tempfile = "3.18" thrift = "0.17.0" tokio = { version = "1.47", default-features = false, features = [ @@ -150,6 +154,7 @@ tokio = { version = "1.47", default-features = false, features = [ toml = "0.8" tracing = "0.1.41" tracing-subscriber = "0.3.20" +trybuild = "1" typed-builder = "0.20" typetag = "0.2" url = "2.5.7" diff --git a/crates/iceberg/src/lib.rs b/crates/iceberg/src/lib.rs index 301992d15e..43473cc35e 100644 --- a/crates/iceberg/src/lib.rs +++ b/crates/iceberg/src/lib.rs @@ -65,6 +65,7 @@ #[macro_use] extern crate derive_builder; extern crate core; +extern crate self as iceberg; mod error; pub use error::{Error, ErrorKind, Result}; diff --git a/crates/property-macro/Cargo.toml b/crates/property-macro/Cargo.toml new file mode 100644 index 0000000000..700b2ecf44 --- /dev/null +++ b/crates/property-macro/Cargo.toml @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +edition = { workspace = true } +homepage = { workspace = true } +name = "iceberg-property-macro" +publish = false +readme = "README.md" +rust-version = { workspace = true } +version = { workspace = true } + +license = { workspace = true } +repository = { workspace = true } + +categories = ["database"] +description = "Property derive macro for Apache Iceberg Rust" +keywords = ["iceberg"] + +[lib] +proc-macro = true + +[dependencies] +proc-macro2 = { workspace = true } +quote = { workspace = true } +syn = { workspace = true, features = ["full"] } + +[dev-dependencies] +iceberg = { workspace = true } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +trybuild = { workspace = true } + +[lints] +workspace = true diff --git a/crates/property-macro/README.md b/crates/property-macro/README.md new file mode 100644 index 0000000000..e85e9194b0 --- /dev/null +++ b/crates/property-macro/README.md @@ -0,0 +1,264 @@ + + +# Iceberg property derive macro + +`Properties` parses a typed struct from a flat `HashMap` and +can generate opt-in read-only getters. It deliberately does not generate +property-map serialization or implement `Default`, `Serialize`, `Deserialize`, +or any other trait. + +## Generated API + +For every annotated struct, `#[derive(Properties)]` generates this inherent +constructor: + +```text +impl MyProperties { + pub fn from_properties( + properties: &HashMap, + ) -> iceberg::Result; +} +``` + +`from_properties` borrows the source map, parses every modeled property, and +uses its annotated default when a property is absent. Unknown keys are ignored. +An invalid value returns an `iceberg::Error` with `ErrorKind::DataInvalid` and a +message containing its primary property key. + +Adding `getter` to a field generates a public immutable accessor with the field +name. Primitive `Copy` types, references, pointers, and compositions of those +types return `T`; other types return `&T`. Because a procedural macro cannot +resolve trait implementations, a user-defined `Copy` type returns `&T`. +Documentation attributes on the field are copied to the generated getter. The +macro generates no setters, backing fields, or conversion back to a property +map. + +## Complete example + +This example covers exact keys and defaults, optional values, case-insensitive +booleans, prefixed maps, nested groups, custom single-value parsing, custom +multi-key parsing, lists of additional keys, read-only getters, ignored unknown +keys, and contextual errors. + +```rust +use std::collections::HashMap; + +use iceberg::{Error, ErrorKind}; +use iceberg_property_macro::Properties; + +const RETRIES: &str = "commit.retry.num-retries"; +const OWNER: &str = "owner"; +const FANOUT: &str = "write.datafusion.fanout.enabled"; +const COLUMN_FPP_PREFIX: &str = "write.parquet.bloom-filter-fpp.column."; +const LOCATION: &str = "write.data.path"; +const WIDTH: &str = "dimensions.width"; +const HEIGHT: &str = "dimensions.height"; +const DEPTH: &str = "dimensions.depth"; + +fn parse_location(value: &str) -> iceberg::Result { + let location = value.trim().trim_end_matches('/'); + if location.is_empty() { + Err(Error::new( + ErrorKind::DataInvalid, + "location must not be empty", + )) + } else { + Ok(location.to_string()) + } +} + +fn parse_dimensions( + properties: &HashMap, + width_key: &str, + additional_keys: &[&str], + default: (u64, u64, u64), +) -> iceberg::Result<(u64, u64, u64)> { + if additional_keys.len() != 2 { + return Err(Error::new( + ErrorKind::DataInvalid, + "dimensions require height and depth keys", + )); + } + let parse = |key: &str, default| { + properties + .get(key) + .map(|value| { + value.parse::().map_err(|error| { + Error::new(ErrorKind::DataInvalid, error.to_string()) + }) + }) + .transpose() + .map(|value| value.unwrap_or(default)) + }; + + Ok(( + parse(width_key, default.0)?, + parse(additional_keys[0], default.1)?, + parse(additional_keys[1], default.2)?, + )) +} + +#[derive(Debug, Properties)] +struct CommitProperties { + /// Maximum number of times to retry a commit. + #[property(key = RETRIES, default = 4, getter)] + retries: usize, +} + +#[derive(Debug, Properties)] +struct TableLikeProperties { + /// Nested groups parse from the same flat property map. + #[property(nested, getter)] + commit: CommitProperties, + + /// Option distinguishes an absent property from a present value. + #[property(key = OWNER, default = None, getter)] + owner: Option, + + /// This DataFusion-specific boolean is parsed case-insensitively. + /// Its `true` default is engine-specific, rather than an Iceberg-wide default. + #[property(key = FANOUT, default = true, getter)] + fanout_enabled: bool, + + /// A prefix captures suffix/value pairs into a typed map. + #[property(prefix = COLUMN_FPP_PREFIX, getter)] + column_fpp: HashMap, + + /// A single-key parser can validate and normalize a property value. + #[property( + key = LOCATION, + default = "warehouse", + parse_with = parse_location, + getter + )] + location: String, + + /// A full-map parser can model one field with multiple property keys. + #[property( + key = WIDTH, + additional_keys = [HEIGHT, DEPTH], + default = (640, 480, 320), + parse_properties_with = parse_dimensions, + getter + )] + dimensions: (u64, u64, u64), +} + +fn main() -> iceberg::Result<()> { + let defaults = TableLikeProperties::from_properties(&HashMap::new())?; + assert_eq!(defaults.commit().retries(), 4); + assert_eq!(defaults.owner(), &None); + assert!(defaults.fanout_enabled()); + assert!(defaults.column_fpp().is_empty()); + assert_eq!(defaults.location(), "warehouse"); + assert_eq!(defaults.dimensions(), (640, 480, 320)); + + let raw = HashMap::from([ + (RETRIES.to_string(), "8".to_string()), + (OWNER.to_string(), "iceberg".to_string()), + (FANOUT.to_string(), "FALSE".to_string()), + (format!("{COLUMN_FPP_PREFIX}id"), "0.01".to_string()), + (LOCATION.to_string(), " s3://bucket/table/ ".to_string()), + (WIDTH.to_string(), "1920".to_string()), + (HEIGHT.to_string(), "1080".to_string()), + (DEPTH.to_string(), "720".to_string()), + ("unmodeled".to_string(), "ignored".to_string()), + ]); + + let properties = TableLikeProperties::from_properties(&raw)?; + assert_eq!(properties.commit().retries(), 8); + assert_eq!(properties.owner().as_deref(), Some("iceberg")); + assert!(!properties.fanout_enabled()); + assert_eq!(properties.column_fpp()["id"], 0.01); + assert_eq!(properties.location(), "s3://bucket/table"); + assert_eq!(properties.dimensions(), (1920, 1080, 720)); + + let error = TableLikeProperties::from_properties(&HashMap::from([( + LOCATION.to_string(), + "/".to_string(), + )])) + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::DataInvalid); + assert!(format!("{error}").contains(LOCATION)); + + Ok(()) +} +``` + +## Using ordinary derives together + +`Properties` does not implicitly derive other traits, so `Default`, +`Serialize`, and `Deserialize` can be selected independently and behave like +ordinary Rust derives: + +```rust +use std::collections::HashMap; + +use iceberg_property_macro::Properties; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Default, Serialize, Deserialize, Properties)] +struct ReadProperties { + #[property( + key = "commit.retry.num-retries", + default = 4, + getter + )] + retries: u64, +} + +fn main() -> Result<(), Box> { + // The property annotation supplies the default used by from_properties. + let properties = ReadProperties::from_properties(&HashMap::new())?; + assert_eq!(properties.retries(), 4); + + // The ordinary Default derive uses the field's Rust default instead. + let defaults = ReadProperties::default(); + assert_eq!(defaults.retries(), 0); + + // Ordinary Serde derives use Rust field names, not property keys. + let json = serde_json::to_string(&properties)?; + assert_eq!(json, r#"{"retries":4}"#); + let decoded: ReadProperties = serde_json::from_str(r#"{"retries":7}"#)?; + assert_eq!(decoded.retries(), 7); + Ok(()) +} +``` + +All field settings must be grouped under `#[property(...)]`. This keeps `key`, +`default`, `prefix`, `nested`, parser hooks, additional keys, and getter +generation in one attribute and avoids collisions with ordinary Rust derives. + +Exact-key fields require a `default`. The `prefix` setting requires +`HashMap` and returns the entries matched by the prefix, or an empty +map when none match. `nested` embeds another `Properties` struct while reading +the same flat map. Neither prefix nor nested fields accept a `default`. +`parse_with` customizes parsing for one exact-key field; on an `Option` field, +the parser produces `T` and the macro wraps a present value in `Some`. +`parse_properties_with` receives the complete property map, and its required +`additional_keys` setting supplies the list of secondary keys. + +Boolean values are parsed case-insensitively. Other values require `FromStr` +unless a custom parser is supplied. String-literal and path defaults are +converted into their field type with `Into`. + +`from_properties` and both custom parser hooks use `iceberg::Result`. Generated +`FromStr` failures use `ErrorKind::DataInvalid`. The macro preserves errors from +custom parsers and adds the primary property key as error context. diff --git a/crates/property-macro/src/lib.rs b/crates/property-macro/src/lib.rs new file mode 100644 index 0000000000..e73b48c0f7 --- /dev/null +++ b/crates/property-macro/src/lib.rs @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#![doc = include_str!("../README.md")] + +use proc_macro::TokenStream; +use syn::{DeriveInput, parse_macro_input}; + +mod properties; + +/// Derives property-map parsing and opt-in read-only accessors for a struct. +#[proc_macro_derive(Properties, attributes(property))] +pub fn derive_properties(input: TokenStream) -> TokenStream { + let input = parse_macro_input!(input as DeriveInput); + + match properties::expand_properties(input) { + Ok(tokens) => tokens.into(), + Err(error) => error.into_compile_error().into(), + } +} diff --git a/crates/property-macro/src/properties.rs b/crates/property-macro/src/properties.rs new file mode 100644 index 0000000000..03820311fb --- /dev/null +++ b/crates/property-macro/src/properties.rs @@ -0,0 +1,657 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use proc_macro2::TokenStream as TokenStream2; +use quote::quote; +use syn::parse::{Parse, ParseStream}; +use syn::punctuated::Punctuated; +use syn::{ + Attribute, Data, DeriveInput, Error, Expr, ExprLit, ExprPath, Field, Fields, GenericArgument, + Ident, Lit, Path, PathArguments, Token, Type, +}; + +struct PropertyField { + ident: Ident, + ty: Type, + key: Option, + additional_keys: Option>, + prefix: Option, + nested: bool, + default: Option, + parse_with: Option, + parse_properties_with: Option, + option_inner_type: Option, + map_value_type: Option, + public_getter: bool, + doc_attributes: Vec, +} + +enum PropertyOption { + Key(Expr), + AdditionalKeys(Vec), + Prefix(Expr), + Nested, + Default(Expr), + ParseWith(Path), + ParsePropertiesWith(Path), + Getter, +} + +#[derive(Default)] +struct PropertyOptions { + key: Option, + additional_keys: Option>, + prefix: Option, + nested: bool, + default: Option, + parse_with: Option, + parse_properties_with: Option, + public_getter: bool, +} + +impl Parse for PropertyOption { + fn parse(input: ParseStream<'_>) -> syn::Result { + let name = input.parse::()?; + let option_name = name.to_string(); + if option_name == "nested" { + return Ok(Self::Nested); + } + if option_name == "getter" { + return Ok(Self::Getter); + } + + input.parse::()?; + let expression = input.parse::()?; + match option_name.as_str() { + "key" => Ok(Self::Key(expression)), + "additional_keys" => { + expression_list(expression, "additional_keys").map(Self::AdditionalKeys) + } + "prefix" => Ok(Self::Prefix(expression)), + "default" => Ok(Self::Default(expression)), + "parse_with" => expression_path(expression, "parse_with").map(Self::ParseWith), + "parse_properties_with" => { + expression_path(expression, "parse_properties_with").map(Self::ParsePropertiesWith) + } + _ => Err(Error::new_spanned(name, "unknown property option")), + } + } +} + +pub(crate) fn expand_properties(input: DeriveInput) -> syn::Result { + let struct_name = input.ident; + let generics = input.generics; + let fields = match input.data { + Data::Struct(data) => match data.fields { + Fields::Named(fields) => fields.named, + _ => { + return Err(Error::new_spanned( + struct_name, + "Properties can only be derived for structs with named fields", + )); + } + }, + _ => { + return Err(Error::new_spanned( + struct_name, + "Properties can only be derived for structs", + )); + } + }; + + let fields = fields + .iter() + .map(|field| parse_property_field(field, property_options(field)?)) + .collect::>>()?; + let parses = fields + .iter() + .map(parse_field) + .collect::>>()?; + let accessors = fields.iter().map(field_getter); + let (impl_generics, type_generics, where_clause) = generics.split_for_impl(); + + Ok(quote! { + impl #impl_generics #struct_name #type_generics #where_clause { + #(#accessors)* + + pub fn from_properties( + properties: &::std::collections::HashMap< + ::std::string::String, + ::std::string::String, + >, + ) -> ::iceberg::Result { + Ok(Self { + #(#parses,)* + }) + } + } + }) +} + +fn parse_property_field( + field: &Field, + property_options: PropertyOptions, +) -> syn::Result { + let ident = field + .ident + .clone() + .ok_or_else(|| Error::new_spanned(field, "Properties fields must be named"))?; + let PropertyOptions { + key, + additional_keys, + prefix, + nested, + default, + parse_with, + parse_properties_with, + public_getter, + } = property_options; + + if usize::from(key.is_some()) + usize::from(prefix.is_some()) + usize::from(nested) != 1 { + return Err(Error::new_spanned( + field, + "Properties fields must declare exactly one of key, prefix, or nested in #[property(...)]", + )); + } + + if nested && default.is_some() { + return Err(Error::new_spanned( + field, + "nested fields obtain defaults from their own property annotations and cannot declare default in #[property(...)]", + )); + } + if prefix.is_some() && default.is_some() { + return Err(Error::new_spanned( + field, + "prefix fields collect matching properties and cannot declare default in #[property(...)]", + )); + } + if key.is_some() && default.is_none() { + return Err(Error::new_spanned( + field, + "Properties key fields must declare default in #[property(...)]", + )); + } + + let map_value_type = hash_map_value_type(&field.ty); + if prefix.is_some() && map_value_type.is_none() { + return Err(Error::new_spanned( + &field.ty, + "property prefix fields must have type HashMap", + )); + } + + if additional_keys.is_some() && parse_properties_with.is_none() { + return Err(Error::new_spanned( + field, + "additional_keys requires parse_properties_with in #[property(...)]", + )); + } + if parse_properties_with.is_some() && additional_keys.is_none() { + return Err(Error::new_spanned( + field, + "parse_properties_with requires additional_keys in #[property(...)]", + )); + } + if (prefix.is_some() || nested) + && (additional_keys.is_some() || parse_with.is_some() || parse_properties_with.is_some()) + { + return Err(Error::new_spanned( + field, + "prefix and nested fields do not support custom parse functions", + )); + } + if parse_with.is_some() && parse_properties_with.is_some() { + return Err(Error::new_spanned( + field, + "fields cannot declare both parse_with and parse_properties_with", + )); + } + Ok(PropertyField { + ident, + ty: field.ty.clone(), + key, + additional_keys, + prefix, + nested, + default, + parse_with, + parse_properties_with, + option_inner_type: option_inner_type(&field.ty), + map_value_type, + public_getter, + doc_attributes: field + .attrs + .iter() + .filter(|attribute| attribute.path().is_ident("doc")) + .cloned() + .collect(), + }) +} + +fn property_options(field: &Field) -> syn::Result { + let Some(attribute) = find_attribute(&field.attrs, "property")? else { + return Err(Error::new_spanned( + field, + "Properties fields must declare #[property(...)]", + )); + }; + + let parsed = + attribute.parse_args_with(Punctuated::::parse_terminated)?; + if parsed.is_empty() { + return Err(Error::new_spanned( + attribute, + "property must declare at least one option", + )); + } + + let mut options = PropertyOptions::default(); + for option in parsed { + match option { + PropertyOption::Key(value) => { + set_property_option(&mut options.key, value, attribute, "key")? + } + PropertyOption::AdditionalKeys(value) => set_property_option( + &mut options.additional_keys, + value, + attribute, + "additional_keys", + )?, + PropertyOption::Prefix(value) => { + set_property_option(&mut options.prefix, value, attribute, "prefix")? + } + PropertyOption::Nested => { + if options.nested { + return Err(Error::new_spanned( + attribute, + "duplicate nested property option", + )); + } + options.nested = true; + } + PropertyOption::Default(value) => { + set_property_option(&mut options.default, value, attribute, "default")? + } + PropertyOption::ParseWith(value) => { + set_property_option(&mut options.parse_with, value, attribute, "parse_with")? + } + PropertyOption::ParsePropertiesWith(value) => set_property_option( + &mut options.parse_properties_with, + value, + attribute, + "parse_properties_with", + )?, + PropertyOption::Getter => { + if options.public_getter { + return Err(Error::new_spanned(attribute, "duplicate property accessor")); + } + options.public_getter = true; + } + } + } + + Ok(options) +} + +fn set_property_option( + target: &mut Option, + value: T, + attribute: &Attribute, + name: &str, +) -> syn::Result<()> { + if target.is_some() { + return Err(Error::new_spanned( + attribute, + format!("duplicate {name} property option"), + )); + } + *target = Some(value); + Ok(()) +} + +fn field_getter(field: &PropertyField) -> TokenStream2 { + if !field.public_getter { + return TokenStream2::new(); + } + let ident = &field.ident; + let ty = &field.ty; + let docs = &field.doc_attributes; + if is_copy_type(ty) { + quote! { + #(#docs)* + pub fn #ident(&self) -> #ty { + self.#ident + } + } + } else { + quote! { + #(#docs)* + pub fn #ident(&self) -> &#ty { + &self.#ident + } + } + } +} + +fn expression_path(expression: Expr, name: &str) -> syn::Result { + match expression { + Expr::Path(ExprPath { path, .. }) => Ok(path), + _ => Err(Error::new_spanned( + expression, + format!("{name} must be a path"), + )), + } +} + +fn expression_list(expression: Expr, name: &str) -> syn::Result> { + let Expr::Array(array) = expression else { + return Err(Error::new_spanned( + expression, + format!("{name} must be an array of keys"), + )); + }; + if array.elems.is_empty() { + return Err(Error::new_spanned( + array, + format!("{name} must contain at least one key"), + )); + } + Ok(array.elems.into_iter().collect()) +} + +fn find_attribute<'a>( + attributes: &'a [Attribute], + name: &str, +) -> syn::Result> { + let mut matching = attributes + .iter() + .filter(|attribute| attribute.path().is_ident(name)); + let first = matching.next(); + if let Some(duplicate) = matching.next() { + return Err(Error::new_spanned( + duplicate, + format!("duplicate #[{name}] attribute"), + )); + } + Ok(first) +} + +fn parse_field(field: &PropertyField) -> syn::Result { + let ident = &field.ident; + if field.nested { + let ty = &field.ty; + return Ok(quote!(#ident: <#ty>::from_properties(properties)?)); + } + + let ty = &field.ty; + + if let Some(parse_properties_with) = &field.parse_properties_with { + let key = field.key.as_ref().ok_or_else(|| { + Error::new_spanned( + &field.ident, + "parse_properties_with fields must declare key", + ) + })?; + let additional_keys = field.additional_keys.as_ref().ok_or_else(|| { + Error::new_spanned( + &field.ident, + "parse_properties_with fields must declare additional_keys", + ) + })?; + let default = typed_default(field)?; + return Ok(quote! { + #ident: { + let parsed: ::iceberg::Result<#ty> = #parse_properties_with( + properties, + #key, + &[#(#additional_keys),*], + #default, + ); + parsed.map_err(|error| error.with_context("property", #key))? + } + }); + } + + if let Some(prefix) = &field.prefix { + let value_type = field.map_value_type.as_ref().ok_or_else(|| { + Error::new_spanned( + &field.ty, + "property prefix fields must have type HashMap", + ) + })?; + let parse = if is_bool(value_type) { + quote!(value.to_ascii_lowercase().parse::<#value_type>()) + } else { + quote!(value.parse::<#value_type>()) + }; + return Ok(quote! { + #ident: properties + .iter() + .filter_map(|(key, value)| { + key.strip_prefix(#prefix).map(|suffix| { + #parse + .map(|parsed| (suffix.to_string(), parsed)) + .map_err(|error| { + ::iceberg::Error::new( + ::iceberg::ErrorKind::DataInvalid, + format!("Invalid value for {key}: {error}"), + ) + }) + }) + }) + .collect::<::iceberg::Result<::std::collections::HashMap<_, _>>>()? + }); + } + + let key = field + .key + .as_ref() + .ok_or_else(|| Error::new_spanned(&field.ident, "property fields must declare key"))?; + let default = typed_default(field)?; + let parse = match (&field.parse_with, &field.option_inner_type) { + (Some(parse_with), Some(inner_type)) => quote! { + { + let parsed: ::iceberg::Result<#inner_type> = #parse_with(value); + Some(parsed.map_err(|error| error.with_context("property", #key))?) + } + }, + (Some(parse_with), None) => quote! { + { + let parsed: ::iceberg::Result<#ty> = #parse_with(value); + parsed.map_err(|error| error.with_context("property", #key))? + } + }, + (None, Some(inner_type)) if is_bool(inner_type) => quote! { + Some(value.to_ascii_lowercase().parse::<#inner_type>().map_err(|error| { + ::iceberg::Error::new( + ::iceberg::ErrorKind::DataInvalid, + format!("Invalid value for {}: {error}", #key), + ) + })?) + }, + (None, Some(inner_type)) => quote! { + Some(value.parse::<#inner_type>().map_err(|error| { + ::iceberg::Error::new( + ::iceberg::ErrorKind::DataInvalid, + format!("Invalid value for {}: {error}", #key), + ) + })?) + }, + (None, None) if is_bool(ty) => quote! { + value.to_ascii_lowercase().parse::<#ty>().map_err(|error| { + ::iceberg::Error::new( + ::iceberg::ErrorKind::DataInvalid, + format!("Invalid value for {}: {error}", #key), + ) + })? + }, + (None, None) => quote! { + value.parse::<#ty>().map_err(|error| { + ::iceberg::Error::new( + ::iceberg::ErrorKind::DataInvalid, + format!("Invalid value for {}: {error}", #key), + ) + })? + }, + }; + + Ok(quote! { + #ident: match properties.get(#key) { + Some(value) => #parse, + None => #default, + } + }) +} + +fn typed_default(field: &PropertyField) -> syn::Result { + let ty = &field.ty; + let default = field.default.as_ref().ok_or_else(|| { + Error::new_spanned(&field.ident, "property key fields must declare default") + })?; + let default = default_value(default, ty); + Ok(quote!({ + let value: #ty = #default; + value + })) +} + +fn default_value(default: &Expr, ty: &Type) -> TokenStream2 { + if matches!( + default, + Expr::Lit(ExprLit { + lit: Lit::Str(_), + .. + }) | Expr::Path(_) + ) { + quote!(::std::convert::Into::<#ty>::into(#default)) + } else { + quote!(#default) + } +} + +fn option_inner_type(ty: &Type) -> Option { + let Type::Path(type_path) = ty else { + return None; + }; + + let segment = type_path.path.segments.last()?; + if segment.ident != "Option" { + return None; + } + + let PathArguments::AngleBracketed(arguments) = &segment.arguments else { + return None; + }; + let Some(GenericArgument::Type(inner_type)) = arguments.args.first() else { + return None; + }; + + Some(inner_type.clone()) +} + +fn hash_map_value_type(ty: &Type) -> Option { + let Type::Path(type_path) = ty else { + return None; + }; + + let segment = type_path.path.segments.last()?; + if segment.ident != "HashMap" { + return None; + } + + let PathArguments::AngleBracketed(arguments) = &segment.arguments else { + return None; + }; + let mut arguments = arguments.args.iter(); + let Some(GenericArgument::Type(key_type)) = arguments.next() else { + return None; + }; + let Some(GenericArgument::Type(value_type)) = arguments.next() else { + return None; + }; + if !is_named_type(key_type, "String") { + return None; + } + + Some(value_type.clone()) +} + +fn is_bool(ty: &Type) -> bool { + is_named_type(ty, "bool") +} + +fn is_copy_type(ty: &Type) -> bool { + match ty { + Type::Array(array) => is_copy_type(&array.elem), + Type::BareFn(_) | Type::Never(_) | Type::Ptr(_) => true, + Type::Group(group) => is_copy_type(&group.elem), + Type::Paren(paren) => is_copy_type(&paren.elem), + Type::Reference(reference) => reference.mutability.is_none(), + Type::Tuple(tuple) => tuple.elems.iter().all(is_copy_type), + Type::Path(type_path) if type_path.qself.is_none() => { + let Some(segment) = type_path.path.segments.last() else { + return false; + }; + if matches!( + segment.ident.to_string().as_str(), + "bool" + | "char" + | "f32" + | "f64" + | "i8" + | "i16" + | "i32" + | "i64" + | "i128" + | "isize" + | "u8" + | "u16" + | "u32" + | "u64" + | "u128" + | "usize" + ) { + return true; + } + if segment.ident != "Option" && segment.ident != "Result" { + return false; + } + let PathArguments::AngleBracketed(arguments) = &segment.arguments else { + return false; + }; + arguments.args.iter().all(|argument| match argument { + GenericArgument::Lifetime(_) => true, + GenericArgument::Type(ty) => is_copy_type(ty), + _ => false, + }) + } + _ => false, + } +} + +fn is_named_type(ty: &Type, name: &str) -> bool { + let Type::Path(type_path) = ty else { + return false; + }; + + type_path + .path + .segments + .last() + .is_some_and(|segment| segment.ident == name) +} diff --git a/crates/property-macro/tests/compile-fail/additional_keys_without_parse_properties.rs b/crates/property-macro/tests/compile-fail/additional_keys_without_parse_properties.rs new file mode 100644 index 0000000000..3036fd3b96 --- /dev/null +++ b/crates/property-macro/tests/compile-fail/additional_keys_without_parse_properties.rs @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +use iceberg_property_macro::Properties; + +#[derive(Properties)] +struct AdditionalKeysWithoutParser { + #[property(key = "value", additional_keys = ["other"], default = 1)] + value: u64, +} + +fn main() {} diff --git a/crates/property-macro/tests/compile-fail/additional_keys_without_parse_properties.stderr b/crates/property-macro/tests/compile-fail/additional_keys_without_parse_properties.stderr new file mode 100644 index 0000000000..905ffb0043 --- /dev/null +++ b/crates/property-macro/tests/compile-fail/additional_keys_without_parse_properties.stderr @@ -0,0 +1,6 @@ +error: additional_keys requires parse_properties_with in #[property(...)] + --> tests/compile-fail/additional_keys_without_parse_properties.rs:23:5 + | +23 | / #[property(key = "value", additional_keys = ["other"], default = 1)] +24 | | value: u64, + | |______________^ diff --git a/crates/property-macro/tests/compile-fail/missing_property.rs b/crates/property-macro/tests/compile-fail/missing_property.rs new file mode 100644 index 0000000000..07ed0f770b --- /dev/null +++ b/crates/property-macro/tests/compile-fail/missing_property.rs @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +use iceberg_property_macro::Properties; + +#[derive(Properties)] +struct MissingProperty { + value: u64, +} + +fn main() {} diff --git a/crates/property-macro/tests/compile-fail/missing_property.stderr b/crates/property-macro/tests/compile-fail/missing_property.stderr new file mode 100644 index 0000000000..3555f484d6 --- /dev/null +++ b/crates/property-macro/tests/compile-fail/missing_property.stderr @@ -0,0 +1,5 @@ +error: Properties fields must declare #[property(...)] + --> tests/compile-fail/missing_property.rs:23:5 + | +23 | value: u64, + | ^^^^^^^^^^ diff --git a/crates/property-macro/tests/compile-fail/multiple_sources.rs b/crates/property-macro/tests/compile-fail/multiple_sources.rs new file mode 100644 index 0000000000..97aefaa567 --- /dev/null +++ b/crates/property-macro/tests/compile-fail/multiple_sources.rs @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +use std::collections::HashMap; + +use iceberg_property_macro::Properties; + +#[derive(Properties)] +struct MultipleSources { + #[property(key = "value", prefix = "values.", default = HashMap::new())] + value: HashMap, +} + +fn main() {} diff --git a/crates/property-macro/tests/compile-fail/multiple_sources.stderr b/crates/property-macro/tests/compile-fail/multiple_sources.stderr new file mode 100644 index 0000000000..2e4c67f868 --- /dev/null +++ b/crates/property-macro/tests/compile-fail/multiple_sources.stderr @@ -0,0 +1,6 @@ +error: Properties fields must declare exactly one of key, prefix, or nested in #[property(...)] + --> tests/compile-fail/multiple_sources.rs:25:5 + | +25 | / #[property(key = "value", prefix = "values.", default = HashMap::new())] +26 | | value: HashMap, + | |_______________________________^ diff --git a/crates/property-macro/tests/compile-fail/nested_default.rs b/crates/property-macro/tests/compile-fail/nested_default.rs new file mode 100644 index 0000000000..601b1df1e4 --- /dev/null +++ b/crates/property-macro/tests/compile-fail/nested_default.rs @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +use iceberg_property_macro::Properties; + +#[derive(Properties)] +struct Inner { + #[property(key = "value", default = 1)] + value: u64, +} + +#[derive(Properties)] +struct NestedDefault { + #[property(nested, default = unreachable!())] + inner: Inner, +} + +fn main() {} diff --git a/crates/property-macro/tests/compile-fail/nested_default.stderr b/crates/property-macro/tests/compile-fail/nested_default.stderr new file mode 100644 index 0000000000..6db683674a --- /dev/null +++ b/crates/property-macro/tests/compile-fail/nested_default.stderr @@ -0,0 +1,6 @@ +error: nested fields obtain defaults from their own property annotations and cannot declare default in #[property(...)] + --> tests/compile-fail/nested_default.rs:29:5 + | +29 | / #[property(nested, default = unreachable!())] +30 | | inner: Inner, + | |________________^ diff --git a/crates/property-macro/tests/compile-fail/parse_properties_without_additional_keys.rs b/crates/property-macro/tests/compile-fail/parse_properties_without_additional_keys.rs new file mode 100644 index 0000000000..96dd5b18d2 --- /dev/null +++ b/crates/property-macro/tests/compile-fail/parse_properties_without_additional_keys.rs @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +use std::collections::HashMap; + +use iceberg_property_macro::Properties; + +fn parse_all( + _properties: &HashMap, + _key: &str, + _additional_keys: &[&str], + default: u64, +) -> iceberg::Result { + Ok(default) +} + +#[derive(Properties)] +struct ParsePropertiesWithoutAdditionalKeys { + #[property(key = "value", default = 1, parse_properties_with = parse_all)] + value: u64, +} + +fn main() {} diff --git a/crates/property-macro/tests/compile-fail/parse_properties_without_additional_keys.stderr b/crates/property-macro/tests/compile-fail/parse_properties_without_additional_keys.stderr new file mode 100644 index 0000000000..ff2168f92f --- /dev/null +++ b/crates/property-macro/tests/compile-fail/parse_properties_without_additional_keys.stderr @@ -0,0 +1,6 @@ +error: parse_properties_with requires additional_keys in #[property(...)] + --> tests/compile-fail/parse_properties_without_additional_keys.rs:34:5 + | +34 | / #[property(key = "value", default = 1, parse_properties_with = parse_all)] +35 | | value: u64, + | |______________^ diff --git a/crates/property-macro/tests/compile-fail/prefix_default.rs b/crates/property-macro/tests/compile-fail/prefix_default.rs new file mode 100644 index 0000000000..d7c4ec3397 --- /dev/null +++ b/crates/property-macro/tests/compile-fail/prefix_default.rs @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +use std::collections::HashMap; + +use iceberg_property_macro::Properties; + +#[derive(Properties)] +struct PrefixDefault { + #[property(prefix = "values.", default = HashMap::new())] + values: HashMap, +} + +fn main() {} diff --git a/crates/property-macro/tests/compile-fail/prefix_default.stderr b/crates/property-macro/tests/compile-fail/prefix_default.stderr new file mode 100644 index 0000000000..9dcddded2d --- /dev/null +++ b/crates/property-macro/tests/compile-fail/prefix_default.stderr @@ -0,0 +1,6 @@ +error: prefix fields collect matching properties and cannot declare default in #[property(...)] + --> tests/compile-fail/prefix_default.rs:25:5 + | +25 | / #[property(prefix = "values.", default = HashMap::new())] +26 | | values: HashMap, + | |________________________________^ diff --git a/crates/property-macro/tests/compile-fail/prefix_non_map.rs b/crates/property-macro/tests/compile-fail/prefix_non_map.rs new file mode 100644 index 0000000000..1d904daece --- /dev/null +++ b/crates/property-macro/tests/compile-fail/prefix_non_map.rs @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +use iceberg_property_macro::Properties; + +#[derive(Properties)] +struct PrefixNonMap { + #[property(prefix = "values.")] + value: u64, +} + +fn main() {} diff --git a/crates/property-macro/tests/compile-fail/prefix_non_map.stderr b/crates/property-macro/tests/compile-fail/prefix_non_map.stderr new file mode 100644 index 0000000000..72627bc592 --- /dev/null +++ b/crates/property-macro/tests/compile-fail/prefix_non_map.stderr @@ -0,0 +1,5 @@ +error: property prefix fields must have type HashMap + --> tests/compile-fail/prefix_non_map.rs:24:12 + | +24 | value: u64, + | ^^^ diff --git a/crates/property-macro/tests/compile_fail.rs b/crates/property-macro/tests/compile_fail.rs new file mode 100644 index 0000000000..3d692ce893 --- /dev/null +++ b/crates/property-macro/tests/compile_fail.rs @@ -0,0 +1,21 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#[test] +fn reports_invalid_property_annotations() { + trybuild::TestCases::new().compile_fail("tests/compile-fail/*.rs"); +} diff --git a/crates/property-macro/tests/properties.rs b/crates/property-macro/tests/properties.rs new file mode 100644 index 0000000000..2ced3b7f73 --- /dev/null +++ b/crates/property-macro/tests/properties.rs @@ -0,0 +1,304 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::HashMap; +use std::str::FromStr; + +use iceberg::{Error, ErrorKind}; +use iceberg_property_macro::Properties; +use serde::{Deserialize, Serialize}; + +const RETRIES: &str = "commit.retry.num-retries"; +const OWNER: &str = "owner"; +const FORMAT: &str = "write.format.default"; +const FANOUT_ENABLED: &str = "write.datafusion.fanout.enabled"; +const COLUMN_FPP_PREFIX: &str = "write.parquet.bloom-filter-fpp.column."; +const WIDTH: &str = "dimensions.width"; +const HEIGHT: &str = "dimensions.height"; +const DEPTH: &str = "dimensions.depth"; + +fn parse_dimensions( + properties: &HashMap, + width_key: &str, + additional_keys: &[&str], + default: (u64, u64, u64), +) -> iceberg::Result<(u64, u64, u64)> { + if additional_keys.len() != 2 { + return Err(Error::new( + ErrorKind::DataInvalid, + "dimensions require height and depth keys", + )); + } + let parse = |property_key: &str, default| { + properties + .get(property_key) + .map(|value| { + value + .parse::() + .map_err(|error| Error::new(ErrorKind::DataInvalid, error.to_string())) + }) + .transpose() + .map(|value| value.unwrap_or(default)) + }; + + Ok(( + parse(width_key, default.0)?, + parse(additional_keys[0], default.1)?, + parse(additional_keys[1], default.2)?, + )) +} + +#[derive(Debug, Properties)] +struct TestProperties { + #[property(key = RETRIES, default = 4, getter)] + retries: u64, + + #[property(key = OWNER, default = None, getter)] + owner: Option, + + #[property(key = FORMAT, default = "parquet", getter)] + format: String, + + #[property(key = FANOUT_ENABLED, default = true, getter)] + fanout_enabled: bool, + + #[property( + prefix = COLUMN_FPP_PREFIX, + getter + )] + column_fpp: HashMap, + + #[property( + key = WIDTH, + additional_keys = [HEIGHT, DEPTH], + default = (640, 480, 320), + parse_properties_with = parse_dimensions, + getter + )] + dimensions: (u64, u64, u64), +} + +#[test] +fn reads_defaults_through_generated_getters() { + let properties = TestProperties::from_properties(&HashMap::new()).unwrap(); + + assert_eq!(properties.retries(), 4); + assert_eq!(properties.owner(), &None); + assert_eq!(properties.format(), "parquet"); + assert!(properties.fanout_enabled()); + assert!(properties.column_fpp().is_empty()); + assert_eq!(properties.dimensions(), (640, 480, 320)); +} + +#[test] +fn reads_overrides_and_ignores_unknown_properties() { + let raw = HashMap::from([ + (RETRIES.to_string(), "8".to_string()), + (OWNER.to_string(), "iceberg".to_string()), + (FORMAT.to_string(), "orc".to_string()), + (FANOUT_ENABLED.to_string(), "FALSE".to_string()), + (format!("{COLUMN_FPP_PREFIX}id"), "0.01".to_string()), + (WIDTH.to_string(), "1920".to_string()), + (HEIGHT.to_string(), "1080".to_string()), + (DEPTH.to_string(), "720".to_string()), + ("unknown".to_string(), "ignored".to_string()), + ]); + let properties = TestProperties::from_properties(&raw).unwrap(); + + assert_eq!(properties.retries(), 8); + assert_eq!(properties.owner().as_deref(), Some("iceberg")); + assert_eq!(properties.format(), "orc"); + assert!(!properties.fanout_enabled()); + assert_eq!(properties.column_fpp()["id"], 0.01); + assert_eq!(properties.dimensions(), (1920, 1080, 720)); +} + +#[test] +fn reports_the_property_with_an_invalid_value() { + let numeric_error = TestProperties::from_properties(&HashMap::from([( + RETRIES.to_string(), + "many".to_string(), + )])) + .unwrap_err(); + assert_eq!(numeric_error.kind(), ErrorKind::DataInvalid); + assert!(numeric_error.message().contains(RETRIES)); + + let boolean_error = TestProperties::from_properties(&HashMap::from([( + FANOUT_ENABLED.to_string(), + "sometimes".to_string(), + )])) + .unwrap_err(); + assert_eq!(boolean_error.kind(), ErrorKind::DataInvalid); + assert!(boolean_error.message().contains(FANOUT_ENABLED)); + + let prefixed_key = format!("{COLUMN_FPP_PREFIX}id"); + let prefix_error = TestProperties::from_properties(&HashMap::from([( + prefixed_key.clone(), + "low".to_string(), + )])) + .unwrap_err(); + assert_eq!(prefix_error.kind(), ErrorKind::DataInvalid); + assert!(prefix_error.message().contains(&prefixed_key)); + + let dimensions_error = + TestProperties::from_properties(&HashMap::from([(WIDTH.to_string(), "wide".to_string())])) + .unwrap_err(); + assert_eq!(dimensions_error.kind(), ErrorKind::DataInvalid); + assert!(format!("{dimensions_error}").contains(WIDTH)); +} + +#[derive(Debug, Properties)] +struct CommitProperties { + /// Maximum number of times to retry a commit. + #[property(key = RETRIES, default = 4, getter)] + retries: u64, +} + +#[derive(Debug, Properties)] +struct NestedProperties { + #[property(nested, getter)] + commit: CommitProperties, +} + +#[test] +fn nested_properties_read_the_same_flat_map() { + let raw = HashMap::from([(RETRIES.to_string(), "9".to_string())]); + let properties = NestedProperties::from_properties(&raw).unwrap(); + + assert_eq!(properties.commit().retries(), 9); +} + +fn parse_non_empty(value: &str) -> iceberg::Result { + let value = value.trim(); + if value.is_empty() { + Err(Error::new( + ErrorKind::DataInvalid, + "value must not be empty", + )) + } else { + Ok(value.to_string()) + } +} + +#[derive(Debug, Properties)] +struct ValidatedProperties { + #[property( + key = "location", + default = "default", + parse_with = parse_non_empty, + getter + )] + location: String, +} + +#[test] +fn custom_single_value_parser_can_validate_and_normalize() { + let defaults = ValidatedProperties::from_properties(&HashMap::new()).unwrap(); + assert_eq!(defaults.location(), "default"); + + let parsed = ValidatedProperties::from_properties(&HashMap::from([( + "location".to_string(), + " path ".to_string(), + )])) + .unwrap(); + assert_eq!(parsed.location(), "path"); + + let error = ValidatedProperties::from_properties(&HashMap::from([( + "location".to_string(), + " ".to_string(), + )])) + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::DataInvalid); + assert_eq!(error.message(), "value must not be empty"); + assert!(format!("{error}").contains("property: location")); +} + +#[derive(Debug, Properties)] +struct OptionalValidatedProperties { + #[property( + key = "optional-location", + default = None, + parse_with = parse_non_empty, + getter + )] + location: Option, +} + +#[test] +fn custom_single_value_parser_wraps_present_optional_values() { + let defaults = OptionalValidatedProperties::from_properties(&HashMap::new()).unwrap(); + assert_eq!(defaults.location(), &None); + + let parsed = OptionalValidatedProperties::from_properties(&HashMap::from([( + "optional-location".to_string(), + " path ".to_string(), + )])) + .unwrap(); + assert_eq!(parsed.location().as_deref(), Some("path")); +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct Meters(u64); + +impl FromStr for Meters { + type Err = ::Err; + + fn from_str(value: &str) -> Result { + value.parse().map(Self) + } +} + +#[derive(Debug, Properties)] +struct CopyGetterProperties { + #[property(key = "distance", default = Meters(1), getter)] + distance: Meters, + + #[property(key = "count", default = Some(2), getter)] + count: Option, +} + +#[test] +fn returns_only_structurally_known_copy_types_by_value() { + let properties = CopyGetterProperties::from_properties(&HashMap::new()).unwrap(); + + let _: &Meters = properties.distance(); + let _: Option = properties.count(); + assert_eq!(properties.distance(), &Meters(1)); + assert_eq!(properties.count(), Some(2)); +} + +#[derive(Debug, Default, Serialize, Deserialize, Properties)] +struct DerivedTraitProperties { + #[property(key = RETRIES, default = 4, getter)] + retries: u64, +} + +#[test] +fn coexists_with_default_serialize_and_deserialize_derives() { + let defaults = DerivedTraitProperties::default(); + assert_eq!(defaults.retries(), 0); + + let properties = DerivedTraitProperties::from_properties(&HashMap::new()).unwrap(); + assert_eq!(properties.retries(), 4); + assert_eq!( + serde_json::to_string(&properties).unwrap(), + r#"{"retries":4}"# + ); + + let decoded: DerivedTraitProperties = serde_json::from_str(r#"{"retries":7}"#).unwrap(); + assert_eq!(decoded.retries(), 7); +}