From f8043ddf613d328ca8f31994566bd46e41c65f85 Mon Sep 17 00:00:00 2001 From: Renjie Liu Date: Tue, 4 Aug 2026 17:37:17 +0800 Subject: [PATCH 01/14] Property framework for iceberg --- Cargo.lock | 10 + Cargo.toml | 1 + crates/iceberg/Cargo.toml | 1 + crates/iceberg/public-api.txt | 45 +- crates/iceberg/src/catalog/utils.rs | 2 +- crates/iceberg/src/encryption/manager.rs | 4 +- crates/iceberg/src/spec/table_metadata.rs | 10 +- crates/iceberg/src/spec/table_properties.rs | 445 ++++++++++-------- .../src/transaction/expire_snapshots.rs | 10 +- crates/iceberg/src/transaction/mod.rs | 10 +- .../src/writer/file_writer/parquet_writer.rs | 8 +- .../datafusion/src/physical_plan/write.rs | 8 +- crates/property-macro/Cargo.toml | 42 ++ crates/property-macro/src/lib.rs | 321 +++++++++++++ 14 files changed, 678 insertions(+), 239 deletions(-) create mode 100644 crates/property-macro/Cargo.toml create mode 100644 crates/property-macro/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index ead0b2e13c..663cae7945 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3779,6 +3779,7 @@ dependencies = [ "flate2", "fnv", "futures", + "iceberg-property-macro", "iceberg_test_utils", "itertools 0.13.0", "minijinja", @@ -3999,6 +4000,15 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "iceberg-property-macro" +version = "0.10.0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "iceberg-sqllogictest" version = "0.10.0" diff --git a/Cargo.toml b/Cargo.toml index c7315a6d13..d315879cf2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ members = [ "crates/catalog/*", "crates/examples", "crates/iceberg", + "crates/property-macro", "crates/integration_tests", "crates/integrations/*", "crates/sqllogictest", diff --git a/crates/iceberg/Cargo.toml b/crates/iceberg/Cargo.toml index c33445c64c..999d23648b 100644 --- a/crates/iceberg/Cargo.toml +++ b/crates/iceberg/Cargo.toml @@ -59,6 +59,7 @@ flate2 = { workspace = true } fnv = { workspace = true } futures = { workspace = true } itertools = { workspace = true } +iceberg-property-macro = { version = "0.10.0", path = "../property-macro" } moka = { version = "0.12.10", features = ["future"] } murmur3 = { workspace = true } once_cell = { workspace = true } diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index f5a54304df..61f68e50a3 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -2735,25 +2735,6 @@ pub fn iceberg::spec::TableMetadataBuilder::clone(&self) -> iceberg::spec::Table impl core::fmt::Debug for iceberg::spec::TableMetadataBuilder pub fn iceberg::spec::TableMetadataBuilder::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result pub struct iceberg::spec::TableProperties -pub iceberg::spec::TableProperties::cdc_enabled: bool -pub iceberg::spec::TableProperties::cdc_max_chunk_size: usize -pub iceberg::spec::TableProperties::cdc_min_chunk_size: usize -pub iceberg::spec::TableProperties::cdc_norm_level: i32 -pub iceberg::spec::TableProperties::commit_max_retry_wait_ms: u64 -pub iceberg::spec::TableProperties::commit_min_retry_wait_ms: u64 -pub iceberg::spec::TableProperties::commit_num_retries: usize -pub iceberg::spec::TableProperties::commit_total_retry_timeout_ms: u64 -pub iceberg::spec::TableProperties::encryption_data_key_length: usize -pub iceberg::spec::TableProperties::encryption_key_id: core::option::Option -pub iceberg::spec::TableProperties::gc_enabled: bool -pub iceberg::spec::TableProperties::max_ref_age_ms: i64 -pub iceberg::spec::TableProperties::max_snapshot_age_ms: i64 -pub iceberg::spec::TableProperties::metadata_compression_codec: iceberg::compression::CompressionCodec -pub iceberg::spec::TableProperties::min_snapshots_to_keep: usize -pub iceberg::spec::TableProperties::write_datafusion_fanout_enabled: bool -pub iceberg::spec::TableProperties::write_format_default: alloc::string::String -pub iceberg::spec::TableProperties::write_metadata_path: core::option::Option -pub iceberg::spec::TableProperties::write_target_file_size_bytes: usize impl iceberg::spec::TableProperties pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS: &str pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT: u64 @@ -2806,11 +2787,37 @@ pub const iceberg::spec::TableProperties::PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT pub const iceberg::spec::TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES: &str pub const iceberg::spec::TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT: usize pub const iceberg::spec::TableProperties::RESERVED_PROPERTIES: [&str; 9] +impl iceberg::spec::TableProperties +pub fn iceberg::spec::TableProperties::cdc_enabled(&self) -> bool +pub fn iceberg::spec::TableProperties::cdc_max_chunk_size(&self) -> usize +pub fn iceberg::spec::TableProperties::cdc_min_chunk_size(&self) -> usize +pub fn iceberg::spec::TableProperties::cdc_norm_level(&self) -> i32 +pub fn iceberg::spec::TableProperties::commit_max_retry_wait_ms(&self) -> u64 +pub fn iceberg::spec::TableProperties::commit_min_retry_wait_ms(&self) -> u64 +pub fn iceberg::spec::TableProperties::commit_num_retries(&self) -> usize +pub fn iceberg::spec::TableProperties::commit_total_retry_timeout_ms(&self) -> u64 +pub fn iceberg::spec::TableProperties::encryption_data_key_length(&self) -> usize +pub fn iceberg::spec::TableProperties::encryption_key_id(&self) -> core::option::Option +pub fn iceberg::spec::TableProperties::gc_enabled(&self) -> bool +pub fn iceberg::spec::TableProperties::max_ref_age_ms(&self) -> i64 +pub fn iceberg::spec::TableProperties::max_snapshot_age_ms(&self) -> i64 +pub fn iceberg::spec::TableProperties::metadata_compression_codec(&self) -> iceberg::compression::CompressionCodec +pub fn iceberg::spec::TableProperties::min_snapshots_to_keep(&self) -> usize +pub fn iceberg::spec::TableProperties::write_datafusion_fanout_enabled(&self) -> bool +pub fn iceberg::spec::TableProperties::write_format_default(&self) -> iceberg::spec::DataFileFormat +pub fn iceberg::spec::TableProperties::write_metadata_path(&self) -> core::option::Option +pub fn iceberg::spec::TableProperties::write_target_file_size_bytes(&self) -> usize impl core::convert::TryFrom<&std::collections::hash::map::HashMap> for iceberg::spec::TableProperties pub type iceberg::spec::TableProperties::Error = iceberg::Error pub fn iceberg::spec::TableProperties::try_from(props: &std::collections::hash::map::HashMap) -> iceberg::Result +impl core::default::Default for iceberg::spec::TableProperties +pub fn iceberg::spec::TableProperties::default() -> Self impl core::fmt::Debug for iceberg::spec::TableProperties pub fn iceberg::spec::TableProperties::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl serde_core::ser::Serialize for iceberg::spec::TableProperties +pub fn iceberg::spec::TableProperties::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> where S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::TableProperties +pub fn iceberg::spec::TableProperties::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> pub struct iceberg::spec::UnboundPartitionField pub iceberg::spec::UnboundPartitionField::field_id: core::option::Option pub iceberg::spec::UnboundPartitionField::name: alloc::string::String diff --git a/crates/iceberg/src/catalog/utils.rs b/crates/iceberg/src/catalog/utils.rs index 853fc8d09a..f38fb57c79 100644 --- a/crates/iceberg/src/catalog/utils.rs +++ b/crates/iceberg/src/catalog/utils.rs @@ -60,7 +60,7 @@ pub async fn drop_table_data(table_info: &Table) -> Result<()> { } // Delete data files only if gc.enabled is true, to avoid corrupting shared tables - if metadata.table_properties()?.gc_enabled { + if metadata.table_properties()?.gc_enabled() { delete_data_files(io, &manifests_to_delete).await?; } diff --git a/crates/iceberg/src/encryption/manager.rs b/crates/iceberg/src/encryption/manager.rs index e2294c2f2c..260202f5bc 100644 --- a/crates/iceberg/src/encryption/manager.rs +++ b/crates/iceberg/src/encryption/manager.rs @@ -119,7 +119,7 @@ impl EncryptionManager { } let table_properties = metadata.table_properties()?; - let Some(table_key_id) = table_properties.encryption_key_id else { + let Some(table_key_id) = table_properties.encryption_key_id() else { if kms_client.is_some() { tracing::warn!( "KeyManagementClient provided but table does not have encryption.key-id set" @@ -140,7 +140,7 @@ impl EncryptionManager { .table_key_id(table_key_id) .encryption_keys(metadata.encryption_keys.clone()) .key_size(AesKeySize::from_key_length( - table_properties.encryption_data_key_length, + table_properties.encryption_data_key_length(), )?) .build(); Ok(Some(Arc::new(em))) diff --git a/crates/iceberg/src/spec/table_metadata.rs b/crates/iceberg/src/spec/table_metadata.rs index ecc0586680..71b794b0c5 100644 --- a/crates/iceberg/src/spec/table_metadata.rs +++ b/crates/iceberg/src/spec/table_metadata.rs @@ -371,7 +371,7 @@ impl TableMetadata { pub fn metadata_location(&self) -> Result { Ok(self .table_properties()? - .write_metadata_path + .write_metadata_path() .unwrap_or_else(|| format!("{}/{}", self.location(), METADATA_FOLDER_NAME))) } @@ -4043,11 +4043,11 @@ mod tests { let props = metadata.table_properties().unwrap(); assert_eq!( - props.commit_num_retries, + props.commit_num_retries(), TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT ); assert_eq!( - props.write_target_file_size_bytes, + props.write_target_file_size_bytes(), TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT ); } @@ -4089,8 +4089,8 @@ mod tests { let props = metadata.table_properties().unwrap(); - assert_eq!(props.commit_num_retries, 10); - assert_eq!(props.write_target_file_size_bytes, 1024); + assert_eq!(props.commit_num_retries(), 10); + assert_eq!(props.write_target_file_size_bytes(), 1024); } #[test] diff --git a/crates/iceberg/src/spec/table_properties.rs b/crates/iceberg/src/spec/table_properties.rs index 379feee5c1..cf4881d4c3 100644 --- a/crates/iceberg/src/spec/table_properties.rs +++ b/crates/iceberg/src/spec/table_properties.rs @@ -16,29 +16,12 @@ // under the License. use std::collections::HashMap; -use std::fmt::Display; -use std::str::FromStr; + +use iceberg_property_macro::Properties; use crate::compression::CompressionCodec; use crate::error::{Error, ErrorKind, Result}; - -fn parse_property( - properties: &HashMap, - key: &str, - default: T, -) -> Result -where - ::Err: Display, -{ - properties.get(key).map_or(Ok(default), |value| { - value.parse::().map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!("Invalid value for {key}: {e}"), - ) - }) - }) -} +use crate::spec::DataFileFormat; /// Strips trailing slashes from a location, preserving a bare URI scheme root fn strip_trailing_slash(path: &str) -> &str { @@ -52,23 +35,16 @@ fn strip_trailing_slash(path: &str) -> &str { path } -fn parse_location_property( - properties: &HashMap, - key: &str, -) -> Result> { - properties - .get(key) - .map(|path| { - if path.is_empty() { - return Err(Error::new( - ErrorKind::DataInvalid, - format!("Invalid value for {key}: path must not be empty"), - )); - } - - Ok(strip_trailing_slash(path).to_string()) - }) - .transpose() +fn parse_metadata_location(value: &str) -> Result> { + if value.is_empty() { + return Err(Error::new(ErrorKind::DataInvalid, "path must not be empty")); + } + + Ok(Some(strip_trailing_slash(value).to_string())) +} + +fn parse_optional_string(value: &str) -> Result> { + Ok(Some(value.to_string())) } /// Parse compression codec for metadata files from table properties. @@ -91,6 +67,10 @@ pub(crate) fn parse_metadata_file_compression( .map(|s| s.as_str()) .unwrap_or(TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC_DEFAULT); + parse_metadata_file_compression_value(value) +} + +fn parse_metadata_file_compression_value(value: &str) -> Result { // Handle empty string as None if value.is_empty() { return Ok(CompressionCodec::None); @@ -128,51 +108,97 @@ pub(crate) fn parse_metadata_file_compression( } } -/// TableProperties that contains the properties of a table. -#[derive(Debug)] +fn serialize_compression_codec(codec: &CompressionCodec) -> String { + codec.name().to_string() +} + +/// Typed table properties parsed from a table's string property map. +#[derive(Debug, Properties)] pub struct TableProperties { /// The number of times to retry a commit. - pub commit_num_retries: usize, + #[key = "commit.retry.num-retries"] + #[default(TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT)] + commit_num_retries: usize, /// The minimum wait time between retries. - pub commit_min_retry_wait_ms: u64, + #[key = "commit.retry.min-wait-ms"] + #[default(TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT)] + commit_min_retry_wait_ms: u64, /// The maximum wait time between retries. - pub commit_max_retry_wait_ms: u64, + #[key = "commit.retry.max-wait-ms"] + #[default(TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT)] + commit_max_retry_wait_ms: u64, /// The total timeout for commit retries. - pub commit_total_retry_timeout_ms: u64, + #[key = "commit.retry.total-timeout-ms"] + #[default(TableProperties::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT)] + commit_total_retry_timeout_ms: u64, /// The default format for files. - pub write_format_default: String, + #[key = "write.format.default"] + #[default(DataFileFormat::Parquet)] + write_format_default: DataFileFormat, /// The target file size for files. - pub write_target_file_size_bytes: usize, + #[key = "write.target-file-size-bytes"] + #[default(TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT)] + write_target_file_size_bytes: usize, /// Base directory for metadata files (manifests, manifest lists), with any /// trailing slash trimmed. `None` if `write.metadata.path` is not set. - pub write_metadata_path: Option, + #[key = "write.metadata.path"] + #[default(None)] + #[parse_with(parse_metadata_location)] + write_metadata_path: Option, /// Compression codec for metadata files (JSON) - pub metadata_compression_codec: CompressionCodec, + #[key = "write.metadata.compression-codec"] + #[default(CompressionCodec::None)] + #[parse_with(parse_metadata_file_compression_value)] + #[serialize_with(serialize_compression_codec)] + metadata_compression_codec: CompressionCodec, /// Whether to use `FanoutWriter` for partitioned tables. - pub write_datafusion_fanout_enabled: bool, + #[key = "write.datafusion.fanout.enabled"] + #[default(TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT)] + write_datafusion_fanout_enabled: bool, /// Whether garbage collection is enabled on drop. /// When `false`, data files will not be deleted when a table is dropped. - pub gc_enabled: bool, + #[key = "gc.enabled"] + #[default(TableProperties::PROPERTY_GC_ENABLED_DEFAULT)] + gc_enabled: bool, /// Default maximum age of a snapshot to keep when expiring snapshots. - pub max_snapshot_age_ms: i64, + #[key = "history.expire.max-snapshot-age-ms"] + #[default(TableProperties::PROPERTY_MAX_SNAPSHOT_AGE_MS_DEFAULT)] + max_snapshot_age_ms: i64, /// Default minimum number of snapshots to keep per branch when expiring snapshots. - pub min_snapshots_to_keep: usize, + #[key = "history.expire.min-snapshots-to-keep"] + #[default(TableProperties::PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT)] + min_snapshots_to_keep: usize, /// Default maximum age of a snapshot reference to keep when expiring snapshots. - pub max_ref_age_ms: i64, + #[key = "history.expire.max-ref-age-ms"] + #[default(TableProperties::PROPERTY_MAX_REF_AGE_MS_DEFAULT)] + max_ref_age_ms: i64, /// Whether content-defined chunking is enabled. /// `true` only when `write.parquet.content-defined-chunking.enabled = "true"`. - pub cdc_enabled: bool, + #[key = "write.parquet.content-defined-chunking.enabled"] + #[default(TableProperties::PROPERTY_PARQUET_CDC_ENABLED_DEFAULT)] + cdc_enabled: bool, /// Content-defined chunking minimum chunk size in bytes. - pub cdc_min_chunk_size: usize, + #[key = "write.parquet.content-defined-chunking.min-chunk-size"] + #[default(TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT)] + cdc_min_chunk_size: usize, /// Content-defined chunking maximum chunk size in bytes. - pub cdc_max_chunk_size: usize, + #[key = "write.parquet.content-defined-chunking.max-chunk-size"] + #[default(TableProperties::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE_DEFAULT)] + cdc_max_chunk_size: usize, /// Content-defined chunking normalization level (gearhash bit adjustment). - pub cdc_norm_level: i32, + #[key = "write.parquet.content-defined-chunking.norm-level"] + #[default(TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL_DEFAULT)] + cdc_norm_level: i32, /// The master key id used to encrypt this table's manifest list and data /// files. `None` if `encryption.key-id` is not set. - pub encryption_key_id: Option, + #[key = "encryption.key-id"] + #[default(None)] + #[parse_with(parse_optional_string)] + encryption_key_id: Option, /// The encryption data encryption key length in bytes. - pub encryption_data_key_length: usize, + #[key = "encryption.data-key-length"] + #[default(TableProperties::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH_DEFAULT)] + encryption_data_key_length: usize, } impl TableProperties { @@ -328,100 +354,10 @@ impl TableProperties { } impl TryFrom<&HashMap> for TableProperties { - // parse by entry key or use default value type Error = Error; fn try_from(props: &HashMap) -> Result { - Ok(TableProperties { - commit_num_retries: parse_property( - props, - TableProperties::PROPERTY_COMMIT_NUM_RETRIES, - TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT, - )?, - commit_min_retry_wait_ms: parse_property( - props, - TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS, - TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT, - )?, - commit_max_retry_wait_ms: parse_property( - props, - TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS, - TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT, - )?, - commit_total_retry_timeout_ms: parse_property( - props, - TableProperties::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS, - TableProperties::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT, - )?, - write_format_default: parse_property( - props, - TableProperties::PROPERTY_DEFAULT_FILE_FORMAT, - TableProperties::PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT.to_string(), - )?, - write_target_file_size_bytes: parse_property( - props, - TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES, - TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT, - )?, - write_metadata_path: parse_location_property( - props, - TableProperties::PROPERTY_WRITE_METADATA_PATH, - )?, - metadata_compression_codec: parse_metadata_file_compression(props)?, - write_datafusion_fanout_enabled: parse_property( - props, - TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED, - TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT, - )?, - gc_enabled: parse_property( - props, - TableProperties::PROPERTY_GC_ENABLED, - TableProperties::PROPERTY_GC_ENABLED_DEFAULT, - )?, - max_snapshot_age_ms: parse_property( - props, - TableProperties::PROPERTY_MAX_SNAPSHOT_AGE_MS, - TableProperties::PROPERTY_MAX_SNAPSHOT_AGE_MS_DEFAULT, - )?, - min_snapshots_to_keep: parse_property( - props, - TableProperties::PROPERTY_MIN_SNAPSHOTS_TO_KEEP, - TableProperties::PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT, - )?, - max_ref_age_ms: parse_property( - props, - TableProperties::PROPERTY_MAX_REF_AGE_MS, - TableProperties::PROPERTY_MAX_REF_AGE_MS_DEFAULT, - )?, - cdc_enabled: parse_property( - props, - TableProperties::PROPERTY_PARQUET_CDC_ENABLED, - TableProperties::PROPERTY_PARQUET_CDC_ENABLED_DEFAULT, - )?, - cdc_min_chunk_size: parse_property( - props, - TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE, - TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT, - )?, - cdc_max_chunk_size: parse_property( - props, - TableProperties::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE, - TableProperties::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE_DEFAULT, - )?, - cdc_norm_level: parse_property( - props, - TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL, - TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL_DEFAULT, - )?, - encryption_key_id: props - .get(TableProperties::PROPERTY_ENCRYPTION_KEY_ID) - .cloned(), - encryption_data_key_length: parse_property( - props, - TableProperties::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH, - TableProperties::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH_DEFAULT, - )?, - }) + Self::from_properties(props).map_err(|error| Error::new(ErrorKind::DataInvalid, error)) } } @@ -432,51 +368,169 @@ mod tests { #[test] fn test_table_properties_default() { - let props = HashMap::new(); - let table_properties = TableProperties::try_from(&props).unwrap(); + let table_properties = TableProperties::default(); assert_eq!( - table_properties.commit_num_retries, + table_properties.commit_num_retries(), TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT ); assert_eq!( - table_properties.commit_min_retry_wait_ms, + table_properties.commit_min_retry_wait_ms(), TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT ); assert_eq!( - table_properties.commit_max_retry_wait_ms, + table_properties.commit_max_retry_wait_ms(), TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT ); assert_eq!( - table_properties.write_format_default, - TableProperties::PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT.to_string() + table_properties.write_format_default(), + DataFileFormat::Parquet ); assert_eq!( - table_properties.write_target_file_size_bytes, + table_properties.write_target_file_size_bytes(), TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT ); // Test compression defaults (none means CompressionCodec::None) assert_eq!( - table_properties.metadata_compression_codec, + table_properties.metadata_compression_codec(), CompressionCodec::None ); assert_eq!( - table_properties.gc_enabled, + table_properties.gc_enabled(), TableProperties::PROPERTY_GC_ENABLED_DEFAULT ); assert_eq!( - table_properties.max_snapshot_age_ms, + table_properties.max_snapshot_age_ms(), TableProperties::PROPERTY_MAX_SNAPSHOT_AGE_MS_DEFAULT ); assert_eq!( - table_properties.min_snapshots_to_keep, + table_properties.min_snapshots_to_keep(), TableProperties::PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT ); assert_eq!( - table_properties.max_ref_age_ms, + table_properties.max_ref_age_ms(), TableProperties::PROPERTY_MAX_REF_AGE_MS_DEFAULT ); } + #[test] + fn test_empty_properties_match_default() { + let parsed = TableProperties::try_from(&HashMap::new()).unwrap(); + let defaults = TableProperties::default(); + + assert_eq!(parsed.commit_num_retries(), defaults.commit_num_retries()); + assert_eq!( + parsed.commit_min_retry_wait_ms(), + defaults.commit_min_retry_wait_ms() + ); + assert_eq!( + parsed.commit_max_retry_wait_ms(), + defaults.commit_max_retry_wait_ms() + ); + assert_eq!( + parsed.commit_total_retry_timeout_ms(), + defaults.commit_total_retry_timeout_ms() + ); + assert_eq!( + parsed.write_format_default(), + defaults.write_format_default() + ); + assert_eq!( + parsed.write_target_file_size_bytes(), + defaults.write_target_file_size_bytes() + ); + assert_eq!(parsed.write_metadata_path(), defaults.write_metadata_path()); + assert_eq!( + parsed.metadata_compression_codec(), + defaults.metadata_compression_codec() + ); + assert_eq!( + parsed.write_datafusion_fanout_enabled(), + defaults.write_datafusion_fanout_enabled() + ); + assert_eq!(parsed.gc_enabled(), defaults.gc_enabled()); + assert_eq!(parsed.max_snapshot_age_ms(), defaults.max_snapshot_age_ms()); + assert_eq!( + parsed.min_snapshots_to_keep(), + defaults.min_snapshots_to_keep() + ); + assert_eq!(parsed.max_ref_age_ms(), defaults.max_ref_age_ms()); + assert_eq!(parsed.cdc_enabled(), defaults.cdc_enabled()); + assert_eq!(parsed.cdc_min_chunk_size(), defaults.cdc_min_chunk_size()); + assert_eq!(parsed.cdc_max_chunk_size(), defaults.cdc_max_chunk_size()); + assert_eq!(parsed.cdc_norm_level(), defaults.cdc_norm_level()); + assert_eq!(parsed.encryption_key_id(), defaults.encryption_key_id()); + assert_eq!( + parsed.encryption_data_key_length(), + defaults.encryption_data_key_length() + ); + } + + #[test] + fn test_table_properties_json_round_trip() { + let properties = TableProperties::try_from(&HashMap::from([ + ( + TableProperties::PROPERTY_DEFAULT_FILE_FORMAT.to_string(), + "ORC".to_string(), + ), + ( + TableProperties::PROPERTY_WRITE_METADATA_PATH.to_string(), + "s3://warehouse/table/metadata/".to_string(), + ), + ( + TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), + "GZIP".to_string(), + ), + ( + TableProperties::PROPERTY_ENCRYPTION_KEY_ID.to_string(), + "table-key".to_string(), + ), + ])) + .unwrap(); + + let json = serde_json::to_value(&properties).unwrap(); + assert_eq!(json[TableProperties::PROPERTY_DEFAULT_FILE_FORMAT], "orc"); + assert_eq!( + json[TableProperties::PROPERTY_WRITE_METADATA_PATH], + "s3://warehouse/table/metadata" + ); + assert_eq!( + json[TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC], + "gzip" + ); + + let decoded: TableProperties = serde_json::from_value(json).unwrap(); + assert_eq!(decoded.write_format_default(), DataFileFormat::Orc); + assert_eq!( + decoded.write_metadata_path(), + Some("s3://warehouse/table/metadata".to_string()) + ); + assert_eq!( + decoded.metadata_compression_codec(), + CompressionCodec::gzip_default() + ); + assert_eq!(decoded.encryption_key_id(), Some("table-key".to_string())); + } + + #[test] + fn test_default_table_properties_json_round_trip() { + let defaults = TableProperties::default(); + let json = serde_json::to_value(&defaults).unwrap(); + + assert!( + json.get(TableProperties::PROPERTY_WRITE_METADATA_PATH) + .is_none() + ); + assert!( + json.get(TableProperties::PROPERTY_ENCRYPTION_KEY_ID) + .is_none() + ); + + let decoded: TableProperties = serde_json::from_value(json).unwrap(); + assert_eq!(decoded.write_format_default(), DataFileFormat::Parquet); + assert_eq!(decoded.write_metadata_path(), None); + assert_eq!(decoded.encryption_key_id(), None); + } + #[test] fn test_table_properties_history_expire_overrides() { let props = HashMap::from([ @@ -494,16 +548,16 @@ mod tests { ), ]); let table_properties = TableProperties::try_from(&props).unwrap(); - assert_eq!(table_properties.max_snapshot_age_ms, 1234); - assert_eq!(table_properties.min_snapshots_to_keep, 7); - assert_eq!(table_properties.max_ref_age_ms, 5678); + assert_eq!(table_properties.max_snapshot_age_ms(), 1234); + assert_eq!(table_properties.min_snapshots_to_keep(), 7); + assert_eq!(table_properties.max_ref_age_ms(), 5678); } #[test] fn test_table_properties_write_metadata_path() { // Test unset let table_properties = TableProperties::try_from(&HashMap::new()).unwrap(); - assert_eq!(table_properties.write_metadata_path, None); + assert_eq!(table_properties.write_metadata_path(), None); // Test empty path is invalid let props = HashMap::from([( @@ -524,8 +578,8 @@ mod tests { )]); let table_properties = TableProperties::try_from(&props).unwrap(); assert_eq!( - table_properties.write_metadata_path.as_deref(), - Some("s3://other-bucket/custom-meta") + table_properties.write_metadata_path(), + Some("s3://other-bucket/custom-meta".to_string()) ); } @@ -553,7 +607,7 @@ mod tests { )]); let table_properties = TableProperties::try_from(&props).unwrap(); assert_eq!( - table_properties.metadata_compression_codec, + table_properties.metadata_compression_codec(), CompressionCodec::gzip_default() ); } @@ -566,7 +620,7 @@ mod tests { )]); let table_properties = TableProperties::try_from(&props).unwrap(); assert_eq!( - table_properties.metadata_compression_codec, + table_properties.metadata_compression_codec(), CompressionCodec::None ); } @@ -580,7 +634,7 @@ mod tests { )]); let table_properties = TableProperties::try_from(&props_upper).unwrap(); assert_eq!( - table_properties.metadata_compression_codec, + table_properties.metadata_compression_codec(), CompressionCodec::gzip_default() ); @@ -591,7 +645,7 @@ mod tests { )]); let table_properties = TableProperties::try_from(&props_mixed).unwrap(); assert_eq!( - table_properties.metadata_compression_codec, + table_properties.metadata_compression_codec(), CompressionCodec::gzip_default() ); @@ -602,7 +656,7 @@ mod tests { )]); let table_properties = TableProperties::try_from(&props_none_upper).unwrap(); assert_eq!( - table_properties.metadata_compression_codec, + table_properties.metadata_compression_codec(), CompressionCodec::None ); } @@ -632,11 +686,14 @@ mod tests { ), ]); let table_properties = TableProperties::try_from(&props).unwrap(); - assert_eq!(table_properties.commit_num_retries, 10); - assert_eq!(table_properties.commit_max_retry_wait_ms, 20); - assert_eq!(table_properties.write_format_default, "avro".to_string()); - assert_eq!(table_properties.write_target_file_size_bytes, 512); - assert!(!table_properties.gc_enabled); + assert_eq!(table_properties.commit_num_retries(), 10); + assert_eq!(table_properties.commit_max_retry_wait_ms(), 20); + assert_eq!( + table_properties.write_format_default(), + DataFileFormat::Avro + ); + assert_eq!(table_properties.write_target_file_size_bytes(), 512); + assert!(!table_properties.gc_enabled()); } #[test] @@ -814,7 +871,7 @@ mod tests { fn test_cdc_disabled_by_default() { let props = HashMap::new(); let tp = TableProperties::try_from(&props).unwrap(); - assert!(!tp.cdc_enabled); + assert!(!tp.cdc_enabled()); } #[test] @@ -824,10 +881,10 @@ mod tests { "true".to_string(), )]); let tp = TableProperties::try_from(&props).unwrap(); - assert!(tp.cdc_enabled); - assert_eq!(tp.cdc_min_chunk_size, 256 * 1024); - assert_eq!(tp.cdc_max_chunk_size, 1024 * 1024); - assert_eq!(tp.cdc_norm_level, 0); + assert!(tp.cdc_enabled()); + assert_eq!(tp.cdc_min_chunk_size(), 256 * 1024); + assert_eq!(tp.cdc_max_chunk_size(), 1024 * 1024); + assert_eq!(tp.cdc_norm_level(), 0); } #[test] @@ -837,7 +894,7 @@ mod tests { "262144".to_string(), )]); let tp = TableProperties::try_from(&props).unwrap(); - assert!(!tp.cdc_enabled); + assert!(!tp.cdc_enabled()); } #[test] @@ -861,10 +918,10 @@ mod tests { ), ]); let tp = TableProperties::try_from(&props).unwrap(); - assert!(tp.cdc_enabled); - assert_eq!(tp.cdc_min_chunk_size, 200000); - assert_eq!(tp.cdc_max_chunk_size, 900000); - assert_eq!(tp.cdc_norm_level, 1); + assert!(tp.cdc_enabled()); + assert_eq!(tp.cdc_min_chunk_size(), 200000); + assert_eq!(tp.cdc_max_chunk_size(), 900000); + assert_eq!(tp.cdc_norm_level(), 1); } #[test] @@ -880,10 +937,10 @@ mod tests { ), ]); let tp = TableProperties::try_from(&props).unwrap(); - assert!(tp.cdc_enabled); - assert_eq!(tp.cdc_min_chunk_size, 256 * 1024); - assert_eq!(tp.cdc_max_chunk_size, 1024 * 1024); - assert_eq!(tp.cdc_norm_level, 2); + assert!(tp.cdc_enabled()); + assert_eq!(tp.cdc_min_chunk_size(), 256 * 1024); + assert_eq!(tp.cdc_max_chunk_size(), 1024 * 1024); + assert_eq!(tp.cdc_norm_level(), 2); } #[test] @@ -899,7 +956,7 @@ mod tests { ), ]); let tp = TableProperties::try_from(&props).unwrap(); - assert_eq!(tp.cdc_norm_level, -2); + assert_eq!(tp.cdc_norm_level(), -2); } #[test] @@ -945,6 +1002,6 @@ mod tests { fn test_cdc_no_properties() { let props = HashMap::from([("some.other.property".to_string(), "value".to_string())]); let tp = TableProperties::try_from(&props).unwrap(); - assert!(!tp.cdc_enabled); + assert!(!tp.cdc_enabled()); } } diff --git a/crates/iceberg/src/transaction/expire_snapshots.rs b/crates/iceberg/src/transaction/expire_snapshots.rs index b2420a1dff..66b0291965 100644 --- a/crates/iceberg/src/transaction/expire_snapshots.rs +++ b/crates/iceberg/src/transaction/expire_snapshots.rs @@ -115,8 +115,10 @@ impl ExpireSnapshotsAction { // days) the age path always runs, so even an explicit-id-only call applies the default cutoff. let default_cutoff = self .older_than_ms - .unwrap_or_else(|| now.saturating_sub(properties.max_snapshot_age_ms)); - let default_min_to_keep = self.retain_last.unwrap_or(properties.min_snapshots_to_keep); + .unwrap_or_else(|| now.saturating_sub(properties.max_snapshot_age_ms())); + let default_min_to_keep = self + .retain_last + .unwrap_or(properties.min_snapshots_to_keep()); // Ref aging: `main` is always kept; any other ref whose head is older than its // `max_ref_age_ms` (defaulting to `history.expire.max-ref-age-ms`) is dropped, like Java's @@ -125,7 +127,7 @@ impl ExpireSnapshotsAction { let mut retained_refs: Vec<&SnapshotReference> = vec![]; for (ref_name, snapshot_ref) in &metadata.refs { if ref_name == MAIN_BRANCH - || !Self::ref_aged_out(metadata, snapshot_ref, now, properties.max_ref_age_ms) + || !Self::ref_aged_out(metadata, snapshot_ref, now, properties.max_ref_age_ms()) { retained_refs.push(snapshot_ref); } else { @@ -302,7 +304,7 @@ impl TransactionAction for ExpireSnapshotsAction { let properties = metadata.table_properties()?; // Expiring metadata defeats a user's explicit decision to disable GC (Java refuses too). - if !properties.gc_enabled { + if !properties.gc_enabled() { return Err(Error::new( ErrorKind::DataInvalid, "Cannot expire snapshots: gc.enabled is false", diff --git a/crates/iceberg/src/transaction/mod.rs b/crates/iceberg/src/transaction/mod.rs index c2fde69fca..07a85360ae 100644 --- a/crates/iceberg/src/transaction/mod.rs +++ b/crates/iceberg/src/transaction/mod.rs @@ -181,7 +181,7 @@ impl Transaction { let table_props = self.table.metadata().table_properties()?; // TODO(https://github.com/apache/iceberg-rust/issues/2034): remove once encrypted writes are supported - if table_props.encryption_key_id.is_some() { + if table_props.encryption_key_id().is_some() { return Err(Error::new( ErrorKind::FeatureUnsupported, "Cannot commit to an encrypted table: encrypted writes are not yet supported", @@ -205,12 +205,12 @@ impl Transaction { fn build_backoff(props: TableProperties) -> Result { Ok(ExponentialBuilder::new() - .with_min_delay(Duration::from_millis(props.commit_min_retry_wait_ms)) - .with_max_delay(Duration::from_millis(props.commit_max_retry_wait_ms)) + .with_min_delay(Duration::from_millis(props.commit_min_retry_wait_ms())) + .with_max_delay(Duration::from_millis(props.commit_max_retry_wait_ms())) .with_total_delay(Some(Duration::from_millis( - props.commit_total_retry_timeout_ms, + props.commit_total_retry_timeout_ms(), ))) - .with_max_times(props.commit_num_retries) + .with_max_times(props.commit_num_retries()) .with_factor(2.0) .build()) } diff --git a/crates/iceberg/src/writer/file_writer/parquet_writer.rs b/crates/iceberg/src/writer/file_writer/parquet_writer.rs index db9f170938..847990f9e2 100644 --- a/crates/iceberg/src/writer/file_writer/parquet_writer.rs +++ b/crates/iceberg/src/writer/file_writer/parquet_writer.rs @@ -86,10 +86,10 @@ impl ParquetWriterBuilder { /// (`write.parquet.content-defined-chunking.*`); other keys fall back to /// parquet-rs defaults. pub fn from_table_properties(table_props: &TableProperties, schema: SchemaRef) -> Self { - let cdc = table_props.cdc_enabled.then_some(CdcOptions { - min_chunk_size: table_props.cdc_min_chunk_size, - max_chunk_size: table_props.cdc_max_chunk_size, - norm_level: table_props.cdc_norm_level, + let cdc = table_props.cdc_enabled().then_some(CdcOptions { + min_chunk_size: table_props.cdc_min_chunk_size(), + max_chunk_size: table_props.cdc_max_chunk_size(), + norm_level: table_props.cdc_norm_level(), }); // TODO: translate the remaining write.parquet.* keys (e.g. compression-codec, // row-group-size-bytes, page-size-bytes). diff --git a/crates/integrations/datafusion/src/physical_plan/write.rs b/crates/integrations/datafusion/src/physical_plan/write.rs index a7d771ec1b..59ac3e047d 100644 --- a/crates/integrations/datafusion/src/physical_plan/write.rs +++ b/crates/integrations/datafusion/src/physical_plan/write.rs @@ -16,7 +16,6 @@ // under the License. use std::fmt::{Debug, Formatter}; -use std::str::FromStr; use std::sync::Arc; use datafusion::arrow::array::{ArrayRef, RecordBatch, StringArray}; @@ -210,8 +209,7 @@ impl ExecutionPlan for IcebergWriteExec { .map_err(to_datafusion_error)?; // Check data file format - let file_format = DataFileFormat::from_str(&table_props.write_format_default) - .map_err(to_datafusion_error)?; + let file_format = table_props.write_format_default(); if file_format != DataFileFormat::Parquet { return Err(to_datafusion_error(Error::new( ErrorKind::FeatureUnsupported, @@ -227,7 +225,7 @@ impl ExecutionPlan for IcebergWriteExec { self.table.metadata().current_schema().clone(), ) .with_match_mode(FieldMatchMode::Name); - let target_file_size = table_props.write_target_file_size_bytes; + let target_file_size = table_props.write_target_file_size_bytes(); let file_io = self.table.file_io().clone(); // todo location_gen and file_name_gen should be configurable @@ -246,7 +244,7 @@ impl ExecutionPlan for IcebergWriteExec { let data_file_writer_builder = DataFileWriterBuilder::new(rolling_writer_builder); // Create TaskWriter - let fanout_enabled = table_props.write_datafusion_fanout_enabled; + let fanout_enabled = table_props.write_datafusion_fanout_enabled(); let schema = self.table.metadata().current_schema().clone(); let partition_spec = self.table.metadata().default_partition_spec().clone(); let task_writer = TaskWriter::try_new( diff --git a/crates/property-macro/Cargo.toml b/crates/property-macro/Cargo.toml new file mode 100644 index 0000000000..cfafd8b24a --- /dev/null +++ b/crates/property-macro/Cargo.toml @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +edition = { workspace = true } +homepage = { workspace = true } +name = "iceberg-property-macro" +publish = true +rust-version = { workspace = true } +version = { workspace = true } + +license = { workspace = true } +repository = { workspace = true } + +categories = ["database"] +description = "Derive macros for Apache Iceberg Rust" +keywords = ["iceberg"] + +[lib] +proc-macro = true + +[dependencies] +proc-macro2 = "1" +quote = "1" +syn = { version = "2", features = ["full"] } + +[lints] +workspace = true diff --git a/crates/property-macro/src/lib.rs b/crates/property-macro/src/lib.rs new file mode 100644 index 0000000000..044d185df9 --- /dev/null +++ b/crates/property-macro/src/lib.rs @@ -0,0 +1,321 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Derives for Iceberg's string-keyed property maps. + +use proc_macro::TokenStream; +use proc_macro2::TokenStream as TokenStream2; +use quote::quote; +use syn::{ + Attribute, Data, DeriveInput, Error, Expr, ExprLit, ExprPath, Field, Fields, Ident, Lit, Meta, + Path, Type, parse_macro_input, +}; + +/// Derive parsing, defaults, JSON serialization, and getters for a typed property map. +/// +/// Each field must declare the table-property key and its default: +/// +/// ```ignore +/// #[derive(Properties)] +/// struct Properties { +/// #[key = "write.format.default"] +/// #[default(DataFileFormat::Parquet)] +/// write_format_default: DataFileFormat, +/// } +/// ``` +/// +/// `parse_with` may be used for property types that do not implement `FromStr` or need +/// validation. `serialize_with` supplies the string representation used in JSON. Optional +/// fields are omitted from JSON when they are `None`. Fields must implement `Clone`; they also +/// need `FromStr` and `ToString` unless the relevant custom parsing or serialization attribute is +/// supplied. +#[proc_macro_derive(Properties, attributes(key, default, parse_with, serialize_with))] +pub fn derive_properties(input: TokenStream) -> TokenStream { + let input = parse_macro_input!(input as DeriveInput); + + match expand_properties(input) { + Ok(tokens) => tokens.into(), + Err(error) => error.into_compile_error().into(), + } +} + +struct PropertyField { + ident: Ident, + ty: Type, + docs: Vec, + key: syn::LitStr, + default: Expr, + parse_with: Option, + serialize_with: Option, + is_option: bool, +} + +fn expand_properties(input: DeriveInput) -> syn::Result { + let struct_name = input.ident; + let fields = match input.data { + Data::Struct(data) => match data.fields { + Fields::Named(fields) => fields.named, + _ => { + return Err(Error::new_spanned( + struct_name, + "Properties can only be derived for structs with named fields", + )); + } + }, + _ => { + return Err(Error::new_spanned( + struct_name, + "Properties can only be derived for structs", + )); + } + }; + + let fields = fields + .iter() + .map(parse_property_field) + .collect::>>()?; + + let defaults = fields.iter().map(|field| { + let ident = &field.ident; + let default = &field.default; + quote!(#ident: #default) + }); + + let getters = fields.iter().map(|field| { + let ident = &field.ident; + let ty = &field.ty; + let docs = &field.docs; + quote! { + #(#docs)* + pub fn #ident(&self) -> #ty { + self.#ident.clone() + } + } + }); + + let parses = fields.iter().map(|field| { + let ident = &field.ident; + let ty = &field.ty; + let key = &field.key; + let default = &field.default; + let parse = match &field.parse_with { + Some(parse_with) => quote! { + #parse_with(value).map_err(|error| { + format!("Invalid value for {}: {error}", #key) + })? + }, + None => quote! { + value.parse::<#ty>().map_err(|error| { + format!("Invalid value for {}: {error}", #key) + })? + }, + }; + + quote! { + #ident: match properties.get(#key) { + Some(value) => #parse, + None => #default, + } + } + }); + + let serializes = fields.iter().map(serialize_field); + + Ok(quote! { + impl ::std::default::Default for #struct_name { + fn default() -> Self { + Self { + #(#defaults,)* + } + } + } + + impl #struct_name { + pub(crate) fn from_properties( + properties: &::std::collections::HashMap<::std::string::String, ::std::string::String>, + ) -> ::std::result::Result { + Ok(Self { + #(#parses,)* + }) + } + + #(#getters)* + } + + impl ::serde::Serialize for #struct_name { + fn serialize(&self, serializer: S) -> ::std::result::Result + where + S: ::serde::Serializer, + { + use ::serde::ser::SerializeMap as _; + + let mut map = serializer.serialize_map(None)?; + #(#serializes)* + map.end() + } + } + + impl<'de> ::serde::Deserialize<'de> for #struct_name { + fn deserialize(deserializer: D) -> ::std::result::Result + where + D: ::serde::Deserializer<'de>, + { + let properties = <::std::collections::HashMap<::std::string::String, ::std::string::String> as ::serde::Deserialize>::deserialize(deserializer)?; + Self::from_properties(&properties).map_err(::serde::de::Error::custom) + } + } + }) +} + +fn parse_property_field(field: &Field) -> syn::Result { + let ident = field + .ident + .clone() + .ok_or_else(|| Error::new_spanned(field, "Properties fields must be named"))?; + let key = attribute_string_value(&field.attrs, "key")?.ok_or_else(|| { + Error::new_spanned(field, "Properties fields must declare #[key = \"...\"]") + })?; + let default = attribute_expression_value(&field.attrs, "default")?.ok_or_else(|| { + Error::new_spanned(field, "Properties fields must declare #[default(...)]") + })?; + + Ok(PropertyField { + ident, + ty: field.ty.clone(), + docs: field + .attrs + .iter() + .filter(|attribute| attribute.path().is_ident("doc")) + .cloned() + .collect(), + key, + default, + parse_with: attribute_path_value(&field.attrs, "parse_with")?, + serialize_with: attribute_path_value(&field.attrs, "serialize_with")?, + is_option: is_option_type(&field.ty), + }) +} + +fn attribute_string_value( + attributes: &[Attribute], + name: &str, +) -> syn::Result> { + let Some(attribute) = find_attribute(attributes, name)? else { + return Ok(None); + }; + + match &attribute.meta { + Meta::NameValue(name_value) => match &name_value.value { + Expr::Lit(ExprLit { + lit: Lit::Str(value), + .. + }) => Ok(Some(value.clone())), + _ => Err(Error::new_spanned( + attribute, + format!("{name} must be a string literal"), + )), + }, + _ => Err(Error::new_spanned( + attribute, + format!("{name} must use the form #[{name} = ...]"), + )), + } +} + +fn attribute_expression_value(attributes: &[Attribute], name: &str) -> syn::Result> { + let Some(attribute) = find_attribute(attributes, name)? else { + return Ok(None); + }; + + match &attribute.meta { + Meta::NameValue(name_value) => Ok(Some(name_value.value.clone())), + Meta::List(_) => attribute.parse_args::().map(Some), + _ => Err(Error::new_spanned( + attribute, + format!("{name} must use the form #[{name}(...)]"), + )), + } +} + +fn attribute_path_value(attributes: &[Attribute], name: &str) -> syn::Result> { + let Some(expression) = attribute_expression_value(attributes, name)? else { + return Ok(None); + }; + + match expression { + Expr::Path(ExprPath { path, .. }) => Ok(Some(path)), + _ => Err(Error::new_spanned( + expression, + format!("{name} must be a path"), + )), + } +} + +fn find_attribute<'a>( + attributes: &'a [Attribute], + name: &str, +) -> syn::Result> { + let mut matching = attributes + .iter() + .filter(|attribute| attribute.path().is_ident(name)); + let first = matching.next(); + if let Some(duplicate) = matching.next() { + return Err(Error::new_spanned( + duplicate, + format!("duplicate #[{name}] attribute"), + )); + } + Ok(first) +} + +fn is_option_type(ty: &Type) -> bool { + let Type::Path(type_path) = ty else { + return false; + }; + + type_path + .path + .segments + .last() + .is_some_and(|segment| segment.ident == "Option") +} + +fn serialize_field(field: &PropertyField) -> TokenStream2 { + let ident = &field.ident; + let key = &field.key; + if field.is_option { + let value = match &field.serialize_with { + Some(serialize_with) => quote!(#serialize_with(&self.#ident)), + None => quote!(::std::string::ToString::to_string( + self.#ident.as_ref().expect("checked is_some above") + )), + }; + quote! { + if self.#ident.is_some() { + map.serialize_entry(#key, &#value)?; + } + } + } else { + let value = match &field.serialize_with { + Some(serialize_with) => quote!(#serialize_with(&self.#ident)), + None => quote!(::std::string::ToString::to_string(&self.#ident)), + }; + quote! { + map.serialize_entry(#key, &#value)?; + } + } +} From d7f7def227f61f626dae4c6f8b9c9e9996d85e18 Mon Sep 17 00:00:00 2001 From: Renjie Liu Date: Tue, 4 Aug 2026 20:34:48 +0800 Subject: [PATCH 02/14] Address comments --- Cargo.lock | 2 + crates/iceberg/public-api.txt | 475 ++++++- crates/iceberg/src/catalog/utils.rs | 2 +- crates/iceberg/src/encryption/manager.rs | 2 +- crates/iceberg/src/spec/table_metadata.rs | 41 +- crates/iceberg/src/spec/table_properties.rs | 1189 +++++++++++++++-- .../src/transaction/expire_snapshots.rs | 6 +- crates/iceberg/src/transaction/mod.rs | 6 +- .../src/writer/file_writer/parquet_writer.rs | 39 +- .../datafusion/src/physical_plan/write.rs | 4 +- crates/property-macro/Cargo.toml | 4 + crates/property-macro/src/lib.rs | 249 ++-- crates/property-macro/tests/properties.rs | 60 + 13 files changed, 1848 insertions(+), 231 deletions(-) create mode 100644 crates/property-macro/tests/properties.rs diff --git a/Cargo.lock b/Cargo.lock index 663cae7945..6e281007fd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4006,6 +4006,8 @@ version = "0.10.0" dependencies = [ "proc-macro2", "quote", + "serde", + "serde_json", "syn", ] diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index 61f68e50a3..e1c0d9583b 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -2244,6 +2244,258 @@ impl serde_core::ser::Serialize for iceberg::spec::NestedField pub fn iceberg::spec::NestedField::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::NestedField pub fn iceberg::spec::NestedField::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::spec::ParsedTableProperties +impl iceberg::spec::ParsedTableProperties +pub fn iceberg::spec::ParsedTableProperties::adaptive_split_size_enabled(&self) -> bool +pub fn iceberg::spec::ParsedTableProperties::avro_compression(&self) -> alloc::string::String +pub fn iceberg::spec::ParsedTableProperties::avro_compression_level(&self) -> core::option::Option +pub fn iceberg::spec::ParsedTableProperties::cdc_enabled(&self) -> bool +pub fn iceberg::spec::ParsedTableProperties::cdc_max_chunk_size(&self) -> usize +pub fn iceberg::spec::ParsedTableProperties::cdc_min_chunk_size(&self) -> usize +pub fn iceberg::spec::ParsedTableProperties::cdc_norm_level(&self) -> i32 +pub fn iceberg::spec::ParsedTableProperties::comment(&self) -> core::option::Option +pub fn iceberg::spec::ParsedTableProperties::commit_max_retry_wait_ms(&self) -> u64 +pub fn iceberg::spec::ParsedTableProperties::commit_min_retry_wait_ms(&self) -> u64 +pub fn iceberg::spec::ParsedTableProperties::commit_num_retries(&self) -> usize +pub fn iceberg::spec::ParsedTableProperties::commit_num_status_checks(&self) -> usize +pub fn iceberg::spec::ParsedTableProperties::commit_status_checks_max_wait_ms(&self) -> u64 +pub fn iceberg::spec::ParsedTableProperties::commit_status_checks_min_wait_ms(&self) -> u64 +pub fn iceberg::spec::ParsedTableProperties::commit_status_checks_total_wait_ms(&self) -> u64 +pub fn iceberg::spec::ParsedTableProperties::commit_total_retry_timeout_ms(&self) -> u64 +pub fn iceberg::spec::ParsedTableProperties::data_planning_mode(&self) -> alloc::string::String +pub fn iceberg::spec::ParsedTableProperties::default_name_mapping(&self) -> core::option::Option +pub fn iceberg::spec::ParsedTableProperties::default_write_metrics_mode(&self) -> alloc::string::String +pub fn iceberg::spec::ParsedTableProperties::delete_avro_compression(&self) -> alloc::string::String +pub fn iceberg::spec::ParsedTableProperties::delete_avro_compression_level(&self) -> core::option::Option +pub fn iceberg::spec::ParsedTableProperties::delete_distribution_mode(&self) -> core::option::Option +pub fn iceberg::spec::ParsedTableProperties::delete_format_default(&self) -> iceberg::spec::DataFileFormat +pub fn iceberg::spec::ParsedTableProperties::delete_granularity(&self) -> alloc::string::String +pub fn iceberg::spec::ParsedTableProperties::delete_isolation_level(&self) -> alloc::string::String +pub fn iceberg::spec::ParsedTableProperties::delete_mode(&self) -> alloc::string::String +pub fn iceberg::spec::ParsedTableProperties::delete_orc_block_size_bytes(&self) -> u64 +pub fn iceberg::spec::ParsedTableProperties::delete_orc_compression(&self) -> alloc::string::String +pub fn iceberg::spec::ParsedTableProperties::delete_orc_compression_strategy(&self) -> alloc::string::String +pub fn iceberg::spec::ParsedTableProperties::delete_orc_stripe_size_bytes(&self) -> u64 +pub fn iceberg::spec::ParsedTableProperties::delete_orc_write_batch_size(&self) -> usize +pub fn iceberg::spec::ParsedTableProperties::delete_parquet_compression(&self) -> alloc::string::String +pub fn iceberg::spec::ParsedTableProperties::delete_parquet_compression_level(&self) -> core::option::Option +pub fn iceberg::spec::ParsedTableProperties::delete_parquet_dict_size_bytes(&self) -> usize +pub fn iceberg::spec::ParsedTableProperties::delete_parquet_page_row_limit(&self) -> usize +pub fn iceberg::spec::ParsedTableProperties::delete_parquet_page_size_bytes(&self) -> usize +pub fn iceberg::spec::ParsedTableProperties::delete_parquet_page_version(&self) -> alloc::string::String +pub fn iceberg::spec::ParsedTableProperties::delete_parquet_row_group_check_max_record_count(&self) -> usize +pub fn iceberg::spec::ParsedTableProperties::delete_parquet_row_group_check_min_record_count(&self) -> usize +pub fn iceberg::spec::ParsedTableProperties::delete_parquet_row_group_size_bytes(&self) -> usize +pub fn iceberg::spec::ParsedTableProperties::delete_planning_mode(&self) -> alloc::string::String +pub fn iceberg::spec::ParsedTableProperties::delete_target_file_size_bytes(&self) -> usize +pub fn iceberg::spec::ParsedTableProperties::encryption_data_key_length(&self) -> usize +pub fn iceberg::spec::ParsedTableProperties::encryption_key_id(&self) -> core::option::Option +pub fn iceberg::spec::ParsedTableProperties::engine_hive_enabled(&self) -> bool +pub fn iceberg::spec::ParsedTableProperties::gc_enabled(&self) -> bool +pub fn iceberg::spec::ParsedTableProperties::hive_lock_enabled(&self) -> bool +pub fn iceberg::spec::ParsedTableProperties::identifier_fields_rely(&self) -> bool +pub fn iceberg::spec::ParsedTableProperties::manifest_compression(&self) -> alloc::string::String +pub fn iceberg::spec::ParsedTableProperties::manifest_compression_level(&self) -> core::option::Option +pub fn iceberg::spec::ParsedTableProperties::manifest_lists_enabled(&self) -> bool +pub fn iceberg::spec::ParsedTableProperties::manifest_merge_enabled(&self) -> bool +pub fn iceberg::spec::ParsedTableProperties::manifest_min_merge_count(&self) -> usize +pub fn iceberg::spec::ParsedTableProperties::manifest_target_size_bytes(&self) -> usize +pub fn iceberg::spec::ParsedTableProperties::max_ref_age_ms(&self) -> i64 +pub fn iceberg::spec::ParsedTableProperties::max_snapshot_age_ms(&self) -> i64 +pub fn iceberg::spec::ParsedTableProperties::merge_distribution_mode(&self) -> core::option::Option +pub fn iceberg::spec::ParsedTableProperties::merge_isolation_level(&self) -> alloc::string::String +pub fn iceberg::spec::ParsedTableProperties::merge_mode(&self) -> alloc::string::String +pub fn iceberg::spec::ParsedTableProperties::metadata_compression_codec(&self) -> iceberg::compression::CompressionCodec +pub fn iceberg::spec::ParsedTableProperties::metadata_delete_after_commit_enabled(&self) -> bool +pub fn iceberg::spec::ParsedTableProperties::metadata_previous_versions_max(&self) -> usize +pub fn iceberg::spec::ParsedTableProperties::metadata_split_size(&self) -> u64 +pub fn iceberg::spec::ParsedTableProperties::metrics_max_inferred_column_defaults(&self) -> usize +pub fn iceberg::spec::ParsedTableProperties::metrics_mode_column_config(&self) -> std::collections::hash::map::HashMap +pub fn iceberg::spec::ParsedTableProperties::min_snapshots_to_keep(&self) -> usize +pub fn iceberg::spec::ParsedTableProperties::object_store_enabled(&self) -> bool +pub fn iceberg::spec::ParsedTableProperties::object_store_path(&self) -> core::option::Option +pub fn iceberg::spec::ParsedTableProperties::orc_batch_size(&self) -> usize +pub fn iceberg::spec::ParsedTableProperties::orc_block_size_bytes(&self) -> u64 +pub fn iceberg::spec::ParsedTableProperties::orc_bloom_filter_columns(&self) -> alloc::string::String +pub fn iceberg::spec::ParsedTableProperties::orc_bloom_filter_fpp(&self) -> f64 +pub fn iceberg::spec::ParsedTableProperties::orc_compression(&self) -> alloc::string::String +pub fn iceberg::spec::ParsedTableProperties::orc_compression_strategy(&self) -> alloc::string::String +pub fn iceberg::spec::ParsedTableProperties::orc_stripe_size_bytes(&self) -> u64 +pub fn iceberg::spec::ParsedTableProperties::orc_vectorization_enabled(&self) -> bool +pub fn iceberg::spec::ParsedTableProperties::orc_write_batch_size(&self) -> usize +pub fn iceberg::spec::ParsedTableProperties::parquet_batch_size(&self) -> usize +pub fn iceberg::spec::ParsedTableProperties::parquet_bloom_filter_adaptive_enabled(&self) -> bool +pub fn iceberg::spec::ParsedTableProperties::parquet_bloom_filter_column_enabled(&self) -> std::collections::hash::map::HashMap +pub fn iceberg::spec::ParsedTableProperties::parquet_bloom_filter_column_fpp(&self) -> std::collections::hash::map::HashMap +pub fn iceberg::spec::ParsedTableProperties::parquet_bloom_filter_column_ndv(&self) -> std::collections::hash::map::HashMap +pub fn iceberg::spec::ParsedTableProperties::parquet_bloom_filter_max_bytes(&self) -> usize +pub fn iceberg::spec::ParsedTableProperties::parquet_column_stats_enabled(&self) -> std::collections::hash::map::HashMap +pub fn iceberg::spec::ParsedTableProperties::parquet_compression(&self) -> alloc::string::String +pub fn iceberg::spec::ParsedTableProperties::parquet_compression_level(&self) -> core::option::Option +pub fn iceberg::spec::ParsedTableProperties::parquet_dict_encoding_enabled_column(&self) -> std::collections::hash::map::HashMap +pub fn iceberg::spec::ParsedTableProperties::parquet_dict_size_bytes(&self) -> usize +pub fn iceberg::spec::ParsedTableProperties::parquet_page_row_limit(&self) -> usize +pub fn iceberg::spec::ParsedTableProperties::parquet_page_size_bytes(&self) -> usize +pub fn iceberg::spec::ParsedTableProperties::parquet_page_version(&self) -> alloc::string::String +pub fn iceberg::spec::ParsedTableProperties::parquet_row_group_check_max_record_count(&self) -> usize +pub fn iceberg::spec::ParsedTableProperties::parquet_row_group_check_min_record_count(&self) -> usize +pub fn iceberg::spec::ParsedTableProperties::parquet_row_group_size_bytes(&self) -> usize +pub fn iceberg::spec::ParsedTableProperties::parquet_row_group_size_track_uncompressed(&self) -> bool +pub fn iceberg::spec::ParsedTableProperties::parquet_shred_variants(&self) -> bool +pub fn iceberg::spec::ParsedTableProperties::parquet_variant_buffer_size(&self) -> usize +pub fn iceberg::spec::ParsedTableProperties::parquet_vectorization_enabled(&self) -> bool +pub fn iceberg::spec::ParsedTableProperties::snapshot_id_inheritance_enabled(&self) -> bool +pub fn iceberg::spec::ParsedTableProperties::spark_write_accept_any_schema(&self) -> bool +pub fn iceberg::spec::ParsedTableProperties::spark_write_advisory_partition_size_bytes(&self) -> core::option::Option +pub fn iceberg::spec::ParsedTableProperties::spark_write_auto_schema_evolution(&self) -> bool +pub fn iceberg::spec::ParsedTableProperties::spark_write_partitioned_fanout_enabled(&self) -> bool +pub fn iceberg::spec::ParsedTableProperties::split_lookback(&self) -> usize +pub fn iceberg::spec::ParsedTableProperties::split_open_file_cost(&self) -> u64 +pub fn iceberg::spec::ParsedTableProperties::split_size(&self) -> u64 +pub fn iceberg::spec::ParsedTableProperties::update_distribution_mode(&self) -> core::option::Option +pub fn iceberg::spec::ParsedTableProperties::update_isolation_level(&self) -> alloc::string::String +pub fn iceberg::spec::ParsedTableProperties::update_mode(&self) -> alloc::string::String +pub fn iceberg::spec::ParsedTableProperties::upsert_enabled(&self) -> bool +pub fn iceberg::spec::ParsedTableProperties::with_adaptive_split_size_enabled(self, value: bool) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_avro_compression(self, value: alloc::string::String) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_avro_compression_level(self, value: core::option::Option) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_cdc_enabled(self, value: bool) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_cdc_max_chunk_size(self, value: usize) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_cdc_min_chunk_size(self, value: usize) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_cdc_norm_level(self, value: i32) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_comment(self, value: core::option::Option) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_commit_max_retry_wait_ms(self, value: u64) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_commit_min_retry_wait_ms(self, value: u64) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_commit_num_retries(self, value: usize) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_commit_num_status_checks(self, value: usize) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_commit_status_checks_max_wait_ms(self, value: u64) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_commit_status_checks_min_wait_ms(self, value: u64) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_commit_status_checks_total_wait_ms(self, value: u64) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_commit_total_retry_timeout_ms(self, value: u64) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_data_planning_mode(self, value: alloc::string::String) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_default_name_mapping(self, value: core::option::Option) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_default_write_metrics_mode(self, value: alloc::string::String) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_delete_avro_compression(self, value: alloc::string::String) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_delete_avro_compression_level(self, value: core::option::Option) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_delete_distribution_mode(self, value: core::option::Option) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_delete_format_default(self, value: iceberg::spec::DataFileFormat) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_delete_granularity(self, value: alloc::string::String) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_delete_isolation_level(self, value: alloc::string::String) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_delete_mode(self, value: alloc::string::String) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_delete_orc_block_size_bytes(self, value: u64) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_delete_orc_compression(self, value: alloc::string::String) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_delete_orc_compression_strategy(self, value: alloc::string::String) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_delete_orc_stripe_size_bytes(self, value: u64) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_delete_orc_write_batch_size(self, value: usize) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_delete_parquet_compression(self, value: alloc::string::String) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_delete_parquet_compression_level(self, value: core::option::Option) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_delete_parquet_dict_size_bytes(self, value: usize) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_delete_parquet_page_row_limit(self, value: usize) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_delete_parquet_page_size_bytes(self, value: usize) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_delete_parquet_page_version(self, value: alloc::string::String) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_delete_parquet_row_group_check_max_record_count(self, value: usize) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_delete_parquet_row_group_check_min_record_count(self, value: usize) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_delete_parquet_row_group_size_bytes(self, value: usize) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_delete_planning_mode(self, value: alloc::string::String) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_delete_target_file_size_bytes(self, value: usize) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_encryption_data_key_length(self, value: usize) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_encryption_key_id(self, value: core::option::Option) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_engine_hive_enabled(self, value: bool) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_gc_enabled(self, value: bool) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_hive_lock_enabled(self, value: bool) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_identifier_fields_rely(self, value: bool) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_manifest_compression(self, value: alloc::string::String) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_manifest_compression_level(self, value: core::option::Option) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_manifest_lists_enabled(self, value: bool) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_manifest_merge_enabled(self, value: bool) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_manifest_min_merge_count(self, value: usize) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_manifest_target_size_bytes(self, value: usize) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_max_ref_age_ms(self, value: i64) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_max_snapshot_age_ms(self, value: i64) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_merge_distribution_mode(self, value: core::option::Option) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_merge_isolation_level(self, value: alloc::string::String) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_merge_mode(self, value: alloc::string::String) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_metadata_compression_codec(self, value: iceberg::compression::CompressionCodec) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_metadata_delete_after_commit_enabled(self, value: bool) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_metadata_previous_versions_max(self, value: usize) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_metadata_split_size(self, value: u64) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_metrics_max_inferred_column_defaults(self, value: usize) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_metrics_mode_column_config(self, value: std::collections::hash::map::HashMap) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_min_snapshots_to_keep(self, value: usize) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_object_store_enabled(self, value: bool) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_object_store_path(self, value: core::option::Option) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_orc_batch_size(self, value: usize) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_orc_block_size_bytes(self, value: u64) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_orc_bloom_filter_columns(self, value: alloc::string::String) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_orc_bloom_filter_fpp(self, value: f64) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_orc_compression(self, value: alloc::string::String) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_orc_compression_strategy(self, value: alloc::string::String) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_orc_stripe_size_bytes(self, value: u64) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_orc_vectorization_enabled(self, value: bool) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_orc_write_batch_size(self, value: usize) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_parquet_batch_size(self, value: usize) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_parquet_bloom_filter_adaptive_enabled(self, value: bool) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_parquet_bloom_filter_column_enabled(self, value: std::collections::hash::map::HashMap) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_parquet_bloom_filter_column_fpp(self, value: std::collections::hash::map::HashMap) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_parquet_bloom_filter_column_ndv(self, value: std::collections::hash::map::HashMap) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_parquet_bloom_filter_max_bytes(self, value: usize) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_parquet_column_stats_enabled(self, value: std::collections::hash::map::HashMap) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_parquet_compression(self, value: alloc::string::String) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_parquet_compression_level(self, value: core::option::Option) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_parquet_dict_encoding_enabled_column(self, value: std::collections::hash::map::HashMap) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_parquet_dict_size_bytes(self, value: usize) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_parquet_page_row_limit(self, value: usize) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_parquet_page_size_bytes(self, value: usize) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_parquet_page_version(self, value: alloc::string::String) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_parquet_row_group_check_max_record_count(self, value: usize) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_parquet_row_group_check_min_record_count(self, value: usize) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_parquet_row_group_size_bytes(self, value: usize) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_parquet_row_group_size_track_uncompressed(self, value: bool) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_parquet_shred_variants(self, value: bool) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_parquet_variant_buffer_size(self, value: usize) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_parquet_vectorization_enabled(self, value: bool) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_snapshot_id_inheritance_enabled(self, value: bool) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_spark_write_accept_any_schema(self, value: bool) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_spark_write_advisory_partition_size_bytes(self, value: core::option::Option) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_spark_write_auto_schema_evolution(self, value: bool) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_spark_write_partitioned_fanout_enabled(self, value: bool) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_split_lookback(self, value: usize) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_split_open_file_cost(self, value: u64) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_split_size(self, value: u64) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_update_distribution_mode(self, value: core::option::Option) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_update_isolation_level(self, value: alloc::string::String) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_update_mode(self, value: alloc::string::String) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_upsert_enabled(self, value: bool) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_write_audit_publish_enabled(self, value: bool) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_write_data_location(self, value: core::option::Option) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_write_datafusion_fanout_enabled(self, value: bool) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_write_distribution_mode(self, value: core::option::Option) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_write_folder_storage_location(self, value: core::option::Option) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_write_format_default(self, value: iceberg::spec::DataFileFormat) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_write_location_provider_impl(self, value: core::option::Option) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_write_metadata_path(self, value: core::option::Option) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_write_object_store_partitioned_paths(self, value: bool) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_write_partition_summary_limit(self, value: u64) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_write_target_file_size_bytes(self, value: usize) -> Self +pub fn iceberg::spec::ParsedTableProperties::write_audit_publish_enabled(&self) -> bool +pub fn iceberg::spec::ParsedTableProperties::write_data_location(&self) -> core::option::Option +pub fn iceberg::spec::ParsedTableProperties::write_datafusion_fanout_enabled(&self) -> bool +pub fn iceberg::spec::ParsedTableProperties::write_distribution_mode(&self) -> core::option::Option +pub fn iceberg::spec::ParsedTableProperties::write_folder_storage_location(&self) -> core::option::Option +pub fn iceberg::spec::ParsedTableProperties::write_format_default(&self) -> iceberg::spec::DataFileFormat +pub fn iceberg::spec::ParsedTableProperties::write_location_provider_impl(&self) -> core::option::Option +pub fn iceberg::spec::ParsedTableProperties::write_metadata_path(&self) -> core::option::Option +pub fn iceberg::spec::ParsedTableProperties::write_object_store_partitioned_paths(&self) -> bool +pub fn iceberg::spec::ParsedTableProperties::write_partition_summary_limit(&self) -> u64 +pub fn iceberg::spec::ParsedTableProperties::write_target_file_size_bytes(&self) -> usize +impl core::default::Default for iceberg::spec::ParsedTableProperties +pub fn iceberg::spec::ParsedTableProperties::default() -> Self +impl core::fmt::Debug for iceberg::spec::ParsedTableProperties +pub fn iceberg::spec::ParsedTableProperties::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl serde_core::ser::Serialize for iceberg::spec::ParsedTableProperties +pub fn iceberg::spec::ParsedTableProperties::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> where S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::ParsedTableProperties +pub fn iceberg::spec::ParsedTableProperties::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> pub struct iceberg::spec::PartitionField pub iceberg::spec::PartitionField::field_id: i32 pub iceberg::spec::PartitionField::name: alloc::string::String @@ -2652,6 +2904,7 @@ pub fn iceberg::spec::TableMetadata::metadata_location(&self) -> iceberg::Result pub fn iceberg::spec::TableMetadata::metadata_log(&self) -> &[iceberg::spec::MetadataLog] pub fn iceberg::spec::TableMetadata::next_row_id(&self) -> u64 pub fn iceberg::spec::TableMetadata::next_sequence_number(&self) -> i64 +pub fn iceberg::spec::TableMetadata::parsed_table_properties(&self) -> iceberg::Result pub fn iceberg::spec::TableMetadata::partition_spec_by_id(&self, spec_id: i32) -> core::option::Option<&iceberg::spec::PartitionSpecRef> pub fn iceberg::spec::TableMetadata::partition_specs_iter(&self) -> impl core::iter::traits::exact_size::ExactSizeIterator pub fn iceberg::spec::TableMetadata::partition_statistics_for_snapshot(&self, snapshot_id: i64) -> core::option::Option<&iceberg::spec::PartitionStatisticsFile> @@ -2735,13 +2988,47 @@ pub fn iceberg::spec::TableMetadataBuilder::clone(&self) -> iceberg::spec::Table impl core::fmt::Debug for iceberg::spec::TableMetadataBuilder pub fn iceberg::spec::TableMetadataBuilder::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result pub struct iceberg::spec::TableProperties +pub iceberg::spec::TableProperties::cdc_enabled: bool +pub iceberg::spec::TableProperties::cdc_max_chunk_size: usize +pub iceberg::spec::TableProperties::cdc_min_chunk_size: usize +pub iceberg::spec::TableProperties::cdc_norm_level: i32 +pub iceberg::spec::TableProperties::commit_max_retry_wait_ms: u64 +pub iceberg::spec::TableProperties::commit_min_retry_wait_ms: u64 +pub iceberg::spec::TableProperties::commit_num_retries: usize +pub iceberg::spec::TableProperties::commit_total_retry_timeout_ms: u64 +pub iceberg::spec::TableProperties::encryption_data_key_length: usize +pub iceberg::spec::TableProperties::encryption_key_id: core::option::Option +pub iceberg::spec::TableProperties::gc_enabled: bool +pub iceberg::spec::TableProperties::max_ref_age_ms: i64 +pub iceberg::spec::TableProperties::max_snapshot_age_ms: i64 +pub iceberg::spec::TableProperties::metadata_compression_codec: iceberg::compression::CompressionCodec +pub iceberg::spec::TableProperties::min_snapshots_to_keep: usize +pub iceberg::spec::TableProperties::write_datafusion_fanout_enabled: bool +pub iceberg::spec::TableProperties::write_format_default: alloc::string::String +pub iceberg::spec::TableProperties::write_metadata_path: core::option::Option +pub iceberg::spec::TableProperties::write_target_file_size_bytes: usize impl iceberg::spec::TableProperties +pub const iceberg::spec::TableProperties::PROPERTY_ADAPTIVE_SPLIT_SIZE_ENABLED: &str +pub const iceberg::spec::TableProperties::PROPERTY_ADAPTIVE_SPLIT_SIZE_ENABLED_DEFAULT: bool +pub const iceberg::spec::TableProperties::PROPERTY_AVRO_COMPRESSION: &str +pub const iceberg::spec::TableProperties::PROPERTY_AVRO_COMPRESSION_DEFAULT: &str +pub const iceberg::spec::TableProperties::PROPERTY_AVRO_COMPRESSION_LEVEL: &str +pub const iceberg::spec::TableProperties::PROPERTY_AVRO_COMPRESSION_LEVEL_DEFAULT: core::option::Option<&str> +pub const iceberg::spec::TableProperties::PROPERTY_COMMENT: &str pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS: &str pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT: u64 pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS: &str pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT: u64 pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_NUM_RETRIES: &str pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT: usize +pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_NUM_STATUS_CHECKS: &str +pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_NUM_STATUS_CHECKS_DEFAULT: usize +pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_STATUS_CHECKS_MAX_WAIT_MS: &str +pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_STATUS_CHECKS_MAX_WAIT_MS_DEFAULT: u64 +pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_STATUS_CHECKS_MIN_WAIT_MS: &str +pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_STATUS_CHECKS_MIN_WAIT_MS_DEFAULT: u64 +pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_STATUS_CHECKS_TOTAL_WAIT_MS: &str +pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_STATUS_CHECKS_TOTAL_WAIT_MS_DEFAULT: u64 pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS: &str pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT: u64 pub const iceberg::spec::TableProperties::PROPERTY_CURRENT_SCHEMA: &str @@ -2750,27 +3037,124 @@ pub const iceberg::spec::TableProperties::PROPERTY_CURRENT_SNAPSHOT_SUMMARY: &st pub const iceberg::spec::TableProperties::PROPERTY_CURRENT_SNAPSHOT_TIMESTAMP: &str pub const iceberg::spec::TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED: &str pub const iceberg::spec::TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT: bool +pub const iceberg::spec::TableProperties::PROPERTY_DATA_PLANNING_MODE: &str pub const iceberg::spec::TableProperties::PROPERTY_DEFAULT_FILE_FORMAT: &str pub const iceberg::spec::TableProperties::PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT: &str +pub const iceberg::spec::TableProperties::PROPERTY_DEFAULT_NAME_MAPPING: &str pub const iceberg::spec::TableProperties::PROPERTY_DEFAULT_PARTITION_SPEC: &str pub const iceberg::spec::TableProperties::PROPERTY_DEFAULT_SORT_ORDER: &str +pub const iceberg::spec::TableProperties::PROPERTY_DEFAULT_WRITE_METRICS_MODE: &str +pub const iceberg::spec::TableProperties::PROPERTY_DEFAULT_WRITE_METRICS_MODE_DEFAULT: &str +pub const iceberg::spec::TableProperties::PROPERTY_DELETE_AVRO_COMPRESSION: &str +pub const iceberg::spec::TableProperties::PROPERTY_DELETE_AVRO_COMPRESSION_LEVEL: &str pub const iceberg::spec::TableProperties::PROPERTY_DELETE_DEFAULT_FILE_FORMAT: &str +pub const iceberg::spec::TableProperties::PROPERTY_DELETE_DISTRIBUTION_MODE: &str +pub const iceberg::spec::TableProperties::PROPERTY_DELETE_GRANULARITY: &str +pub const iceberg::spec::TableProperties::PROPERTY_DELETE_GRANULARITY_DEFAULT: &str +pub const iceberg::spec::TableProperties::PROPERTY_DELETE_ISOLATION_LEVEL: &str +pub const iceberg::spec::TableProperties::PROPERTY_DELETE_ISOLATION_LEVEL_DEFAULT: &str +pub const iceberg::spec::TableProperties::PROPERTY_DELETE_MODE: &str +pub const iceberg::spec::TableProperties::PROPERTY_DELETE_MODE_DEFAULT: &str +pub const iceberg::spec::TableProperties::PROPERTY_DELETE_ORC_BLOCK_SIZE_BYTES: &str +pub const iceberg::spec::TableProperties::PROPERTY_DELETE_ORC_COMPRESSION: &str +pub const iceberg::spec::TableProperties::PROPERTY_DELETE_ORC_COMPRESSION_STRATEGY: &str +pub const iceberg::spec::TableProperties::PROPERTY_DELETE_ORC_STRIPE_SIZE_BYTES: &str +pub const iceberg::spec::TableProperties::PROPERTY_DELETE_ORC_WRITE_BATCH_SIZE: &str +pub const iceberg::spec::TableProperties::PROPERTY_DELETE_PARQUET_COMPRESSION: &str +pub const iceberg::spec::TableProperties::PROPERTY_DELETE_PARQUET_COMPRESSION_LEVEL: &str +pub const iceberg::spec::TableProperties::PROPERTY_DELETE_PARQUET_DICT_SIZE_BYTES: &str +pub const iceberg::spec::TableProperties::PROPERTY_DELETE_PARQUET_PAGE_ROW_LIMIT: &str +pub const iceberg::spec::TableProperties::PROPERTY_DELETE_PARQUET_PAGE_SIZE_BYTES: &str +pub const iceberg::spec::TableProperties::PROPERTY_DELETE_PARQUET_PAGE_VERSION: &str +pub const iceberg::spec::TableProperties::PROPERTY_DELETE_PARQUET_ROW_GROUP_CHECK_MAX_RECORD_COUNT: &str +pub const iceberg::spec::TableProperties::PROPERTY_DELETE_PARQUET_ROW_GROUP_CHECK_MIN_RECORD_COUNT: &str +pub const iceberg::spec::TableProperties::PROPERTY_DELETE_PARQUET_ROW_GROUP_SIZE_BYTES: &str +pub const iceberg::spec::TableProperties::PROPERTY_DELETE_PLANNING_MODE: &str +pub const iceberg::spec::TableProperties::PROPERTY_DELETE_TARGET_FILE_SIZE_BYTES: &str +pub const iceberg::spec::TableProperties::PROPERTY_DELETE_TARGET_FILE_SIZE_BYTES_DEFAULT: usize +pub const iceberg::spec::TableProperties::PROPERTY_ENCRYPTION_AAD_LENGTH_DEFAULT: usize pub const iceberg::spec::TableProperties::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH: &str pub const iceberg::spec::TableProperties::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH_DEFAULT: usize +pub const iceberg::spec::TableProperties::PROPERTY_ENCRYPTION_DEK_LENGTH: &str +pub const iceberg::spec::TableProperties::PROPERTY_ENCRYPTION_DEK_LENGTH_DEFAULT: usize pub const iceberg::spec::TableProperties::PROPERTY_ENCRYPTION_KEY_ID: &str +pub const iceberg::spec::TableProperties::PROPERTY_ENCRYPTION_TABLE_KEY: &str +pub const iceberg::spec::TableProperties::PROPERTY_ENGINE_HIVE_ENABLED: &str +pub const iceberg::spec::TableProperties::PROPERTY_ENGINE_HIVE_ENABLED_DEFAULT: bool pub const iceberg::spec::TableProperties::PROPERTY_FORMAT_VERSION: &str pub const iceberg::spec::TableProperties::PROPERTY_GC_ENABLED: &str pub const iceberg::spec::TableProperties::PROPERTY_GC_ENABLED_DEFAULT: bool +pub const iceberg::spec::TableProperties::PROPERTY_HIVE_LOCK_ENABLED: &str +pub const iceberg::spec::TableProperties::PROPERTY_HIVE_LOCK_ENABLED_DEFAULT: bool +pub const iceberg::spec::TableProperties::PROPERTY_IDENTIFIER_FIELDS_RELY: &str +pub const iceberg::spec::TableProperties::PROPERTY_IDENTIFIER_FIELDS_RELY_DEFAULT: bool +pub const iceberg::spec::TableProperties::PROPERTY_MANIFEST_COMPRESSION: &str +pub const iceberg::spec::TableProperties::PROPERTY_MANIFEST_COMPRESSION_DEFAULT: &str +pub const iceberg::spec::TableProperties::PROPERTY_MANIFEST_COMPRESSION_LEVEL: &str +pub const iceberg::spec::TableProperties::PROPERTY_MANIFEST_COMPRESSION_LEVEL_DEFAULT: core::option::Option<&str> +pub const iceberg::spec::TableProperties::PROPERTY_MANIFEST_LISTS_ENABLED: &str +pub const iceberg::spec::TableProperties::PROPERTY_MANIFEST_LISTS_ENABLED_DEFAULT: bool +pub const iceberg::spec::TableProperties::PROPERTY_MANIFEST_MERGE_ENABLED: &str +pub const iceberg::spec::TableProperties::PROPERTY_MANIFEST_MERGE_ENABLED_DEFAULT: bool +pub const iceberg::spec::TableProperties::PROPERTY_MANIFEST_MIN_MERGE_COUNT: &str +pub const iceberg::spec::TableProperties::PROPERTY_MANIFEST_MIN_MERGE_COUNT_DEFAULT: usize +pub const iceberg::spec::TableProperties::PROPERTY_MANIFEST_TARGET_SIZE_BYTES: &str +pub const iceberg::spec::TableProperties::PROPERTY_MANIFEST_TARGET_SIZE_BYTES_DEFAULT: usize pub const iceberg::spec::TableProperties::PROPERTY_MAX_REF_AGE_MS: &str pub const iceberg::spec::TableProperties::PROPERTY_MAX_REF_AGE_MS_DEFAULT: i64 pub const iceberg::spec::TableProperties::PROPERTY_MAX_SNAPSHOT_AGE_MS: &str pub const iceberg::spec::TableProperties::PROPERTY_MAX_SNAPSHOT_AGE_MS_DEFAULT: i64 +pub const iceberg::spec::TableProperties::PROPERTY_MERGE_DISTRIBUTION_MODE: &str +pub const iceberg::spec::TableProperties::PROPERTY_MERGE_ISOLATION_LEVEL: &str +pub const iceberg::spec::TableProperties::PROPERTY_MERGE_ISOLATION_LEVEL_DEFAULT: &str +pub const iceberg::spec::TableProperties::PROPERTY_MERGE_MODE: &str +pub const iceberg::spec::TableProperties::PROPERTY_MERGE_MODE_DEFAULT: &str +pub const iceberg::spec::TableProperties::PROPERTY_METADATA_COMPRESSION: &str pub const iceberg::spec::TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC: &str pub const iceberg::spec::TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC_DEFAULT: &str +pub const iceberg::spec::TableProperties::PROPERTY_METADATA_COMPRESSION_DEFAULT: &str +pub const iceberg::spec::TableProperties::PROPERTY_METADATA_DELETE_AFTER_COMMIT_ENABLED: &str +pub const iceberg::spec::TableProperties::PROPERTY_METADATA_DELETE_AFTER_COMMIT_ENABLED_DEFAULT: bool pub const iceberg::spec::TableProperties::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX: &str pub const iceberg::spec::TableProperties::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX_DEFAULT: usize +pub const iceberg::spec::TableProperties::PROPERTY_METADATA_SPLIT_SIZE: &str +pub const iceberg::spec::TableProperties::PROPERTY_METADATA_SPLIT_SIZE_DEFAULT: u64 +pub const iceberg::spec::TableProperties::PROPERTY_METRICS_MAX_INFERRED_COLUMN_DEFAULTS: &str +pub const iceberg::spec::TableProperties::PROPERTY_METRICS_MAX_INFERRED_COLUMN_DEFAULTS_DEFAULT: usize +pub const iceberg::spec::TableProperties::PROPERTY_METRICS_MODE_COLUMN_CONF_PREFIX: &str pub const iceberg::spec::TableProperties::PROPERTY_MIN_SNAPSHOTS_TO_KEEP: &str pub const iceberg::spec::TableProperties::PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT: usize +pub const iceberg::spec::TableProperties::PROPERTY_OBJECT_STORE_ENABLED: &str +pub const iceberg::spec::TableProperties::PROPERTY_OBJECT_STORE_ENABLED_DEFAULT: bool +pub const iceberg::spec::TableProperties::PROPERTY_OBJECT_STORE_PATH: &str +pub const iceberg::spec::TableProperties::PROPERTY_ORC_BATCH_SIZE: &str +pub const iceberg::spec::TableProperties::PROPERTY_ORC_BATCH_SIZE_DEFAULT: usize +pub const iceberg::spec::TableProperties::PROPERTY_ORC_BLOCK_SIZE_BYTES: &str +pub const iceberg::spec::TableProperties::PROPERTY_ORC_BLOCK_SIZE_BYTES_DEFAULT: u64 +pub const iceberg::spec::TableProperties::PROPERTY_ORC_BLOOM_FILTER_COLUMNS: &str +pub const iceberg::spec::TableProperties::PROPERTY_ORC_BLOOM_FILTER_COLUMNS_DEFAULT: &str +pub const iceberg::spec::TableProperties::PROPERTY_ORC_BLOOM_FILTER_FPP: &str +pub const iceberg::spec::TableProperties::PROPERTY_ORC_BLOOM_FILTER_FPP_DEFAULT: f64 +pub const iceberg::spec::TableProperties::PROPERTY_ORC_COMPRESSION: &str +pub const iceberg::spec::TableProperties::PROPERTY_ORC_COMPRESSION_DEFAULT: &str +pub const iceberg::spec::TableProperties::PROPERTY_ORC_COMPRESSION_STRATEGY: &str +pub const iceberg::spec::TableProperties::PROPERTY_ORC_COMPRESSION_STRATEGY_DEFAULT: &str +pub const iceberg::spec::TableProperties::PROPERTY_ORC_STRIPE_SIZE_BYTES: &str +pub const iceberg::spec::TableProperties::PROPERTY_ORC_STRIPE_SIZE_BYTES_DEFAULT: u64 +pub const iceberg::spec::TableProperties::PROPERTY_ORC_VECTORIZATION_ENABLED: &str +pub const iceberg::spec::TableProperties::PROPERTY_ORC_VECTORIZATION_ENABLED_DEFAULT: bool +pub const iceberg::spec::TableProperties::PROPERTY_ORC_WRITE_BATCH_SIZE: &str +pub const iceberg::spec::TableProperties::PROPERTY_ORC_WRITE_BATCH_SIZE_DEFAULT: usize +pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_BATCH_SIZE: &str +pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_BATCH_SIZE_DEFAULT: usize +pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_BLOOM_FILTER_ADAPTIVE_ENABLED: &str +pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_BLOOM_FILTER_ADAPTIVE_ENABLED_DEFAULT: bool +pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX: &str +pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_BLOOM_FILTER_COLUMN_FPP_DEFAULT: f64 +pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_BLOOM_FILTER_COLUMN_FPP_PREFIX: &str +pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_BLOOM_FILTER_COLUMN_NDV_PREFIX: &str +pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_BLOOM_FILTER_MAX_BYTES: &str +pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_BLOOM_FILTER_MAX_BYTES_DEFAULT: usize pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_ENABLED: &str pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_ENABLED_DEFAULT: bool pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE: &str @@ -2779,45 +3163,83 @@ pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE: & pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT: usize pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL: &str pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL_DEFAULT: i32 +pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_COLUMN_STATS_ENABLED_PREFIX: &str +pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_COMPRESSION: &str +pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_COMPRESSION_DEFAULT: &str +pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_COMPRESSION_DEFAULT_SINCE_1_4_0: &str +pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_COMPRESSION_LEVEL: &str +pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_COMPRESSION_LEVEL_DEFAULT: core::option::Option<&str> +pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_DICT_ENCODING_ENABLED_COLUMN_PREFIX: &str +pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_DICT_SIZE_BYTES: &str +pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_DICT_SIZE_BYTES_DEFAULT: usize +pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_PAGE_ROW_LIMIT: &str +pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_PAGE_ROW_LIMIT_DEFAULT: usize +pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_PAGE_SIZE_BYTES: &str +pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_PAGE_SIZE_BYTES_DEFAULT: usize +pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_PAGE_VERSION: &str +pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_PAGE_VERSION_DEFAULT: &str +pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_ROW_GROUP_CHECK_MAX_RECORD_COUNT: &str +pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_ROW_GROUP_CHECK_MAX_RECORD_COUNT_DEFAULT: usize +pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_ROW_GROUP_CHECK_MIN_RECORD_COUNT: &str +pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_ROW_GROUP_CHECK_MIN_RECORD_COUNT_DEFAULT: usize +pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES: &str +pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT: usize +pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_TRACK_UNCOMPRESSED: &str +pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_TRACK_UNCOMPRESSED_DEFAULT: bool +pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_SHRED_VARIANTS: &str +pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_SHRED_VARIANTS_DEFAULT: bool +pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_VARIANT_BUFFER_SIZE: &str +pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_VARIANT_BUFFER_SIZE_DEFAULT: usize +pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_VECTORIZATION_ENABLED: &str +pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_VECTORIZATION_ENABLED_DEFAULT: bool +pub const iceberg::spec::TableProperties::PROPERTY_PLANNING_MODE_DEFAULT: &str pub const iceberg::spec::TableProperties::PROPERTY_SNAPSHOT_COUNT: &str +pub const iceberg::spec::TableProperties::PROPERTY_SNAPSHOT_ID_INHERITANCE_ENABLED: &str +pub const iceberg::spec::TableProperties::PROPERTY_SNAPSHOT_ID_INHERITANCE_ENABLED_DEFAULT: bool +pub const iceberg::spec::TableProperties::PROPERTY_SPARK_WRITE_ACCEPT_ANY_SCHEMA: &str +pub const iceberg::spec::TableProperties::PROPERTY_SPARK_WRITE_ACCEPT_ANY_SCHEMA_DEFAULT: bool +pub const iceberg::spec::TableProperties::PROPERTY_SPARK_WRITE_ADVISORY_PARTITION_SIZE_BYTES: &str +pub const iceberg::spec::TableProperties::PROPERTY_SPARK_WRITE_AUTO_SCHEMA_EVOLUTION: &str +pub const iceberg::spec::TableProperties::PROPERTY_SPARK_WRITE_AUTO_SCHEMA_EVOLUTION_DEFAULT: bool +pub const iceberg::spec::TableProperties::PROPERTY_SPARK_WRITE_PARTITIONED_FANOUT_ENABLED: &str +pub const iceberg::spec::TableProperties::PROPERTY_SPARK_WRITE_PARTITIONED_FANOUT_ENABLED_DEFAULT: bool +pub const iceberg::spec::TableProperties::PROPERTY_SPLIT_LOOKBACK: &str +pub const iceberg::spec::TableProperties::PROPERTY_SPLIT_LOOKBACK_DEFAULT: usize +pub const iceberg::spec::TableProperties::PROPERTY_SPLIT_OPEN_FILE_COST: &str +pub const iceberg::spec::TableProperties::PROPERTY_SPLIT_OPEN_FILE_COST_DEFAULT: u64 +pub const iceberg::spec::TableProperties::PROPERTY_SPLIT_SIZE: &str +pub const iceberg::spec::TableProperties::PROPERTY_SPLIT_SIZE_DEFAULT: u64 +pub const iceberg::spec::TableProperties::PROPERTY_UPDATE_DISTRIBUTION_MODE: &str +pub const iceberg::spec::TableProperties::PROPERTY_UPDATE_ISOLATION_LEVEL: &str +pub const iceberg::spec::TableProperties::PROPERTY_UPDATE_ISOLATION_LEVEL_DEFAULT: &str +pub const iceberg::spec::TableProperties::PROPERTY_UPDATE_MODE: &str +pub const iceberg::spec::TableProperties::PROPERTY_UPDATE_MODE_DEFAULT: &str +pub const iceberg::spec::TableProperties::PROPERTY_UPSERT_ENABLED: &str +pub const iceberg::spec::TableProperties::PROPERTY_UPSERT_ENABLED_DEFAULT: bool pub const iceberg::spec::TableProperties::PROPERTY_UUID: &str +pub const iceberg::spec::TableProperties::PROPERTY_WRITE_AUDIT_PUBLISH_ENABLED: &str +pub const iceberg::spec::TableProperties::PROPERTY_WRITE_AUDIT_PUBLISH_ENABLED_DEFAULT: bool +pub const iceberg::spec::TableProperties::PROPERTY_WRITE_DATA_LOCATION: &str +pub const iceberg::spec::TableProperties::PROPERTY_WRITE_DISTRIBUTION_MODE: &str +pub const iceberg::spec::TableProperties::PROPERTY_WRITE_DISTRIBUTION_MODE_HASH: &str +pub const iceberg::spec::TableProperties::PROPERTY_WRITE_DISTRIBUTION_MODE_NONE: &str +pub const iceberg::spec::TableProperties::PROPERTY_WRITE_DISTRIBUTION_MODE_RANGE: &str +pub const iceberg::spec::TableProperties::PROPERTY_WRITE_FOLDER_STORAGE_LOCATION: &str +pub const iceberg::spec::TableProperties::PROPERTY_WRITE_LOCATION_PROVIDER_IMPL: &str +pub const iceberg::spec::TableProperties::PROPERTY_WRITE_METADATA_LOCATION: &str pub const iceberg::spec::TableProperties::PROPERTY_WRITE_METADATA_PATH: &str +pub const iceberg::spec::TableProperties::PROPERTY_WRITE_OBJECT_STORE_PARTITIONED_PATHS: &str +pub const iceberg::spec::TableProperties::PROPERTY_WRITE_OBJECT_STORE_PARTITIONED_PATHS_DEFAULT: bool pub const iceberg::spec::TableProperties::PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT: &str pub const iceberg::spec::TableProperties::PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT_DEFAULT: u64 pub const iceberg::spec::TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES: &str pub const iceberg::spec::TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT: usize pub const iceberg::spec::TableProperties::RESERVED_PROPERTIES: [&str; 9] -impl iceberg::spec::TableProperties -pub fn iceberg::spec::TableProperties::cdc_enabled(&self) -> bool -pub fn iceberg::spec::TableProperties::cdc_max_chunk_size(&self) -> usize -pub fn iceberg::spec::TableProperties::cdc_min_chunk_size(&self) -> usize -pub fn iceberg::spec::TableProperties::cdc_norm_level(&self) -> i32 -pub fn iceberg::spec::TableProperties::commit_max_retry_wait_ms(&self) -> u64 -pub fn iceberg::spec::TableProperties::commit_min_retry_wait_ms(&self) -> u64 -pub fn iceberg::spec::TableProperties::commit_num_retries(&self) -> usize -pub fn iceberg::spec::TableProperties::commit_total_retry_timeout_ms(&self) -> u64 -pub fn iceberg::spec::TableProperties::encryption_data_key_length(&self) -> usize -pub fn iceberg::spec::TableProperties::encryption_key_id(&self) -> core::option::Option -pub fn iceberg::spec::TableProperties::gc_enabled(&self) -> bool -pub fn iceberg::spec::TableProperties::max_ref_age_ms(&self) -> i64 -pub fn iceberg::spec::TableProperties::max_snapshot_age_ms(&self) -> i64 -pub fn iceberg::spec::TableProperties::metadata_compression_codec(&self) -> iceberg::compression::CompressionCodec -pub fn iceberg::spec::TableProperties::min_snapshots_to_keep(&self) -> usize -pub fn iceberg::spec::TableProperties::write_datafusion_fanout_enabled(&self) -> bool -pub fn iceberg::spec::TableProperties::write_format_default(&self) -> iceberg::spec::DataFileFormat -pub fn iceberg::spec::TableProperties::write_metadata_path(&self) -> core::option::Option -pub fn iceberg::spec::TableProperties::write_target_file_size_bytes(&self) -> usize impl core::convert::TryFrom<&std::collections::hash::map::HashMap> for iceberg::spec::TableProperties pub type iceberg::spec::TableProperties::Error = iceberg::Error pub fn iceberg::spec::TableProperties::try_from(props: &std::collections::hash::map::HashMap) -> iceberg::Result -impl core::default::Default for iceberg::spec::TableProperties -pub fn iceberg::spec::TableProperties::default() -> Self impl core::fmt::Debug for iceberg::spec::TableProperties pub fn iceberg::spec::TableProperties::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl serde_core::ser::Serialize for iceberg::spec::TableProperties -pub fn iceberg::spec::TableProperties::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> where S: serde_core::ser::Serializer -impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::TableProperties -pub fn iceberg::spec::TableProperties::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> pub struct iceberg::spec::UnboundPartitionField pub iceberg::spec::UnboundPartitionField::field_id: core::option::Option pub iceberg::spec::UnboundPartitionField::name: alloc::string::String @@ -3277,6 +3699,7 @@ pub async fn iceberg::writer::file_writer::ParquetWriter::close(self) -> iceberg pub async fn iceberg::writer::file_writer::ParquetWriter::write(&mut self, batch: &arrow_array::record_batch::RecordBatch) -> iceberg::Result<()> pub struct iceberg::writer::file_writer::ParquetWriterBuilder impl iceberg::writer::file_writer::ParquetWriterBuilder +pub fn iceberg::writer::file_writer::ParquetWriterBuilder::from_parsed_table_properties(table_props: &iceberg::spec::ParsedTableProperties, schema: iceberg::spec::SchemaRef) -> Self pub fn iceberg::writer::file_writer::ParquetWriterBuilder::from_table_properties(table_props: &iceberg::spec::TableProperties, schema: iceberg::spec::SchemaRef) -> Self pub fn iceberg::writer::file_writer::ParquetWriterBuilder::new(props: parquet::file::properties::WriterProperties, schema: iceberg::spec::SchemaRef) -> Self pub fn iceberg::writer::file_writer::ParquetWriterBuilder::new_with_match_mode(props: parquet::file::properties::WriterProperties, schema: iceberg::spec::SchemaRef, match_mode: iceberg::arrow::FieldMatchMode) -> Self diff --git a/crates/iceberg/src/catalog/utils.rs b/crates/iceberg/src/catalog/utils.rs index f38fb57c79..d214c4f0fe 100644 --- a/crates/iceberg/src/catalog/utils.rs +++ b/crates/iceberg/src/catalog/utils.rs @@ -60,7 +60,7 @@ pub async fn drop_table_data(table_info: &Table) -> Result<()> { } // Delete data files only if gc.enabled is true, to avoid corrupting shared tables - if metadata.table_properties()?.gc_enabled() { + if metadata.parsed_table_properties()?.gc_enabled() { delete_data_files(io, &manifests_to_delete).await?; } diff --git a/crates/iceberg/src/encryption/manager.rs b/crates/iceberg/src/encryption/manager.rs index 260202f5bc..2393f7e65d 100644 --- a/crates/iceberg/src/encryption/manager.rs +++ b/crates/iceberg/src/encryption/manager.rs @@ -118,7 +118,7 @@ impl EncryptionManager { return Ok(None); } - let table_properties = metadata.table_properties()?; + let table_properties = metadata.parsed_table_properties()?; let Some(table_key_id) = table_properties.encryption_key_id() else { if kms_client.is_some() { tracing::warn!( diff --git a/crates/iceberg/src/spec/table_metadata.rs b/crates/iceberg/src/spec/table_metadata.rs index 71b794b0c5..9e019ea6ad 100644 --- a/crates/iceberg/src/spec/table_metadata.rs +++ b/crates/iceberg/src/spec/table_metadata.rs @@ -33,9 +33,9 @@ use uuid::Uuid; use super::snapshot::SnapshotReference; pub use super::table_metadata_builder::{TableMetadataBuildResult, TableMetadataBuilder}; use super::{ - DEFAULT_PARTITION_SPEC_ID, PartitionSpecRef, PartitionStatisticsFile, SchemaId, SchemaRef, - SnapshotRef, SnapshotRetention, SortOrder, SortOrderRef, StatisticsFile, StructType, - TableProperties, parse_metadata_file_compression, + DEFAULT_PARTITION_SPEC_ID, ParsedTableProperties, PartitionSpecRef, PartitionStatisticsFile, + SchemaId, SchemaRef, SnapshotRef, SnapshotRetention, SortOrder, SortOrderRef, StatisticsFile, + StructType, TableProperties, parse_metadata_file_compression, }; use crate::catalog::{METADATA_FOLDER_NAME, MetadataLocation}; use crate::compression::CompressionCodec; @@ -371,7 +371,7 @@ impl TableMetadata { pub fn metadata_location(&self) -> Result { Ok(self .table_properties()? - .write_metadata_path() + .write_metadata_path .unwrap_or_else(|| format!("{}/{}", self.location(), METADATA_FOLDER_NAME))) } @@ -387,13 +387,22 @@ impl TableMetadata { parse_metadata_file_compression(&self.properties) } - /// Returns typed table properties parsed from the raw properties map with defaults. + /// Returns the existing typed table properties parsed from the raw property map. pub fn table_properties(&self) -> Result { - TableProperties::try_from(&self.properties).map_err(|e| { - Error::new(ErrorKind::DataInvalid, "Invalid table properties").with_source(e) + TableProperties::try_from(&self.properties).map_err(|error| { + Error::new(ErrorKind::DataInvalid, "Invalid table properties").with_source(error) }) } + /// Returns all supported table properties parsed by the property framework. + pub fn parsed_table_properties(&self) -> Result { + serde_json::to_value(&self.properties) + .and_then(serde_json::from_value) + .map_err(|error| { + Error::new(ErrorKind::DataInvalid, "Invalid table properties").with_source(error) + }) + } + /// Return location of statistics files. #[inline] pub fn statistics_iter(&self) -> impl ExactSizeIterator { @@ -4017,7 +4026,7 @@ mod tests { } #[test] - fn test_table_properties_with_defaults() { + fn test_parsed_table_properties_with_defaults() { use crate::spec::TableProperties; let schema = Schema::builder() @@ -4040,7 +4049,7 @@ mod tests { .unwrap() .metadata; - let props = metadata.table_properties().unwrap(); + let props = metadata.parsed_table_properties().unwrap(); assert_eq!( props.commit_num_retries(), @@ -4050,10 +4059,16 @@ mod tests { props.write_target_file_size_bytes(), TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT ); + + let existing_props = metadata.table_properties().unwrap(); + assert_eq!( + existing_props.commit_num_retries, + TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT + ); } #[test] - fn test_table_properties_with_custom_values() { + fn test_parsed_table_properties_with_custom_values() { use crate::spec::TableProperties; let schema = Schema::builder() @@ -4087,14 +4102,14 @@ mod tests { .unwrap() .metadata; - let props = metadata.table_properties().unwrap(); + let props = metadata.parsed_table_properties().unwrap(); assert_eq!(props.commit_num_retries(), 10); assert_eq!(props.write_target_file_size_bytes(), 1024); } #[test] - fn test_table_properties_with_invalid_value() { + fn test_parsed_table_properties_with_invalid_value() { let schema = Schema::builder() .with_fields(vec![ NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(), @@ -4120,7 +4135,7 @@ mod tests { .unwrap() .metadata; - let err = metadata.table_properties().unwrap_err(); + let err = metadata.parsed_table_properties().unwrap_err(); assert_eq!(err.kind(), ErrorKind::DataInvalid); assert!(err.message().contains("Invalid table properties")); } diff --git a/crates/iceberg/src/spec/table_properties.rs b/crates/iceberg/src/spec/table_properties.rs index cf4881d4c3..4409a4c5de 100644 --- a/crates/iceberg/src/spec/table_properties.rs +++ b/crates/iceberg/src/spec/table_properties.rs @@ -16,6 +16,8 @@ // under the License. use std::collections::HashMap; +use std::fmt::Display; +use std::str::FromStr; use iceberg_property_macro::Properties; @@ -23,6 +25,24 @@ use crate::compression::CompressionCodec; use crate::error::{Error, ErrorKind, Result}; use crate::spec::DataFileFormat; +fn parse_property( + properties: &HashMap, + key: &str, + default: T, +) -> Result +where + ::Err: Display, +{ + properties.get(key).map_or(Ok(default), |value| { + value.parse::().map_err(|e| { + Error::new( + ErrorKind::DataInvalid, + format!("Invalid value for {key}: {e}"), + ) + }) + }) +} + /// Strips trailing slashes from a location, preserving a bare URI scheme root fn strip_trailing_slash(path: &str) -> &str { let mut path = path; @@ -43,31 +63,23 @@ fn parse_metadata_location(value: &str) -> Result> { Ok(Some(strip_trailing_slash(value).to_string())) } -fn parse_optional_string(value: &str) -> Result> { - Ok(Some(value.to_string())) -} - -/// Parse compression codec for metadata files from table properties. -/// Retrieves the compression codec property, applies defaults, and parses the value. -/// Only "none" (or empty string) and "gzip" are supported for metadata compression. -/// -/// # Arguments -/// -/// * `properties` - HashMap containing table properties -/// -/// # Errors -/// -/// Returns an error if the codec is not "none", "", or "gzip" (case-insensitive). -/// Lz4 and Zstd are not supported for metadata file compression. -pub(crate) fn parse_metadata_file_compression( +fn parse_location_property( properties: &HashMap, -) -> Result { - let value = properties - .get(TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC) - .map(|s| s.as_str()) - .unwrap_or(TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC_DEFAULT); - - parse_metadata_file_compression_value(value) + key: &str, +) -> Result> { + properties + .get(key) + .map(|path| { + if path.is_empty() { + return Err(Error::new( + ErrorKind::DataInvalid, + format!("Invalid value for {key}: path must not be empty"), + )); + } + + Ok(strip_trailing_slash(path).to_string()) + }) + .transpose() } fn parse_metadata_file_compression_value(value: &str) -> Result { @@ -108,99 +120,469 @@ fn parse_metadata_file_compression_value(value: &str) -> Result, +) -> Result { + parse_metadata_file_compression_value( + properties + .get(TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC) + .map(String::as_str) + .unwrap_or(TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC_DEFAULT), + ) +} + fn serialize_compression_codec(codec: &CompressionCodec) -> String { codec.name().to_string() } /// Typed table properties parsed from a table's string property map. +/// +/// This includes the properties defined by the pinned [Java `TableProperties`] reference as well +/// as Iceberg Rust-specific properties. +/// +/// [Java `TableProperties`]: https://github.com/apache/iceberg/blob/d8c10a1608170f0ba83be740d6ab0b6a3757cb3e/core/src/main/java/org/apache/iceberg/TableProperties.java #[derive(Debug, Properties)] -pub struct TableProperties { +pub struct ParsedTableProperties { /// The number of times to retry a commit. - #[key = "commit.retry.num-retries"] + #[key(TableProperties::PROPERTY_COMMIT_NUM_RETRIES)] #[default(TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT)] commit_num_retries: usize, /// The minimum wait time between retries. - #[key = "commit.retry.min-wait-ms"] + #[key(TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS)] #[default(TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT)] commit_min_retry_wait_ms: u64, /// The maximum wait time between retries. - #[key = "commit.retry.max-wait-ms"] + #[key(TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS)] #[default(TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT)] commit_max_retry_wait_ms: u64, /// The total timeout for commit retries. - #[key = "commit.retry.total-timeout-ms"] + #[key(TableProperties::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS)] #[default(TableProperties::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT)] commit_total_retry_timeout_ms: u64, + #[key(TableProperties::PROPERTY_COMMENT)] + #[default(None)] + #[doc = "The table's business meaning and usage context."] + comment: Option, + #[key(TableProperties::PROPERTY_COMMIT_NUM_STATUS_CHECKS)] + #[default(TableProperties::PROPERTY_COMMIT_NUM_STATUS_CHECKS_DEFAULT)] + commit_num_status_checks: usize, + #[key(TableProperties::PROPERTY_COMMIT_STATUS_CHECKS_MIN_WAIT_MS)] + #[default(TableProperties::PROPERTY_COMMIT_STATUS_CHECKS_MIN_WAIT_MS_DEFAULT)] + commit_status_checks_min_wait_ms: u64, + #[key(TableProperties::PROPERTY_COMMIT_STATUS_CHECKS_MAX_WAIT_MS)] + #[default(TableProperties::PROPERTY_COMMIT_STATUS_CHECKS_MAX_WAIT_MS_DEFAULT)] + commit_status_checks_max_wait_ms: u64, + #[key(TableProperties::PROPERTY_COMMIT_STATUS_CHECKS_TOTAL_WAIT_MS)] + #[default(TableProperties::PROPERTY_COMMIT_STATUS_CHECKS_TOTAL_WAIT_MS_DEFAULT)] + commit_status_checks_total_wait_ms: u64, + #[key(TableProperties::PROPERTY_MANIFEST_TARGET_SIZE_BYTES)] + #[default(TableProperties::PROPERTY_MANIFEST_TARGET_SIZE_BYTES_DEFAULT)] + manifest_target_size_bytes: usize, + #[key(TableProperties::PROPERTY_MANIFEST_MIN_MERGE_COUNT)] + #[default(TableProperties::PROPERTY_MANIFEST_MIN_MERGE_COUNT_DEFAULT)] + manifest_min_merge_count: usize, + #[key(TableProperties::PROPERTY_MANIFEST_MERGE_ENABLED)] + #[default(TableProperties::PROPERTY_MANIFEST_MERGE_ENABLED_DEFAULT)] + manifest_merge_enabled: bool, /// The default format for files. - #[key = "write.format.default"] + #[key(TableProperties::PROPERTY_DEFAULT_FILE_FORMAT)] #[default(DataFileFormat::Parquet)] write_format_default: DataFileFormat, + #[key(TableProperties::PROPERTY_DELETE_DEFAULT_FILE_FORMAT)] + #[default(DataFileFormat::Parquet)] + delete_format_default: DataFileFormat, + #[key(TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES)] + #[default(TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT)] + parquet_row_group_size_bytes: usize, + #[key(TableProperties::PROPERTY_DELETE_PARQUET_ROW_GROUP_SIZE_BYTES)] + #[default(TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT)] + delete_parquet_row_group_size_bytes: usize, + #[key(TableProperties::PROPERTY_PARQUET_PAGE_SIZE_BYTES)] + #[default(TableProperties::PROPERTY_PARQUET_PAGE_SIZE_BYTES_DEFAULT)] + parquet_page_size_bytes: usize, + #[key(TableProperties::PROPERTY_DELETE_PARQUET_PAGE_SIZE_BYTES)] + #[default(TableProperties::PROPERTY_PARQUET_PAGE_SIZE_BYTES_DEFAULT)] + delete_parquet_page_size_bytes: usize, + #[key(TableProperties::PROPERTY_PARQUET_PAGE_VERSION)] + #[default(TableProperties::PROPERTY_PARQUET_PAGE_VERSION_DEFAULT.to_string())] + parquet_page_version: String, + #[key(TableProperties::PROPERTY_DELETE_PARQUET_PAGE_VERSION)] + #[default(TableProperties::PROPERTY_PARQUET_PAGE_VERSION_DEFAULT.to_string())] + delete_parquet_page_version: String, + #[key(TableProperties::PROPERTY_PARQUET_PAGE_ROW_LIMIT)] + #[default(TableProperties::PROPERTY_PARQUET_PAGE_ROW_LIMIT_DEFAULT)] + parquet_page_row_limit: usize, + #[key(TableProperties::PROPERTY_DELETE_PARQUET_PAGE_ROW_LIMIT)] + #[default(TableProperties::PROPERTY_PARQUET_PAGE_ROW_LIMIT_DEFAULT)] + delete_parquet_page_row_limit: usize, + #[key(TableProperties::PROPERTY_PARQUET_DICT_SIZE_BYTES)] + #[default(TableProperties::PROPERTY_PARQUET_DICT_SIZE_BYTES_DEFAULT)] + parquet_dict_size_bytes: usize, + #[key(TableProperties::PROPERTY_DELETE_PARQUET_DICT_SIZE_BYTES)] + #[default(TableProperties::PROPERTY_PARQUET_DICT_SIZE_BYTES_DEFAULT)] + delete_parquet_dict_size_bytes: usize, + #[key(TableProperties::PROPERTY_PARQUET_COMPRESSION)] + #[default(TableProperties::PROPERTY_PARQUET_COMPRESSION_DEFAULT_SINCE_1_4_0.to_string())] + parquet_compression: String, + #[key(TableProperties::PROPERTY_DELETE_PARQUET_COMPRESSION)] + #[default(TableProperties::PROPERTY_PARQUET_COMPRESSION_DEFAULT_SINCE_1_4_0.to_string())] + delete_parquet_compression: String, + #[key(TableProperties::PROPERTY_PARQUET_COMPRESSION_LEVEL)] + #[default(None)] + parquet_compression_level: Option, + #[key(TableProperties::PROPERTY_DELETE_PARQUET_COMPRESSION_LEVEL)] + #[default(None)] + delete_parquet_compression_level: Option, + #[key(TableProperties::PROPERTY_PARQUET_SHRED_VARIANTS)] + #[default(TableProperties::PROPERTY_PARQUET_SHRED_VARIANTS_DEFAULT)] + parquet_shred_variants: bool, + #[key(TableProperties::PROPERTY_PARQUET_VARIANT_BUFFER_SIZE)] + #[default(TableProperties::PROPERTY_PARQUET_VARIANT_BUFFER_SIZE_DEFAULT)] + parquet_variant_buffer_size: usize, + #[key(TableProperties::PROPERTY_PARQUET_ROW_GROUP_CHECK_MIN_RECORD_COUNT)] + #[default(TableProperties::PROPERTY_PARQUET_ROW_GROUP_CHECK_MIN_RECORD_COUNT_DEFAULT)] + parquet_row_group_check_min_record_count: usize, + #[key(TableProperties::PROPERTY_DELETE_PARQUET_ROW_GROUP_CHECK_MIN_RECORD_COUNT)] + #[default(TableProperties::PROPERTY_PARQUET_ROW_GROUP_CHECK_MIN_RECORD_COUNT_DEFAULT)] + delete_parquet_row_group_check_min_record_count: usize, + #[key(TableProperties::PROPERTY_PARQUET_ROW_GROUP_CHECK_MAX_RECORD_COUNT)] + #[default(TableProperties::PROPERTY_PARQUET_ROW_GROUP_CHECK_MAX_RECORD_COUNT_DEFAULT)] + parquet_row_group_check_max_record_count: usize, + #[key(TableProperties::PROPERTY_DELETE_PARQUET_ROW_GROUP_CHECK_MAX_RECORD_COUNT)] + #[default(TableProperties::PROPERTY_PARQUET_ROW_GROUP_CHECK_MAX_RECORD_COUNT_DEFAULT)] + delete_parquet_row_group_check_max_record_count: usize, + #[key(TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_TRACK_UNCOMPRESSED)] + #[default(TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_TRACK_UNCOMPRESSED_DEFAULT)] + parquet_row_group_size_track_uncompressed: bool, + #[key(TableProperties::PROPERTY_PARQUET_BLOOM_FILTER_MAX_BYTES)] + #[default(TableProperties::PROPERTY_PARQUET_BLOOM_FILTER_MAX_BYTES_DEFAULT)] + parquet_bloom_filter_max_bytes: usize, + #[key(TableProperties::PROPERTY_PARQUET_BLOOM_FILTER_ADAPTIVE_ENABLED)] + #[default(TableProperties::PROPERTY_PARQUET_BLOOM_FILTER_ADAPTIVE_ENABLED_DEFAULT)] + parquet_bloom_filter_adaptive_enabled: bool, + #[prefix(TableProperties::PROPERTY_PARQUET_BLOOM_FILTER_COLUMN_FPP_PREFIX)] + #[default(HashMap::new())] + parquet_bloom_filter_column_fpp: HashMap, + #[prefix(TableProperties::PROPERTY_PARQUET_BLOOM_FILTER_COLUMN_NDV_PREFIX)] + #[default(HashMap::new())] + parquet_bloom_filter_column_ndv: HashMap, + #[prefix(TableProperties::PROPERTY_PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX)] + #[default(HashMap::new())] + parquet_bloom_filter_column_enabled: HashMap, + #[prefix(TableProperties::PROPERTY_PARQUET_COLUMN_STATS_ENABLED_PREFIX)] + #[default(HashMap::new())] + parquet_column_stats_enabled: HashMap, + #[prefix(TableProperties::PROPERTY_PARQUET_DICT_ENCODING_ENABLED_COLUMN_PREFIX)] + #[default(HashMap::new())] + parquet_dict_encoding_enabled_column: HashMap, + #[key(TableProperties::PROPERTY_AVRO_COMPRESSION)] + #[default(TableProperties::PROPERTY_AVRO_COMPRESSION_DEFAULT.to_string())] + avro_compression: String, + #[key(TableProperties::PROPERTY_DELETE_AVRO_COMPRESSION)] + #[default(TableProperties::PROPERTY_AVRO_COMPRESSION_DEFAULT.to_string())] + delete_avro_compression: String, + #[key(TableProperties::PROPERTY_AVRO_COMPRESSION_LEVEL)] + #[default(None)] + avro_compression_level: Option, + #[key(TableProperties::PROPERTY_DELETE_AVRO_COMPRESSION_LEVEL)] + #[default(None)] + delete_avro_compression_level: Option, + #[key(TableProperties::PROPERTY_MANIFEST_COMPRESSION)] + #[default(TableProperties::PROPERTY_MANIFEST_COMPRESSION_DEFAULT.to_string())] + manifest_compression: String, + #[key(TableProperties::PROPERTY_MANIFEST_COMPRESSION_LEVEL)] + #[default(None)] + manifest_compression_level: Option, + #[key(TableProperties::PROPERTY_ORC_STRIPE_SIZE_BYTES)] + #[default(TableProperties::PROPERTY_ORC_STRIPE_SIZE_BYTES_DEFAULT)] + orc_stripe_size_bytes: u64, + #[key(TableProperties::PROPERTY_DELETE_ORC_STRIPE_SIZE_BYTES)] + #[default(TableProperties::PROPERTY_ORC_STRIPE_SIZE_BYTES_DEFAULT)] + delete_orc_stripe_size_bytes: u64, + #[key(TableProperties::PROPERTY_ORC_BLOOM_FILTER_COLUMNS)] + #[default(TableProperties::PROPERTY_ORC_BLOOM_FILTER_COLUMNS_DEFAULT.to_string())] + orc_bloom_filter_columns: String, + #[key(TableProperties::PROPERTY_ORC_BLOOM_FILTER_FPP)] + #[default(TableProperties::PROPERTY_ORC_BLOOM_FILTER_FPP_DEFAULT)] + orc_bloom_filter_fpp: f64, + #[key(TableProperties::PROPERTY_ORC_BLOCK_SIZE_BYTES)] + #[default(TableProperties::PROPERTY_ORC_BLOCK_SIZE_BYTES_DEFAULT)] + orc_block_size_bytes: u64, + #[key(TableProperties::PROPERTY_DELETE_ORC_BLOCK_SIZE_BYTES)] + #[default(TableProperties::PROPERTY_ORC_BLOCK_SIZE_BYTES_DEFAULT)] + delete_orc_block_size_bytes: u64, + #[key(TableProperties::PROPERTY_ORC_WRITE_BATCH_SIZE)] + #[default(TableProperties::PROPERTY_ORC_WRITE_BATCH_SIZE_DEFAULT)] + orc_write_batch_size: usize, + #[key(TableProperties::PROPERTY_DELETE_ORC_WRITE_BATCH_SIZE)] + #[default(TableProperties::PROPERTY_ORC_WRITE_BATCH_SIZE_DEFAULT)] + delete_orc_write_batch_size: usize, + #[key(TableProperties::PROPERTY_ORC_COMPRESSION)] + #[default(TableProperties::PROPERTY_ORC_COMPRESSION_DEFAULT.to_string())] + orc_compression: String, + #[key(TableProperties::PROPERTY_DELETE_ORC_COMPRESSION)] + #[default(TableProperties::PROPERTY_ORC_COMPRESSION_DEFAULT.to_string())] + delete_orc_compression: String, + #[key(TableProperties::PROPERTY_ORC_COMPRESSION_STRATEGY)] + #[default(TableProperties::PROPERTY_ORC_COMPRESSION_STRATEGY_DEFAULT.to_string())] + orc_compression_strategy: String, + #[key(TableProperties::PROPERTY_DELETE_ORC_COMPRESSION_STRATEGY)] + #[default(TableProperties::PROPERTY_ORC_COMPRESSION_STRATEGY_DEFAULT.to_string())] + delete_orc_compression_strategy: String, + #[key(TableProperties::PROPERTY_SPLIT_SIZE)] + #[default(TableProperties::PROPERTY_SPLIT_SIZE_DEFAULT)] + split_size: u64, + #[key(TableProperties::PROPERTY_METADATA_SPLIT_SIZE)] + #[default(TableProperties::PROPERTY_METADATA_SPLIT_SIZE_DEFAULT)] + metadata_split_size: u64, + #[key(TableProperties::PROPERTY_SPLIT_LOOKBACK)] + #[default(TableProperties::PROPERTY_SPLIT_LOOKBACK_DEFAULT)] + split_lookback: usize, + #[key(TableProperties::PROPERTY_SPLIT_OPEN_FILE_COST)] + #[default(TableProperties::PROPERTY_SPLIT_OPEN_FILE_COST_DEFAULT)] + split_open_file_cost: u64, + #[key(TableProperties::PROPERTY_ADAPTIVE_SPLIT_SIZE_ENABLED)] + #[default(TableProperties::PROPERTY_ADAPTIVE_SPLIT_SIZE_ENABLED_DEFAULT)] + adaptive_split_size_enabled: bool, + #[key(TableProperties::PROPERTY_PARQUET_VECTORIZATION_ENABLED)] + #[default(TableProperties::PROPERTY_PARQUET_VECTORIZATION_ENABLED_DEFAULT)] + parquet_vectorization_enabled: bool, + #[key(TableProperties::PROPERTY_PARQUET_BATCH_SIZE)] + #[default(TableProperties::PROPERTY_PARQUET_BATCH_SIZE_DEFAULT)] + parquet_batch_size: usize, + #[key(TableProperties::PROPERTY_ORC_VECTORIZATION_ENABLED)] + #[default(TableProperties::PROPERTY_ORC_VECTORIZATION_ENABLED_DEFAULT)] + orc_vectorization_enabled: bool, + #[key(TableProperties::PROPERTY_ORC_BATCH_SIZE)] + #[default(TableProperties::PROPERTY_ORC_BATCH_SIZE_DEFAULT)] + orc_batch_size: usize, + #[key(TableProperties::PROPERTY_DATA_PLANNING_MODE)] + #[default(TableProperties::PROPERTY_PLANNING_MODE_DEFAULT.to_string())] + data_planning_mode: String, + #[key(TableProperties::PROPERTY_DELETE_PLANNING_MODE)] + #[default(TableProperties::PROPERTY_PLANNING_MODE_DEFAULT.to_string())] + delete_planning_mode: String, + #[key(TableProperties::PROPERTY_IDENTIFIER_FIELDS_RELY)] + #[default(TableProperties::PROPERTY_IDENTIFIER_FIELDS_RELY_DEFAULT)] + identifier_fields_rely: bool, /// The target file size for files. - #[key = "write.target-file-size-bytes"] + #[key(TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES)] #[default(TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT)] write_target_file_size_bytes: usize, + #[key(TableProperties::PROPERTY_DELETE_TARGET_FILE_SIZE_BYTES)] + #[default(TableProperties::PROPERTY_DELETE_TARGET_FILE_SIZE_BYTES_DEFAULT)] + delete_target_file_size_bytes: usize, + #[key(TableProperties::PROPERTY_OBJECT_STORE_ENABLED)] + #[default(TableProperties::PROPERTY_OBJECT_STORE_ENABLED_DEFAULT)] + object_store_enabled: bool, + #[key(TableProperties::PROPERTY_WRITE_OBJECT_STORE_PARTITIONED_PATHS)] + #[default(TableProperties::PROPERTY_WRITE_OBJECT_STORE_PARTITIONED_PATHS_DEFAULT)] + write_object_store_partitioned_paths: bool, + #[key(TableProperties::PROPERTY_OBJECT_STORE_PATH)] + #[default(None)] + object_store_path: Option, + #[key(TableProperties::PROPERTY_WRITE_LOCATION_PROVIDER_IMPL)] + #[default(None)] + write_location_provider_impl: Option, + #[key(TableProperties::PROPERTY_WRITE_FOLDER_STORAGE_LOCATION)] + #[default(None)] + write_folder_storage_location: Option, + #[key(TableProperties::PROPERTY_WRITE_DATA_LOCATION)] + #[default(None)] + write_data_location: Option, /// Base directory for metadata files (manifests, manifest lists), with any /// trailing slash trimmed. `None` if `write.metadata.path` is not set. - #[key = "write.metadata.path"] + #[key(TableProperties::PROPERTY_WRITE_METADATA_PATH)] #[default(None)] #[parse_with(parse_metadata_location)] write_metadata_path: Option, + #[key(TableProperties::PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT)] + #[default(TableProperties::PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT_DEFAULT)] + write_partition_summary_limit: u64, + #[key(TableProperties::PROPERTY_MANIFEST_LISTS_ENABLED)] + #[default(TableProperties::PROPERTY_MANIFEST_LISTS_ENABLED_DEFAULT)] + manifest_lists_enabled: bool, /// Compression codec for metadata files (JSON) - #[key = "write.metadata.compression-codec"] + #[key(TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC)] #[default(CompressionCodec::None)] #[parse_with(parse_metadata_file_compression_value)] #[serialize_with(serialize_compression_codec)] metadata_compression_codec: CompressionCodec, + #[key(TableProperties::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX)] + #[default(TableProperties::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX_DEFAULT)] + metadata_previous_versions_max: usize, + #[key(TableProperties::PROPERTY_METADATA_DELETE_AFTER_COMMIT_ENABLED)] + #[default(TableProperties::PROPERTY_METADATA_DELETE_AFTER_COMMIT_ENABLED_DEFAULT)] + metadata_delete_after_commit_enabled: bool, + #[key(TableProperties::PROPERTY_METRICS_MAX_INFERRED_COLUMN_DEFAULTS)] + #[default(TableProperties::PROPERTY_METRICS_MAX_INFERRED_COLUMN_DEFAULTS_DEFAULT)] + metrics_max_inferred_column_defaults: usize, + #[prefix(TableProperties::PROPERTY_METRICS_MODE_COLUMN_CONF_PREFIX)] + #[default(HashMap::new())] + metrics_mode_column_config: HashMap, + #[key(TableProperties::PROPERTY_DEFAULT_WRITE_METRICS_MODE)] + #[default(TableProperties::PROPERTY_DEFAULT_WRITE_METRICS_MODE_DEFAULT.to_string())] + default_write_metrics_mode: String, + #[key(TableProperties::PROPERTY_DEFAULT_NAME_MAPPING)] + #[default(None)] + default_name_mapping: Option, + #[key(TableProperties::PROPERTY_WRITE_AUDIT_PUBLISH_ENABLED)] + #[default(TableProperties::PROPERTY_WRITE_AUDIT_PUBLISH_ENABLED_DEFAULT)] + write_audit_publish_enabled: bool, + #[key(TableProperties::PROPERTY_SPARK_WRITE_PARTITIONED_FANOUT_ENABLED)] + #[default(TableProperties::PROPERTY_SPARK_WRITE_PARTITIONED_FANOUT_ENABLED_DEFAULT)] + spark_write_partitioned_fanout_enabled: bool, + #[key(TableProperties::PROPERTY_SPARK_WRITE_ACCEPT_ANY_SCHEMA)] + #[default(TableProperties::PROPERTY_SPARK_WRITE_ACCEPT_ANY_SCHEMA_DEFAULT)] + spark_write_accept_any_schema: bool, + #[key(TableProperties::PROPERTY_SPARK_WRITE_AUTO_SCHEMA_EVOLUTION)] + #[default(TableProperties::PROPERTY_SPARK_WRITE_AUTO_SCHEMA_EVOLUTION_DEFAULT)] + spark_write_auto_schema_evolution: bool, + #[key(TableProperties::PROPERTY_SPARK_WRITE_ADVISORY_PARTITION_SIZE_BYTES)] + #[default(None)] + spark_write_advisory_partition_size_bytes: Option, + #[key(TableProperties::PROPERTY_SNAPSHOT_ID_INHERITANCE_ENABLED)] + #[default(TableProperties::PROPERTY_SNAPSHOT_ID_INHERITANCE_ENABLED_DEFAULT)] + snapshot_id_inheritance_enabled: bool, + #[key(TableProperties::PROPERTY_ENGINE_HIVE_ENABLED)] + #[default(TableProperties::PROPERTY_ENGINE_HIVE_ENABLED_DEFAULT)] + engine_hive_enabled: bool, + #[key(TableProperties::PROPERTY_HIVE_LOCK_ENABLED)] + #[default(TableProperties::PROPERTY_HIVE_LOCK_ENABLED_DEFAULT)] + hive_lock_enabled: bool, + #[key(TableProperties::PROPERTY_WRITE_DISTRIBUTION_MODE)] + #[default(None)] + write_distribution_mode: Option, /// Whether to use `FanoutWriter` for partitioned tables. - #[key = "write.datafusion.fanout.enabled"] + #[key(TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED)] #[default(TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT)] write_datafusion_fanout_enabled: bool, /// Whether garbage collection is enabled on drop. /// When `false`, data files will not be deleted when a table is dropped. - #[key = "gc.enabled"] + #[key(TableProperties::PROPERTY_GC_ENABLED)] #[default(TableProperties::PROPERTY_GC_ENABLED_DEFAULT)] gc_enabled: bool, /// Default maximum age of a snapshot to keep when expiring snapshots. - #[key = "history.expire.max-snapshot-age-ms"] + #[key(TableProperties::PROPERTY_MAX_SNAPSHOT_AGE_MS)] #[default(TableProperties::PROPERTY_MAX_SNAPSHOT_AGE_MS_DEFAULT)] max_snapshot_age_ms: i64, /// Default minimum number of snapshots to keep per branch when expiring snapshots. - #[key = "history.expire.min-snapshots-to-keep"] + #[key(TableProperties::PROPERTY_MIN_SNAPSHOTS_TO_KEEP)] #[default(TableProperties::PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT)] min_snapshots_to_keep: usize, /// Default maximum age of a snapshot reference to keep when expiring snapshots. - #[key = "history.expire.max-ref-age-ms"] + #[key(TableProperties::PROPERTY_MAX_REF_AGE_MS)] #[default(TableProperties::PROPERTY_MAX_REF_AGE_MS_DEFAULT)] max_ref_age_ms: i64, + #[key(TableProperties::PROPERTY_DELETE_GRANULARITY)] + #[default(TableProperties::PROPERTY_DELETE_GRANULARITY_DEFAULT.to_string())] + delete_granularity: String, + #[key(TableProperties::PROPERTY_DELETE_ISOLATION_LEVEL)] + #[default(TableProperties::PROPERTY_DELETE_ISOLATION_LEVEL_DEFAULT.to_string())] + delete_isolation_level: String, + #[key(TableProperties::PROPERTY_DELETE_MODE)] + #[default(TableProperties::PROPERTY_DELETE_MODE_DEFAULT.to_string())] + delete_mode: String, + #[key(TableProperties::PROPERTY_DELETE_DISTRIBUTION_MODE)] + #[default(None)] + delete_distribution_mode: Option, + #[key(TableProperties::PROPERTY_UPDATE_ISOLATION_LEVEL)] + #[default(TableProperties::PROPERTY_UPDATE_ISOLATION_LEVEL_DEFAULT.to_string())] + update_isolation_level: String, + #[key(TableProperties::PROPERTY_UPDATE_MODE)] + #[default(TableProperties::PROPERTY_UPDATE_MODE_DEFAULT.to_string())] + update_mode: String, + #[key(TableProperties::PROPERTY_UPDATE_DISTRIBUTION_MODE)] + #[default(None)] + update_distribution_mode: Option, + #[key(TableProperties::PROPERTY_MERGE_ISOLATION_LEVEL)] + #[default(TableProperties::PROPERTY_MERGE_ISOLATION_LEVEL_DEFAULT.to_string())] + merge_isolation_level: String, + #[key(TableProperties::PROPERTY_MERGE_MODE)] + #[default(TableProperties::PROPERTY_MERGE_MODE_DEFAULT.to_string())] + merge_mode: String, + #[key(TableProperties::PROPERTY_MERGE_DISTRIBUTION_MODE)] + #[default(None)] + merge_distribution_mode: Option, + #[key(TableProperties::PROPERTY_UPSERT_ENABLED)] + #[default(TableProperties::PROPERTY_UPSERT_ENABLED_DEFAULT)] + upsert_enabled: bool, /// Whether content-defined chunking is enabled. /// `true` only when `write.parquet.content-defined-chunking.enabled = "true"`. - #[key = "write.parquet.content-defined-chunking.enabled"] + #[key(TableProperties::PROPERTY_PARQUET_CDC_ENABLED)] #[default(TableProperties::PROPERTY_PARQUET_CDC_ENABLED_DEFAULT)] cdc_enabled: bool, /// Content-defined chunking minimum chunk size in bytes. - #[key = "write.parquet.content-defined-chunking.min-chunk-size"] + #[key(TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE)] #[default(TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT)] cdc_min_chunk_size: usize, /// Content-defined chunking maximum chunk size in bytes. - #[key = "write.parquet.content-defined-chunking.max-chunk-size"] + #[key(TableProperties::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE)] #[default(TableProperties::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE_DEFAULT)] cdc_max_chunk_size: usize, /// Content-defined chunking normalization level (gearhash bit adjustment). - #[key = "write.parquet.content-defined-chunking.norm-level"] + #[key(TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL)] #[default(TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL_DEFAULT)] cdc_norm_level: i32, /// The master key id used to encrypt this table's manifest list and data /// files. `None` if `encryption.key-id` is not set. - #[key = "encryption.key-id"] + #[key(TableProperties::PROPERTY_ENCRYPTION_KEY_ID)] #[default(None)] - #[parse_with(parse_optional_string)] encryption_key_id: Option, /// The encryption data encryption key length in bytes. - #[key = "encryption.data-key-length"] + #[key(TableProperties::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH)] #[default(TableProperties::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH_DEFAULT)] encryption_data_key_length: usize, } +/// TableProperties that contains the properties of a table. +#[derive(Debug)] +pub struct TableProperties { + /// The number of times to retry a commit. + pub commit_num_retries: usize, + /// The minimum wait time between retries. + pub commit_min_retry_wait_ms: u64, + /// The maximum wait time between retries. + pub commit_max_retry_wait_ms: u64, + /// The total timeout for commit retries. + pub commit_total_retry_timeout_ms: u64, + /// The default format for files. + pub write_format_default: String, + /// The target file size for files. + pub write_target_file_size_bytes: usize, + /// Base directory for metadata files (manifests, manifest lists), with any + /// trailing slash trimmed. `None` if `write.metadata.path` is not set. + pub write_metadata_path: Option, + /// Compression codec for metadata files (JSON) + pub metadata_compression_codec: CompressionCodec, + /// Whether to use `FanoutWriter` for partitioned tables. + pub write_datafusion_fanout_enabled: bool, + /// Whether garbage collection is enabled on drop. + /// When `false`, data files will not be deleted when a table is dropped. + pub gc_enabled: bool, + /// Default maximum age of a snapshot to keep when expiring snapshots. + pub max_snapshot_age_ms: i64, + /// Default minimum number of snapshots to keep per branch when expiring snapshots. + pub min_snapshots_to_keep: usize, + /// Default maximum age of a snapshot reference to keep when expiring snapshots. + pub max_ref_age_ms: i64, + /// Whether content-defined chunking is enabled. + /// `true` only when `write.parquet.content-defined-chunking.enabled = "true"`. + pub cdc_enabled: bool, + /// Content-defined chunking minimum chunk size in bytes. + pub cdc_min_chunk_size: usize, + /// Content-defined chunking maximum chunk size in bytes. + pub cdc_max_chunk_size: usize, + /// Content-defined chunking normalization level (gearhash bit adjustment). + pub cdc_norm_level: i32, + /// The master key id used to encrypt this table's manifest list and data + /// files. `None` if `encryption.key-id` is not set. + pub encryption_key_id: Option, + /// The encryption data encryption key length in bytes. + pub encryption_data_key_length: usize, +} + impl TableProperties { /// Reserved table property for table format version. /// @@ -351,24 +733,508 @@ impl TableProperties { pub const PROPERTY_ENCRYPTION_DATA_KEY_LENGTH: &str = "encryption.data-key-length"; /// Default value for the encryption DEK length (16 bytes = AES-128). pub const PROPERTY_ENCRYPTION_DATA_KEY_LENGTH_DEFAULT: usize = 16; + + /// Table property documenting the business meaning and usage context of the table. + pub const PROPERTY_COMMENT: &str = "comment"; + + /// Property key for commit status-check retries. + pub const PROPERTY_COMMIT_NUM_STATUS_CHECKS: &str = "commit.status-check.num-retries"; + /// Default number of commit status-check retries. + pub const PROPERTY_COMMIT_NUM_STATUS_CHECKS_DEFAULT: usize = 3; + /// Property key for minimum wait between commit status checks. + pub const PROPERTY_COMMIT_STATUS_CHECKS_MIN_WAIT_MS: &str = "commit.status-check.min-wait-ms"; + /// Default minimum wait between commit status checks. + pub const PROPERTY_COMMIT_STATUS_CHECKS_MIN_WAIT_MS_DEFAULT: u64 = 1000; + /// Property key for maximum wait between commit status checks. + pub const PROPERTY_COMMIT_STATUS_CHECKS_MAX_WAIT_MS: &str = "commit.status-check.max-wait-ms"; + /// Default maximum wait between commit status checks. + pub const PROPERTY_COMMIT_STATUS_CHECKS_MAX_WAIT_MS_DEFAULT: u64 = 60 * 1000; + /// Property key for total commit status-check timeout. + pub const PROPERTY_COMMIT_STATUS_CHECKS_TOTAL_WAIT_MS: &str = + "commit.status-check.total-timeout-ms"; + /// Default total commit status-check timeout. + pub const PROPERTY_COMMIT_STATUS_CHECKS_TOTAL_WAIT_MS_DEFAULT: u64 = 30 * 60 * 1000; + + /// Property key for the target manifest file size. + pub const PROPERTY_MANIFEST_TARGET_SIZE_BYTES: &str = "commit.manifest.target-size-bytes"; + /// Default target manifest file size. + pub const PROPERTY_MANIFEST_TARGET_SIZE_BYTES_DEFAULT: usize = 8 * 1024 * 1024; + /// Property key for the minimum manifest count before merging. + pub const PROPERTY_MANIFEST_MIN_MERGE_COUNT: &str = "commit.manifest.min-count-to-merge"; + /// Default minimum manifest count before merging. + pub const PROPERTY_MANIFEST_MIN_MERGE_COUNT_DEFAULT: usize = 100; + /// Property key controlling automatic manifest merging. + pub const PROPERTY_MANIFEST_MERGE_ENABLED: &str = "commit.manifest-merge.enabled"; + /// Default automatic manifest merging setting. + pub const PROPERTY_MANIFEST_MERGE_ENABLED_DEFAULT: bool = true; + + /// Property key for Parquet row group size. + pub const PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES: &str = "write.parquet.row-group-size-bytes"; + /// Property key for delete-file Parquet row group size. + pub const PROPERTY_DELETE_PARQUET_ROW_GROUP_SIZE_BYTES: &str = + "write.delete.parquet.row-group-size-bytes"; + /// Default Parquet row group size. + pub const PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT: usize = 128 * 1024 * 1024; + /// Property key for Parquet page size. + pub const PROPERTY_PARQUET_PAGE_SIZE_BYTES: &str = "write.parquet.page-size-bytes"; + /// Property key for delete-file Parquet page size. + pub const PROPERTY_DELETE_PARQUET_PAGE_SIZE_BYTES: &str = + "write.delete.parquet.page-size-bytes"; + /// Default Parquet page size. + pub const PROPERTY_PARQUET_PAGE_SIZE_BYTES_DEFAULT: usize = 1024 * 1024; + /// Property key for Parquet page version. + pub const PROPERTY_PARQUET_PAGE_VERSION: &str = "write.parquet.page-version"; + /// Property key for delete-file Parquet page version. + pub const PROPERTY_DELETE_PARQUET_PAGE_VERSION: &str = "write.delete.parquet.page-version"; + /// Default Parquet page version. + pub const PROPERTY_PARQUET_PAGE_VERSION_DEFAULT: &str = "v1"; + /// Property key for the Parquet page row limit. + pub const PROPERTY_PARQUET_PAGE_ROW_LIMIT: &str = "write.parquet.page-row-limit"; + /// Property key for the delete-file Parquet page row limit. + pub const PROPERTY_DELETE_PARQUET_PAGE_ROW_LIMIT: &str = "write.delete.parquet.page-row-limit"; + /// Default Parquet page row limit. + pub const PROPERTY_PARQUET_PAGE_ROW_LIMIT_DEFAULT: usize = 20_000; + /// Property key for Parquet dictionary size. + pub const PROPERTY_PARQUET_DICT_SIZE_BYTES: &str = "write.parquet.dict-size-bytes"; + /// Property key for delete-file Parquet dictionary size. + pub const PROPERTY_DELETE_PARQUET_DICT_SIZE_BYTES: &str = + "write.delete.parquet.dict-size-bytes"; + /// Default Parquet dictionary size. + pub const PROPERTY_PARQUET_DICT_SIZE_BYTES_DEFAULT: usize = 2 * 1024 * 1024; + /// Property key for Parquet compression codec. + pub const PROPERTY_PARQUET_COMPRESSION: &str = "write.parquet.compression-codec"; + /// Property key for delete-file Parquet compression codec. + pub const PROPERTY_DELETE_PARQUET_COMPRESSION: &str = "write.delete.parquet.compression-codec"; + /// Original default Parquet compression codec. + pub const PROPERTY_PARQUET_COMPRESSION_DEFAULT: &str = "gzip"; + /// Default Parquet compression codec since Iceberg 1.4.0. + pub const PROPERTY_PARQUET_COMPRESSION_DEFAULT_SINCE_1_4_0: &str = "zstd"; + /// Property key for Parquet compression level. + pub const PROPERTY_PARQUET_COMPRESSION_LEVEL: &str = "write.parquet.compression-level"; + /// Property key for delete-file Parquet compression level. + pub const PROPERTY_DELETE_PARQUET_COMPRESSION_LEVEL: &str = + "write.delete.parquet.compression-level"; + /// Default Parquet compression level. + pub const PROPERTY_PARQUET_COMPRESSION_LEVEL_DEFAULT: Option<&str> = None; + /// Property key controlling Parquet variant shredding. + pub const PROPERTY_PARQUET_SHRED_VARIANTS: &str = "write.parquet.shred-variants"; + /// Default Parquet variant shredding setting. + pub const PROPERTY_PARQUET_SHRED_VARIANTS_DEFAULT: bool = false; + /// Property key for Parquet variant inference buffer size. + pub const PROPERTY_PARQUET_VARIANT_BUFFER_SIZE: &str = + "write.parquet.variant-inference-buffer-size"; + /// Default Parquet variant inference buffer size. + pub const PROPERTY_PARQUET_VARIANT_BUFFER_SIZE_DEFAULT: usize = 100; + /// Property key for minimum record-count row group checks. + pub const PROPERTY_PARQUET_ROW_GROUP_CHECK_MIN_RECORD_COUNT: &str = + "write.parquet.row-group-check-min-record-count"; + /// Delete-file property key for minimum record-count row group checks. + pub const PROPERTY_DELETE_PARQUET_ROW_GROUP_CHECK_MIN_RECORD_COUNT: &str = + "write.delete.parquet.row-group-check-min-record-count"; + /// Default minimum record-count row group check. + pub const PROPERTY_PARQUET_ROW_GROUP_CHECK_MIN_RECORD_COUNT_DEFAULT: usize = 100; + /// Property key for maximum record-count row group checks. + pub const PROPERTY_PARQUET_ROW_GROUP_CHECK_MAX_RECORD_COUNT: &str = + "write.parquet.row-group-check-max-record-count"; + /// Delete-file property key for maximum record-count row group checks. + pub const PROPERTY_DELETE_PARQUET_ROW_GROUP_CHECK_MAX_RECORD_COUNT: &str = + "write.delete.parquet.row-group-check-max-record-count"; + /// Default maximum record-count row group check. + pub const PROPERTY_PARQUET_ROW_GROUP_CHECK_MAX_RECORD_COUNT_DEFAULT: usize = 10_000; + /// Property key for tracking uncompressed Parquet row group size. + pub const PROPERTY_PARQUET_ROW_GROUP_SIZE_TRACK_UNCOMPRESSED: &str = + "write.parquet.row-group-size-track-uncompressed"; + /// Default uncompressed row group size tracking setting. + pub const PROPERTY_PARQUET_ROW_GROUP_SIZE_TRACK_UNCOMPRESSED_DEFAULT: bool = false; + /// Property key for maximum Parquet bloom filter bytes. + pub const PROPERTY_PARQUET_BLOOM_FILTER_MAX_BYTES: &str = + "write.parquet.bloom-filter-max-bytes"; + /// Default maximum Parquet bloom filter bytes. + pub const PROPERTY_PARQUET_BLOOM_FILTER_MAX_BYTES_DEFAULT: usize = 1024 * 1024; + /// Property key for adaptive Parquet bloom filters. + pub const PROPERTY_PARQUET_BLOOM_FILTER_ADAPTIVE_ENABLED: &str = + "write.parquet.bloom-filter-adaptive-enabled"; + /// Default adaptive Parquet bloom filter setting. + pub const PROPERTY_PARQUET_BLOOM_FILTER_ADAPTIVE_ENABLED_DEFAULT: bool = false; + /// Prefix for per-column Parquet bloom filter false-positive probability. + pub const PROPERTY_PARQUET_BLOOM_FILTER_COLUMN_FPP_PREFIX: &str = + "write.parquet.bloom-filter-fpp.column."; + /// Default per-column Parquet bloom filter false-positive probability. + pub const PROPERTY_PARQUET_BLOOM_FILTER_COLUMN_FPP_DEFAULT: f64 = 0.01; + /// Prefix for per-column Parquet bloom filter distinct-value counts. + pub const PROPERTY_PARQUET_BLOOM_FILTER_COLUMN_NDV_PREFIX: &str = + "write.parquet.bloom-filter-ndv.column."; + /// Prefix for enabling Parquet bloom filters by column. + pub const PROPERTY_PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX: &str = + "write.parquet.bloom-filter-enabled.column."; + /// Prefix for enabling Parquet column statistics. + pub const PROPERTY_PARQUET_COLUMN_STATS_ENABLED_PREFIX: &str = + "write.parquet.stats-enabled.column."; + /// Prefix for enabling Parquet dictionary encoding by column. + pub const PROPERTY_PARQUET_DICT_ENCODING_ENABLED_COLUMN_PREFIX: &str = + "write.parquet.dict-encoding-enabled.column."; + + /// Property key for Avro compression codec. + pub const PROPERTY_AVRO_COMPRESSION: &str = "write.avro.compression-codec"; + /// Property key for delete-file Avro compression codec. + pub const PROPERTY_DELETE_AVRO_COMPRESSION: &str = "write.delete.avro.compression-codec"; + /// Default Avro compression codec. + pub const PROPERTY_AVRO_COMPRESSION_DEFAULT: &str = "gzip"; + /// Property key for Avro compression level. + pub const PROPERTY_AVRO_COMPRESSION_LEVEL: &str = "write.avro.compression-level"; + /// Property key for delete-file Avro compression level. + pub const PROPERTY_DELETE_AVRO_COMPRESSION_LEVEL: &str = "write.delete.avro.compression-level"; + /// Default Avro compression level. + pub const PROPERTY_AVRO_COMPRESSION_LEVEL_DEFAULT: Option<&str> = None; + /// Property key for manifest compression codec. + pub const PROPERTY_MANIFEST_COMPRESSION: &str = "write.manifest.compression-codec"; + /// Default manifest compression codec. + pub const PROPERTY_MANIFEST_COMPRESSION_DEFAULT: &str = "gzip"; + /// Property key for manifest compression level. + pub const PROPERTY_MANIFEST_COMPRESSION_LEVEL: &str = "write.manifest.compression-level"; + /// Default manifest compression level. + pub const PROPERTY_MANIFEST_COMPRESSION_LEVEL_DEFAULT: Option<&str> = None; + + /// Property key for ORC stripe size. + pub const PROPERTY_ORC_STRIPE_SIZE_BYTES: &str = "write.orc.stripe-size-bytes"; + /// Property key for delete-file ORC stripe size. + pub const PROPERTY_DELETE_ORC_STRIPE_SIZE_BYTES: &str = "write.delete.orc.stripe-size-bytes"; + /// Default ORC stripe size. + pub const PROPERTY_ORC_STRIPE_SIZE_BYTES_DEFAULT: u64 = 64 * 1024 * 1024; + /// Property key for ORC bloom filter columns. + pub const PROPERTY_ORC_BLOOM_FILTER_COLUMNS: &str = "write.orc.bloom.filter.columns"; + /// Default ORC bloom filter columns. + pub const PROPERTY_ORC_BLOOM_FILTER_COLUMNS_DEFAULT: &str = ""; + /// Property key for ORC bloom filter false-positive probability. + pub const PROPERTY_ORC_BLOOM_FILTER_FPP: &str = "write.orc.bloom.filter.fpp"; + /// Default ORC bloom filter false-positive probability. + pub const PROPERTY_ORC_BLOOM_FILTER_FPP_DEFAULT: f64 = 0.05; + /// Property key for ORC block size. + pub const PROPERTY_ORC_BLOCK_SIZE_BYTES: &str = "write.orc.block-size-bytes"; + /// Property key for delete-file ORC block size. + pub const PROPERTY_DELETE_ORC_BLOCK_SIZE_BYTES: &str = "write.delete.orc.block-size-bytes"; + /// Default ORC block size. + pub const PROPERTY_ORC_BLOCK_SIZE_BYTES_DEFAULT: u64 = 256 * 1024 * 1024; + /// Property key for ORC vectorized write batch size. + pub const PROPERTY_ORC_WRITE_BATCH_SIZE: &str = "write.orc.vectorized.batch-size"; + /// Property key for delete-file ORC vectorized write batch size. + pub const PROPERTY_DELETE_ORC_WRITE_BATCH_SIZE: &str = "write.delete.orc.vectorized.batch-size"; + /// Default ORC vectorized write batch size. + pub const PROPERTY_ORC_WRITE_BATCH_SIZE_DEFAULT: usize = 1024; + /// Property key for ORC compression codec. + pub const PROPERTY_ORC_COMPRESSION: &str = "write.orc.compression-codec"; + /// Property key for delete-file ORC compression codec. + pub const PROPERTY_DELETE_ORC_COMPRESSION: &str = "write.delete.orc.compression-codec"; + /// Default ORC compression codec. + pub const PROPERTY_ORC_COMPRESSION_DEFAULT: &str = "zlib"; + /// Property key for ORC compression strategy. + pub const PROPERTY_ORC_COMPRESSION_STRATEGY: &str = "write.orc.compression-strategy"; + /// Property key for delete-file ORC compression strategy. + pub const PROPERTY_DELETE_ORC_COMPRESSION_STRATEGY: &str = + "write.delete.orc.compression-strategy"; + /// Default ORC compression strategy. + pub const PROPERTY_ORC_COMPRESSION_STRATEGY_DEFAULT: &str = "speed"; + + /// Property key for read split target size. + pub const PROPERTY_SPLIT_SIZE: &str = "read.split.target-size"; + /// Default read split target size. + pub const PROPERTY_SPLIT_SIZE_DEFAULT: u64 = 128 * 1024 * 1024; + /// Property key for metadata split target size. + pub const PROPERTY_METADATA_SPLIT_SIZE: &str = "read.split.metadata-target-size"; + /// Default metadata split target size. + pub const PROPERTY_METADATA_SPLIT_SIZE_DEFAULT: u64 = 32 * 1024 * 1024; + /// Property key for split planning lookback. + pub const PROPERTY_SPLIT_LOOKBACK: &str = "read.split.planning-lookback"; + /// Default split planning lookback. + pub const PROPERTY_SPLIT_LOOKBACK_DEFAULT: usize = 10; + /// Property key for split open-file cost. + pub const PROPERTY_SPLIT_OPEN_FILE_COST: &str = "read.split.open-file-cost"; + /// Default split open-file cost. + pub const PROPERTY_SPLIT_OPEN_FILE_COST_DEFAULT: u64 = 4 * 1024 * 1024; + /// Property key controlling adaptive split sizing. + pub const PROPERTY_ADAPTIVE_SPLIT_SIZE_ENABLED: &str = "read.split.adaptive-size.enabled"; + /// Default adaptive split sizing setting. + pub const PROPERTY_ADAPTIVE_SPLIT_SIZE_ENABLED_DEFAULT: bool = true; + /// Property key controlling Parquet vectorized reads. + pub const PROPERTY_PARQUET_VECTORIZATION_ENABLED: &str = "read.parquet.vectorization.enabled"; + /// Default Parquet vectorized read setting. + pub const PROPERTY_PARQUET_VECTORIZATION_ENABLED_DEFAULT: bool = true; + /// Property key for Parquet vectorized read batch size. + pub const PROPERTY_PARQUET_BATCH_SIZE: &str = "read.parquet.vectorization.batch-size"; + /// Default Parquet vectorized read batch size. + pub const PROPERTY_PARQUET_BATCH_SIZE_DEFAULT: usize = 5000; + /// Property key controlling ORC vectorized reads. + pub const PROPERTY_ORC_VECTORIZATION_ENABLED: &str = "read.orc.vectorization.enabled"; + /// Default ORC vectorized read setting. + pub const PROPERTY_ORC_VECTORIZATION_ENABLED_DEFAULT: bool = false; + /// Property key for ORC vectorized read batch size. + pub const PROPERTY_ORC_BATCH_SIZE: &str = "read.orc.vectorization.batch-size"; + /// Default ORC vectorized read batch size. + pub const PROPERTY_ORC_BATCH_SIZE_DEFAULT: usize = 5000; + /// Property key for data planning mode. + pub const PROPERTY_DATA_PLANNING_MODE: &str = "read.data-planning-mode"; + /// Property key for delete planning mode. + pub const PROPERTY_DELETE_PLANNING_MODE: &str = "read.delete-planning-mode"; + /// Default planning mode. + pub const PROPERTY_PLANNING_MODE_DEFAULT: &str = "auto"; + /// Property key declaring identifier fields reliable. + pub const PROPERTY_IDENTIFIER_FIELDS_RELY: &str = "identifier-fields.rely"; + /// Default identifier-field reliability setting. + pub const PROPERTY_IDENTIFIER_FIELDS_RELY_DEFAULT: bool = false; + + /// Property key controlling object-store locations. + pub const PROPERTY_OBJECT_STORE_ENABLED: &str = "write.object-storage.enabled"; + /// Default object-store location setting. + pub const PROPERTY_OBJECT_STORE_ENABLED_DEFAULT: bool = false; + /// Property key controlling partitioned paths for object storage. + pub const PROPERTY_WRITE_OBJECT_STORE_PARTITIONED_PATHS: &str = + "write.object-storage.partitioned-paths"; + /// Default partitioned-path setting for object storage. + pub const PROPERTY_WRITE_OBJECT_STORE_PARTITIONED_PATHS_DEFAULT: bool = true; + /// Deprecated object-store path property. + pub const PROPERTY_OBJECT_STORE_PATH: &str = "write.object-storage.path"; + /// Property key for a custom location provider implementation. + pub const PROPERTY_WRITE_LOCATION_PROVIDER_IMPL: &str = "write.location-provider.impl"; + /// Deprecated folder-storage location property. + pub const PROPERTY_WRITE_FOLDER_STORAGE_LOCATION: &str = "write.folder-storage.path"; + /// Property key for the table data location. + pub const PROPERTY_WRITE_DATA_LOCATION: &str = "write.data.path"; + /// Alias matching Java's metadata location property name. + pub const PROPERTY_WRITE_METADATA_LOCATION: &str = Self::PROPERTY_WRITE_METADATA_PATH; + /// Deprecated property controlling manifest-list writes. + pub const PROPERTY_MANIFEST_LISTS_ENABLED: &str = "write.manifest-lists.enabled"; + /// Default manifest-list write setting. + pub const PROPERTY_MANIFEST_LISTS_ENABLED_DEFAULT: bool = true; + /// Alias matching Java's metadata compression property name. + pub const PROPERTY_METADATA_COMPRESSION: &str = Self::PROPERTY_METADATA_COMPRESSION_CODEC; + /// Alias matching Java's metadata compression default name. + pub const PROPERTY_METADATA_COMPRESSION_DEFAULT: &str = + Self::PROPERTY_METADATA_COMPRESSION_CODEC_DEFAULT; + /// Property key controlling deletion of old metadata after commit. + pub const PROPERTY_METADATA_DELETE_AFTER_COMMIT_ENABLED: &str = + "write.metadata.delete-after-commit.enabled"; + /// Default old-metadata deletion setting. + pub const PROPERTY_METADATA_DELETE_AFTER_COMMIT_ENABLED_DEFAULT: bool = false; + /// Property key for the maximum inferred column metric defaults. + pub const PROPERTY_METRICS_MAX_INFERRED_COLUMN_DEFAULTS: &str = + "write.metadata.metrics.max-inferred-column-defaults"; + /// Default maximum inferred column metric defaults. + pub const PROPERTY_METRICS_MAX_INFERRED_COLUMN_DEFAULTS_DEFAULT: usize = 100; + /// Prefix for per-column metrics modes. + pub const PROPERTY_METRICS_MODE_COLUMN_CONF_PREFIX: &str = "write.metadata.metrics.column."; + /// Property key for the default write metrics mode. + pub const PROPERTY_DEFAULT_WRITE_METRICS_MODE: &str = "write.metadata.metrics.default"; + /// Default write metrics mode. + pub const PROPERTY_DEFAULT_WRITE_METRICS_MODE_DEFAULT: &str = "truncate(16)"; + /// Property key for the default schema name mapping. + pub const PROPERTY_DEFAULT_NAME_MAPPING: &str = "schema.name-mapping.default"; + /// Property key enabling write-audit-publish behavior. + pub const PROPERTY_WRITE_AUDIT_PUBLISH_ENABLED: &str = "write.wap.enabled"; + /// Default write-audit-publish setting. + pub const PROPERTY_WRITE_AUDIT_PUBLISH_ENABLED_DEFAULT: bool = false; + /// Property key for delete-file target size. + pub const PROPERTY_DELETE_TARGET_FILE_SIZE_BYTES: &str = "write.delete.target-file-size-bytes"; + /// Default delete-file target size. + pub const PROPERTY_DELETE_TARGET_FILE_SIZE_BYTES_DEFAULT: usize = 64 * 1024 * 1024; + + /// Deprecated Spark fanout writer property. + pub const PROPERTY_SPARK_WRITE_PARTITIONED_FANOUT_ENABLED: &str = "write.spark.fanout.enabled"; + /// Default deprecated Spark fanout writer setting. + pub const PROPERTY_SPARK_WRITE_PARTITIONED_FANOUT_ENABLED_DEFAULT: bool = false; + /// Deprecated Spark accept-any-schema property. + pub const PROPERTY_SPARK_WRITE_ACCEPT_ANY_SCHEMA: &str = "write.spark.accept-any-schema"; + /// Default deprecated Spark accept-any-schema setting. + pub const PROPERTY_SPARK_WRITE_ACCEPT_ANY_SCHEMA_DEFAULT: bool = false; + /// Deprecated Spark automatic schema evolution property. + pub const PROPERTY_SPARK_WRITE_AUTO_SCHEMA_EVOLUTION: &str = + "write.spark.auto-schema-evolution.enabled"; + /// Default deprecated Spark automatic schema evolution setting. + pub const PROPERTY_SPARK_WRITE_AUTO_SCHEMA_EVOLUTION_DEFAULT: bool = true; + /// Deprecated Spark advisory partition size property. + pub const PROPERTY_SPARK_WRITE_ADVISORY_PARTITION_SIZE_BYTES: &str = + "write.spark.advisory-partition-size-bytes"; + /// Property key for snapshot ID inheritance compatibility. + pub const PROPERTY_SNAPSHOT_ID_INHERITANCE_ENABLED: &str = + "compatibility.snapshot-id-inheritance.enabled"; + /// Default snapshot ID inheritance setting. + pub const PROPERTY_SNAPSHOT_ID_INHERITANCE_ENABLED_DEFAULT: bool = false; + /// Property key enabling Hive engine behavior. + pub const PROPERTY_ENGINE_HIVE_ENABLED: &str = "engine.hive.enabled"; + /// Default Hive engine setting. + pub const PROPERTY_ENGINE_HIVE_ENABLED_DEFAULT: bool = false; + /// Property key enabling Hive locking. + pub const PROPERTY_HIVE_LOCK_ENABLED: &str = "engine.hive.lock-enabled"; + /// Default Hive lock setting. + pub const PROPERTY_HIVE_LOCK_ENABLED_DEFAULT: bool = true; + /// Property key for write distribution mode. + pub const PROPERTY_WRITE_DISTRIBUTION_MODE: &str = "write.distribution-mode"; + /// No-distribution mode value. + pub const PROPERTY_WRITE_DISTRIBUTION_MODE_NONE: &str = "none"; + /// Hash-distribution mode value. + pub const PROPERTY_WRITE_DISTRIBUTION_MODE_HASH: &str = "hash"; + /// Range-distribution mode value. + pub const PROPERTY_WRITE_DISTRIBUTION_MODE_RANGE: &str = "range"; + + /// Property key for delete granularity. + pub const PROPERTY_DELETE_GRANULARITY: &str = "write.delete.granularity"; + /// Default delete granularity. + pub const PROPERTY_DELETE_GRANULARITY_DEFAULT: &str = "partition"; + /// Property key for delete isolation level. + pub const PROPERTY_DELETE_ISOLATION_LEVEL: &str = "write.delete.isolation-level"; + /// Default delete isolation level. + pub const PROPERTY_DELETE_ISOLATION_LEVEL_DEFAULT: &str = "serializable"; + /// Property key for delete operation mode. + pub const PROPERTY_DELETE_MODE: &str = "write.delete.mode"; + /// Default delete operation mode. + pub const PROPERTY_DELETE_MODE_DEFAULT: &str = "copy-on-write"; + /// Property key for delete distribution mode. + pub const PROPERTY_DELETE_DISTRIBUTION_MODE: &str = "write.delete.distribution-mode"; + /// Property key for update isolation level. + pub const PROPERTY_UPDATE_ISOLATION_LEVEL: &str = "write.update.isolation-level"; + /// Default update isolation level. + pub const PROPERTY_UPDATE_ISOLATION_LEVEL_DEFAULT: &str = "serializable"; + /// Property key for update operation mode. + pub const PROPERTY_UPDATE_MODE: &str = "write.update.mode"; + /// Default update operation mode. + pub const PROPERTY_UPDATE_MODE_DEFAULT: &str = "copy-on-write"; + /// Property key for update distribution mode. + pub const PROPERTY_UPDATE_DISTRIBUTION_MODE: &str = "write.update.distribution-mode"; + /// Property key for merge isolation level. + pub const PROPERTY_MERGE_ISOLATION_LEVEL: &str = "write.merge.isolation-level"; + /// Default merge isolation level. + pub const PROPERTY_MERGE_ISOLATION_LEVEL_DEFAULT: &str = "serializable"; + /// Property key for merge operation mode. + pub const PROPERTY_MERGE_MODE: &str = "write.merge.mode"; + /// Default merge operation mode. + pub const PROPERTY_MERGE_MODE_DEFAULT: &str = "copy-on-write"; + /// Property key for merge distribution mode. + pub const PROPERTY_MERGE_DISTRIBUTION_MODE: &str = "write.merge.distribution-mode"; + /// Property key enabling upserts. + pub const PROPERTY_UPSERT_ENABLED: &str = "write.upsert.enabled"; + /// Default upsert setting. + pub const PROPERTY_UPSERT_ENABLED_DEFAULT: bool = false; + /// Alias matching Java's encryption table key property name. + pub const PROPERTY_ENCRYPTION_TABLE_KEY: &str = Self::PROPERTY_ENCRYPTION_KEY_ID; + /// Alias matching Java's encryption DEK length property name. + pub const PROPERTY_ENCRYPTION_DEK_LENGTH: &str = Self::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH; + /// Alias matching Java's encryption DEK length default name. + pub const PROPERTY_ENCRYPTION_DEK_LENGTH_DEFAULT: usize = + Self::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH_DEFAULT; + /// Default encryption AAD length. + pub const PROPERTY_ENCRYPTION_AAD_LENGTH_DEFAULT: usize = 16; } impl TryFrom<&HashMap> for TableProperties { type Error = Error; fn try_from(props: &HashMap) -> Result { - Self::from_properties(props).map_err(|error| Error::new(ErrorKind::DataInvalid, error)) + Ok(Self { + commit_num_retries: parse_property( + props, + Self::PROPERTY_COMMIT_NUM_RETRIES, + Self::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT, + )?, + commit_min_retry_wait_ms: parse_property( + props, + Self::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS, + Self::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT, + )?, + commit_max_retry_wait_ms: parse_property( + props, + Self::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS, + Self::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT, + )?, + commit_total_retry_timeout_ms: parse_property( + props, + Self::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS, + Self::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT, + )?, + write_format_default: parse_property( + props, + Self::PROPERTY_DEFAULT_FILE_FORMAT, + Self::PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT.to_string(), + )?, + write_target_file_size_bytes: parse_property( + props, + Self::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES, + Self::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT, + )?, + write_metadata_path: parse_location_property( + props, + Self::PROPERTY_WRITE_METADATA_PATH, + )?, + metadata_compression_codec: parse_metadata_file_compression(props)?, + write_datafusion_fanout_enabled: parse_property( + props, + Self::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED, + Self::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT, + )?, + gc_enabled: parse_property( + props, + Self::PROPERTY_GC_ENABLED, + Self::PROPERTY_GC_ENABLED_DEFAULT, + )?, + max_snapshot_age_ms: parse_property( + props, + Self::PROPERTY_MAX_SNAPSHOT_AGE_MS, + Self::PROPERTY_MAX_SNAPSHOT_AGE_MS_DEFAULT, + )?, + min_snapshots_to_keep: parse_property( + props, + Self::PROPERTY_MIN_SNAPSHOTS_TO_KEEP, + Self::PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT, + )?, + max_ref_age_ms: parse_property( + props, + Self::PROPERTY_MAX_REF_AGE_MS, + Self::PROPERTY_MAX_REF_AGE_MS_DEFAULT, + )?, + cdc_enabled: parse_property( + props, + Self::PROPERTY_PARQUET_CDC_ENABLED, + Self::PROPERTY_PARQUET_CDC_ENABLED_DEFAULT, + )?, + cdc_min_chunk_size: parse_property( + props, + Self::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE, + Self::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT, + )?, + cdc_max_chunk_size: parse_property( + props, + Self::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE, + Self::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE_DEFAULT, + )?, + cdc_norm_level: parse_property( + props, + Self::PROPERTY_PARQUET_CDC_NORM_LEVEL, + Self::PROPERTY_PARQUET_CDC_NORM_LEVEL_DEFAULT, + )?, + encryption_key_id: props.get(Self::PROPERTY_ENCRYPTION_KEY_ID).cloned(), + encryption_data_key_length: parse_property( + props, + Self::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH, + Self::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH_DEFAULT, + )?, + }) } } #[cfg(test)] mod tests { + use std::collections::HashMap; + use super::*; use crate::compression::CompressionCodec; + fn parse(properties: &HashMap) -> Result { + serde_json::to_value(properties) + .and_then(serde_json::from_value) + .map_err(|error| Error::new(ErrorKind::DataInvalid, error.to_string())) + } + #[test] - fn test_table_properties_default() { - let table_properties = TableProperties::default(); + fn test_parsed_table_properties_default() { + let table_properties = ParsedTableProperties::default(); assert_eq!( table_properties.commit_num_retries(), TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT @@ -412,10 +1278,139 @@ mod tests { ); } + #[test] + fn test_existing_table_properties_api() { + let properties = HashMap::from([ + ( + TableProperties::PROPERTY_COMMIT_NUM_RETRIES.to_string(), + "8".to_string(), + ), + ( + TableProperties::PROPERTY_DEFAULT_FILE_FORMAT.to_string(), + "orc".to_string(), + ), + ( + TableProperties::PROPERTY_WRITE_METADATA_PATH.to_string(), + "s3://warehouse/table/metadata/".to_string(), + ), + ]); + + let parsed = TableProperties::try_from(&properties).unwrap(); + assert_eq!(parsed.commit_num_retries, 8); + assert_eq!(parsed.write_format_default, "orc"); + assert_eq!( + parsed.write_metadata_path.as_deref(), + Some("s3://warehouse/table/metadata") + ); + } + + #[test] + fn test_parsed_table_properties_modifiers() { + let table_properties = ParsedTableProperties::default() + .with_commit_num_retries(9) + .with_comment(Some("orders table".to_string())) + .with_write_format_default(DataFileFormat::Avro); + + assert_eq!(table_properties.commit_num_retries(), 9); + assert_eq!(table_properties.comment(), Some("orders table".to_string())); + assert_eq!( + table_properties.write_format_default(), + DataFileFormat::Avro + ); + } + + #[test] + fn test_properties_from_java_table_properties() { + let properties = parse(&HashMap::from([ + ( + TableProperties::PROPERTY_COMMIT_NUM_STATUS_CHECKS.to_string(), + "7".to_string(), + ), + ( + TableProperties::PROPERTY_DELETE_DEFAULT_FILE_FORMAT.to_string(), + "orc".to_string(), + ), + ( + TableProperties::PROPERTY_PARQUET_PAGE_ROW_LIMIT.to_string(), + "1000".to_string(), + ), + ( + format!( + "{}customer_id", + TableProperties::PROPERTY_PARQUET_BLOOM_FILTER_COLUMN_FPP_PREFIX + ), + "0.02".to_string(), + ), + ( + TableProperties::PROPERTY_DELETE_AVRO_COMPRESSION.to_string(), + "snappy".to_string(), + ), + ( + TableProperties::PROPERTY_ORC_BLOOM_FILTER_FPP.to_string(), + "0.1".to_string(), + ), + ( + TableProperties::PROPERTY_SPLIT_LOOKBACK.to_string(), + "25".to_string(), + ), + ( + TableProperties::PROPERTY_WRITE_OBJECT_STORE_PARTITIONED_PATHS.to_string(), + "false".to_string(), + ), + ( + TableProperties::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX.to_string(), + "20".to_string(), + ), + ( + format!( + "{}customer_id", + TableProperties::PROPERTY_METRICS_MODE_COLUMN_CONF_PREFIX + ), + "full".to_string(), + ), + ( + TableProperties::PROPERTY_SPARK_WRITE_ADVISORY_PARTITION_SIZE_BYTES.to_string(), + "4096".to_string(), + ), + ( + TableProperties::PROPERTY_DELETE_MODE.to_string(), + "merge-on-read".to_string(), + ), + ( + TableProperties::PROPERTY_UPSERT_ENABLED.to_string(), + "true".to_string(), + ), + ])) + .unwrap(); + + assert_eq!(properties.commit_num_status_checks(), 7); + assert_eq!(properties.delete_format_default(), DataFileFormat::Orc); + assert_eq!(properties.parquet_page_row_limit(), 1000); + assert_eq!( + properties.parquet_bloom_filter_column_fpp()["customer_id"], + 0.02 + ); + assert_eq!(properties.delete_avro_compression(), "snappy"); + assert_eq!(properties.orc_bloom_filter_fpp(), 0.1); + assert_eq!(properties.split_lookback(), 25); + assert!(!properties.write_object_store_partitioned_paths()); + assert_eq!(properties.metadata_previous_versions_max(), 20); + assert_eq!( + properties.metrics_mode_column_config()["customer_id"], + "full" + ); + assert_eq!( + properties.spark_write_advisory_partition_size_bytes(), + Some(4096) + ); + assert_eq!(properties.delete_mode(), "merge-on-read"); + assert!(properties.upsert_enabled()); + } + #[test] fn test_empty_properties_match_default() { - let parsed = TableProperties::try_from(&HashMap::new()).unwrap(); - let defaults = TableProperties::default(); + let parsed = parse(&HashMap::new()).unwrap(); + let defaults = ParsedTableProperties::default(); assert_eq!(parsed.commit_num_retries(), defaults.commit_num_retries()); assert_eq!( @@ -466,8 +1461,8 @@ mod tests { } #[test] - fn test_table_properties_json_round_trip() { - let properties = TableProperties::try_from(&HashMap::from([ + fn test_parsed_table_properties_json_round_trip() { + let properties = parse(&HashMap::from([ ( TableProperties::PROPERTY_DEFAULT_FILE_FORMAT.to_string(), "ORC".to_string(), @@ -498,7 +1493,7 @@ mod tests { "gzip" ); - let decoded: TableProperties = serde_json::from_value(json).unwrap(); + let decoded: ParsedTableProperties = serde_json::from_value(json).unwrap(); assert_eq!(decoded.write_format_default(), DataFileFormat::Orc); assert_eq!( decoded.write_metadata_path(), @@ -512,8 +1507,8 @@ mod tests { } #[test] - fn test_default_table_properties_json_round_trip() { - let defaults = TableProperties::default(); + fn test_default_parsed_table_properties_json_round_trip() { + let defaults = ParsedTableProperties::default(); let json = serde_json::to_value(&defaults).unwrap(); assert!( @@ -525,14 +1520,14 @@ mod tests { .is_none() ); - let decoded: TableProperties = serde_json::from_value(json).unwrap(); + let decoded: ParsedTableProperties = serde_json::from_value(json).unwrap(); assert_eq!(decoded.write_format_default(), DataFileFormat::Parquet); assert_eq!(decoded.write_metadata_path(), None); assert_eq!(decoded.encryption_key_id(), None); } #[test] - fn test_table_properties_history_expire_overrides() { + fn test_parsed_table_properties_history_expire_overrides() { let props = HashMap::from([ ( TableProperties::PROPERTY_MAX_SNAPSHOT_AGE_MS.to_string(), @@ -547,16 +1542,16 @@ mod tests { "5678".to_string(), ), ]); - let table_properties = TableProperties::try_from(&props).unwrap(); + let table_properties = parse(&props).unwrap(); assert_eq!(table_properties.max_snapshot_age_ms(), 1234); assert_eq!(table_properties.min_snapshots_to_keep(), 7); assert_eq!(table_properties.max_ref_age_ms(), 5678); } #[test] - fn test_table_properties_write_metadata_path() { + fn test_parsed_table_properties_write_metadata_path() { // Test unset - let table_properties = TableProperties::try_from(&HashMap::new()).unwrap(); + let table_properties = parse(&HashMap::new()).unwrap(); assert_eq!(table_properties.write_metadata_path(), None); // Test empty path is invalid @@ -564,7 +1559,7 @@ mod tests { TableProperties::PROPERTY_WRITE_METADATA_PATH.to_string(), String::new(), )]); - let error = TableProperties::try_from(&props).unwrap_err(); + let error = parse(&props).unwrap_err(); assert_eq!(error.kind(), ErrorKind::DataInvalid); assert!( error @@ -576,7 +1571,7 @@ mod tests { TableProperties::PROPERTY_WRITE_METADATA_PATH.to_string(), "s3://other-bucket/custom-meta/".to_string(), )]); - let table_properties = TableProperties::try_from(&props).unwrap(); + let table_properties = parse(&props).unwrap(); assert_eq!( table_properties.write_metadata_path(), Some("s3://other-bucket/custom-meta".to_string()) @@ -600,12 +1595,12 @@ mod tests { } #[test] - fn test_table_properties_compression() { + fn test_parsed_table_properties_compression() { let props = HashMap::from([( TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), "gzip".to_string(), )]); - let table_properties = TableProperties::try_from(&props).unwrap(); + let table_properties = parse(&props).unwrap(); assert_eq!( table_properties.metadata_compression_codec(), CompressionCodec::gzip_default() @@ -613,12 +1608,12 @@ mod tests { } #[test] - fn test_table_properties_compression_none() { + fn test_parsed_table_properties_compression_none() { let props = HashMap::from([( TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), "none".to_string(), )]); - let table_properties = TableProperties::try_from(&props).unwrap(); + let table_properties = parse(&props).unwrap(); assert_eq!( table_properties.metadata_compression_codec(), CompressionCodec::None @@ -626,13 +1621,13 @@ mod tests { } #[test] - fn test_table_properties_compression_case_insensitive() { + fn test_parsed_table_properties_compression_case_insensitive() { // Test uppercase let props_upper = HashMap::from([( TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), "GZIP".to_string(), )]); - let table_properties = TableProperties::try_from(&props_upper).unwrap(); + let table_properties = parse(&props_upper).unwrap(); assert_eq!( table_properties.metadata_compression_codec(), CompressionCodec::gzip_default() @@ -643,7 +1638,7 @@ mod tests { TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), "GzIp".to_string(), )]); - let table_properties = TableProperties::try_from(&props_mixed).unwrap(); + let table_properties = parse(&props_mixed).unwrap(); assert_eq!( table_properties.metadata_compression_codec(), CompressionCodec::gzip_default() @@ -654,7 +1649,7 @@ mod tests { TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), "NONE".to_string(), )]); - let table_properties = TableProperties::try_from(&props_none_upper).unwrap(); + let table_properties = parse(&props_none_upper).unwrap(); assert_eq!( table_properties.metadata_compression_codec(), CompressionCodec::None @@ -662,7 +1657,7 @@ mod tests { } #[test] - fn test_table_properties_valid() { + fn test_parsed_table_properties_valid() { let props = HashMap::from([ ( TableProperties::PROPERTY_COMMIT_NUM_RETRIES.to_string(), @@ -685,7 +1680,7 @@ mod tests { "false".to_string(), ), ]); - let table_properties = TableProperties::try_from(&props).unwrap(); + let table_properties = parse(&props).unwrap(); assert_eq!(table_properties.commit_num_retries(), 10); assert_eq!(table_properties.commit_max_retry_wait_ms(), 20); assert_eq!( @@ -697,13 +1692,13 @@ mod tests { } #[test] - fn test_table_properties_invalid() { + fn test_parsed_table_properties_invalid() { let invalid_retries = HashMap::from([( TableProperties::PROPERTY_COMMIT_NUM_RETRIES.to_string(), "abc".to_string(), )]); - let table_properties = TableProperties::try_from(&invalid_retries).unwrap_err(); + let table_properties = parse(&invalid_retries).unwrap_err(); assert!( table_properties.to_string().contains( "Invalid value for commit.retry.num-retries: invalid digit found in string" @@ -714,7 +1709,7 @@ mod tests { TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS.to_string(), "abc".to_string(), )]); - let table_properties = TableProperties::try_from(&invalid_min_wait).unwrap_err(); + let table_properties = parse(&invalid_min_wait).unwrap_err(); assert!( table_properties.to_string().contains( "Invalid value for commit.retry.min-wait-ms: invalid digit found in string" @@ -725,7 +1720,7 @@ mod tests { TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS.to_string(), "abc".to_string(), )]); - let table_properties = TableProperties::try_from(&invalid_max_wait).unwrap_err(); + let table_properties = parse(&invalid_max_wait).unwrap_err(); assert!( table_properties.to_string().contains( "Invalid value for commit.retry.max-wait-ms: invalid digit found in string" @@ -736,7 +1731,7 @@ mod tests { TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES.to_string(), "abc".to_string(), )]); - let table_properties = TableProperties::try_from(&invalid_target_size).unwrap_err(); + let table_properties = parse(&invalid_target_size).unwrap_err(); assert!(table_properties.to_string().contains( "Invalid value for write.target-file-size-bytes: invalid digit found in string" )); @@ -745,7 +1740,7 @@ mod tests { TableProperties::PROPERTY_GC_ENABLED.to_string(), "notabool".to_string(), )]); - let table_properties = TableProperties::try_from(&invalid_gc_enabled).unwrap_err(); + let table_properties = parse(&invalid_gc_enabled).unwrap_err(); assert!( table_properties .to_string() @@ -754,7 +1749,7 @@ mod tests { } #[test] - fn test_table_properties_compression_invalid_rejected() { + fn test_parsed_table_properties_compression_invalid_rejected() { let invalid_codecs = ["lz4", "zstd", "snappy"]; for codec in invalid_codecs { @@ -762,7 +1757,7 @@ mod tests { TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), codec.to_string(), )]); - let err = TableProperties::try_from(&props).unwrap_err(); + let err = parse(&props).unwrap_err(); let err_msg = err.to_string(); assert!( err_msg.contains(&format!("Invalid metadata compression codec: {codec}")), @@ -783,7 +1778,7 @@ mod tests { "none".to_string(), )]); assert_eq!( - parse_metadata_file_compression(&props).unwrap(), + parse(&props).unwrap().metadata_compression_codec(), CompressionCodec::None ); @@ -793,7 +1788,7 @@ mod tests { "".to_string(), )]); assert_eq!( - parse_metadata_file_compression(&props).unwrap(), + parse(&props).unwrap().metadata_compression_codec(), CompressionCodec::None ); @@ -803,7 +1798,7 @@ mod tests { "gzip".to_string(), )]); assert_eq!( - parse_metadata_file_compression(&props).unwrap(), + parse(&props).unwrap().metadata_compression_codec(), CompressionCodec::gzip_default() ); @@ -813,7 +1808,7 @@ mod tests { "NONE".to_string(), )]); assert_eq!( - parse_metadata_file_compression(&props).unwrap(), + parse(&props).unwrap().metadata_compression_codec(), CompressionCodec::None ); @@ -823,7 +1818,7 @@ mod tests { "GZIP".to_string(), )]); assert_eq!( - parse_metadata_file_compression(&props).unwrap(), + parse(&props).unwrap().metadata_compression_codec(), CompressionCodec::gzip_default() ); @@ -833,14 +1828,14 @@ mod tests { "GzIp".to_string(), )]); assert_eq!( - parse_metadata_file_compression(&props).unwrap(), + parse(&props).unwrap().metadata_compression_codec(), CompressionCodec::gzip_default() ); // Test default when property is missing let props = HashMap::new(); assert_eq!( - parse_metadata_file_compression(&props).unwrap(), + parse(&props).unwrap().metadata_compression_codec(), CompressionCodec::None ); } @@ -854,7 +1849,7 @@ mod tests { TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), codec.to_string(), )]); - let err = parse_metadata_file_compression(&props).unwrap_err(); + let err = parse(&props).unwrap_err(); let err_msg = err.to_string(); assert!( err_msg.contains("Invalid metadata compression codec"), @@ -870,7 +1865,7 @@ mod tests { #[test] fn test_cdc_disabled_by_default() { let props = HashMap::new(); - let tp = TableProperties::try_from(&props).unwrap(); + let tp = parse(&props).unwrap(); assert!(!tp.cdc_enabled()); } @@ -880,7 +1875,7 @@ mod tests { TableProperties::PROPERTY_PARQUET_CDC_ENABLED.to_string(), "true".to_string(), )]); - let tp = TableProperties::try_from(&props).unwrap(); + let tp = parse(&props).unwrap(); assert!(tp.cdc_enabled()); assert_eq!(tp.cdc_min_chunk_size(), 256 * 1024); assert_eq!(tp.cdc_max_chunk_size(), 1024 * 1024); @@ -893,7 +1888,7 @@ mod tests { TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE.to_string(), "262144".to_string(), )]); - let tp = TableProperties::try_from(&props).unwrap(); + let tp = parse(&props).unwrap(); assert!(!tp.cdc_enabled()); } @@ -917,7 +1912,7 @@ mod tests { "1".to_string(), ), ]); - let tp = TableProperties::try_from(&props).unwrap(); + let tp = parse(&props).unwrap(); assert!(tp.cdc_enabled()); assert_eq!(tp.cdc_min_chunk_size(), 200000); assert_eq!(tp.cdc_max_chunk_size(), 900000); @@ -936,7 +1931,7 @@ mod tests { "2".to_string(), ), ]); - let tp = TableProperties::try_from(&props).unwrap(); + let tp = parse(&props).unwrap(); assert!(tp.cdc_enabled()); assert_eq!(tp.cdc_min_chunk_size(), 256 * 1024); assert_eq!(tp.cdc_max_chunk_size(), 1024 * 1024); @@ -955,7 +1950,7 @@ mod tests { "-2".to_string(), ), ]); - let tp = TableProperties::try_from(&props).unwrap(); + let tp = parse(&props).unwrap(); assert_eq!(tp.cdc_norm_level(), -2); } @@ -971,7 +1966,7 @@ mod tests { "not_a_number".to_string(), ), ]); - let err = TableProperties::try_from(&props).unwrap_err(); + let err = parse(&props).unwrap_err(); assert!( err.to_string().contains( "Invalid value for write.parquet.content-defined-chunking.min-chunk-size" @@ -991,7 +1986,7 @@ mod tests { "not_a_number".to_string(), ), ]); - let err = TableProperties::try_from(&props).unwrap_err(); + let err = parse(&props).unwrap_err(); assert!( err.to_string() .contains("Invalid value for write.parquet.content-defined-chunking.norm-level") @@ -1001,7 +1996,7 @@ mod tests { #[test] fn test_cdc_no_properties() { let props = HashMap::from([("some.other.property".to_string(), "value".to_string())]); - let tp = TableProperties::try_from(&props).unwrap(); + let tp = parse(&props).unwrap(); assert!(!tp.cdc_enabled()); } } diff --git a/crates/iceberg/src/transaction/expire_snapshots.rs b/crates/iceberg/src/transaction/expire_snapshots.rs index 66b0291965..5178cb6bc8 100644 --- a/crates/iceberg/src/transaction/expire_snapshots.rs +++ b/crates/iceberg/src/transaction/expire_snapshots.rs @@ -22,7 +22,7 @@ use async_trait::async_trait; use chrono::Utc; use crate::spec::{ - MAIN_BRANCH, SnapshotReference, SnapshotRetention, TableMetadata, TableProperties, + MAIN_BRANCH, ParsedTableProperties, SnapshotReference, SnapshotRetention, TableMetadata, }; use crate::table::Table; use crate::transaction::action::{ActionCommit, TransactionAction}; @@ -99,7 +99,7 @@ impl ExpireSnapshotsAction { } /// Resolves the snapshots and refs to remove, following Java `RemoveSnapshots.internalApply`. - fn plan(&self, table: &Table, properties: &TableProperties) -> Result { + fn plan(&self, table: &Table, properties: &ParsedTableProperties) -> Result { // Matches Java `RemoveSnapshots.retainLast`, which requires at least one snapshot. if self.retain_last == Some(0) { return Err(Error::new( @@ -301,7 +301,7 @@ struct ExpirePlan { impl TransactionAction for ExpireSnapshotsAction { async fn commit(self: Arc, table: &Table) -> Result { let metadata = table.metadata(); - let properties = metadata.table_properties()?; + let properties = metadata.parsed_table_properties()?; // Expiring metadata defeats a user's explicit decision to disable GC (Java refuses too). if !properties.gc_enabled() { diff --git a/crates/iceberg/src/transaction/mod.rs b/crates/iceberg/src/transaction/mod.rs index 07a85360ae..69015d4002 100644 --- a/crates/iceberg/src/transaction/mod.rs +++ b/crates/iceberg/src/transaction/mod.rs @@ -70,7 +70,7 @@ use backon::{BackoffBuilder, ExponentialBackoff, ExponentialBuilder, RetryableWi pub use update_schema::AddColumn; use crate::error::Result; -use crate::spec::TableProperties; +use crate::spec::ParsedTableProperties; use crate::table::Table; use crate::transaction::action::BoxedTransactionAction; use crate::transaction::append::FastAppendAction; @@ -178,7 +178,7 @@ impl Transaction { return Ok(self.table); } - let table_props = self.table.metadata().table_properties()?; + let table_props = self.table.metadata().parsed_table_properties()?; // TODO(https://github.com/apache/iceberg-rust/issues/2034): remove once encrypted writes are supported if table_props.encryption_key_id().is_some() { @@ -203,7 +203,7 @@ impl Transaction { .1 } - fn build_backoff(props: TableProperties) -> Result { + fn build_backoff(props: ParsedTableProperties) -> Result { Ok(ExponentialBuilder::new() .with_min_delay(Duration::from_millis(props.commit_min_retry_wait_ms())) .with_max_delay(Duration::from_millis(props.commit_max_retry_wait_ms())) diff --git a/crates/iceberg/src/writer/file_writer/parquet_writer.rs b/crates/iceberg/src/writer/file_writer/parquet_writer.rs index 847990f9e2..53f531910a 100644 --- a/crates/iceberg/src/writer/file_writer/parquet_writer.rs +++ b/crates/iceberg/src/writer/file_writer/parquet_writer.rs @@ -39,8 +39,9 @@ use crate::arrow::{ use crate::io::{FileIO, FileWrite, OutputFile}; use crate::spec::{ DataContentType, DataFileBuilder, DataFileFormat, Datum, ListType, Literal, MapType, - NestedFieldRef, PartitionSpec, PrimitiveType, Schema, SchemaRef, SchemaVisitor, Struct, - StructType, TableMetadata, TableProperties, Type, VariantType, visit_schema, + NestedFieldRef, ParsedTableProperties, PartitionSpec, PrimitiveType, Schema, SchemaRef, + SchemaVisitor, Struct, StructType, TableMetadata, TableProperties, Type, VariantType, + visit_schema, }; use crate::transform::create_transform_function; use crate::writer::{CurrentFileStatus, DataFile}; @@ -86,11 +87,28 @@ impl ParquetWriterBuilder { /// (`write.parquet.content-defined-chunking.*`); other keys fall back to /// parquet-rs defaults. pub fn from_table_properties(table_props: &TableProperties, schema: SchemaRef) -> Self { + let cdc = table_props.cdc_enabled.then_some(CdcOptions { + min_chunk_size: table_props.cdc_min_chunk_size, + max_chunk_size: table_props.cdc_max_chunk_size, + norm_level: table_props.cdc_norm_level, + }); + Self::from_cdc_options(cdc, schema) + } + + /// Build a `ParquetWriterBuilder` from properties parsed by the property framework. + pub fn from_parsed_table_properties( + table_props: &ParsedTableProperties, + schema: SchemaRef, + ) -> Self { let cdc = table_props.cdc_enabled().then_some(CdcOptions { min_chunk_size: table_props.cdc_min_chunk_size(), max_chunk_size: table_props.cdc_max_chunk_size(), norm_level: table_props.cdc_norm_level(), }); + Self::from_cdc_options(cdc, schema) + } + + fn from_cdc_options(cdc: Option, schema: SchemaRef) -> Self { // TODO: translate the remaining write.parquet.* keys (e.g. compression-codec, // row-group-size-bytes, page-size-bytes). // This constructor is intended to be the single place that maps them. @@ -2342,7 +2360,7 @@ mod tests { } // ----------------------------------------------------------------- - // ParquetWriterBuilder::from_table_properties + // ParquetWriterBuilder property constructors // ----------------------------------------------------------------- fn cdc_test_schema() -> SchemaRef { @@ -2359,13 +2377,20 @@ mod tests { ) } - fn table_props(entries: HashMap) -> TableProperties { - TableProperties::try_from(&entries).unwrap() + fn table_props(entries: HashMap) -> ParsedTableProperties { + serde_json::from_value(serde_json::to_value(entries).unwrap()).unwrap() } #[test] fn test_from_table_properties_no_cdc_by_default() { let tp = table_props(HashMap::new()); + let builder = ParquetWriterBuilder::from_parsed_table_properties(&tp, cdc_test_schema()); + assert!(builder.props.content_defined_chunking().is_none()); + } + + #[test] + fn test_existing_table_properties_constructor() { + let tp = TableProperties::try_from(&HashMap::new()).unwrap(); let builder = ParquetWriterBuilder::from_table_properties(&tp, cdc_test_schema()); assert!(builder.props.content_defined_chunking().is_none()); } @@ -2374,7 +2399,7 @@ mod tests { async fn test_from_table_properties_propagate_to_writer() { // `build()` must carry the translated `WriterProperties` through to the // `ParquetWriter` unchanged — otherwise the `write.parquet.*` settings - // derived in `from_table_properties` would never reach parquet-rs. + // derived in `from_parsed_table_properties` would never reach parquet-rs. // // Asserting on the writer's `WriterProperties` (rather than re-reading a // written file) keeps this a direct propagation check: every future @@ -2403,7 +2428,7 @@ mod tests { let output = FileIO::new_with_fs() .new_output(format!("{}/cdc.parquet", tmp.path().to_str().unwrap())) .unwrap(); - let writer = ParquetWriterBuilder::from_table_properties(&tp, cdc_test_schema()) + let writer = ParquetWriterBuilder::from_parsed_table_properties(&tp, cdc_test_schema()) .build(output) .await .unwrap(); diff --git a/crates/integrations/datafusion/src/physical_plan/write.rs b/crates/integrations/datafusion/src/physical_plan/write.rs index 59ac3e047d..312f525256 100644 --- a/crates/integrations/datafusion/src/physical_plan/write.rs +++ b/crates/integrations/datafusion/src/physical_plan/write.rs @@ -205,7 +205,7 @@ impl ExecutionPlan for IcebergWriteExec { let table_props = self .table .metadata() - .table_properties() + .parsed_table_properties() .map_err(to_datafusion_error)?; // Check data file format @@ -220,7 +220,7 @@ impl ExecutionPlan for IcebergWriteExec { // Build the writer from the already-parsed table properties so it honors // `write.parquet.*` settings (e.g. CDC). Arrow batches flowing through // DataFusion carry no field-id metadata, so match fields by name. - let parquet_file_writer_builder = ParquetWriterBuilder::from_table_properties( + let parquet_file_writer_builder = ParquetWriterBuilder::from_parsed_table_properties( &table_props, self.table.metadata().current_schema().clone(), ) diff --git a/crates/property-macro/Cargo.toml b/crates/property-macro/Cargo.toml index cfafd8b24a..f67c8aabf5 100644 --- a/crates/property-macro/Cargo.toml +++ b/crates/property-macro/Cargo.toml @@ -38,5 +38,9 @@ proc-macro2 = "1" quote = "1" syn = { version = "2", features = ["full"] } +[dev-dependencies] +serde = { workspace = true } +serde_json = { workspace = true } + [lints] workspace = true diff --git a/crates/property-macro/src/lib.rs b/crates/property-macro/src/lib.rs index 044d185df9..5672d6450f 100644 --- a/crates/property-macro/src/lib.rs +++ b/crates/property-macro/src/lib.rs @@ -19,10 +19,10 @@ use proc_macro::TokenStream; use proc_macro2::TokenStream as TokenStream2; -use quote::quote; +use quote::{format_ident, quote}; use syn::{ - Attribute, Data, DeriveInput, Error, Expr, ExprLit, ExprPath, Field, Fields, Ident, Lit, Meta, - Path, Type, parse_macro_input, + Attribute, Data, DeriveInput, Error, Expr, ExprPath, Field, Fields, Ident, Meta, Path, Type, + parse_macro_input, }; /// Derive parsing, defaults, JSON serialization, and getters for a typed property map. @@ -32,18 +32,23 @@ use syn::{ /// ```ignore /// #[derive(Properties)] /// struct Properties { -/// #[key = "write.format.default"] +/// #[key(TableProperties::DEFAULT_FILE_FORMAT)] /// #[default(DataFileFormat::Parquet)] +/// #[doc = "Default file format"] /// write_format_default: DataFileFormat, /// } /// ``` /// -/// `parse_with` may be used for property types that do not implement `FromStr` or need -/// validation. `serialize_with` supplies the string representation used in JSON. Optional -/// fields are omitted from JSON when they are `None`. Fields must implement `Clone`; they also -/// need `FromStr` and `ToString` unless the relevant custom parsing or serialization attribute is -/// supplied. -#[proc_macro_derive(Properties, attributes(key, default, parse_with, serialize_with))] +/// `prefix` captures a family of properties in a `HashMap`, keyed by the suffix after +/// the declared prefix. `parse_with` may be used for exact-key property types that do not implement +/// `FromStr` or need validation. `serialize_with` supplies their string representation in JSON. +/// Optional fields are omitted from JSON when they are `None`. Fields must implement `Clone`; they +/// also need `FromStr` and `ToString` unless the relevant custom parsing or serialization attribute +/// is supplied. +#[proc_macro_derive( + Properties, + attributes(key, prefix, default, parse_with, serialize_with) +)] pub fn derive_properties(input: TokenStream) -> TokenStream { let input = parse_macro_input!(input as DeriveInput); @@ -57,11 +62,13 @@ struct PropertyField { ident: Ident, ty: Type, docs: Vec, - key: syn::LitStr, + key: Option, + prefix: Option, default: Expr, parse_with: Option, serialize_with: Option, - is_option: bool, + option_inner_type: Option, + map_value_type: Option, } fn expand_properties(input: DeriveInput) -> syn::Result { @@ -95,44 +102,34 @@ fn expand_properties(input: DeriveInput) -> syn::Result { quote!(#ident: #default) }); - let getters = fields.iter().map(|field| { + let accessors = fields.iter().map(|field| { let ident = &field.ident; let ty = &field.ty; let docs = &field.docs; + let with_ident = format_ident!("with_{ident}"); + let getter_doc = format!("Returns the `{ident}` property."); + let with_doc = format!("Sets the `{ident}` property."); + let getter_docs = if docs.is_empty() { + quote!(#[doc = #getter_doc]) + } else { + quote!(#(#docs)*) + }; quote! { - #(#docs)* + #getter_docs pub fn #ident(&self) -> #ty { self.#ident.clone() } - } - }); - let parses = fields.iter().map(|field| { - let ident = &field.ident; - let ty = &field.ty; - let key = &field.key; - let default = &field.default; - let parse = match &field.parse_with { - Some(parse_with) => quote! { - #parse_with(value).map_err(|error| { - format!("Invalid value for {}: {error}", #key) - })? - }, - None => quote! { - value.parse::<#ty>().map_err(|error| { - format!("Invalid value for {}: {error}", #key) - })? - }, - }; - - quote! { - #ident: match properties.get(#key) { - Some(value) => #parse, - None => #default, + #[doc = #with_doc] + pub fn #with_ident(mut self, value: #ty) -> Self { + self.#ident = value; + self } } }); + let parses = fields.iter().map(parse_field); + let serializes = fields.iter().map(serialize_field); Ok(quote! { @@ -153,7 +150,7 @@ fn expand_properties(input: DeriveInput) -> syn::Result { }) } - #(#getters)* + #(#accessors)* } impl ::serde::Serialize for #struct_name { @@ -186,13 +183,35 @@ fn parse_property_field(field: &Field) -> syn::Result { .ident .clone() .ok_or_else(|| Error::new_spanned(field, "Properties fields must be named"))?; - let key = attribute_string_value(&field.attrs, "key")?.ok_or_else(|| { - Error::new_spanned(field, "Properties fields must declare #[key = \"...\"]") - })?; + let key = attribute_expression_value(&field.attrs, "key")?; + let prefix = attribute_expression_value(&field.attrs, "prefix")?; + if key.is_some() == prefix.is_some() { + return Err(Error::new_spanned( + field, + "Properties fields must declare exactly one of #[key(...)] or #[prefix(...)]", + )); + } let default = attribute_expression_value(&field.attrs, "default")?.ok_or_else(|| { Error::new_spanned(field, "Properties fields must declare #[default(...)]") })?; + let map_value_type = map_value_type(&field.ty); + if prefix.is_some() && map_value_type.is_none() { + return Err(Error::new_spanned( + &field.ty, + "#[prefix(...)] fields must have type HashMap", + )); + } + if prefix.is_some() + && (attribute_path_value(&field.attrs, "parse_with")?.is_some() + || attribute_path_value(&field.attrs, "serialize_with")?.is_some()) + { + return Err(Error::new_spanned( + field, + "#[prefix(...)] fields do not support parse_with or serialize_with", + )); + } + Ok(PropertyField { ident, ty: field.ty.clone(), @@ -203,39 +222,15 @@ fn parse_property_field(field: &Field) -> syn::Result { .cloned() .collect(), key, + prefix, default, parse_with: attribute_path_value(&field.attrs, "parse_with")?, serialize_with: attribute_path_value(&field.attrs, "serialize_with")?, - is_option: is_option_type(&field.ty), + option_inner_type: option_inner_type(&field.ty), + map_value_type, }) } -fn attribute_string_value( - attributes: &[Attribute], - name: &str, -) -> syn::Result> { - let Some(attribute) = find_attribute(attributes, name)? else { - return Ok(None); - }; - - match &attribute.meta { - Meta::NameValue(name_value) => match &name_value.value { - Expr::Lit(ExprLit { - lit: Lit::Str(value), - .. - }) => Ok(Some(value.clone())), - _ => Err(Error::new_spanned( - attribute, - format!("{name} must be a string literal"), - )), - }, - _ => Err(Error::new_spanned( - attribute, - format!("{name} must use the form #[{name} = ...]"), - )), - } -} - fn attribute_expression_value(attributes: &[Attribute], name: &str) -> syn::Result> { let Some(attribute) = find_attribute(attributes, name)? else { return Ok(None); @@ -282,22 +277,120 @@ fn find_attribute<'a>( Ok(first) } -fn is_option_type(ty: &Type) -> bool { +fn parse_field(field: &PropertyField) -> TokenStream2 { + let ident = &field.ident; + let default = &field.default; + + if let Some(prefix) = &field.prefix { + let value_type = field + .map_value_type + .as_ref() + .expect("prefix fields are validated as maps"); + return quote! { + #ident: { + let parsed = properties + .iter() + .filter_map(|(key, value)| { + key.strip_prefix(#prefix).map(|suffix| { + value.parse::<#value_type>() + .map(|parsed| (suffix.to_string(), parsed)) + .map_err(|error| format!("Invalid value for {key}: {error}")) + }) + }) + .collect::<::std::result::Result< + ::std::collections::HashMap<_, _>, + ::std::string::String, + >>()?; + if parsed.is_empty() { + #default + } else { + parsed + } + } + }; + } + + let ty = &field.ty; + let key = field.key.as_ref().expect("exact-key fields have a key"); + let parse = match (&field.parse_with, &field.option_inner_type) { + (Some(parse_with), _) => quote! { + #parse_with(value).map_err(|error| { + format!("Invalid value for {}: {error}", #key) + })? + }, + (None, Some(inner_type)) => quote! { + Some(value.parse::<#inner_type>().map_err(|error| { + format!("Invalid value for {}: {error}", #key) + })?) + }, + (None, None) => quote! { + value.parse::<#ty>().map_err(|error| { + format!("Invalid value for {}: {error}", #key) + })? + }, + }; + + quote! { + #ident: match properties.get(#key) { + Some(value) => #parse, + None => #default, + } + } +} + +fn option_inner_type(ty: &Type) -> Option { let Type::Path(type_path) = ty else { - return false; + return None; }; - type_path - .path - .segments - .last() - .is_some_and(|segment| segment.ident == "Option") + let segment = type_path.path.segments.last()?; + if segment.ident != "Option" { + return None; + } + + let syn::PathArguments::AngleBracketed(arguments) = &segment.arguments else { + return None; + }; + let Some(syn::GenericArgument::Type(inner_type)) = arguments.args.first() else { + return None; + }; + + Some(inner_type.clone()) +} + +fn map_value_type(ty: &Type) -> Option { + let Type::Path(type_path) = ty else { + return None; + }; + + let segment = type_path.path.segments.last()?; + if segment.ident != "HashMap" { + return None; + } + + let syn::PathArguments::AngleBracketed(arguments) = &segment.arguments else { + return None; + }; + let Some(syn::GenericArgument::Type(value_type)) = arguments.args.iter().nth(1) else { + return None; + }; + + Some(value_type.clone()) } fn serialize_field(field: &PropertyField) -> TokenStream2 { let ident = &field.ident; - let key = &field.key; - if field.is_option { + if let Some(prefix) = &field.prefix { + return quote! { + for (suffix, value) in &self.#ident { + let key = format!("{}{}", #prefix, suffix); + map.serialize_entry(&key, &::std::string::ToString::to_string(value))?; + } + }; + } + + let key = field.key.as_ref().expect("exact-key fields have a key"); + if field.option_inner_type.is_some() { let value = match &field.serialize_with { Some(serialize_with) => quote!(#serialize_with(&self.#ident)), None => quote!(::std::string::ToString::to_string( diff --git a/crates/property-macro/tests/properties.rs b/crates/property-macro/tests/properties.rs new file mode 100644 index 0000000000..7f4fa14663 --- /dev/null +++ b/crates/property-macro/tests/properties.rs @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::HashMap; + +use iceberg_property_macro::Properties; + +const RETRIES: &str = "commit.retry.num-retries"; +const OWNER: &str = "owner"; +const COLUMN_FPP_PREFIX: &str = "write.parquet.bloom-filter-fpp.column."; + +#[derive(Debug, Properties)] +struct TestProperties { + #[key(RETRIES)] + #[default(4)] + #[doc = "Number of retries."] + retries: u64, + #[key(OWNER)] + #[default(None)] + owner: Option, + #[prefix(COLUMN_FPP_PREFIX)] + #[default(HashMap::new())] + column_fpp: HashMap, +} + +#[test] +fn generates_defaults_getters_modifiers_and_serde() { + let properties = TestProperties::default() + .with_retries(8) + .with_owner(Some("iceberg".to_string())) + .with_column_fpp(HashMap::from([("id".to_string(), 0.01)])); + + assert_eq!(properties.retries(), 8); + assert_eq!(properties.owner(), Some("iceberg".to_string())); + assert_eq!(properties.column_fpp()["id"], 0.01); + + let json = serde_json::to_value(&properties).unwrap(); + assert_eq!(json[RETRIES], "8"); + assert_eq!(json[OWNER], "iceberg"); + assert_eq!(json[format!("{COLUMN_FPP_PREFIX}id")], "0.01"); + + let decoded: TestProperties = serde_json::from_value(json).unwrap(); + assert_eq!(decoded.retries(), 8); + assert_eq!(decoded.owner(), Some("iceberg".to_string())); + assert_eq!(decoded.column_fpp()["id"], 0.01); +} From 5bfe098fc144093e3be6fb90518f54c30ca53835 Mon Sep 17 00:00:00 2001 From: Renjie Liu Date: Wed, 5 Aug 2026 07:30:10 +0800 Subject: [PATCH 03/14] Address comments --- crates/iceberg/public-api.txt | 844 ++++++----- crates/iceberg/src/catalog/utils.rs | 2 +- crates/iceberg/src/encryption/manager.rs | 7 +- crates/iceberg/src/spec/mod.rs | 2 + crates/iceberg/src/spec/parsed_table_prop.rs | 848 +++++++++++ crates/iceberg/src/spec/table_metadata.rs | 8 +- crates/iceberg/src/spec/table_properties.rs | 1342 ++--------------- .../src/transaction/expire_snapshots.rs | 11 +- crates/iceberg/src/transaction/mod.rs | 13 +- .../src/writer/file_writer/parquet_writer.rs | 13 +- .../datafusion/src/physical_plan/write.rs | 7 +- crates/property-macro/src/lib.rs | 112 +- crates/property-macro/tests/properties.rs | 27 + 13 files changed, 1565 insertions(+), 1671 deletions(-) create mode 100644 crates/iceberg/src/spec/parsed_table_prop.rs diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index e1c0d9583b..4f9fe5df5f 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -2246,248 +2246,34 @@ impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::NestedField pub fn iceberg::spec::NestedField::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> pub struct iceberg::spec::ParsedTableProperties impl iceberg::spec::ParsedTableProperties -pub fn iceberg::spec::ParsedTableProperties::adaptive_split_size_enabled(&self) -> bool -pub fn iceberg::spec::ParsedTableProperties::avro_compression(&self) -> alloc::string::String -pub fn iceberg::spec::ParsedTableProperties::avro_compression_level(&self) -> core::option::Option -pub fn iceberg::spec::ParsedTableProperties::cdc_enabled(&self) -> bool -pub fn iceberg::spec::ParsedTableProperties::cdc_max_chunk_size(&self) -> usize -pub fn iceberg::spec::ParsedTableProperties::cdc_min_chunk_size(&self) -> usize -pub fn iceberg::spec::ParsedTableProperties::cdc_norm_level(&self) -> i32 -pub fn iceberg::spec::ParsedTableProperties::comment(&self) -> core::option::Option -pub fn iceberg::spec::ParsedTableProperties::commit_max_retry_wait_ms(&self) -> u64 -pub fn iceberg::spec::ParsedTableProperties::commit_min_retry_wait_ms(&self) -> u64 -pub fn iceberg::spec::ParsedTableProperties::commit_num_retries(&self) -> usize -pub fn iceberg::spec::ParsedTableProperties::commit_num_status_checks(&self) -> usize -pub fn iceberg::spec::ParsedTableProperties::commit_status_checks_max_wait_ms(&self) -> u64 -pub fn iceberg::spec::ParsedTableProperties::commit_status_checks_min_wait_ms(&self) -> u64 -pub fn iceberg::spec::ParsedTableProperties::commit_status_checks_total_wait_ms(&self) -> u64 -pub fn iceberg::spec::ParsedTableProperties::commit_total_retry_timeout_ms(&self) -> u64 -pub fn iceberg::spec::ParsedTableProperties::data_planning_mode(&self) -> alloc::string::String -pub fn iceberg::spec::ParsedTableProperties::default_name_mapping(&self) -> core::option::Option -pub fn iceberg::spec::ParsedTableProperties::default_write_metrics_mode(&self) -> alloc::string::String -pub fn iceberg::spec::ParsedTableProperties::delete_avro_compression(&self) -> alloc::string::String -pub fn iceberg::spec::ParsedTableProperties::delete_avro_compression_level(&self) -> core::option::Option -pub fn iceberg::spec::ParsedTableProperties::delete_distribution_mode(&self) -> core::option::Option -pub fn iceberg::spec::ParsedTableProperties::delete_format_default(&self) -> iceberg::spec::DataFileFormat -pub fn iceberg::spec::ParsedTableProperties::delete_granularity(&self) -> alloc::string::String -pub fn iceberg::spec::ParsedTableProperties::delete_isolation_level(&self) -> alloc::string::String -pub fn iceberg::spec::ParsedTableProperties::delete_mode(&self) -> alloc::string::String -pub fn iceberg::spec::ParsedTableProperties::delete_orc_block_size_bytes(&self) -> u64 -pub fn iceberg::spec::ParsedTableProperties::delete_orc_compression(&self) -> alloc::string::String -pub fn iceberg::spec::ParsedTableProperties::delete_orc_compression_strategy(&self) -> alloc::string::String -pub fn iceberg::spec::ParsedTableProperties::delete_orc_stripe_size_bytes(&self) -> u64 -pub fn iceberg::spec::ParsedTableProperties::delete_orc_write_batch_size(&self) -> usize -pub fn iceberg::spec::ParsedTableProperties::delete_parquet_compression(&self) -> alloc::string::String -pub fn iceberg::spec::ParsedTableProperties::delete_parquet_compression_level(&self) -> core::option::Option -pub fn iceberg::spec::ParsedTableProperties::delete_parquet_dict_size_bytes(&self) -> usize -pub fn iceberg::spec::ParsedTableProperties::delete_parquet_page_row_limit(&self) -> usize -pub fn iceberg::spec::ParsedTableProperties::delete_parquet_page_size_bytes(&self) -> usize -pub fn iceberg::spec::ParsedTableProperties::delete_parquet_page_version(&self) -> alloc::string::String -pub fn iceberg::spec::ParsedTableProperties::delete_parquet_row_group_check_max_record_count(&self) -> usize -pub fn iceberg::spec::ParsedTableProperties::delete_parquet_row_group_check_min_record_count(&self) -> usize -pub fn iceberg::spec::ParsedTableProperties::delete_parquet_row_group_size_bytes(&self) -> usize -pub fn iceberg::spec::ParsedTableProperties::delete_planning_mode(&self) -> alloc::string::String -pub fn iceberg::spec::ParsedTableProperties::delete_target_file_size_bytes(&self) -> usize -pub fn iceberg::spec::ParsedTableProperties::encryption_data_key_length(&self) -> usize -pub fn iceberg::spec::ParsedTableProperties::encryption_key_id(&self) -> core::option::Option -pub fn iceberg::spec::ParsedTableProperties::engine_hive_enabled(&self) -> bool -pub fn iceberg::spec::ParsedTableProperties::gc_enabled(&self) -> bool -pub fn iceberg::spec::ParsedTableProperties::hive_lock_enabled(&self) -> bool -pub fn iceberg::spec::ParsedTableProperties::identifier_fields_rely(&self) -> bool -pub fn iceberg::spec::ParsedTableProperties::manifest_compression(&self) -> alloc::string::String -pub fn iceberg::spec::ParsedTableProperties::manifest_compression_level(&self) -> core::option::Option -pub fn iceberg::spec::ParsedTableProperties::manifest_lists_enabled(&self) -> bool -pub fn iceberg::spec::ParsedTableProperties::manifest_merge_enabled(&self) -> bool -pub fn iceberg::spec::ParsedTableProperties::manifest_min_merge_count(&self) -> usize -pub fn iceberg::spec::ParsedTableProperties::manifest_target_size_bytes(&self) -> usize -pub fn iceberg::spec::ParsedTableProperties::max_ref_age_ms(&self) -> i64 -pub fn iceberg::spec::ParsedTableProperties::max_snapshot_age_ms(&self) -> i64 -pub fn iceberg::spec::ParsedTableProperties::merge_distribution_mode(&self) -> core::option::Option -pub fn iceberg::spec::ParsedTableProperties::merge_isolation_level(&self) -> alloc::string::String -pub fn iceberg::spec::ParsedTableProperties::merge_mode(&self) -> alloc::string::String -pub fn iceberg::spec::ParsedTableProperties::metadata_compression_codec(&self) -> iceberg::compression::CompressionCodec -pub fn iceberg::spec::ParsedTableProperties::metadata_delete_after_commit_enabled(&self) -> bool -pub fn iceberg::spec::ParsedTableProperties::metadata_previous_versions_max(&self) -> usize -pub fn iceberg::spec::ParsedTableProperties::metadata_split_size(&self) -> u64 -pub fn iceberg::spec::ParsedTableProperties::metrics_max_inferred_column_defaults(&self) -> usize -pub fn iceberg::spec::ParsedTableProperties::metrics_mode_column_config(&self) -> std::collections::hash::map::HashMap -pub fn iceberg::spec::ParsedTableProperties::min_snapshots_to_keep(&self) -> usize -pub fn iceberg::spec::ParsedTableProperties::object_store_enabled(&self) -> bool -pub fn iceberg::spec::ParsedTableProperties::object_store_path(&self) -> core::option::Option -pub fn iceberg::spec::ParsedTableProperties::orc_batch_size(&self) -> usize -pub fn iceberg::spec::ParsedTableProperties::orc_block_size_bytes(&self) -> u64 -pub fn iceberg::spec::ParsedTableProperties::orc_bloom_filter_columns(&self) -> alloc::string::String -pub fn iceberg::spec::ParsedTableProperties::orc_bloom_filter_fpp(&self) -> f64 -pub fn iceberg::spec::ParsedTableProperties::orc_compression(&self) -> alloc::string::String -pub fn iceberg::spec::ParsedTableProperties::orc_compression_strategy(&self) -> alloc::string::String -pub fn iceberg::spec::ParsedTableProperties::orc_stripe_size_bytes(&self) -> u64 -pub fn iceberg::spec::ParsedTableProperties::orc_vectorization_enabled(&self) -> bool -pub fn iceberg::spec::ParsedTableProperties::orc_write_batch_size(&self) -> usize -pub fn iceberg::spec::ParsedTableProperties::parquet_batch_size(&self) -> usize -pub fn iceberg::spec::ParsedTableProperties::parquet_bloom_filter_adaptive_enabled(&self) -> bool -pub fn iceberg::spec::ParsedTableProperties::parquet_bloom_filter_column_enabled(&self) -> std::collections::hash::map::HashMap -pub fn iceberg::spec::ParsedTableProperties::parquet_bloom_filter_column_fpp(&self) -> std::collections::hash::map::HashMap -pub fn iceberg::spec::ParsedTableProperties::parquet_bloom_filter_column_ndv(&self) -> std::collections::hash::map::HashMap -pub fn iceberg::spec::ParsedTableProperties::parquet_bloom_filter_max_bytes(&self) -> usize -pub fn iceberg::spec::ParsedTableProperties::parquet_column_stats_enabled(&self) -> std::collections::hash::map::HashMap -pub fn iceberg::spec::ParsedTableProperties::parquet_compression(&self) -> alloc::string::String -pub fn iceberg::spec::ParsedTableProperties::parquet_compression_level(&self) -> core::option::Option -pub fn iceberg::spec::ParsedTableProperties::parquet_dict_encoding_enabled_column(&self) -> std::collections::hash::map::HashMap -pub fn iceberg::spec::ParsedTableProperties::parquet_dict_size_bytes(&self) -> usize -pub fn iceberg::spec::ParsedTableProperties::parquet_page_row_limit(&self) -> usize -pub fn iceberg::spec::ParsedTableProperties::parquet_page_size_bytes(&self) -> usize -pub fn iceberg::spec::ParsedTableProperties::parquet_page_version(&self) -> alloc::string::String -pub fn iceberg::spec::ParsedTableProperties::parquet_row_group_check_max_record_count(&self) -> usize -pub fn iceberg::spec::ParsedTableProperties::parquet_row_group_check_min_record_count(&self) -> usize -pub fn iceberg::spec::ParsedTableProperties::parquet_row_group_size_bytes(&self) -> usize -pub fn iceberg::spec::ParsedTableProperties::parquet_row_group_size_track_uncompressed(&self) -> bool -pub fn iceberg::spec::ParsedTableProperties::parquet_shred_variants(&self) -> bool -pub fn iceberg::spec::ParsedTableProperties::parquet_variant_buffer_size(&self) -> usize -pub fn iceberg::spec::ParsedTableProperties::parquet_vectorization_enabled(&self) -> bool -pub fn iceberg::spec::ParsedTableProperties::snapshot_id_inheritance_enabled(&self) -> bool -pub fn iceberg::spec::ParsedTableProperties::spark_write_accept_any_schema(&self) -> bool -pub fn iceberg::spec::ParsedTableProperties::spark_write_advisory_partition_size_bytes(&self) -> core::option::Option -pub fn iceberg::spec::ParsedTableProperties::spark_write_auto_schema_evolution(&self) -> bool -pub fn iceberg::spec::ParsedTableProperties::spark_write_partitioned_fanout_enabled(&self) -> bool -pub fn iceberg::spec::ParsedTableProperties::split_lookback(&self) -> usize -pub fn iceberg::spec::ParsedTableProperties::split_open_file_cost(&self) -> u64 -pub fn iceberg::spec::ParsedTableProperties::split_size(&self) -> u64 -pub fn iceberg::spec::ParsedTableProperties::update_distribution_mode(&self) -> core::option::Option -pub fn iceberg::spec::ParsedTableProperties::update_isolation_level(&self) -> alloc::string::String -pub fn iceberg::spec::ParsedTableProperties::update_mode(&self) -> alloc::string::String -pub fn iceberg::spec::ParsedTableProperties::upsert_enabled(&self) -> bool -pub fn iceberg::spec::ParsedTableProperties::with_adaptive_split_size_enabled(self, value: bool) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_avro_compression(self, value: alloc::string::String) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_avro_compression_level(self, value: core::option::Option) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_cdc_enabled(self, value: bool) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_cdc_max_chunk_size(self, value: usize) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_cdc_min_chunk_size(self, value: usize) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_cdc_norm_level(self, value: i32) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_comment(self, value: core::option::Option) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_commit_max_retry_wait_ms(self, value: u64) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_commit_min_retry_wait_ms(self, value: u64) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_commit_num_retries(self, value: usize) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_commit_num_status_checks(self, value: usize) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_commit_status_checks_max_wait_ms(self, value: u64) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_commit_status_checks_min_wait_ms(self, value: u64) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_commit_status_checks_total_wait_ms(self, value: u64) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_commit_total_retry_timeout_ms(self, value: u64) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_data_planning_mode(self, value: alloc::string::String) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_default_name_mapping(self, value: core::option::Option) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_default_write_metrics_mode(self, value: alloc::string::String) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_delete_avro_compression(self, value: alloc::string::String) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_delete_avro_compression_level(self, value: core::option::Option) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_delete_distribution_mode(self, value: core::option::Option) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_delete_format_default(self, value: iceberg::spec::DataFileFormat) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_delete_granularity(self, value: alloc::string::String) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_delete_isolation_level(self, value: alloc::string::String) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_delete_mode(self, value: alloc::string::String) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_delete_orc_block_size_bytes(self, value: u64) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_delete_orc_compression(self, value: alloc::string::String) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_delete_orc_compression_strategy(self, value: alloc::string::String) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_delete_orc_stripe_size_bytes(self, value: u64) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_delete_orc_write_batch_size(self, value: usize) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_delete_parquet_compression(self, value: alloc::string::String) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_delete_parquet_compression_level(self, value: core::option::Option) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_delete_parquet_dict_size_bytes(self, value: usize) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_delete_parquet_page_row_limit(self, value: usize) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_delete_parquet_page_size_bytes(self, value: usize) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_delete_parquet_page_version(self, value: alloc::string::String) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_delete_parquet_row_group_check_max_record_count(self, value: usize) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_delete_parquet_row_group_check_min_record_count(self, value: usize) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_delete_parquet_row_group_size_bytes(self, value: usize) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_delete_planning_mode(self, value: alloc::string::String) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_delete_target_file_size_bytes(self, value: usize) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_encryption_data_key_length(self, value: usize) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_encryption_key_id(self, value: core::option::Option) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_engine_hive_enabled(self, value: bool) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_gc_enabled(self, value: bool) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_hive_lock_enabled(self, value: bool) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_identifier_fields_rely(self, value: bool) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_manifest_compression(self, value: alloc::string::String) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_manifest_compression_level(self, value: core::option::Option) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_manifest_lists_enabled(self, value: bool) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_manifest_merge_enabled(self, value: bool) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_manifest_min_merge_count(self, value: usize) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_manifest_target_size_bytes(self, value: usize) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_max_ref_age_ms(self, value: i64) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_max_snapshot_age_ms(self, value: i64) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_merge_distribution_mode(self, value: core::option::Option) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_merge_isolation_level(self, value: alloc::string::String) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_merge_mode(self, value: alloc::string::String) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_metadata_compression_codec(self, value: iceberg::compression::CompressionCodec) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_metadata_delete_after_commit_enabled(self, value: bool) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_metadata_previous_versions_max(self, value: usize) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_metadata_split_size(self, value: u64) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_metrics_max_inferred_column_defaults(self, value: usize) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_metrics_mode_column_config(self, value: std::collections::hash::map::HashMap) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_min_snapshots_to_keep(self, value: usize) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_object_store_enabled(self, value: bool) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_object_store_path(self, value: core::option::Option) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_orc_batch_size(self, value: usize) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_orc_block_size_bytes(self, value: u64) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_orc_bloom_filter_columns(self, value: alloc::string::String) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_orc_bloom_filter_fpp(self, value: f64) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_orc_compression(self, value: alloc::string::String) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_orc_compression_strategy(self, value: alloc::string::String) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_orc_stripe_size_bytes(self, value: u64) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_orc_vectorization_enabled(self, value: bool) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_orc_write_batch_size(self, value: usize) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_parquet_batch_size(self, value: usize) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_parquet_bloom_filter_adaptive_enabled(self, value: bool) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_parquet_bloom_filter_column_enabled(self, value: std::collections::hash::map::HashMap) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_parquet_bloom_filter_column_fpp(self, value: std::collections::hash::map::HashMap) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_parquet_bloom_filter_column_ndv(self, value: std::collections::hash::map::HashMap) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_parquet_bloom_filter_max_bytes(self, value: usize) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_parquet_column_stats_enabled(self, value: std::collections::hash::map::HashMap) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_parquet_compression(self, value: alloc::string::String) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_parquet_compression_level(self, value: core::option::Option) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_parquet_dict_encoding_enabled_column(self, value: std::collections::hash::map::HashMap) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_parquet_dict_size_bytes(self, value: usize) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_parquet_page_row_limit(self, value: usize) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_parquet_page_size_bytes(self, value: usize) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_parquet_page_version(self, value: alloc::string::String) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_parquet_row_group_check_max_record_count(self, value: usize) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_parquet_row_group_check_min_record_count(self, value: usize) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_parquet_row_group_size_bytes(self, value: usize) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_parquet_row_group_size_track_uncompressed(self, value: bool) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_parquet_shred_variants(self, value: bool) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_parquet_variant_buffer_size(self, value: usize) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_parquet_vectorization_enabled(self, value: bool) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_snapshot_id_inheritance_enabled(self, value: bool) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_spark_write_accept_any_schema(self, value: bool) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_spark_write_advisory_partition_size_bytes(self, value: core::option::Option) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_spark_write_auto_schema_evolution(self, value: bool) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_spark_write_partitioned_fanout_enabled(self, value: bool) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_split_lookback(self, value: usize) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_split_open_file_cost(self, value: u64) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_split_size(self, value: u64) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_update_distribution_mode(self, value: core::option::Option) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_update_isolation_level(self, value: alloc::string::String) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_update_mode(self, value: alloc::string::String) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_upsert_enabled(self, value: bool) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_write_audit_publish_enabled(self, value: bool) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_write_data_location(self, value: core::option::Option) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_write_datafusion_fanout_enabled(self, value: bool) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_write_distribution_mode(self, value: core::option::Option) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_write_folder_storage_location(self, value: core::option::Option) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_write_format_default(self, value: iceberg::spec::DataFileFormat) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_write_location_provider_impl(self, value: core::option::Option) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_write_metadata_path(self, value: core::option::Option) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_write_object_store_partitioned_paths(self, value: bool) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_write_partition_summary_limit(self, value: u64) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_write_target_file_size_bytes(self, value: usize) -> Self -pub fn iceberg::spec::ParsedTableProperties::write_audit_publish_enabled(&self) -> bool -pub fn iceberg::spec::ParsedTableProperties::write_data_location(&self) -> core::option::Option -pub fn iceberg::spec::ParsedTableProperties::write_datafusion_fanout_enabled(&self) -> bool -pub fn iceberg::spec::ParsedTableProperties::write_distribution_mode(&self) -> core::option::Option -pub fn iceberg::spec::ParsedTableProperties::write_folder_storage_location(&self) -> core::option::Option -pub fn iceberg::spec::ParsedTableProperties::write_format_default(&self) -> iceberg::spec::DataFileFormat -pub fn iceberg::spec::ParsedTableProperties::write_location_provider_impl(&self) -> core::option::Option -pub fn iceberg::spec::ParsedTableProperties::write_metadata_path(&self) -> core::option::Option -pub fn iceberg::spec::ParsedTableProperties::write_object_store_partitioned_paths(&self) -> bool -pub fn iceberg::spec::ParsedTableProperties::write_partition_summary_limit(&self) -> u64 -pub fn iceberg::spec::ParsedTableProperties::write_target_file_size_bytes(&self) -> usize +pub fn iceberg::spec::ParsedTableProperties::avro(&self) -> iceberg::spec::TableAvroProperties +pub fn iceberg::spec::ParsedTableProperties::commit(&self) -> iceberg::spec::TableCommitProperties +pub fn iceberg::spec::ParsedTableProperties::compatibility(&self) -> iceberg::spec::TableCompatibilityProperties +pub fn iceberg::spec::ParsedTableProperties::encryption(&self) -> iceberg::spec::TableEncryptionProperties +pub fn iceberg::spec::ParsedTableProperties::general(&self) -> iceberg::spec::TableGeneralProperties +pub fn iceberg::spec::ParsedTableProperties::history(&self) -> iceberg::spec::TableHistoryProperties +pub fn iceberg::spec::ParsedTableProperties::manifest(&self) -> iceberg::spec::TableManifestProperties +pub fn iceberg::spec::ParsedTableProperties::metadata(&self) -> iceberg::spec::TableMetadataProperties +pub fn iceberg::spec::ParsedTableProperties::orc(&self) -> iceberg::spec::TableOrcProperties +pub fn iceberg::spec::ParsedTableProperties::parquet(&self) -> iceberg::spec::TableParquetProperties +pub fn iceberg::spec::ParsedTableProperties::read(&self) -> iceberg::spec::TableReadProperties +pub fn iceberg::spec::ParsedTableProperties::row_level(&self) -> iceberg::spec::TableRowLevelProperties +pub fn iceberg::spec::ParsedTableProperties::with_avro(self, value: iceberg::spec::TableAvroProperties) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_commit(self, value: iceberg::spec::TableCommitProperties) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_compatibility(self, value: iceberg::spec::TableCompatibilityProperties) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_encryption(self, value: iceberg::spec::TableEncryptionProperties) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_general(self, value: iceberg::spec::TableGeneralProperties) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_history(self, value: iceberg::spec::TableHistoryProperties) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_manifest(self, value: iceberg::spec::TableManifestProperties) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_metadata(self, value: iceberg::spec::TableMetadataProperties) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_orc(self, value: iceberg::spec::TableOrcProperties) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_parquet(self, value: iceberg::spec::TableParquetProperties) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_read(self, value: iceberg::spec::TableReadProperties) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_row_level(self, value: iceberg::spec::TableRowLevelProperties) -> Self +pub fn iceberg::spec::ParsedTableProperties::with_write(self, value: iceberg::spec::TableWriteProperties) -> Self +pub fn iceberg::spec::ParsedTableProperties::write(&self) -> iceberg::spec::TableWriteProperties +impl core::clone::Clone for iceberg::spec::ParsedTableProperties +pub fn iceberg::spec::ParsedTableProperties::clone(&self) -> iceberg::spec::ParsedTableProperties impl core::default::Default for iceberg::spec::ParsedTableProperties pub fn iceberg::spec::ParsedTableProperties::default() -> Self impl core::fmt::Debug for iceberg::spec::ParsedTableProperties @@ -2877,6 +2663,156 @@ impl serde_core::ser::Serialize for iceberg::spec::Summary pub fn iceberg::spec::Summary::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::Summary pub fn iceberg::spec::Summary::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::spec::TableAvroProperties +impl iceberg::spec::TableAvroProperties +pub fn iceberg::spec::TableAvroProperties::compression(&self) -> alloc::string::String +pub fn iceberg::spec::TableAvroProperties::compression_level(&self) -> core::option::Option +pub fn iceberg::spec::TableAvroProperties::delete_compression(&self) -> alloc::string::String +pub fn iceberg::spec::TableAvroProperties::delete_compression_level(&self) -> core::option::Option +pub fn iceberg::spec::TableAvroProperties::with_compression(self, value: alloc::string::String) -> Self +pub fn iceberg::spec::TableAvroProperties::with_compression_level(self, value: core::option::Option) -> Self +pub fn iceberg::spec::TableAvroProperties::with_delete_compression(self, value: alloc::string::String) -> Self +pub fn iceberg::spec::TableAvroProperties::with_delete_compression_level(self, value: core::option::Option) -> Self +impl core::clone::Clone for iceberg::spec::TableAvroProperties +pub fn iceberg::spec::TableAvroProperties::clone(&self) -> iceberg::spec::TableAvroProperties +impl core::default::Default for iceberg::spec::TableAvroProperties +pub fn iceberg::spec::TableAvroProperties::default() -> Self +impl core::fmt::Debug for iceberg::spec::TableAvroProperties +pub fn iceberg::spec::TableAvroProperties::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl serde_core::ser::Serialize for iceberg::spec::TableAvroProperties +pub fn iceberg::spec::TableAvroProperties::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> where S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::TableAvroProperties +pub fn iceberg::spec::TableAvroProperties::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> +pub struct iceberg::spec::TableCommitProperties +impl iceberg::spec::TableCommitProperties +pub fn iceberg::spec::TableCommitProperties::max_retry_wait_ms(&self) -> u64 +pub fn iceberg::spec::TableCommitProperties::min_retry_wait_ms(&self) -> u64 +pub fn iceberg::spec::TableCommitProperties::num_retries(&self) -> usize +pub fn iceberg::spec::TableCommitProperties::num_status_checks(&self) -> usize +pub fn iceberg::spec::TableCommitProperties::status_checks_max_wait_ms(&self) -> u64 +pub fn iceberg::spec::TableCommitProperties::status_checks_min_wait_ms(&self) -> u64 +pub fn iceberg::spec::TableCommitProperties::status_checks_total_wait_ms(&self) -> u64 +pub fn iceberg::spec::TableCommitProperties::total_retry_timeout_ms(&self) -> u64 +pub fn iceberg::spec::TableCommitProperties::with_max_retry_wait_ms(self, value: u64) -> Self +pub fn iceberg::spec::TableCommitProperties::with_min_retry_wait_ms(self, value: u64) -> Self +pub fn iceberg::spec::TableCommitProperties::with_num_retries(self, value: usize) -> Self +pub fn iceberg::spec::TableCommitProperties::with_num_status_checks(self, value: usize) -> Self +pub fn iceberg::spec::TableCommitProperties::with_status_checks_max_wait_ms(self, value: u64) -> Self +pub fn iceberg::spec::TableCommitProperties::with_status_checks_min_wait_ms(self, value: u64) -> Self +pub fn iceberg::spec::TableCommitProperties::with_status_checks_total_wait_ms(self, value: u64) -> Self +pub fn iceberg::spec::TableCommitProperties::with_total_retry_timeout_ms(self, value: u64) -> Self +impl core::clone::Clone for iceberg::spec::TableCommitProperties +pub fn iceberg::spec::TableCommitProperties::clone(&self) -> iceberg::spec::TableCommitProperties +impl core::default::Default for iceberg::spec::TableCommitProperties +pub fn iceberg::spec::TableCommitProperties::default() -> Self +impl core::fmt::Debug for iceberg::spec::TableCommitProperties +pub fn iceberg::spec::TableCommitProperties::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl serde_core::ser::Serialize for iceberg::spec::TableCommitProperties +pub fn iceberg::spec::TableCommitProperties::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> where S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::TableCommitProperties +pub fn iceberg::spec::TableCommitProperties::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> +pub struct iceberg::spec::TableCompatibilityProperties +impl iceberg::spec::TableCompatibilityProperties +pub fn iceberg::spec::TableCompatibilityProperties::engine_hive_enabled(&self) -> bool +pub fn iceberg::spec::TableCompatibilityProperties::hive_lock_enabled(&self) -> bool +pub fn iceberg::spec::TableCompatibilityProperties::snapshot_id_inheritance_enabled(&self) -> bool +pub fn iceberg::spec::TableCompatibilityProperties::spark_write_accept_any_schema(&self) -> bool +pub fn iceberg::spec::TableCompatibilityProperties::spark_write_advisory_partition_size_bytes(&self) -> core::option::Option +pub fn iceberg::spec::TableCompatibilityProperties::spark_write_auto_schema_evolution(&self) -> bool +pub fn iceberg::spec::TableCompatibilityProperties::spark_write_partitioned_fanout_enabled(&self) -> bool +pub fn iceberg::spec::TableCompatibilityProperties::with_engine_hive_enabled(self, value: bool) -> Self +pub fn iceberg::spec::TableCompatibilityProperties::with_hive_lock_enabled(self, value: bool) -> Self +pub fn iceberg::spec::TableCompatibilityProperties::with_snapshot_id_inheritance_enabled(self, value: bool) -> Self +pub fn iceberg::spec::TableCompatibilityProperties::with_spark_write_accept_any_schema(self, value: bool) -> Self +pub fn iceberg::spec::TableCompatibilityProperties::with_spark_write_advisory_partition_size_bytes(self, value: core::option::Option) -> Self +pub fn iceberg::spec::TableCompatibilityProperties::with_spark_write_auto_schema_evolution(self, value: bool) -> Self +pub fn iceberg::spec::TableCompatibilityProperties::with_spark_write_partitioned_fanout_enabled(self, value: bool) -> Self +impl core::clone::Clone for iceberg::spec::TableCompatibilityProperties +pub fn iceberg::spec::TableCompatibilityProperties::clone(&self) -> iceberg::spec::TableCompatibilityProperties +impl core::default::Default for iceberg::spec::TableCompatibilityProperties +pub fn iceberg::spec::TableCompatibilityProperties::default() -> Self +impl core::fmt::Debug for iceberg::spec::TableCompatibilityProperties +pub fn iceberg::spec::TableCompatibilityProperties::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl serde_core::ser::Serialize for iceberg::spec::TableCompatibilityProperties +pub fn iceberg::spec::TableCompatibilityProperties::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> where S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::TableCompatibilityProperties +pub fn iceberg::spec::TableCompatibilityProperties::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> +pub struct iceberg::spec::TableEncryptionProperties +impl iceberg::spec::TableEncryptionProperties +pub fn iceberg::spec::TableEncryptionProperties::data_key_length(&self) -> usize +pub fn iceberg::spec::TableEncryptionProperties::key_id(&self) -> core::option::Option +pub fn iceberg::spec::TableEncryptionProperties::with_data_key_length(self, value: usize) -> Self +pub fn iceberg::spec::TableEncryptionProperties::with_key_id(self, value: core::option::Option) -> Self +impl core::clone::Clone for iceberg::spec::TableEncryptionProperties +pub fn iceberg::spec::TableEncryptionProperties::clone(&self) -> iceberg::spec::TableEncryptionProperties +impl core::default::Default for iceberg::spec::TableEncryptionProperties +pub fn iceberg::spec::TableEncryptionProperties::default() -> Self +impl core::fmt::Debug for iceberg::spec::TableEncryptionProperties +pub fn iceberg::spec::TableEncryptionProperties::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl serde_core::ser::Serialize for iceberg::spec::TableEncryptionProperties +pub fn iceberg::spec::TableEncryptionProperties::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> where S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::TableEncryptionProperties +pub fn iceberg::spec::TableEncryptionProperties::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> +pub struct iceberg::spec::TableGeneralProperties +impl iceberg::spec::TableGeneralProperties +pub fn iceberg::spec::TableGeneralProperties::comment(&self) -> core::option::Option +pub fn iceberg::spec::TableGeneralProperties::identifier_fields_rely(&self) -> bool +pub fn iceberg::spec::TableGeneralProperties::with_comment(self, value: core::option::Option) -> Self +pub fn iceberg::spec::TableGeneralProperties::with_identifier_fields_rely(self, value: bool) -> Self +impl core::clone::Clone for iceberg::spec::TableGeneralProperties +pub fn iceberg::spec::TableGeneralProperties::clone(&self) -> iceberg::spec::TableGeneralProperties +impl core::default::Default for iceberg::spec::TableGeneralProperties +pub fn iceberg::spec::TableGeneralProperties::default() -> Self +impl core::fmt::Debug for iceberg::spec::TableGeneralProperties +pub fn iceberg::spec::TableGeneralProperties::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl serde_core::ser::Serialize for iceberg::spec::TableGeneralProperties +pub fn iceberg::spec::TableGeneralProperties::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> where S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::TableGeneralProperties +pub fn iceberg::spec::TableGeneralProperties::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> +pub struct iceberg::spec::TableHistoryProperties +impl iceberg::spec::TableHistoryProperties +pub fn iceberg::spec::TableHistoryProperties::gc_enabled(&self) -> bool +pub fn iceberg::spec::TableHistoryProperties::max_ref_age_ms(&self) -> i64 +pub fn iceberg::spec::TableHistoryProperties::max_snapshot_age_ms(&self) -> i64 +pub fn iceberg::spec::TableHistoryProperties::min_snapshots_to_keep(&self) -> usize +pub fn iceberg::spec::TableHistoryProperties::with_gc_enabled(self, value: bool) -> Self +pub fn iceberg::spec::TableHistoryProperties::with_max_ref_age_ms(self, value: i64) -> Self +pub fn iceberg::spec::TableHistoryProperties::with_max_snapshot_age_ms(self, value: i64) -> Self +pub fn iceberg::spec::TableHistoryProperties::with_min_snapshots_to_keep(self, value: usize) -> Self +impl core::clone::Clone for iceberg::spec::TableHistoryProperties +pub fn iceberg::spec::TableHistoryProperties::clone(&self) -> iceberg::spec::TableHistoryProperties +impl core::default::Default for iceberg::spec::TableHistoryProperties +pub fn iceberg::spec::TableHistoryProperties::default() -> Self +impl core::fmt::Debug for iceberg::spec::TableHistoryProperties +pub fn iceberg::spec::TableHistoryProperties::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl serde_core::ser::Serialize for iceberg::spec::TableHistoryProperties +pub fn iceberg::spec::TableHistoryProperties::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> where S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::TableHistoryProperties +pub fn iceberg::spec::TableHistoryProperties::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> +pub struct iceberg::spec::TableManifestProperties +impl iceberg::spec::TableManifestProperties +pub fn iceberg::spec::TableManifestProperties::compression(&self) -> alloc::string::String +pub fn iceberg::spec::TableManifestProperties::compression_level(&self) -> core::option::Option +pub fn iceberg::spec::TableManifestProperties::lists_enabled(&self) -> bool +pub fn iceberg::spec::TableManifestProperties::merge_enabled(&self) -> bool +pub fn iceberg::spec::TableManifestProperties::min_merge_count(&self) -> usize +pub fn iceberg::spec::TableManifestProperties::target_size_bytes(&self) -> usize +pub fn iceberg::spec::TableManifestProperties::with_compression(self, value: alloc::string::String) -> Self +pub fn iceberg::spec::TableManifestProperties::with_compression_level(self, value: core::option::Option) -> Self +pub fn iceberg::spec::TableManifestProperties::with_lists_enabled(self, value: bool) -> Self +pub fn iceberg::spec::TableManifestProperties::with_merge_enabled(self, value: bool) -> Self +pub fn iceberg::spec::TableManifestProperties::with_min_merge_count(self, value: usize) -> Self +pub fn iceberg::spec::TableManifestProperties::with_target_size_bytes(self, value: usize) -> Self +impl core::clone::Clone for iceberg::spec::TableManifestProperties +pub fn iceberg::spec::TableManifestProperties::clone(&self) -> iceberg::spec::TableManifestProperties +impl core::default::Default for iceberg::spec::TableManifestProperties +pub fn iceberg::spec::TableManifestProperties::default() -> Self +impl core::fmt::Debug for iceberg::spec::TableManifestProperties +pub fn iceberg::spec::TableManifestProperties::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl serde_core::ser::Serialize for iceberg::spec::TableManifestProperties +pub fn iceberg::spec::TableManifestProperties::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> where S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::TableManifestProperties +pub fn iceberg::spec::TableManifestProperties::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> pub struct iceberg::spec::TableMetadata impl iceberg::spec::TableMetadata pub fn iceberg::spec::TableMetadata::current_schema(&self) -> &iceberg::spec::SchemaRef @@ -2987,6 +2923,148 @@ impl core::clone::Clone for iceberg::spec::TableMetadataBuilder pub fn iceberg::spec::TableMetadataBuilder::clone(&self) -> iceberg::spec::TableMetadataBuilder impl core::fmt::Debug for iceberg::spec::TableMetadataBuilder pub fn iceberg::spec::TableMetadataBuilder::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg::spec::TableMetadataProperties +impl iceberg::spec::TableMetadataProperties +pub fn iceberg::spec::TableMetadataProperties::compression_codec(&self) -> iceberg::compression::CompressionCodec +pub fn iceberg::spec::TableMetadataProperties::default_metrics_mode(&self) -> alloc::string::String +pub fn iceberg::spec::TableMetadataProperties::default_name_mapping(&self) -> core::option::Option +pub fn iceberg::spec::TableMetadataProperties::delete_after_commit_enabled(&self) -> bool +pub fn iceberg::spec::TableMetadataProperties::metrics_max_inferred_column_defaults(&self) -> usize +pub fn iceberg::spec::TableMetadataProperties::metrics_mode_by_column(&self) -> std::collections::hash::map::HashMap +pub fn iceberg::spec::TableMetadataProperties::partition_summary_limit(&self) -> u64 +pub fn iceberg::spec::TableMetadataProperties::path(&self) -> core::option::Option +pub fn iceberg::spec::TableMetadataProperties::previous_versions_max(&self) -> usize +pub fn iceberg::spec::TableMetadataProperties::with_compression_codec(self, value: iceberg::compression::CompressionCodec) -> Self +pub fn iceberg::spec::TableMetadataProperties::with_default_metrics_mode(self, value: alloc::string::String) -> Self +pub fn iceberg::spec::TableMetadataProperties::with_default_name_mapping(self, value: core::option::Option) -> Self +pub fn iceberg::spec::TableMetadataProperties::with_delete_after_commit_enabled(self, value: bool) -> Self +pub fn iceberg::spec::TableMetadataProperties::with_metrics_max_inferred_column_defaults(self, value: usize) -> Self +pub fn iceberg::spec::TableMetadataProperties::with_metrics_mode_by_column(self, value: std::collections::hash::map::HashMap) -> Self +pub fn iceberg::spec::TableMetadataProperties::with_partition_summary_limit(self, value: u64) -> Self +pub fn iceberg::spec::TableMetadataProperties::with_path(self, value: core::option::Option) -> Self +pub fn iceberg::spec::TableMetadataProperties::with_previous_versions_max(self, value: usize) -> Self +impl core::clone::Clone for iceberg::spec::TableMetadataProperties +pub fn iceberg::spec::TableMetadataProperties::clone(&self) -> iceberg::spec::TableMetadataProperties +impl core::default::Default for iceberg::spec::TableMetadataProperties +pub fn iceberg::spec::TableMetadataProperties::default() -> Self +impl core::fmt::Debug for iceberg::spec::TableMetadataProperties +pub fn iceberg::spec::TableMetadataProperties::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl serde_core::ser::Serialize for iceberg::spec::TableMetadataProperties +pub fn iceberg::spec::TableMetadataProperties::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> where S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::TableMetadataProperties +pub fn iceberg::spec::TableMetadataProperties::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> +pub struct iceberg::spec::TableOrcProperties +impl iceberg::spec::TableOrcProperties +pub fn iceberg::spec::TableOrcProperties::block_size_bytes(&self) -> u64 +pub fn iceberg::spec::TableOrcProperties::bloom_filter_columns(&self) -> alloc::string::String +pub fn iceberg::spec::TableOrcProperties::bloom_filter_fpp(&self) -> f64 +pub fn iceberg::spec::TableOrcProperties::compression(&self) -> alloc::string::String +pub fn iceberg::spec::TableOrcProperties::compression_strategy(&self) -> alloc::string::String +pub fn iceberg::spec::TableOrcProperties::delete_block_size_bytes(&self) -> u64 +pub fn iceberg::spec::TableOrcProperties::delete_compression(&self) -> alloc::string::String +pub fn iceberg::spec::TableOrcProperties::delete_compression_strategy(&self) -> alloc::string::String +pub fn iceberg::spec::TableOrcProperties::delete_stripe_size_bytes(&self) -> u64 +pub fn iceberg::spec::TableOrcProperties::delete_write_batch_size(&self) -> usize +pub fn iceberg::spec::TableOrcProperties::stripe_size_bytes(&self) -> u64 +pub fn iceberg::spec::TableOrcProperties::with_block_size_bytes(self, value: u64) -> Self +pub fn iceberg::spec::TableOrcProperties::with_bloom_filter_columns(self, value: alloc::string::String) -> Self +pub fn iceberg::spec::TableOrcProperties::with_bloom_filter_fpp(self, value: f64) -> Self +pub fn iceberg::spec::TableOrcProperties::with_compression(self, value: alloc::string::String) -> Self +pub fn iceberg::spec::TableOrcProperties::with_compression_strategy(self, value: alloc::string::String) -> Self +pub fn iceberg::spec::TableOrcProperties::with_delete_block_size_bytes(self, value: u64) -> Self +pub fn iceberg::spec::TableOrcProperties::with_delete_compression(self, value: alloc::string::String) -> Self +pub fn iceberg::spec::TableOrcProperties::with_delete_compression_strategy(self, value: alloc::string::String) -> Self +pub fn iceberg::spec::TableOrcProperties::with_delete_stripe_size_bytes(self, value: u64) -> Self +pub fn iceberg::spec::TableOrcProperties::with_delete_write_batch_size(self, value: usize) -> Self +pub fn iceberg::spec::TableOrcProperties::with_stripe_size_bytes(self, value: u64) -> Self +pub fn iceberg::spec::TableOrcProperties::with_write_batch_size(self, value: usize) -> Self +pub fn iceberg::spec::TableOrcProperties::write_batch_size(&self) -> usize +impl core::clone::Clone for iceberg::spec::TableOrcProperties +pub fn iceberg::spec::TableOrcProperties::clone(&self) -> iceberg::spec::TableOrcProperties +impl core::default::Default for iceberg::spec::TableOrcProperties +pub fn iceberg::spec::TableOrcProperties::default() -> Self +impl core::fmt::Debug for iceberg::spec::TableOrcProperties +pub fn iceberg::spec::TableOrcProperties::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl serde_core::ser::Serialize for iceberg::spec::TableOrcProperties +pub fn iceberg::spec::TableOrcProperties::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> where S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::TableOrcProperties +pub fn iceberg::spec::TableOrcProperties::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> +pub struct iceberg::spec::TableParquetProperties +impl iceberg::spec::TableParquetProperties +pub fn iceberg::spec::TableParquetProperties::bloom_filter_adaptive_enabled(&self) -> bool +pub fn iceberg::spec::TableParquetProperties::bloom_filter_column_enabled(&self) -> std::collections::hash::map::HashMap +pub fn iceberg::spec::TableParquetProperties::bloom_filter_column_fpp(&self) -> std::collections::hash::map::HashMap +pub fn iceberg::spec::TableParquetProperties::bloom_filter_column_ndv(&self) -> std::collections::hash::map::HashMap +pub fn iceberg::spec::TableParquetProperties::bloom_filter_max_bytes(&self) -> usize +pub fn iceberg::spec::TableParquetProperties::column_stats_enabled(&self) -> std::collections::hash::map::HashMap +pub fn iceberg::spec::TableParquetProperties::compression(&self) -> alloc::string::String +pub fn iceberg::spec::TableParquetProperties::compression_level(&self) -> core::option::Option +pub fn iceberg::spec::TableParquetProperties::content_defined_chunking_enabled(&self) -> bool +pub fn iceberg::spec::TableParquetProperties::content_defined_chunking_max_chunk_size(&self) -> usize +pub fn iceberg::spec::TableParquetProperties::content_defined_chunking_min_chunk_size(&self) -> usize +pub fn iceberg::spec::TableParquetProperties::content_defined_chunking_norm_level(&self) -> i32 +pub fn iceberg::spec::TableParquetProperties::delete_compression(&self) -> alloc::string::String +pub fn iceberg::spec::TableParquetProperties::delete_compression_level(&self) -> core::option::Option +pub fn iceberg::spec::TableParquetProperties::delete_dict_size_bytes(&self) -> usize +pub fn iceberg::spec::TableParquetProperties::delete_page_row_limit(&self) -> usize +pub fn iceberg::spec::TableParquetProperties::delete_page_size_bytes(&self) -> usize +pub fn iceberg::spec::TableParquetProperties::delete_page_version(&self) -> alloc::string::String +pub fn iceberg::spec::TableParquetProperties::delete_row_group_check_max_record_count(&self) -> usize +pub fn iceberg::spec::TableParquetProperties::delete_row_group_check_min_record_count(&self) -> usize +pub fn iceberg::spec::TableParquetProperties::delete_row_group_size_bytes(&self) -> usize +pub fn iceberg::spec::TableParquetProperties::dict_encoding_enabled_by_column(&self) -> std::collections::hash::map::HashMap +pub fn iceberg::spec::TableParquetProperties::dict_size_bytes(&self) -> usize +pub fn iceberg::spec::TableParquetProperties::page_row_limit(&self) -> usize +pub fn iceberg::spec::TableParquetProperties::page_size_bytes(&self) -> usize +pub fn iceberg::spec::TableParquetProperties::page_version(&self) -> alloc::string::String +pub fn iceberg::spec::TableParquetProperties::row_group_check_max_record_count(&self) -> usize +pub fn iceberg::spec::TableParquetProperties::row_group_check_min_record_count(&self) -> usize +pub fn iceberg::spec::TableParquetProperties::row_group_size_bytes(&self) -> usize +pub fn iceberg::spec::TableParquetProperties::row_group_size_track_uncompressed(&self) -> bool +pub fn iceberg::spec::TableParquetProperties::shred_variants(&self) -> bool +pub fn iceberg::spec::TableParquetProperties::variant_inference_buffer_size(&self) -> usize +pub fn iceberg::spec::TableParquetProperties::with_bloom_filter_adaptive_enabled(self, value: bool) -> Self +pub fn iceberg::spec::TableParquetProperties::with_bloom_filter_column_enabled(self, value: std::collections::hash::map::HashMap) -> Self +pub fn iceberg::spec::TableParquetProperties::with_bloom_filter_column_fpp(self, value: std::collections::hash::map::HashMap) -> Self +pub fn iceberg::spec::TableParquetProperties::with_bloom_filter_column_ndv(self, value: std::collections::hash::map::HashMap) -> Self +pub fn iceberg::spec::TableParquetProperties::with_bloom_filter_max_bytes(self, value: usize) -> Self +pub fn iceberg::spec::TableParquetProperties::with_column_stats_enabled(self, value: std::collections::hash::map::HashMap) -> Self +pub fn iceberg::spec::TableParquetProperties::with_compression(self, value: alloc::string::String) -> Self +pub fn iceberg::spec::TableParquetProperties::with_compression_level(self, value: core::option::Option) -> Self +pub fn iceberg::spec::TableParquetProperties::with_content_defined_chunking_enabled(self, value: bool) -> Self +pub fn iceberg::spec::TableParquetProperties::with_content_defined_chunking_max_chunk_size(self, value: usize) -> Self +pub fn iceberg::spec::TableParquetProperties::with_content_defined_chunking_min_chunk_size(self, value: usize) -> Self +pub fn iceberg::spec::TableParquetProperties::with_content_defined_chunking_norm_level(self, value: i32) -> Self +pub fn iceberg::spec::TableParquetProperties::with_delete_compression(self, value: alloc::string::String) -> Self +pub fn iceberg::spec::TableParquetProperties::with_delete_compression_level(self, value: core::option::Option) -> Self +pub fn iceberg::spec::TableParquetProperties::with_delete_dict_size_bytes(self, value: usize) -> Self +pub fn iceberg::spec::TableParquetProperties::with_delete_page_row_limit(self, value: usize) -> Self +pub fn iceberg::spec::TableParquetProperties::with_delete_page_size_bytes(self, value: usize) -> Self +pub fn iceberg::spec::TableParquetProperties::with_delete_page_version(self, value: alloc::string::String) -> Self +pub fn iceberg::spec::TableParquetProperties::with_delete_row_group_check_max_record_count(self, value: usize) -> Self +pub fn iceberg::spec::TableParquetProperties::with_delete_row_group_check_min_record_count(self, value: usize) -> Self +pub fn iceberg::spec::TableParquetProperties::with_delete_row_group_size_bytes(self, value: usize) -> Self +pub fn iceberg::spec::TableParquetProperties::with_dict_encoding_enabled_by_column(self, value: std::collections::hash::map::HashMap) -> Self +pub fn iceberg::spec::TableParquetProperties::with_dict_size_bytes(self, value: usize) -> Self +pub fn iceberg::spec::TableParquetProperties::with_page_row_limit(self, value: usize) -> Self +pub fn iceberg::spec::TableParquetProperties::with_page_size_bytes(self, value: usize) -> Self +pub fn iceberg::spec::TableParquetProperties::with_page_version(self, value: alloc::string::String) -> Self +pub fn iceberg::spec::TableParquetProperties::with_row_group_check_max_record_count(self, value: usize) -> Self +pub fn iceberg::spec::TableParquetProperties::with_row_group_check_min_record_count(self, value: usize) -> Self +pub fn iceberg::spec::TableParquetProperties::with_row_group_size_bytes(self, value: usize) -> Self +pub fn iceberg::spec::TableParquetProperties::with_row_group_size_track_uncompressed(self, value: bool) -> Self +pub fn iceberg::spec::TableParquetProperties::with_shred_variants(self, value: bool) -> Self +pub fn iceberg::spec::TableParquetProperties::with_variant_inference_buffer_size(self, value: usize) -> Self +impl core::clone::Clone for iceberg::spec::TableParquetProperties +pub fn iceberg::spec::TableParquetProperties::clone(&self) -> iceberg::spec::TableParquetProperties +impl core::default::Default for iceberg::spec::TableParquetProperties +pub fn iceberg::spec::TableParquetProperties::default() -> Self +impl core::fmt::Debug for iceberg::spec::TableParquetProperties +pub fn iceberg::spec::TableParquetProperties::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl serde_core::ser::Serialize for iceberg::spec::TableParquetProperties +pub fn iceberg::spec::TableParquetProperties::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> where S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::TableParquetProperties +pub fn iceberg::spec::TableParquetProperties::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> pub struct iceberg::spec::TableProperties pub iceberg::spec::TableProperties::cdc_enabled: bool pub iceberg::spec::TableProperties::cdc_max_chunk_size: usize @@ -3008,27 +3086,12 @@ pub iceberg::spec::TableProperties::write_format_default: alloc::string::String pub iceberg::spec::TableProperties::write_metadata_path: core::option::Option pub iceberg::spec::TableProperties::write_target_file_size_bytes: usize impl iceberg::spec::TableProperties -pub const iceberg::spec::TableProperties::PROPERTY_ADAPTIVE_SPLIT_SIZE_ENABLED: &str -pub const iceberg::spec::TableProperties::PROPERTY_ADAPTIVE_SPLIT_SIZE_ENABLED_DEFAULT: bool -pub const iceberg::spec::TableProperties::PROPERTY_AVRO_COMPRESSION: &str -pub const iceberg::spec::TableProperties::PROPERTY_AVRO_COMPRESSION_DEFAULT: &str -pub const iceberg::spec::TableProperties::PROPERTY_AVRO_COMPRESSION_LEVEL: &str -pub const iceberg::spec::TableProperties::PROPERTY_AVRO_COMPRESSION_LEVEL_DEFAULT: core::option::Option<&str> -pub const iceberg::spec::TableProperties::PROPERTY_COMMENT: &str pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS: &str pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT: u64 pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS: &str pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT: u64 pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_NUM_RETRIES: &str pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT: usize -pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_NUM_STATUS_CHECKS: &str -pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_NUM_STATUS_CHECKS_DEFAULT: usize -pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_STATUS_CHECKS_MAX_WAIT_MS: &str -pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_STATUS_CHECKS_MAX_WAIT_MS_DEFAULT: u64 -pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_STATUS_CHECKS_MIN_WAIT_MS: &str -pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_STATUS_CHECKS_MIN_WAIT_MS_DEFAULT: u64 -pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_STATUS_CHECKS_TOTAL_WAIT_MS: &str -pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_STATUS_CHECKS_TOTAL_WAIT_MS_DEFAULT: u64 pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS: &str pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT: u64 pub const iceberg::spec::TableProperties::PROPERTY_CURRENT_SCHEMA: &str @@ -3037,124 +3100,27 @@ pub const iceberg::spec::TableProperties::PROPERTY_CURRENT_SNAPSHOT_SUMMARY: &st pub const iceberg::spec::TableProperties::PROPERTY_CURRENT_SNAPSHOT_TIMESTAMP: &str pub const iceberg::spec::TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED: &str pub const iceberg::spec::TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT: bool -pub const iceberg::spec::TableProperties::PROPERTY_DATA_PLANNING_MODE: &str pub const iceberg::spec::TableProperties::PROPERTY_DEFAULT_FILE_FORMAT: &str pub const iceberg::spec::TableProperties::PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT: &str -pub const iceberg::spec::TableProperties::PROPERTY_DEFAULT_NAME_MAPPING: &str pub const iceberg::spec::TableProperties::PROPERTY_DEFAULT_PARTITION_SPEC: &str pub const iceberg::spec::TableProperties::PROPERTY_DEFAULT_SORT_ORDER: &str -pub const iceberg::spec::TableProperties::PROPERTY_DEFAULT_WRITE_METRICS_MODE: &str -pub const iceberg::spec::TableProperties::PROPERTY_DEFAULT_WRITE_METRICS_MODE_DEFAULT: &str -pub const iceberg::spec::TableProperties::PROPERTY_DELETE_AVRO_COMPRESSION: &str -pub const iceberg::spec::TableProperties::PROPERTY_DELETE_AVRO_COMPRESSION_LEVEL: &str pub const iceberg::spec::TableProperties::PROPERTY_DELETE_DEFAULT_FILE_FORMAT: &str -pub const iceberg::spec::TableProperties::PROPERTY_DELETE_DISTRIBUTION_MODE: &str -pub const iceberg::spec::TableProperties::PROPERTY_DELETE_GRANULARITY: &str -pub const iceberg::spec::TableProperties::PROPERTY_DELETE_GRANULARITY_DEFAULT: &str -pub const iceberg::spec::TableProperties::PROPERTY_DELETE_ISOLATION_LEVEL: &str -pub const iceberg::spec::TableProperties::PROPERTY_DELETE_ISOLATION_LEVEL_DEFAULT: &str -pub const iceberg::spec::TableProperties::PROPERTY_DELETE_MODE: &str -pub const iceberg::spec::TableProperties::PROPERTY_DELETE_MODE_DEFAULT: &str -pub const iceberg::spec::TableProperties::PROPERTY_DELETE_ORC_BLOCK_SIZE_BYTES: &str -pub const iceberg::spec::TableProperties::PROPERTY_DELETE_ORC_COMPRESSION: &str -pub const iceberg::spec::TableProperties::PROPERTY_DELETE_ORC_COMPRESSION_STRATEGY: &str -pub const iceberg::spec::TableProperties::PROPERTY_DELETE_ORC_STRIPE_SIZE_BYTES: &str -pub const iceberg::spec::TableProperties::PROPERTY_DELETE_ORC_WRITE_BATCH_SIZE: &str -pub const iceberg::spec::TableProperties::PROPERTY_DELETE_PARQUET_COMPRESSION: &str -pub const iceberg::spec::TableProperties::PROPERTY_DELETE_PARQUET_COMPRESSION_LEVEL: &str -pub const iceberg::spec::TableProperties::PROPERTY_DELETE_PARQUET_DICT_SIZE_BYTES: &str -pub const iceberg::spec::TableProperties::PROPERTY_DELETE_PARQUET_PAGE_ROW_LIMIT: &str -pub const iceberg::spec::TableProperties::PROPERTY_DELETE_PARQUET_PAGE_SIZE_BYTES: &str -pub const iceberg::spec::TableProperties::PROPERTY_DELETE_PARQUET_PAGE_VERSION: &str -pub const iceberg::spec::TableProperties::PROPERTY_DELETE_PARQUET_ROW_GROUP_CHECK_MAX_RECORD_COUNT: &str -pub const iceberg::spec::TableProperties::PROPERTY_DELETE_PARQUET_ROW_GROUP_CHECK_MIN_RECORD_COUNT: &str -pub const iceberg::spec::TableProperties::PROPERTY_DELETE_PARQUET_ROW_GROUP_SIZE_BYTES: &str -pub const iceberg::spec::TableProperties::PROPERTY_DELETE_PLANNING_MODE: &str -pub const iceberg::spec::TableProperties::PROPERTY_DELETE_TARGET_FILE_SIZE_BYTES: &str -pub const iceberg::spec::TableProperties::PROPERTY_DELETE_TARGET_FILE_SIZE_BYTES_DEFAULT: usize -pub const iceberg::spec::TableProperties::PROPERTY_ENCRYPTION_AAD_LENGTH_DEFAULT: usize pub const iceberg::spec::TableProperties::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH: &str pub const iceberg::spec::TableProperties::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH_DEFAULT: usize -pub const iceberg::spec::TableProperties::PROPERTY_ENCRYPTION_DEK_LENGTH: &str -pub const iceberg::spec::TableProperties::PROPERTY_ENCRYPTION_DEK_LENGTH_DEFAULT: usize pub const iceberg::spec::TableProperties::PROPERTY_ENCRYPTION_KEY_ID: &str -pub const iceberg::spec::TableProperties::PROPERTY_ENCRYPTION_TABLE_KEY: &str -pub const iceberg::spec::TableProperties::PROPERTY_ENGINE_HIVE_ENABLED: &str -pub const iceberg::spec::TableProperties::PROPERTY_ENGINE_HIVE_ENABLED_DEFAULT: bool pub const iceberg::spec::TableProperties::PROPERTY_FORMAT_VERSION: &str pub const iceberg::spec::TableProperties::PROPERTY_GC_ENABLED: &str pub const iceberg::spec::TableProperties::PROPERTY_GC_ENABLED_DEFAULT: bool -pub const iceberg::spec::TableProperties::PROPERTY_HIVE_LOCK_ENABLED: &str -pub const iceberg::spec::TableProperties::PROPERTY_HIVE_LOCK_ENABLED_DEFAULT: bool -pub const iceberg::spec::TableProperties::PROPERTY_IDENTIFIER_FIELDS_RELY: &str -pub const iceberg::spec::TableProperties::PROPERTY_IDENTIFIER_FIELDS_RELY_DEFAULT: bool -pub const iceberg::spec::TableProperties::PROPERTY_MANIFEST_COMPRESSION: &str -pub const iceberg::spec::TableProperties::PROPERTY_MANIFEST_COMPRESSION_DEFAULT: &str -pub const iceberg::spec::TableProperties::PROPERTY_MANIFEST_COMPRESSION_LEVEL: &str -pub const iceberg::spec::TableProperties::PROPERTY_MANIFEST_COMPRESSION_LEVEL_DEFAULT: core::option::Option<&str> -pub const iceberg::spec::TableProperties::PROPERTY_MANIFEST_LISTS_ENABLED: &str -pub const iceberg::spec::TableProperties::PROPERTY_MANIFEST_LISTS_ENABLED_DEFAULT: bool -pub const iceberg::spec::TableProperties::PROPERTY_MANIFEST_MERGE_ENABLED: &str -pub const iceberg::spec::TableProperties::PROPERTY_MANIFEST_MERGE_ENABLED_DEFAULT: bool -pub const iceberg::spec::TableProperties::PROPERTY_MANIFEST_MIN_MERGE_COUNT: &str -pub const iceberg::spec::TableProperties::PROPERTY_MANIFEST_MIN_MERGE_COUNT_DEFAULT: usize -pub const iceberg::spec::TableProperties::PROPERTY_MANIFEST_TARGET_SIZE_BYTES: &str -pub const iceberg::spec::TableProperties::PROPERTY_MANIFEST_TARGET_SIZE_BYTES_DEFAULT: usize pub const iceberg::spec::TableProperties::PROPERTY_MAX_REF_AGE_MS: &str pub const iceberg::spec::TableProperties::PROPERTY_MAX_REF_AGE_MS_DEFAULT: i64 pub const iceberg::spec::TableProperties::PROPERTY_MAX_SNAPSHOT_AGE_MS: &str pub const iceberg::spec::TableProperties::PROPERTY_MAX_SNAPSHOT_AGE_MS_DEFAULT: i64 -pub const iceberg::spec::TableProperties::PROPERTY_MERGE_DISTRIBUTION_MODE: &str -pub const iceberg::spec::TableProperties::PROPERTY_MERGE_ISOLATION_LEVEL: &str -pub const iceberg::spec::TableProperties::PROPERTY_MERGE_ISOLATION_LEVEL_DEFAULT: &str -pub const iceberg::spec::TableProperties::PROPERTY_MERGE_MODE: &str -pub const iceberg::spec::TableProperties::PROPERTY_MERGE_MODE_DEFAULT: &str -pub const iceberg::spec::TableProperties::PROPERTY_METADATA_COMPRESSION: &str pub const iceberg::spec::TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC: &str pub const iceberg::spec::TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC_DEFAULT: &str -pub const iceberg::spec::TableProperties::PROPERTY_METADATA_COMPRESSION_DEFAULT: &str -pub const iceberg::spec::TableProperties::PROPERTY_METADATA_DELETE_AFTER_COMMIT_ENABLED: &str -pub const iceberg::spec::TableProperties::PROPERTY_METADATA_DELETE_AFTER_COMMIT_ENABLED_DEFAULT: bool pub const iceberg::spec::TableProperties::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX: &str pub const iceberg::spec::TableProperties::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX_DEFAULT: usize -pub const iceberg::spec::TableProperties::PROPERTY_METADATA_SPLIT_SIZE: &str -pub const iceberg::spec::TableProperties::PROPERTY_METADATA_SPLIT_SIZE_DEFAULT: u64 -pub const iceberg::spec::TableProperties::PROPERTY_METRICS_MAX_INFERRED_COLUMN_DEFAULTS: &str -pub const iceberg::spec::TableProperties::PROPERTY_METRICS_MAX_INFERRED_COLUMN_DEFAULTS_DEFAULT: usize -pub const iceberg::spec::TableProperties::PROPERTY_METRICS_MODE_COLUMN_CONF_PREFIX: &str pub const iceberg::spec::TableProperties::PROPERTY_MIN_SNAPSHOTS_TO_KEEP: &str pub const iceberg::spec::TableProperties::PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT: usize -pub const iceberg::spec::TableProperties::PROPERTY_OBJECT_STORE_ENABLED: &str -pub const iceberg::spec::TableProperties::PROPERTY_OBJECT_STORE_ENABLED_DEFAULT: bool -pub const iceberg::spec::TableProperties::PROPERTY_OBJECT_STORE_PATH: &str -pub const iceberg::spec::TableProperties::PROPERTY_ORC_BATCH_SIZE: &str -pub const iceberg::spec::TableProperties::PROPERTY_ORC_BATCH_SIZE_DEFAULT: usize -pub const iceberg::spec::TableProperties::PROPERTY_ORC_BLOCK_SIZE_BYTES: &str -pub const iceberg::spec::TableProperties::PROPERTY_ORC_BLOCK_SIZE_BYTES_DEFAULT: u64 -pub const iceberg::spec::TableProperties::PROPERTY_ORC_BLOOM_FILTER_COLUMNS: &str -pub const iceberg::spec::TableProperties::PROPERTY_ORC_BLOOM_FILTER_COLUMNS_DEFAULT: &str -pub const iceberg::spec::TableProperties::PROPERTY_ORC_BLOOM_FILTER_FPP: &str -pub const iceberg::spec::TableProperties::PROPERTY_ORC_BLOOM_FILTER_FPP_DEFAULT: f64 -pub const iceberg::spec::TableProperties::PROPERTY_ORC_COMPRESSION: &str -pub const iceberg::spec::TableProperties::PROPERTY_ORC_COMPRESSION_DEFAULT: &str -pub const iceberg::spec::TableProperties::PROPERTY_ORC_COMPRESSION_STRATEGY: &str -pub const iceberg::spec::TableProperties::PROPERTY_ORC_COMPRESSION_STRATEGY_DEFAULT: &str -pub const iceberg::spec::TableProperties::PROPERTY_ORC_STRIPE_SIZE_BYTES: &str -pub const iceberg::spec::TableProperties::PROPERTY_ORC_STRIPE_SIZE_BYTES_DEFAULT: u64 -pub const iceberg::spec::TableProperties::PROPERTY_ORC_VECTORIZATION_ENABLED: &str -pub const iceberg::spec::TableProperties::PROPERTY_ORC_VECTORIZATION_ENABLED_DEFAULT: bool -pub const iceberg::spec::TableProperties::PROPERTY_ORC_WRITE_BATCH_SIZE: &str -pub const iceberg::spec::TableProperties::PROPERTY_ORC_WRITE_BATCH_SIZE_DEFAULT: usize -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_BATCH_SIZE: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_BATCH_SIZE_DEFAULT: usize -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_BLOOM_FILTER_ADAPTIVE_ENABLED: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_BLOOM_FILTER_ADAPTIVE_ENABLED_DEFAULT: bool -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_BLOOM_FILTER_COLUMN_FPP_DEFAULT: f64 -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_BLOOM_FILTER_COLUMN_FPP_PREFIX: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_BLOOM_FILTER_COLUMN_NDV_PREFIX: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_BLOOM_FILTER_MAX_BYTES: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_BLOOM_FILTER_MAX_BYTES_DEFAULT: usize pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_ENABLED: &str pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_ENABLED_DEFAULT: bool pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE: &str @@ -3163,73 +3129,9 @@ pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE: & pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT: usize pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL: &str pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL_DEFAULT: i32 -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_COLUMN_STATS_ENABLED_PREFIX: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_COMPRESSION: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_COMPRESSION_DEFAULT: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_COMPRESSION_DEFAULT_SINCE_1_4_0: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_COMPRESSION_LEVEL: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_COMPRESSION_LEVEL_DEFAULT: core::option::Option<&str> -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_DICT_ENCODING_ENABLED_COLUMN_PREFIX: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_DICT_SIZE_BYTES: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_DICT_SIZE_BYTES_DEFAULT: usize -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_PAGE_ROW_LIMIT: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_PAGE_ROW_LIMIT_DEFAULT: usize -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_PAGE_SIZE_BYTES: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_PAGE_SIZE_BYTES_DEFAULT: usize -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_PAGE_VERSION: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_PAGE_VERSION_DEFAULT: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_ROW_GROUP_CHECK_MAX_RECORD_COUNT: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_ROW_GROUP_CHECK_MAX_RECORD_COUNT_DEFAULT: usize -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_ROW_GROUP_CHECK_MIN_RECORD_COUNT: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_ROW_GROUP_CHECK_MIN_RECORD_COUNT_DEFAULT: usize -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT: usize -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_TRACK_UNCOMPRESSED: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_TRACK_UNCOMPRESSED_DEFAULT: bool -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_SHRED_VARIANTS: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_SHRED_VARIANTS_DEFAULT: bool -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_VARIANT_BUFFER_SIZE: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_VARIANT_BUFFER_SIZE_DEFAULT: usize -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_VECTORIZATION_ENABLED: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_VECTORIZATION_ENABLED_DEFAULT: bool -pub const iceberg::spec::TableProperties::PROPERTY_PLANNING_MODE_DEFAULT: &str pub const iceberg::spec::TableProperties::PROPERTY_SNAPSHOT_COUNT: &str -pub const iceberg::spec::TableProperties::PROPERTY_SNAPSHOT_ID_INHERITANCE_ENABLED: &str -pub const iceberg::spec::TableProperties::PROPERTY_SNAPSHOT_ID_INHERITANCE_ENABLED_DEFAULT: bool -pub const iceberg::spec::TableProperties::PROPERTY_SPARK_WRITE_ACCEPT_ANY_SCHEMA: &str -pub const iceberg::spec::TableProperties::PROPERTY_SPARK_WRITE_ACCEPT_ANY_SCHEMA_DEFAULT: bool -pub const iceberg::spec::TableProperties::PROPERTY_SPARK_WRITE_ADVISORY_PARTITION_SIZE_BYTES: &str -pub const iceberg::spec::TableProperties::PROPERTY_SPARK_WRITE_AUTO_SCHEMA_EVOLUTION: &str -pub const iceberg::spec::TableProperties::PROPERTY_SPARK_WRITE_AUTO_SCHEMA_EVOLUTION_DEFAULT: bool -pub const iceberg::spec::TableProperties::PROPERTY_SPARK_WRITE_PARTITIONED_FANOUT_ENABLED: &str -pub const iceberg::spec::TableProperties::PROPERTY_SPARK_WRITE_PARTITIONED_FANOUT_ENABLED_DEFAULT: bool -pub const iceberg::spec::TableProperties::PROPERTY_SPLIT_LOOKBACK: &str -pub const iceberg::spec::TableProperties::PROPERTY_SPLIT_LOOKBACK_DEFAULT: usize -pub const iceberg::spec::TableProperties::PROPERTY_SPLIT_OPEN_FILE_COST: &str -pub const iceberg::spec::TableProperties::PROPERTY_SPLIT_OPEN_FILE_COST_DEFAULT: u64 -pub const iceberg::spec::TableProperties::PROPERTY_SPLIT_SIZE: &str -pub const iceberg::spec::TableProperties::PROPERTY_SPLIT_SIZE_DEFAULT: u64 -pub const iceberg::spec::TableProperties::PROPERTY_UPDATE_DISTRIBUTION_MODE: &str -pub const iceberg::spec::TableProperties::PROPERTY_UPDATE_ISOLATION_LEVEL: &str -pub const iceberg::spec::TableProperties::PROPERTY_UPDATE_ISOLATION_LEVEL_DEFAULT: &str -pub const iceberg::spec::TableProperties::PROPERTY_UPDATE_MODE: &str -pub const iceberg::spec::TableProperties::PROPERTY_UPDATE_MODE_DEFAULT: &str -pub const iceberg::spec::TableProperties::PROPERTY_UPSERT_ENABLED: &str -pub const iceberg::spec::TableProperties::PROPERTY_UPSERT_ENABLED_DEFAULT: bool pub const iceberg::spec::TableProperties::PROPERTY_UUID: &str -pub const iceberg::spec::TableProperties::PROPERTY_WRITE_AUDIT_PUBLISH_ENABLED: &str -pub const iceberg::spec::TableProperties::PROPERTY_WRITE_AUDIT_PUBLISH_ENABLED_DEFAULT: bool -pub const iceberg::spec::TableProperties::PROPERTY_WRITE_DATA_LOCATION: &str -pub const iceberg::spec::TableProperties::PROPERTY_WRITE_DISTRIBUTION_MODE: &str -pub const iceberg::spec::TableProperties::PROPERTY_WRITE_DISTRIBUTION_MODE_HASH: &str -pub const iceberg::spec::TableProperties::PROPERTY_WRITE_DISTRIBUTION_MODE_NONE: &str -pub const iceberg::spec::TableProperties::PROPERTY_WRITE_DISTRIBUTION_MODE_RANGE: &str -pub const iceberg::spec::TableProperties::PROPERTY_WRITE_FOLDER_STORAGE_LOCATION: &str -pub const iceberg::spec::TableProperties::PROPERTY_WRITE_LOCATION_PROVIDER_IMPL: &str -pub const iceberg::spec::TableProperties::PROPERTY_WRITE_METADATA_LOCATION: &str pub const iceberg::spec::TableProperties::PROPERTY_WRITE_METADATA_PATH: &str -pub const iceberg::spec::TableProperties::PROPERTY_WRITE_OBJECT_STORE_PARTITIONED_PATHS: &str -pub const iceberg::spec::TableProperties::PROPERTY_WRITE_OBJECT_STORE_PARTITIONED_PATHS_DEFAULT: bool pub const iceberg::spec::TableProperties::PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT: &str pub const iceberg::spec::TableProperties::PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT_DEFAULT: u64 pub const iceberg::spec::TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES: &str @@ -3240,6 +3142,112 @@ pub type iceberg::spec::TableProperties::Error = iceberg::Error pub fn iceberg::spec::TableProperties::try_from(props: &std::collections::hash::map::HashMap) -> iceberg::Result impl core::fmt::Debug for iceberg::spec::TableProperties pub fn iceberg::spec::TableProperties::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg::spec::TableReadProperties +impl iceberg::spec::TableReadProperties +pub fn iceberg::spec::TableReadProperties::adaptive_split_size_enabled(&self) -> bool +pub fn iceberg::spec::TableReadProperties::data_planning_mode(&self) -> alloc::string::String +pub fn iceberg::spec::TableReadProperties::delete_planning_mode(&self) -> alloc::string::String +pub fn iceberg::spec::TableReadProperties::metadata_split_target_size(&self) -> u64 +pub fn iceberg::spec::TableReadProperties::orc_batch_size(&self) -> usize +pub fn iceberg::spec::TableReadProperties::orc_vectorization_enabled(&self) -> bool +pub fn iceberg::spec::TableReadProperties::parquet_batch_size(&self) -> usize +pub fn iceberg::spec::TableReadProperties::parquet_vectorization_enabled(&self) -> bool +pub fn iceberg::spec::TableReadProperties::split_open_file_cost(&self) -> u64 +pub fn iceberg::spec::TableReadProperties::split_planning_lookback(&self) -> usize +pub fn iceberg::spec::TableReadProperties::split_target_size(&self) -> u64 +pub fn iceberg::spec::TableReadProperties::with_adaptive_split_size_enabled(self, value: bool) -> Self +pub fn iceberg::spec::TableReadProperties::with_data_planning_mode(self, value: alloc::string::String) -> Self +pub fn iceberg::spec::TableReadProperties::with_delete_planning_mode(self, value: alloc::string::String) -> Self +pub fn iceberg::spec::TableReadProperties::with_metadata_split_target_size(self, value: u64) -> Self +pub fn iceberg::spec::TableReadProperties::with_orc_batch_size(self, value: usize) -> Self +pub fn iceberg::spec::TableReadProperties::with_orc_vectorization_enabled(self, value: bool) -> Self +pub fn iceberg::spec::TableReadProperties::with_parquet_batch_size(self, value: usize) -> Self +pub fn iceberg::spec::TableReadProperties::with_parquet_vectorization_enabled(self, value: bool) -> Self +pub fn iceberg::spec::TableReadProperties::with_split_open_file_cost(self, value: u64) -> Self +pub fn iceberg::spec::TableReadProperties::with_split_planning_lookback(self, value: usize) -> Self +pub fn iceberg::spec::TableReadProperties::with_split_target_size(self, value: u64) -> Self +impl core::clone::Clone for iceberg::spec::TableReadProperties +pub fn iceberg::spec::TableReadProperties::clone(&self) -> iceberg::spec::TableReadProperties +impl core::default::Default for iceberg::spec::TableReadProperties +pub fn iceberg::spec::TableReadProperties::default() -> Self +impl core::fmt::Debug for iceberg::spec::TableReadProperties +pub fn iceberg::spec::TableReadProperties::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl serde_core::ser::Serialize for iceberg::spec::TableReadProperties +pub fn iceberg::spec::TableReadProperties::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> where S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::TableReadProperties +pub fn iceberg::spec::TableReadProperties::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> +pub struct iceberg::spec::TableRowLevelProperties +impl iceberg::spec::TableRowLevelProperties +pub fn iceberg::spec::TableRowLevelProperties::delete_distribution_mode(&self) -> core::option::Option +pub fn iceberg::spec::TableRowLevelProperties::delete_granularity(&self) -> alloc::string::String +pub fn iceberg::spec::TableRowLevelProperties::delete_isolation_level(&self) -> alloc::string::String +pub fn iceberg::spec::TableRowLevelProperties::delete_mode(&self) -> alloc::string::String +pub fn iceberg::spec::TableRowLevelProperties::merge_distribution_mode(&self) -> core::option::Option +pub fn iceberg::spec::TableRowLevelProperties::merge_isolation_level(&self) -> alloc::string::String +pub fn iceberg::spec::TableRowLevelProperties::merge_mode(&self) -> alloc::string::String +pub fn iceberg::spec::TableRowLevelProperties::update_distribution_mode(&self) -> core::option::Option +pub fn iceberg::spec::TableRowLevelProperties::update_isolation_level(&self) -> alloc::string::String +pub fn iceberg::spec::TableRowLevelProperties::update_mode(&self) -> alloc::string::String +pub fn iceberg::spec::TableRowLevelProperties::upsert_enabled(&self) -> bool +pub fn iceberg::spec::TableRowLevelProperties::with_delete_distribution_mode(self, value: core::option::Option) -> Self +pub fn iceberg::spec::TableRowLevelProperties::with_delete_granularity(self, value: alloc::string::String) -> Self +pub fn iceberg::spec::TableRowLevelProperties::with_delete_isolation_level(self, value: alloc::string::String) -> Self +pub fn iceberg::spec::TableRowLevelProperties::with_delete_mode(self, value: alloc::string::String) -> Self +pub fn iceberg::spec::TableRowLevelProperties::with_merge_distribution_mode(self, value: core::option::Option) -> Self +pub fn iceberg::spec::TableRowLevelProperties::with_merge_isolation_level(self, value: alloc::string::String) -> Self +pub fn iceberg::spec::TableRowLevelProperties::with_merge_mode(self, value: alloc::string::String) -> Self +pub fn iceberg::spec::TableRowLevelProperties::with_update_distribution_mode(self, value: core::option::Option) -> Self +pub fn iceberg::spec::TableRowLevelProperties::with_update_isolation_level(self, value: alloc::string::String) -> Self +pub fn iceberg::spec::TableRowLevelProperties::with_update_mode(self, value: alloc::string::String) -> Self +pub fn iceberg::spec::TableRowLevelProperties::with_upsert_enabled(self, value: bool) -> Self +impl core::clone::Clone for iceberg::spec::TableRowLevelProperties +pub fn iceberg::spec::TableRowLevelProperties::clone(&self) -> iceberg::spec::TableRowLevelProperties +impl core::default::Default for iceberg::spec::TableRowLevelProperties +pub fn iceberg::spec::TableRowLevelProperties::default() -> Self +impl core::fmt::Debug for iceberg::spec::TableRowLevelProperties +pub fn iceberg::spec::TableRowLevelProperties::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl serde_core::ser::Serialize for iceberg::spec::TableRowLevelProperties +pub fn iceberg::spec::TableRowLevelProperties::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> where S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::TableRowLevelProperties +pub fn iceberg::spec::TableRowLevelProperties::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> +pub struct iceberg::spec::TableWriteProperties +impl iceberg::spec::TableWriteProperties +pub fn iceberg::spec::TableWriteProperties::audit_publish_enabled(&self) -> bool +pub fn iceberg::spec::TableWriteProperties::data_location(&self) -> core::option::Option +pub fn iceberg::spec::TableWriteProperties::datafusion_fanout_enabled(&self) -> bool +pub fn iceberg::spec::TableWriteProperties::delete_format_default(&self) -> iceberg::spec::DataFileFormat +pub fn iceberg::spec::TableWriteProperties::delete_target_file_size_bytes(&self) -> usize +pub fn iceberg::spec::TableWriteProperties::distribution_mode(&self) -> core::option::Option +pub fn iceberg::spec::TableWriteProperties::folder_storage_location(&self) -> core::option::Option +pub fn iceberg::spec::TableWriteProperties::format_default(&self) -> iceberg::spec::DataFileFormat +pub fn iceberg::spec::TableWriteProperties::location_provider_impl(&self) -> core::option::Option +pub fn iceberg::spec::TableWriteProperties::object_store_enabled(&self) -> bool +pub fn iceberg::spec::TableWriteProperties::object_store_partitioned_paths(&self) -> bool +pub fn iceberg::spec::TableWriteProperties::object_store_path(&self) -> core::option::Option +pub fn iceberg::spec::TableWriteProperties::target_file_size_bytes(&self) -> usize +pub fn iceberg::spec::TableWriteProperties::with_audit_publish_enabled(self, value: bool) -> Self +pub fn iceberg::spec::TableWriteProperties::with_data_location(self, value: core::option::Option) -> Self +pub fn iceberg::spec::TableWriteProperties::with_datafusion_fanout_enabled(self, value: bool) -> Self +pub fn iceberg::spec::TableWriteProperties::with_delete_format_default(self, value: iceberg::spec::DataFileFormat) -> Self +pub fn iceberg::spec::TableWriteProperties::with_delete_target_file_size_bytes(self, value: usize) -> Self +pub fn iceberg::spec::TableWriteProperties::with_distribution_mode(self, value: core::option::Option) -> Self +pub fn iceberg::spec::TableWriteProperties::with_folder_storage_location(self, value: core::option::Option) -> Self +pub fn iceberg::spec::TableWriteProperties::with_format_default(self, value: iceberg::spec::DataFileFormat) -> Self +pub fn iceberg::spec::TableWriteProperties::with_location_provider_impl(self, value: core::option::Option) -> Self +pub fn iceberg::spec::TableWriteProperties::with_object_store_enabled(self, value: bool) -> Self +pub fn iceberg::spec::TableWriteProperties::with_object_store_partitioned_paths(self, value: bool) -> Self +pub fn iceberg::spec::TableWriteProperties::with_object_store_path(self, value: core::option::Option) -> Self +pub fn iceberg::spec::TableWriteProperties::with_target_file_size_bytes(self, value: usize) -> Self +impl core::clone::Clone for iceberg::spec::TableWriteProperties +pub fn iceberg::spec::TableWriteProperties::clone(&self) -> iceberg::spec::TableWriteProperties +impl core::default::Default for iceberg::spec::TableWriteProperties +pub fn iceberg::spec::TableWriteProperties::default() -> Self +impl core::fmt::Debug for iceberg::spec::TableWriteProperties +pub fn iceberg::spec::TableWriteProperties::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl serde_core::ser::Serialize for iceberg::spec::TableWriteProperties +pub fn iceberg::spec::TableWriteProperties::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> where S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::TableWriteProperties +pub fn iceberg::spec::TableWriteProperties::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> pub struct iceberg::spec::UnboundPartitionField pub iceberg::spec::UnboundPartitionField::field_id: core::option::Option pub iceberg::spec::UnboundPartitionField::name: alloc::string::String diff --git a/crates/iceberg/src/catalog/utils.rs b/crates/iceberg/src/catalog/utils.rs index d214c4f0fe..8ed4c51678 100644 --- a/crates/iceberg/src/catalog/utils.rs +++ b/crates/iceberg/src/catalog/utils.rs @@ -60,7 +60,7 @@ pub async fn drop_table_data(table_info: &Table) -> Result<()> { } // Delete data files only if gc.enabled is true, to avoid corrupting shared tables - if metadata.parsed_table_properties()?.gc_enabled() { + if metadata.parsed_table_properties()?.history().gc_enabled() { delete_data_files(io, &manifests_to_delete).await?; } diff --git a/crates/iceberg/src/encryption/manager.rs b/crates/iceberg/src/encryption/manager.rs index 2393f7e65d..59ef661d95 100644 --- a/crates/iceberg/src/encryption/manager.rs +++ b/crates/iceberg/src/encryption/manager.rs @@ -119,7 +119,8 @@ impl EncryptionManager { } let table_properties = metadata.parsed_table_properties()?; - let Some(table_key_id) = table_properties.encryption_key_id() else { + let encryption = table_properties.encryption(); + let Some(table_key_id) = encryption.key_id() else { if kms_client.is_some() { tracing::warn!( "KeyManagementClient provided but table does not have encryption.key-id set" @@ -139,9 +140,7 @@ impl EncryptionManager { .kms_client(Arc::clone(kms_client)) .table_key_id(table_key_id) .encryption_keys(metadata.encryption_keys.clone()) - .key_size(AesKeySize::from_key_length( - table_properties.encryption_data_key_length(), - )?) + .key_size(AesKeySize::from_key_length(encryption.data_key_length())?) .build(); Ok(Some(Arc::new(em))) } diff --git a/crates/iceberg/src/spec/mod.rs b/crates/iceberg/src/spec/mod.rs index b23ca1eda0..dc9d0f8586 100644 --- a/crates/iceberg/src/spec/mod.rs +++ b/crates/iceberg/src/spec/mod.rs @@ -22,6 +22,7 @@ mod encrypted_key; mod manifest; mod manifest_list; mod name_mapping; +mod parsed_table_prop; mod partition; mod schema; mod snapshot; @@ -42,6 +43,7 @@ pub use encrypted_key::*; pub use manifest::*; pub use manifest_list::*; pub use name_mapping::*; +pub use parsed_table_prop::*; pub use partition::*; pub use schema::*; pub use snapshot::*; diff --git a/crates/iceberg/src/spec/parsed_table_prop.rs b/crates/iceberg/src/spec/parsed_table_prop.rs new file mode 100644 index 0000000000..1e482b15ab --- /dev/null +++ b/crates/iceberg/src/spec/parsed_table_prop.rs @@ -0,0 +1,848 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::HashMap; + +use iceberg_property_macro::Properties; + +use crate::compression::CompressionCodec; +use crate::error::{Error, ErrorKind, Result}; +use crate::spec::DataFileFormat; + +/// Strips trailing slashes from a location, preserving a bare URI scheme root. +fn strip_trailing_slash(path: &str) -> &str { + let mut path = path; + while !path.ends_with("://") { + let Some(stripped) = path.strip_suffix('/') else { + break; + }; + path = stripped; + } + path +} + +fn parse_metadata_location(value: &str) -> Result> { + if value.is_empty() { + return Err(Error::new(ErrorKind::DataInvalid, "path must not be empty")); + } + + Ok(Some(strip_trailing_slash(value).to_string())) +} + +fn parse_metadata_file_compression(value: &str) -> Result { + if value.is_empty() { + return Ok(CompressionCodec::None); + } + + let codec: CompressionCodec = serde_json::from_value(serde_json::Value::String( + value.to_lowercase(), + )) + .map_err(|_| { + Error::new( + ErrorKind::DataInvalid, + format!( + "Invalid metadata compression codec: {value}. Only '{}' and '{}' are supported.", + CompressionCodec::None.name(), + CompressionCodec::gzip_default().name() + ), + ) + })?; + + match codec { + CompressionCodec::None | CompressionCodec::Gzip(_) => Ok(codec), + _ => Err(Error::new( + ErrorKind::DataInvalid, + format!( + "Invalid metadata compression codec: {value}. Only '{}' and '{}' are supported for metadata files.", + CompressionCodec::None.name(), + CompressionCodec::gzip_default().name() + ), + )), + } +} + +fn serialize_compression_codec(codec: &CompressionCodec) -> String { + codec.name().to_string() +} + +/// Typed Iceberg table properties organized by behavior and file format. +/// +/// Serde represents this hierarchy as Iceberg's flat string-to-string property map. Property +/// definitions and descriptions are based on the pinned [Java TableProperties implementation] +/// and [Apache Iceberg configuration documentation]. +/// +/// [Java TableProperties implementation]: https://github.com/apache/iceberg/blob/d8c10a1608170f0ba83be740d6ab0b6a3757cb3e/core/src/main/java/org/apache/iceberg/TableProperties.java +/// [Apache Iceberg configuration documentation]: https://github.com/apache/iceberg/blob/d8c10a1608170f0ba83be740d6ab0b6a3757cb3e/docs/docs/configuration.md +#[derive(Clone, Debug, Properties)] +pub struct ParsedTableProperties { + #[nested] + #[doc = "Informational properties and identifier-field semantics."] + general: TableGeneralProperties, + #[nested] + #[doc = "Commit retry and commit status-check behavior."] + commit: TableCommitProperties, + #[nested] + #[doc = "Manifest creation, merging, and compression behavior."] + manifest: TableManifestProperties, + #[nested] + #[doc = "General table write behavior and output locations."] + write: TableWriteProperties, + #[nested] + #[doc = "Parquet data and delete file behavior."] + parquet: TableParquetProperties, + #[nested] + #[doc = "Avro data and delete file behavior."] + avro: TableAvroProperties, + #[nested] + #[doc = "ORC data and delete file behavior."] + orc: TableOrcProperties, + #[nested] + #[doc = "Read split planning and vectorization behavior."] + read: TableReadProperties, + #[nested] + #[doc = "Table metadata retention, compression, and metrics behavior."] + metadata: TableMetadataProperties, + #[nested] + #[doc = "Engine-specific and compatibility behavior."] + compatibility: TableCompatibilityProperties, + #[nested] + #[doc = "Garbage collection and snapshot retention behavior."] + history: TableHistoryProperties, + #[nested] + #[doc = "Delete, update, merge, and upsert behavior."] + row_level: TableRowLevelProperties, + #[nested] + #[doc = "Table encryption behavior."] + encryption: TableEncryptionProperties, +} + +/// Informational properties and identifier-field semantics. +#[derive(Clone, Debug, Properties)] +pub struct TableGeneralProperties { + #[key = "comment"] + #[default(None)] + #[doc = "Table-level description of the table's business meaning and usage context."] + comment: Option, + #[key = "identifier-fields.rely"] + #[default(false)] + #[doc = "Whether query engines may rely on identifier fields as a primary key for optimization; this is not enforced on writes."] + identifier_fields_rely: bool, +} + +/// Commit retries and post-failure status checks. +#[derive(Clone, Debug, Properties)] +pub struct TableCommitProperties { + #[key = "commit.retry.num-retries"] + #[default(4)] + #[doc = "Number of times to retry a commit before failing."] + num_retries: usize, + #[key = "commit.retry.min-wait-ms"] + #[default(100)] + #[doc = "Minimum time in milliseconds to wait before retrying a commit."] + min_retry_wait_ms: u64, + #[key = "commit.retry.max-wait-ms"] + #[default(60 * 1000)] + #[doc = "Maximum time in milliseconds to wait before retrying a commit."] + max_retry_wait_ms: u64, + #[key = "commit.retry.total-timeout-ms"] + #[default(30 * 60 * 1000)] + #[doc = "Total commit retry timeout in milliseconds."] + total_retry_timeout_ms: u64, + #[key = "commit.status-check.num-retries"] + #[default(3)] + #[doc = "Number of times to check whether a commit succeeded after connectivity is lost."] + num_status_checks: usize, + #[key = "commit.status-check.min-wait-ms"] + #[default(1000)] + #[doc = "Minimum time in milliseconds to wait before retrying a commit status check."] + status_checks_min_wait_ms: u64, + #[key = "commit.status-check.max-wait-ms"] + #[default(60 * 1000)] + #[doc = "Maximum time in milliseconds to wait before retrying a commit status check."] + status_checks_max_wait_ms: u64, + #[key = "commit.status-check.total-timeout-ms"] + #[default(30 * 60 * 1000)] + #[doc = "Total timeout in milliseconds in which commit status checking must succeed."] + status_checks_total_wait_ms: u64, +} + +/// Manifest creation, merging, and compression properties. +#[derive(Clone, Debug, Properties)] +pub struct TableManifestProperties { + #[key = "commit.manifest.target-size-bytes"] + #[default(8 * 1024 * 1024)] + #[doc = "Target size in bytes when merging manifest files."] + target_size_bytes: usize, + #[key = "commit.manifest.min-count-to-merge"] + #[default(100)] + #[doc = "Minimum number of manifests to accumulate before merging."] + min_merge_count: usize, + #[key = "commit.manifest-merge.enabled"] + #[default(true)] + #[doc = "Whether manifests are automatically merged during writes."] + merge_enabled: bool, + #[key = "write.manifest.compression-codec"] + #[default("gzip".to_string())] + #[doc = "Compression codec used for manifest files."] + compression: String, + #[key = "write.manifest.compression-level"] + #[default(None)] + #[doc = "Optional compression level used for manifest files."] + compression_level: Option, + #[key = "write.manifest-lists.enabled"] + #[default(true)] + #[doc = "Deprecated flag for writing manifest lists; manifest lists are always enabled."] + lists_enabled: bool, +} + +/// General write properties and output locations. +#[derive(Clone, Debug, Properties)] +pub struct TableWriteProperties { + #[key = "write.format.default"] + #[default(DataFileFormat::Parquet)] + #[doc = "Default data file format: Parquet, Avro, or ORC."] + format_default: DataFileFormat, + #[key = "write.delete.format.default"] + #[default(DataFileFormat::Parquet)] + #[doc = "Default delete file format: Parquet, Avro, or ORC."] + delete_format_default: DataFileFormat, + #[key = "write.target-file-size-bytes"] + #[default(512 * 1024 * 1024)] + #[doc = "Target size in bytes for generated data files."] + target_file_size_bytes: usize, + #[key = "write.delete.target-file-size-bytes"] + #[default(64 * 1024 * 1024)] + #[doc = "Target size in bytes for generated delete files."] + delete_target_file_size_bytes: usize, + #[key = "write.object-storage.enabled"] + #[default(false)] + #[doc = "Whether the object-storage location provider adds a hash component to file paths."] + object_store_enabled: bool, + #[key = "write.object-storage.partitioned-paths"] + #[default(true)] + #[doc = "Whether object-storage file paths include partition values."] + object_store_partitioned_paths: bool, + #[key = "write.object-storage.path"] + #[default(None)] + #[doc = "Deprecated base object-storage path; use write.data.path instead."] + object_store_path: Option, + #[key = "write.location-provider.impl"] + #[default(None)] + #[doc = "Optional custom location provider implementation."] + location_provider_impl: Option, + #[key = "write.folder-storage.path"] + #[default(None)] + #[doc = "Deprecated base folder-storage path; use write.data.path instead."] + folder_storage_location: Option, + #[key = "write.data.path"] + #[default(None)] + #[doc = "Base location for data files written after this property is set."] + data_location: Option, + #[key = "write.wap.enabled"] + #[default(false)] + #[doc = "Whether write-audit-publish writes are enabled."] + audit_publish_enabled: bool, + #[key = "write.distribution-mode"] + #[default(None)] + #[doc = "Optional write distribution mode: none, hash, or range."] + distribution_mode: Option, + #[key = "write.datafusion.fanout.enabled"] + #[default(true)] + #[doc = "Whether DataFusion uses a fanout writer for partitioned tables."] + datafusion_fanout_enabled: bool, +} + +/// Parquet data and delete file properties. +#[derive(Clone, Debug, Properties)] +pub struct TableParquetProperties { + #[key = "write.parquet.row-group-size-bytes"] + #[default(128 * 1024 * 1024)] + #[doc = "Parquet row group size in bytes for data files."] + row_group_size_bytes: usize, + #[key = "write.delete.parquet.row-group-size-bytes"] + #[default(128 * 1024 * 1024)] + #[doc = "Parquet row group size in bytes for delete files."] + delete_row_group_size_bytes: usize, + #[key = "write.parquet.page-size-bytes"] + #[default(1024 * 1024)] + #[doc = "Parquet page size in bytes for data files."] + page_size_bytes: usize, + #[key = "write.delete.parquet.page-size-bytes"] + #[default(1024 * 1024)] + #[doc = "Parquet page size in bytes for delete files."] + delete_page_size_bytes: usize, + #[key = "write.parquet.page-version"] + #[default("v1".to_string())] + #[doc = "Parquet data page version for data files: v1 or v2."] + page_version: String, + #[key = "write.delete.parquet.page-version"] + #[default("v1".to_string())] + #[doc = "Parquet data page version for delete files: v1 or v2."] + delete_page_version: String, + #[key = "write.parquet.page-row-limit"] + #[default(20_000)] + #[doc = "Maximum number of rows per Parquet page in data files."] + page_row_limit: usize, + #[key = "write.delete.parquet.page-row-limit"] + #[default(20_000)] + #[doc = "Maximum number of rows per Parquet page in delete files."] + delete_page_row_limit: usize, + #[key = "write.parquet.dict-size-bytes"] + #[default(2 * 1024 * 1024)] + #[doc = "Parquet dictionary page size in bytes for data files."] + dict_size_bytes: usize, + #[key = "write.delete.parquet.dict-size-bytes"] + #[default(2 * 1024 * 1024)] + #[doc = "Parquet dictionary page size in bytes for delete files."] + delete_dict_size_bytes: usize, + #[key = "write.parquet.compression-codec"] + #[default("zstd".to_string())] + #[doc = "Parquet compression codec used for data files."] + compression: String, + #[key = "write.delete.parquet.compression-codec"] + #[default("zstd".to_string())] + #[doc = "Parquet compression codec used for delete files."] + delete_compression: String, + #[key = "write.parquet.compression-level"] + #[default(None)] + #[doc = "Optional Parquet compression level for data files."] + compression_level: Option, + #[key = "write.delete.parquet.compression-level"] + #[default(None)] + #[doc = "Optional Parquet compression level for delete files."] + delete_compression_level: Option, + #[key = "write.parquet.shred-variants"] + #[default(false)] + #[doc = "Whether variant columns use shredded Parquet encoding for improved query performance."] + shred_variants: bool, + #[key = "write.parquet.variant-inference-buffer-size"] + #[default(100)] + #[doc = "Number of rows buffered for schema inference when variant shredding is enabled."] + variant_inference_buffer_size: usize, + #[key = "write.parquet.row-group-check-min-record-count"] + #[default(100)] + #[doc = "Minimum record count between Parquet data-file row group size checks."] + row_group_check_min_record_count: usize, + #[key = "write.delete.parquet.row-group-check-min-record-count"] + #[default(100)] + #[doc = "Minimum record count between Parquet delete-file row group size checks."] + delete_row_group_check_min_record_count: usize, + #[key = "write.parquet.row-group-check-max-record-count"] + #[default(10_000)] + #[doc = "Maximum record count between Parquet data-file row group size checks."] + row_group_check_max_record_count: usize, + #[key = "write.delete.parquet.row-group-check-max-record-count"] + #[default(10_000)] + #[doc = "Maximum record count between Parquet delete-file row group size checks."] + delete_row_group_check_max_record_count: usize, + #[key = "write.parquet.row-group-size-track-uncompressed"] + #[default(false)] + #[doc = "Whether uncompressed data size is tracked to enforce the Parquet row group target."] + row_group_size_track_uncompressed: bool, + #[key = "write.parquet.bloom-filter-max-bytes"] + #[default(1024 * 1024)] + #[doc = "Maximum number of bytes for a Parquet bloom filter bitset."] + bloom_filter_max_bytes: usize, + #[key = "write.parquet.bloom-filter-adaptive-enabled"] + #[default(false)] + #[doc = "Whether adaptive Parquet bloom filter sizing selects the smallest suitable filter."] + bloom_filter_adaptive_enabled: bool, + #[prefix = "write.parquet.bloom-filter-fpp.column."] + #[default(HashMap::new())] + #[doc = "Per-column Parquet bloom filter false-positive probabilities, keyed by column name."] + bloom_filter_column_fpp: HashMap, + #[prefix = "write.parquet.bloom-filter-ndv.column."] + #[default(HashMap::new())] + #[doc = "Per-column expected distinct-value counts for Parquet bloom filters."] + bloom_filter_column_ndv: HashMap, + #[prefix = "write.parquet.bloom-filter-enabled.column."] + #[default(HashMap::new())] + #[doc = "Per-column flags controlling whether Parquet bloom filters are written."] + bloom_filter_column_enabled: HashMap, + #[prefix = "write.parquet.stats-enabled.column."] + #[default(HashMap::new())] + #[doc = "Per-column flags controlling whether Parquet column statistics are collected."] + column_stats_enabled: HashMap, + #[prefix = "write.parquet.dict-encoding-enabled.column."] + #[default(HashMap::new())] + #[doc = "Per-column flags controlling whether Parquet dictionary encoding is used."] + dict_encoding_enabled_by_column: HashMap, + #[key = "write.parquet.content-defined-chunking.enabled"] + #[default(false)] + #[doc = "Whether Parquet content-defined chunking is enabled."] + content_defined_chunking_enabled: bool, + #[key = "write.parquet.content-defined-chunking.min-chunk-size"] + #[default(256 * 1024)] + #[doc = "Minimum Parquet content-defined chunk size in bytes."] + content_defined_chunking_min_chunk_size: usize, + #[key = "write.parquet.content-defined-chunking.max-chunk-size"] + #[default(1024 * 1024)] + #[doc = "Maximum Parquet content-defined chunk size in bytes."] + content_defined_chunking_max_chunk_size: usize, + #[key = "write.parquet.content-defined-chunking.norm-level"] + #[default(0)] + #[doc = "Gearhash normalization level used by Parquet content-defined chunking."] + content_defined_chunking_norm_level: i32, +} + +/// Avro data and delete file properties. +#[derive(Clone, Debug, Properties)] +pub struct TableAvroProperties { + #[key = "write.avro.compression-codec"] + #[default("gzip".to_string())] + #[doc = "Avro compression codec used for data files."] + compression: String, + #[key = "write.delete.avro.compression-codec"] + #[default("gzip".to_string())] + #[doc = "Avro compression codec used for delete files."] + delete_compression: String, + #[key = "write.avro.compression-level"] + #[default(None)] + #[doc = "Optional Avro compression level for data files."] + compression_level: Option, + #[key = "write.delete.avro.compression-level"] + #[default(None)] + #[doc = "Optional Avro compression level for delete files."] + delete_compression_level: Option, +} + +/// ORC data and delete file properties. +#[derive(Clone, Debug, Properties)] +pub struct TableOrcProperties { + #[key = "write.orc.stripe-size-bytes"] + #[default(64 * 1024 * 1024)] + #[doc = "Default ORC stripe size in bytes for data files."] + stripe_size_bytes: u64, + #[key = "write.delete.orc.stripe-size-bytes"] + #[default(64 * 1024 * 1024)] + #[doc = "Default ORC stripe size in bytes for delete files."] + delete_stripe_size_bytes: u64, + #[key = "write.orc.bloom.filter.columns"] + #[default(String::new())] + #[doc = "Comma-separated column names for which ORC bloom filters are created."] + bloom_filter_columns: String, + #[key = "write.orc.bloom.filter.fpp"] + #[default(0.05)] + #[doc = "False-positive probability for ORC bloom filters."] + bloom_filter_fpp: f64, + #[key = "write.orc.block-size-bytes"] + #[default(256 * 1024 * 1024)] + #[doc = "Default file-system block size in bytes for ORC data files."] + block_size_bytes: u64, + #[key = "write.delete.orc.block-size-bytes"] + #[default(256 * 1024 * 1024)] + #[doc = "Default file-system block size in bytes for ORC delete files."] + delete_block_size_bytes: u64, + #[key = "write.orc.vectorized.batch-size"] + #[default(1024)] + #[doc = "ORC vectorized write batch size for data files."] + write_batch_size: usize, + #[key = "write.delete.orc.vectorized.batch-size"] + #[default(1024)] + #[doc = "ORC vectorized write batch size for delete files."] + delete_write_batch_size: usize, + #[key = "write.orc.compression-codec"] + #[default("zlib".to_string())] + #[doc = "ORC compression codec used for data files."] + compression: String, + #[key = "write.delete.orc.compression-codec"] + #[default("zlib".to_string())] + #[doc = "ORC compression codec used for delete files."] + delete_compression: String, + #[key = "write.orc.compression-strategy"] + #[default("speed".to_string())] + #[doc = "ORC compression strategy for data files: speed or compression."] + compression_strategy: String, + #[key = "write.delete.orc.compression-strategy"] + #[default("speed".to_string())] + #[doc = "ORC compression strategy for delete files: speed or compression."] + delete_compression_strategy: String, +} + +/// Read split planning and vectorization properties. +#[derive(Clone, Debug, Properties)] +pub struct TableReadProperties { + #[key = "read.split.target-size"] + #[default(128 * 1024 * 1024)] + #[doc = "Target size in bytes when combining data input splits."] + split_target_size: u64, + #[key = "read.split.metadata-target-size"] + #[default(32 * 1024 * 1024)] + #[doc = "Target size in bytes when combining metadata input splits."] + metadata_split_target_size: u64, + #[key = "read.split.planning-lookback"] + #[default(10)] + #[doc = "Number of bins considered when combining input splits."] + split_planning_lookback: usize, + #[key = "read.split.open-file-cost"] + #[default(4 * 1024 * 1024)] + #[doc = "Estimated file-open cost used as a minimum weight when combining splits."] + split_open_file_cost: u64, + #[key = "read.split.adaptive-size.enabled"] + #[default(true)] + #[doc = "Whether split size is adapted to the scan size."] + adaptive_split_size_enabled: bool, + #[key = "read.parquet.vectorization.enabled"] + #[default(true)] + #[doc = "Whether Parquet vectorized reads are enabled."] + parquet_vectorization_enabled: bool, + #[key = "read.parquet.vectorization.batch-size"] + #[default(5000)] + #[doc = "Batch size for Parquet vectorized reads."] + parquet_batch_size: usize, + #[key = "read.orc.vectorization.enabled"] + #[default(false)] + #[doc = "Whether ORC vectorized reads are enabled."] + orc_vectorization_enabled: bool, + #[key = "read.orc.vectorization.batch-size"] + #[default(5000)] + #[doc = "Batch size for ORC vectorized reads."] + orc_batch_size: usize, + #[key = "read.data-planning-mode"] + #[default("auto".to_string())] + #[doc = "Planning mode used for data files."] + data_planning_mode: String, + #[key = "read.delete-planning-mode"] + #[default("auto".to_string())] + #[doc = "Planning mode used for delete files."] + delete_planning_mode: String, +} + +/// Table metadata retention, compression, and metrics properties. +#[derive(Clone, Debug, Properties)] +pub struct TableMetadataProperties { + #[key = "write.metadata.path"] + #[default(None)] + #[parse_with(parse_metadata_location)] + #[doc = "Base location for metadata files written after this property is set, with trailing slashes removed."] + path: Option, + #[key = "write.summary.partition-limit"] + #[default(0)] + #[doc = "Maximum changed-partition count for including partition-level statistics in snapshot summaries."] + partition_summary_limit: u64, + #[key = "write.metadata.compression-codec"] + #[default(CompressionCodec::None)] + #[parse_with(parse_metadata_file_compression)] + #[serialize_with(serialize_compression_codec)] + #[doc = "Compression codec for metadata JSON files: none or gzip."] + compression_codec: CompressionCodec, + #[key = "write.metadata.previous-versions-max"] + #[default(100)] + #[doc = "Maximum number of previous metadata file versions to track."] + previous_versions_max: usize, + #[key = "write.metadata.delete-after-commit.enabled"] + #[default(false)] + #[doc = "Whether the oldest tracked metadata file is deleted after each commit."] + delete_after_commit_enabled: bool, + #[key = "write.metadata.metrics.max-inferred-column-defaults"] + #[default(100)] + #[doc = "Maximum number of columns that receive inferred metrics defaults."] + metrics_max_inferred_column_defaults: usize, + #[prefix = "write.metadata.metrics.column."] + #[default(HashMap::new())] + #[doc = "Per-column metrics modes keyed by column name."] + metrics_mode_by_column: HashMap, + #[key = "write.metadata.metrics.default"] + #[default("truncate(16)".to_string())] + #[doc = "Default metrics mode for table columns."] + default_metrics_mode: String, + #[key = "schema.name-mapping.default"] + #[default(None)] + #[doc = "Default JSON name mapping used to resolve columns in files without field IDs."] + default_name_mapping: Option, +} + +/// Engine-specific and compatibility properties. +#[derive(Clone, Debug, Properties)] +pub struct TableCompatibilityProperties { + #[key = "write.spark.fanout.enabled"] + #[default(false)] + #[doc = "Deprecated Spark fanout-writer flag; the fanout writer accepts unclustered data but uses more memory."] + spark_write_partitioned_fanout_enabled: bool, + #[key = "write.spark.accept-any-schema"] + #[default(false)] + #[doc = "Deprecated Spark flag allowing writes with any compatible schema."] + spark_write_accept_any_schema: bool, + #[key = "write.spark.auto-schema-evolution.enabled"] + #[default(true)] + #[doc = "Deprecated Spark flag enabling automatic schema evolution during writes."] + spark_write_auto_schema_evolution: bool, + #[key = "write.spark.advisory-partition-size-bytes"] + #[default(None)] + #[doc = "Deprecated Spark advisory partition size in bytes."] + spark_write_advisory_partition_size_bytes: Option, + #[key = "compatibility.snapshot-id-inheritance.enabled"] + #[default(false)] + #[doc = "Whether snapshots may be committed without explicit snapshot IDs; format version 2 and later always allow this."] + snapshot_id_inheritance_enabled: bool, + #[key = "engine.hive.enabled"] + #[default(false)] + #[doc = "Whether Hive engine integration behavior is enabled."] + engine_hive_enabled: bool, + #[key = "engine.hive.lock-enabled"] + #[default(true)] + #[doc = "Whether Hive locking is enabled."] + hive_lock_enabled: bool, +} + +/// Garbage collection and snapshot retention properties. +#[derive(Clone, Debug, Properties)] +pub struct TableHistoryProperties { + #[key = "gc.enabled"] + #[default(true)] + #[doc = "Whether garbage collection operations such as snapshot expiration and orphan-file removal are allowed."] + gc_enabled: bool, + #[key = "history.expire.max-snapshot-age-ms"] + #[default(5 * 24 * 60 * 60 * 1000)] + #[doc = "Default maximum snapshot age in milliseconds while expiring snapshots."] + max_snapshot_age_ms: i64, + #[key = "history.expire.min-snapshots-to-keep"] + #[default(1)] + #[doc = "Default minimum number of snapshots retained per branch while expiring snapshots."] + min_snapshots_to_keep: usize, + #[key = "history.expire.max-ref-age-ms"] + #[default(i64::MAX)] + #[doc = "Default maximum age in milliseconds for snapshot references other than the main branch."] + max_ref_age_ms: i64, +} + +/// Delete, update, merge, and upsert properties. +#[derive(Clone, Debug, Properties)] +pub struct TableRowLevelProperties { + #[key = "write.delete.granularity"] + #[default("partition".to_string())] + #[doc = "Granularity of generated delete files: partition or file."] + delete_granularity: String, + #[key = "write.delete.isolation-level"] + #[default("serializable".to_string())] + #[doc = "Isolation level for delete commands: serializable or snapshot."] + delete_isolation_level: String, + #[key = "write.delete.mode"] + #[default("copy-on-write".to_string())] + #[doc = "Execution mode for delete commands: copy-on-write or merge-on-read."] + delete_mode: String, + #[key = "write.delete.distribution-mode"] + #[default(None)] + #[doc = "Optional distribution mode for delete command data."] + delete_distribution_mode: Option, + #[key = "write.update.isolation-level"] + #[default("serializable".to_string())] + #[doc = "Isolation level for update commands: serializable or snapshot."] + update_isolation_level: String, + #[key = "write.update.mode"] + #[default("copy-on-write".to_string())] + #[doc = "Execution mode for update commands: copy-on-write or merge-on-read."] + update_mode: String, + #[key = "write.update.distribution-mode"] + #[default(None)] + #[doc = "Optional distribution mode for update command data."] + update_distribution_mode: Option, + #[key = "write.merge.isolation-level"] + #[default("serializable".to_string())] + #[doc = "Isolation level for merge commands: serializable or snapshot."] + merge_isolation_level: String, + #[key = "write.merge.mode"] + #[default("copy-on-write".to_string())] + #[doc = "Execution mode for merge commands: copy-on-write or merge-on-read."] + merge_mode: String, + #[key = "write.merge.distribution-mode"] + #[default(None)] + #[doc = "Optional distribution mode for merge command data."] + merge_distribution_mode: Option, + #[key = "write.upsert.enabled"] + #[default(false)] + #[doc = "Whether upsert behavior is enabled."] + upsert_enabled: bool, +} + +/// Table encryption properties. +#[derive(Clone, Debug, Properties)] +pub struct TableEncryptionProperties { + #[key = "encryption.key-id"] + #[default(None)] + #[doc = "Identifier of the table's master encryption key."] + key_id: Option, + #[key = "encryption.data-key-length"] + #[default(16)] + #[doc = "Length in bytes of data-encryption keys; valid AES lengths are 16, 24, and 32 bytes."] + data_key_length: usize, +} + +#[cfg(test)] +mod tests { + use super::*; + + fn parse(properties: HashMap) -> Result { + serde_json::from_value(serde_json::to_value(properties).unwrap()) + .map_err(|error| Error::new(ErrorKind::DataInvalid, error.to_string())) + } + + #[test] + fn defaults_are_organized_by_property_group() { + let properties = ParsedTableProperties::default(); + + assert_eq!(properties.commit().num_retries(), 4); + assert_eq!(properties.write().format_default(), DataFileFormat::Parquet); + assert_eq!( + properties.parquet().row_group_size_bytes(), + 128 * 1024 * 1024 + ); + assert_eq!(properties.read().split_target_size(), 128 * 1024 * 1024); + assert!(properties.history().gc_enabled()); + assert_eq!(properties.encryption().data_key_length(), 16); + } + + #[test] + fn nested_properties_serialize_to_a_flat_json_map() { + let properties = ParsedTableProperties::default() + .with_commit(TableCommitProperties::default().with_num_retries(9)) + .with_write( + TableWriteProperties::default() + .with_format_default(DataFileFormat::Orc) + .with_data_location(Some("s3://warehouse/table/data".to_string())), + ) + .with_parquet( + TableParquetProperties::default().with_bloom_filter_column_fpp(HashMap::from([( + "customer_id".to_string(), + 0.02, + )])), + ); + + let json = serde_json::to_value(&properties).unwrap(); + assert_eq!(json["commit.retry.num-retries"], "9"); + assert_eq!(json["write.format.default"], "orc"); + assert_eq!(json["write.data.path"], "s3://warehouse/table/data"); + assert_eq!( + json["write.parquet.bloom-filter-fpp.column.customer_id"], + "0.02" + ); + assert!(json.get("commit").is_none()); + assert!(json.get("write").is_none()); + + let decoded: ParsedTableProperties = serde_json::from_value(json).unwrap(); + assert_eq!(decoded.commit().num_retries(), 9); + assert_eq!(decoded.write().format_default(), DataFileFormat::Orc); + assert_eq!( + decoded.parquet().bloom_filter_column_fpp()["customer_id"], + 0.02 + ); + } + + #[test] + fn every_default_round_trips() { + let defaults = ParsedTableProperties::default(); + let decoded: ParsedTableProperties = + serde_json::from_value(serde_json::to_value(&defaults).unwrap()).unwrap(); + + assert_eq!( + decoded.commit().num_retries(), + defaults.commit().num_retries() + ); + assert_eq!( + decoded.metadata().compression_codec(), + defaults.metadata().compression_codec() + ); + assert_eq!( + decoded.parquet().content_defined_chunking_max_chunk_size(), + defaults.parquet().content_defined_chunking_max_chunk_size() + ); + assert_eq!( + decoded.row_level().merge_mode(), + defaults.row_level().merge_mode() + ); + } + + #[test] + fn parses_values_across_groups() { + let properties = parse(HashMap::from([ + ("comment".to_string(), "orders table".to_string()), + ( + "commit.status-check.num-retries".to_string(), + "7".to_string(), + ), + ( + "write.delete.avro.compression-codec".to_string(), + "snappy".to_string(), + ), + ("read.split.planning-lookback".to_string(), "25".to_string()), + ( + "history.expire.min-snapshots-to-keep".to_string(), + "4".to_string(), + ), + ("write.delete.mode".to_string(), "merge-on-read".to_string()), + ("encryption.data-key-length".to_string(), "32".to_string()), + ])) + .unwrap(); + + assert_eq!( + properties.general().comment(), + Some("orders table".to_string()) + ); + assert_eq!(properties.commit().num_status_checks(), 7); + assert_eq!(properties.avro().delete_compression(), "snappy"); + assert_eq!(properties.read().split_planning_lookback(), 25); + assert_eq!(properties.history().min_snapshots_to_keep(), 4); + assert_eq!(properties.row_level().delete_mode(), "merge-on-read"); + assert_eq!(properties.encryption().data_key_length(), 32); + } + + #[test] + fn metadata_values_are_normalized_and_validated() { + let properties = parse(HashMap::from([ + ( + "write.metadata.path".to_string(), + "s3://warehouse/table/metadata/".to_string(), + ), + ( + "write.metadata.compression-codec".to_string(), + "GZIP".to_string(), + ), + ])) + .unwrap(); + + assert_eq!( + properties.metadata().path(), + Some("s3://warehouse/table/metadata".to_string()) + ); + assert_eq!( + properties.metadata().compression_codec(), + CompressionCodec::gzip_default() + ); + + let error = parse(HashMap::from([( + "write.metadata.compression-codec".to_string(), + "zstd".to_string(), + )])) + .unwrap_err(); + assert!( + error + .message() + .contains("Invalid metadata compression codec") + ); + } + + #[test] + fn invalid_leaf_value_reports_its_property_key() { + let error = parse(HashMap::from([( + "commit.retry.num-retries".to_string(), + "not-a-number".to_string(), + )])) + .unwrap_err(); + + assert!(error.message().contains("commit.retry.num-retries")); + } +} diff --git a/crates/iceberg/src/spec/table_metadata.rs b/crates/iceberg/src/spec/table_metadata.rs index 9e019ea6ad..97da227616 100644 --- a/crates/iceberg/src/spec/table_metadata.rs +++ b/crates/iceberg/src/spec/table_metadata.rs @@ -4052,11 +4052,11 @@ mod tests { let props = metadata.parsed_table_properties().unwrap(); assert_eq!( - props.commit_num_retries(), + props.commit().num_retries(), TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT ); assert_eq!( - props.write_target_file_size_bytes(), + props.write().target_file_size_bytes(), TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT ); @@ -4104,8 +4104,8 @@ mod tests { let props = metadata.parsed_table_properties().unwrap(); - assert_eq!(props.commit_num_retries(), 10); - assert_eq!(props.write_target_file_size_bytes(), 1024); + assert_eq!(props.commit().num_retries(), 10); + assert_eq!(props.write().target_file_size_bytes(), 1024); } #[test] diff --git a/crates/iceberg/src/spec/table_properties.rs b/crates/iceberg/src/spec/table_properties.rs index 4409a4c5de..379feee5c1 100644 --- a/crates/iceberg/src/spec/table_properties.rs +++ b/crates/iceberg/src/spec/table_properties.rs @@ -19,11 +19,8 @@ use std::collections::HashMap; use std::fmt::Display; use std::str::FromStr; -use iceberg_property_macro::Properties; - use crate::compression::CompressionCodec; use crate::error::{Error, ErrorKind, Result}; -use crate::spec::DataFileFormat; fn parse_property( properties: &HashMap, @@ -55,14 +52,6 @@ fn strip_trailing_slash(path: &str) -> &str { path } -fn parse_metadata_location(value: &str) -> Result> { - if value.is_empty() { - return Err(Error::new(ErrorKind::DataInvalid, "path must not be empty")); - } - - Ok(Some(strip_trailing_slash(value).to_string())) -} - fn parse_location_property( properties: &HashMap, key: &str, @@ -82,7 +71,26 @@ fn parse_location_property( .transpose() } -fn parse_metadata_file_compression_value(value: &str) -> Result { +/// Parse compression codec for metadata files from table properties. +/// Retrieves the compression codec property, applies defaults, and parses the value. +/// Only "none" (or empty string) and "gzip" are supported for metadata compression. +/// +/// # Arguments +/// +/// * `properties` - HashMap containing table properties +/// +/// # Errors +/// +/// Returns an error if the codec is not "none", "", or "gzip" (case-insensitive). +/// Lz4 and Zstd are not supported for metadata file compression. +pub(crate) fn parse_metadata_file_compression( + properties: &HashMap, +) -> Result { + let value = properties + .get(TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC) + .map(|s| s.as_str()) + .unwrap_or(TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC_DEFAULT); + // Handle empty string as None if value.is_empty() { return Ok(CompressionCodec::None); @@ -120,422 +128,6 @@ fn parse_metadata_file_compression_value(value: &str) -> Result, -) -> Result { - parse_metadata_file_compression_value( - properties - .get(TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC) - .map(String::as_str) - .unwrap_or(TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC_DEFAULT), - ) -} - -fn serialize_compression_codec(codec: &CompressionCodec) -> String { - codec.name().to_string() -} - -/// Typed table properties parsed from a table's string property map. -/// -/// This includes the properties defined by the pinned [Java `TableProperties`] reference as well -/// as Iceberg Rust-specific properties. -/// -/// [Java `TableProperties`]: https://github.com/apache/iceberg/blob/d8c10a1608170f0ba83be740d6ab0b6a3757cb3e/core/src/main/java/org/apache/iceberg/TableProperties.java -#[derive(Debug, Properties)] -pub struct ParsedTableProperties { - /// The number of times to retry a commit. - #[key(TableProperties::PROPERTY_COMMIT_NUM_RETRIES)] - #[default(TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT)] - commit_num_retries: usize, - /// The minimum wait time between retries. - #[key(TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS)] - #[default(TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT)] - commit_min_retry_wait_ms: u64, - /// The maximum wait time between retries. - #[key(TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS)] - #[default(TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT)] - commit_max_retry_wait_ms: u64, - /// The total timeout for commit retries. - #[key(TableProperties::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS)] - #[default(TableProperties::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT)] - commit_total_retry_timeout_ms: u64, - #[key(TableProperties::PROPERTY_COMMENT)] - #[default(None)] - #[doc = "The table's business meaning and usage context."] - comment: Option, - #[key(TableProperties::PROPERTY_COMMIT_NUM_STATUS_CHECKS)] - #[default(TableProperties::PROPERTY_COMMIT_NUM_STATUS_CHECKS_DEFAULT)] - commit_num_status_checks: usize, - #[key(TableProperties::PROPERTY_COMMIT_STATUS_CHECKS_MIN_WAIT_MS)] - #[default(TableProperties::PROPERTY_COMMIT_STATUS_CHECKS_MIN_WAIT_MS_DEFAULT)] - commit_status_checks_min_wait_ms: u64, - #[key(TableProperties::PROPERTY_COMMIT_STATUS_CHECKS_MAX_WAIT_MS)] - #[default(TableProperties::PROPERTY_COMMIT_STATUS_CHECKS_MAX_WAIT_MS_DEFAULT)] - commit_status_checks_max_wait_ms: u64, - #[key(TableProperties::PROPERTY_COMMIT_STATUS_CHECKS_TOTAL_WAIT_MS)] - #[default(TableProperties::PROPERTY_COMMIT_STATUS_CHECKS_TOTAL_WAIT_MS_DEFAULT)] - commit_status_checks_total_wait_ms: u64, - #[key(TableProperties::PROPERTY_MANIFEST_TARGET_SIZE_BYTES)] - #[default(TableProperties::PROPERTY_MANIFEST_TARGET_SIZE_BYTES_DEFAULT)] - manifest_target_size_bytes: usize, - #[key(TableProperties::PROPERTY_MANIFEST_MIN_MERGE_COUNT)] - #[default(TableProperties::PROPERTY_MANIFEST_MIN_MERGE_COUNT_DEFAULT)] - manifest_min_merge_count: usize, - #[key(TableProperties::PROPERTY_MANIFEST_MERGE_ENABLED)] - #[default(TableProperties::PROPERTY_MANIFEST_MERGE_ENABLED_DEFAULT)] - manifest_merge_enabled: bool, - /// The default format for files. - #[key(TableProperties::PROPERTY_DEFAULT_FILE_FORMAT)] - #[default(DataFileFormat::Parquet)] - write_format_default: DataFileFormat, - #[key(TableProperties::PROPERTY_DELETE_DEFAULT_FILE_FORMAT)] - #[default(DataFileFormat::Parquet)] - delete_format_default: DataFileFormat, - #[key(TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES)] - #[default(TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT)] - parquet_row_group_size_bytes: usize, - #[key(TableProperties::PROPERTY_DELETE_PARQUET_ROW_GROUP_SIZE_BYTES)] - #[default(TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT)] - delete_parquet_row_group_size_bytes: usize, - #[key(TableProperties::PROPERTY_PARQUET_PAGE_SIZE_BYTES)] - #[default(TableProperties::PROPERTY_PARQUET_PAGE_SIZE_BYTES_DEFAULT)] - parquet_page_size_bytes: usize, - #[key(TableProperties::PROPERTY_DELETE_PARQUET_PAGE_SIZE_BYTES)] - #[default(TableProperties::PROPERTY_PARQUET_PAGE_SIZE_BYTES_DEFAULT)] - delete_parquet_page_size_bytes: usize, - #[key(TableProperties::PROPERTY_PARQUET_PAGE_VERSION)] - #[default(TableProperties::PROPERTY_PARQUET_PAGE_VERSION_DEFAULT.to_string())] - parquet_page_version: String, - #[key(TableProperties::PROPERTY_DELETE_PARQUET_PAGE_VERSION)] - #[default(TableProperties::PROPERTY_PARQUET_PAGE_VERSION_DEFAULT.to_string())] - delete_parquet_page_version: String, - #[key(TableProperties::PROPERTY_PARQUET_PAGE_ROW_LIMIT)] - #[default(TableProperties::PROPERTY_PARQUET_PAGE_ROW_LIMIT_DEFAULT)] - parquet_page_row_limit: usize, - #[key(TableProperties::PROPERTY_DELETE_PARQUET_PAGE_ROW_LIMIT)] - #[default(TableProperties::PROPERTY_PARQUET_PAGE_ROW_LIMIT_DEFAULT)] - delete_parquet_page_row_limit: usize, - #[key(TableProperties::PROPERTY_PARQUET_DICT_SIZE_BYTES)] - #[default(TableProperties::PROPERTY_PARQUET_DICT_SIZE_BYTES_DEFAULT)] - parquet_dict_size_bytes: usize, - #[key(TableProperties::PROPERTY_DELETE_PARQUET_DICT_SIZE_BYTES)] - #[default(TableProperties::PROPERTY_PARQUET_DICT_SIZE_BYTES_DEFAULT)] - delete_parquet_dict_size_bytes: usize, - #[key(TableProperties::PROPERTY_PARQUET_COMPRESSION)] - #[default(TableProperties::PROPERTY_PARQUET_COMPRESSION_DEFAULT_SINCE_1_4_0.to_string())] - parquet_compression: String, - #[key(TableProperties::PROPERTY_DELETE_PARQUET_COMPRESSION)] - #[default(TableProperties::PROPERTY_PARQUET_COMPRESSION_DEFAULT_SINCE_1_4_0.to_string())] - delete_parquet_compression: String, - #[key(TableProperties::PROPERTY_PARQUET_COMPRESSION_LEVEL)] - #[default(None)] - parquet_compression_level: Option, - #[key(TableProperties::PROPERTY_DELETE_PARQUET_COMPRESSION_LEVEL)] - #[default(None)] - delete_parquet_compression_level: Option, - #[key(TableProperties::PROPERTY_PARQUET_SHRED_VARIANTS)] - #[default(TableProperties::PROPERTY_PARQUET_SHRED_VARIANTS_DEFAULT)] - parquet_shred_variants: bool, - #[key(TableProperties::PROPERTY_PARQUET_VARIANT_BUFFER_SIZE)] - #[default(TableProperties::PROPERTY_PARQUET_VARIANT_BUFFER_SIZE_DEFAULT)] - parquet_variant_buffer_size: usize, - #[key(TableProperties::PROPERTY_PARQUET_ROW_GROUP_CHECK_MIN_RECORD_COUNT)] - #[default(TableProperties::PROPERTY_PARQUET_ROW_GROUP_CHECK_MIN_RECORD_COUNT_DEFAULT)] - parquet_row_group_check_min_record_count: usize, - #[key(TableProperties::PROPERTY_DELETE_PARQUET_ROW_GROUP_CHECK_MIN_RECORD_COUNT)] - #[default(TableProperties::PROPERTY_PARQUET_ROW_GROUP_CHECK_MIN_RECORD_COUNT_DEFAULT)] - delete_parquet_row_group_check_min_record_count: usize, - #[key(TableProperties::PROPERTY_PARQUET_ROW_GROUP_CHECK_MAX_RECORD_COUNT)] - #[default(TableProperties::PROPERTY_PARQUET_ROW_GROUP_CHECK_MAX_RECORD_COUNT_DEFAULT)] - parquet_row_group_check_max_record_count: usize, - #[key(TableProperties::PROPERTY_DELETE_PARQUET_ROW_GROUP_CHECK_MAX_RECORD_COUNT)] - #[default(TableProperties::PROPERTY_PARQUET_ROW_GROUP_CHECK_MAX_RECORD_COUNT_DEFAULT)] - delete_parquet_row_group_check_max_record_count: usize, - #[key(TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_TRACK_UNCOMPRESSED)] - #[default(TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_TRACK_UNCOMPRESSED_DEFAULT)] - parquet_row_group_size_track_uncompressed: bool, - #[key(TableProperties::PROPERTY_PARQUET_BLOOM_FILTER_MAX_BYTES)] - #[default(TableProperties::PROPERTY_PARQUET_BLOOM_FILTER_MAX_BYTES_DEFAULT)] - parquet_bloom_filter_max_bytes: usize, - #[key(TableProperties::PROPERTY_PARQUET_BLOOM_FILTER_ADAPTIVE_ENABLED)] - #[default(TableProperties::PROPERTY_PARQUET_BLOOM_FILTER_ADAPTIVE_ENABLED_DEFAULT)] - parquet_bloom_filter_adaptive_enabled: bool, - #[prefix(TableProperties::PROPERTY_PARQUET_BLOOM_FILTER_COLUMN_FPP_PREFIX)] - #[default(HashMap::new())] - parquet_bloom_filter_column_fpp: HashMap, - #[prefix(TableProperties::PROPERTY_PARQUET_BLOOM_FILTER_COLUMN_NDV_PREFIX)] - #[default(HashMap::new())] - parquet_bloom_filter_column_ndv: HashMap, - #[prefix(TableProperties::PROPERTY_PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX)] - #[default(HashMap::new())] - parquet_bloom_filter_column_enabled: HashMap, - #[prefix(TableProperties::PROPERTY_PARQUET_COLUMN_STATS_ENABLED_PREFIX)] - #[default(HashMap::new())] - parquet_column_stats_enabled: HashMap, - #[prefix(TableProperties::PROPERTY_PARQUET_DICT_ENCODING_ENABLED_COLUMN_PREFIX)] - #[default(HashMap::new())] - parquet_dict_encoding_enabled_column: HashMap, - #[key(TableProperties::PROPERTY_AVRO_COMPRESSION)] - #[default(TableProperties::PROPERTY_AVRO_COMPRESSION_DEFAULT.to_string())] - avro_compression: String, - #[key(TableProperties::PROPERTY_DELETE_AVRO_COMPRESSION)] - #[default(TableProperties::PROPERTY_AVRO_COMPRESSION_DEFAULT.to_string())] - delete_avro_compression: String, - #[key(TableProperties::PROPERTY_AVRO_COMPRESSION_LEVEL)] - #[default(None)] - avro_compression_level: Option, - #[key(TableProperties::PROPERTY_DELETE_AVRO_COMPRESSION_LEVEL)] - #[default(None)] - delete_avro_compression_level: Option, - #[key(TableProperties::PROPERTY_MANIFEST_COMPRESSION)] - #[default(TableProperties::PROPERTY_MANIFEST_COMPRESSION_DEFAULT.to_string())] - manifest_compression: String, - #[key(TableProperties::PROPERTY_MANIFEST_COMPRESSION_LEVEL)] - #[default(None)] - manifest_compression_level: Option, - #[key(TableProperties::PROPERTY_ORC_STRIPE_SIZE_BYTES)] - #[default(TableProperties::PROPERTY_ORC_STRIPE_SIZE_BYTES_DEFAULT)] - orc_stripe_size_bytes: u64, - #[key(TableProperties::PROPERTY_DELETE_ORC_STRIPE_SIZE_BYTES)] - #[default(TableProperties::PROPERTY_ORC_STRIPE_SIZE_BYTES_DEFAULT)] - delete_orc_stripe_size_bytes: u64, - #[key(TableProperties::PROPERTY_ORC_BLOOM_FILTER_COLUMNS)] - #[default(TableProperties::PROPERTY_ORC_BLOOM_FILTER_COLUMNS_DEFAULT.to_string())] - orc_bloom_filter_columns: String, - #[key(TableProperties::PROPERTY_ORC_BLOOM_FILTER_FPP)] - #[default(TableProperties::PROPERTY_ORC_BLOOM_FILTER_FPP_DEFAULT)] - orc_bloom_filter_fpp: f64, - #[key(TableProperties::PROPERTY_ORC_BLOCK_SIZE_BYTES)] - #[default(TableProperties::PROPERTY_ORC_BLOCK_SIZE_BYTES_DEFAULT)] - orc_block_size_bytes: u64, - #[key(TableProperties::PROPERTY_DELETE_ORC_BLOCK_SIZE_BYTES)] - #[default(TableProperties::PROPERTY_ORC_BLOCK_SIZE_BYTES_DEFAULT)] - delete_orc_block_size_bytes: u64, - #[key(TableProperties::PROPERTY_ORC_WRITE_BATCH_SIZE)] - #[default(TableProperties::PROPERTY_ORC_WRITE_BATCH_SIZE_DEFAULT)] - orc_write_batch_size: usize, - #[key(TableProperties::PROPERTY_DELETE_ORC_WRITE_BATCH_SIZE)] - #[default(TableProperties::PROPERTY_ORC_WRITE_BATCH_SIZE_DEFAULT)] - delete_orc_write_batch_size: usize, - #[key(TableProperties::PROPERTY_ORC_COMPRESSION)] - #[default(TableProperties::PROPERTY_ORC_COMPRESSION_DEFAULT.to_string())] - orc_compression: String, - #[key(TableProperties::PROPERTY_DELETE_ORC_COMPRESSION)] - #[default(TableProperties::PROPERTY_ORC_COMPRESSION_DEFAULT.to_string())] - delete_orc_compression: String, - #[key(TableProperties::PROPERTY_ORC_COMPRESSION_STRATEGY)] - #[default(TableProperties::PROPERTY_ORC_COMPRESSION_STRATEGY_DEFAULT.to_string())] - orc_compression_strategy: String, - #[key(TableProperties::PROPERTY_DELETE_ORC_COMPRESSION_STRATEGY)] - #[default(TableProperties::PROPERTY_ORC_COMPRESSION_STRATEGY_DEFAULT.to_string())] - delete_orc_compression_strategy: String, - #[key(TableProperties::PROPERTY_SPLIT_SIZE)] - #[default(TableProperties::PROPERTY_SPLIT_SIZE_DEFAULT)] - split_size: u64, - #[key(TableProperties::PROPERTY_METADATA_SPLIT_SIZE)] - #[default(TableProperties::PROPERTY_METADATA_SPLIT_SIZE_DEFAULT)] - metadata_split_size: u64, - #[key(TableProperties::PROPERTY_SPLIT_LOOKBACK)] - #[default(TableProperties::PROPERTY_SPLIT_LOOKBACK_DEFAULT)] - split_lookback: usize, - #[key(TableProperties::PROPERTY_SPLIT_OPEN_FILE_COST)] - #[default(TableProperties::PROPERTY_SPLIT_OPEN_FILE_COST_DEFAULT)] - split_open_file_cost: u64, - #[key(TableProperties::PROPERTY_ADAPTIVE_SPLIT_SIZE_ENABLED)] - #[default(TableProperties::PROPERTY_ADAPTIVE_SPLIT_SIZE_ENABLED_DEFAULT)] - adaptive_split_size_enabled: bool, - #[key(TableProperties::PROPERTY_PARQUET_VECTORIZATION_ENABLED)] - #[default(TableProperties::PROPERTY_PARQUET_VECTORIZATION_ENABLED_DEFAULT)] - parquet_vectorization_enabled: bool, - #[key(TableProperties::PROPERTY_PARQUET_BATCH_SIZE)] - #[default(TableProperties::PROPERTY_PARQUET_BATCH_SIZE_DEFAULT)] - parquet_batch_size: usize, - #[key(TableProperties::PROPERTY_ORC_VECTORIZATION_ENABLED)] - #[default(TableProperties::PROPERTY_ORC_VECTORIZATION_ENABLED_DEFAULT)] - orc_vectorization_enabled: bool, - #[key(TableProperties::PROPERTY_ORC_BATCH_SIZE)] - #[default(TableProperties::PROPERTY_ORC_BATCH_SIZE_DEFAULT)] - orc_batch_size: usize, - #[key(TableProperties::PROPERTY_DATA_PLANNING_MODE)] - #[default(TableProperties::PROPERTY_PLANNING_MODE_DEFAULT.to_string())] - data_planning_mode: String, - #[key(TableProperties::PROPERTY_DELETE_PLANNING_MODE)] - #[default(TableProperties::PROPERTY_PLANNING_MODE_DEFAULT.to_string())] - delete_planning_mode: String, - #[key(TableProperties::PROPERTY_IDENTIFIER_FIELDS_RELY)] - #[default(TableProperties::PROPERTY_IDENTIFIER_FIELDS_RELY_DEFAULT)] - identifier_fields_rely: bool, - /// The target file size for files. - #[key(TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES)] - #[default(TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT)] - write_target_file_size_bytes: usize, - #[key(TableProperties::PROPERTY_DELETE_TARGET_FILE_SIZE_BYTES)] - #[default(TableProperties::PROPERTY_DELETE_TARGET_FILE_SIZE_BYTES_DEFAULT)] - delete_target_file_size_bytes: usize, - #[key(TableProperties::PROPERTY_OBJECT_STORE_ENABLED)] - #[default(TableProperties::PROPERTY_OBJECT_STORE_ENABLED_DEFAULT)] - object_store_enabled: bool, - #[key(TableProperties::PROPERTY_WRITE_OBJECT_STORE_PARTITIONED_PATHS)] - #[default(TableProperties::PROPERTY_WRITE_OBJECT_STORE_PARTITIONED_PATHS_DEFAULT)] - write_object_store_partitioned_paths: bool, - #[key(TableProperties::PROPERTY_OBJECT_STORE_PATH)] - #[default(None)] - object_store_path: Option, - #[key(TableProperties::PROPERTY_WRITE_LOCATION_PROVIDER_IMPL)] - #[default(None)] - write_location_provider_impl: Option, - #[key(TableProperties::PROPERTY_WRITE_FOLDER_STORAGE_LOCATION)] - #[default(None)] - write_folder_storage_location: Option, - #[key(TableProperties::PROPERTY_WRITE_DATA_LOCATION)] - #[default(None)] - write_data_location: Option, - /// Base directory for metadata files (manifests, manifest lists), with any - /// trailing slash trimmed. `None` if `write.metadata.path` is not set. - #[key(TableProperties::PROPERTY_WRITE_METADATA_PATH)] - #[default(None)] - #[parse_with(parse_metadata_location)] - write_metadata_path: Option, - #[key(TableProperties::PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT)] - #[default(TableProperties::PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT_DEFAULT)] - write_partition_summary_limit: u64, - #[key(TableProperties::PROPERTY_MANIFEST_LISTS_ENABLED)] - #[default(TableProperties::PROPERTY_MANIFEST_LISTS_ENABLED_DEFAULT)] - manifest_lists_enabled: bool, - /// Compression codec for metadata files (JSON) - #[key(TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC)] - #[default(CompressionCodec::None)] - #[parse_with(parse_metadata_file_compression_value)] - #[serialize_with(serialize_compression_codec)] - metadata_compression_codec: CompressionCodec, - #[key(TableProperties::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX)] - #[default(TableProperties::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX_DEFAULT)] - metadata_previous_versions_max: usize, - #[key(TableProperties::PROPERTY_METADATA_DELETE_AFTER_COMMIT_ENABLED)] - #[default(TableProperties::PROPERTY_METADATA_DELETE_AFTER_COMMIT_ENABLED_DEFAULT)] - metadata_delete_after_commit_enabled: bool, - #[key(TableProperties::PROPERTY_METRICS_MAX_INFERRED_COLUMN_DEFAULTS)] - #[default(TableProperties::PROPERTY_METRICS_MAX_INFERRED_COLUMN_DEFAULTS_DEFAULT)] - metrics_max_inferred_column_defaults: usize, - #[prefix(TableProperties::PROPERTY_METRICS_MODE_COLUMN_CONF_PREFIX)] - #[default(HashMap::new())] - metrics_mode_column_config: HashMap, - #[key(TableProperties::PROPERTY_DEFAULT_WRITE_METRICS_MODE)] - #[default(TableProperties::PROPERTY_DEFAULT_WRITE_METRICS_MODE_DEFAULT.to_string())] - default_write_metrics_mode: String, - #[key(TableProperties::PROPERTY_DEFAULT_NAME_MAPPING)] - #[default(None)] - default_name_mapping: Option, - #[key(TableProperties::PROPERTY_WRITE_AUDIT_PUBLISH_ENABLED)] - #[default(TableProperties::PROPERTY_WRITE_AUDIT_PUBLISH_ENABLED_DEFAULT)] - write_audit_publish_enabled: bool, - #[key(TableProperties::PROPERTY_SPARK_WRITE_PARTITIONED_FANOUT_ENABLED)] - #[default(TableProperties::PROPERTY_SPARK_WRITE_PARTITIONED_FANOUT_ENABLED_DEFAULT)] - spark_write_partitioned_fanout_enabled: bool, - #[key(TableProperties::PROPERTY_SPARK_WRITE_ACCEPT_ANY_SCHEMA)] - #[default(TableProperties::PROPERTY_SPARK_WRITE_ACCEPT_ANY_SCHEMA_DEFAULT)] - spark_write_accept_any_schema: bool, - #[key(TableProperties::PROPERTY_SPARK_WRITE_AUTO_SCHEMA_EVOLUTION)] - #[default(TableProperties::PROPERTY_SPARK_WRITE_AUTO_SCHEMA_EVOLUTION_DEFAULT)] - spark_write_auto_schema_evolution: bool, - #[key(TableProperties::PROPERTY_SPARK_WRITE_ADVISORY_PARTITION_SIZE_BYTES)] - #[default(None)] - spark_write_advisory_partition_size_bytes: Option, - #[key(TableProperties::PROPERTY_SNAPSHOT_ID_INHERITANCE_ENABLED)] - #[default(TableProperties::PROPERTY_SNAPSHOT_ID_INHERITANCE_ENABLED_DEFAULT)] - snapshot_id_inheritance_enabled: bool, - #[key(TableProperties::PROPERTY_ENGINE_HIVE_ENABLED)] - #[default(TableProperties::PROPERTY_ENGINE_HIVE_ENABLED_DEFAULT)] - engine_hive_enabled: bool, - #[key(TableProperties::PROPERTY_HIVE_LOCK_ENABLED)] - #[default(TableProperties::PROPERTY_HIVE_LOCK_ENABLED_DEFAULT)] - hive_lock_enabled: bool, - #[key(TableProperties::PROPERTY_WRITE_DISTRIBUTION_MODE)] - #[default(None)] - write_distribution_mode: Option, - /// Whether to use `FanoutWriter` for partitioned tables. - #[key(TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED)] - #[default(TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT)] - write_datafusion_fanout_enabled: bool, - /// Whether garbage collection is enabled on drop. - /// When `false`, data files will not be deleted when a table is dropped. - #[key(TableProperties::PROPERTY_GC_ENABLED)] - #[default(TableProperties::PROPERTY_GC_ENABLED_DEFAULT)] - gc_enabled: bool, - /// Default maximum age of a snapshot to keep when expiring snapshots. - #[key(TableProperties::PROPERTY_MAX_SNAPSHOT_AGE_MS)] - #[default(TableProperties::PROPERTY_MAX_SNAPSHOT_AGE_MS_DEFAULT)] - max_snapshot_age_ms: i64, - /// Default minimum number of snapshots to keep per branch when expiring snapshots. - #[key(TableProperties::PROPERTY_MIN_SNAPSHOTS_TO_KEEP)] - #[default(TableProperties::PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT)] - min_snapshots_to_keep: usize, - /// Default maximum age of a snapshot reference to keep when expiring snapshots. - #[key(TableProperties::PROPERTY_MAX_REF_AGE_MS)] - #[default(TableProperties::PROPERTY_MAX_REF_AGE_MS_DEFAULT)] - max_ref_age_ms: i64, - #[key(TableProperties::PROPERTY_DELETE_GRANULARITY)] - #[default(TableProperties::PROPERTY_DELETE_GRANULARITY_DEFAULT.to_string())] - delete_granularity: String, - #[key(TableProperties::PROPERTY_DELETE_ISOLATION_LEVEL)] - #[default(TableProperties::PROPERTY_DELETE_ISOLATION_LEVEL_DEFAULT.to_string())] - delete_isolation_level: String, - #[key(TableProperties::PROPERTY_DELETE_MODE)] - #[default(TableProperties::PROPERTY_DELETE_MODE_DEFAULT.to_string())] - delete_mode: String, - #[key(TableProperties::PROPERTY_DELETE_DISTRIBUTION_MODE)] - #[default(None)] - delete_distribution_mode: Option, - #[key(TableProperties::PROPERTY_UPDATE_ISOLATION_LEVEL)] - #[default(TableProperties::PROPERTY_UPDATE_ISOLATION_LEVEL_DEFAULT.to_string())] - update_isolation_level: String, - #[key(TableProperties::PROPERTY_UPDATE_MODE)] - #[default(TableProperties::PROPERTY_UPDATE_MODE_DEFAULT.to_string())] - update_mode: String, - #[key(TableProperties::PROPERTY_UPDATE_DISTRIBUTION_MODE)] - #[default(None)] - update_distribution_mode: Option, - #[key(TableProperties::PROPERTY_MERGE_ISOLATION_LEVEL)] - #[default(TableProperties::PROPERTY_MERGE_ISOLATION_LEVEL_DEFAULT.to_string())] - merge_isolation_level: String, - #[key(TableProperties::PROPERTY_MERGE_MODE)] - #[default(TableProperties::PROPERTY_MERGE_MODE_DEFAULT.to_string())] - merge_mode: String, - #[key(TableProperties::PROPERTY_MERGE_DISTRIBUTION_MODE)] - #[default(None)] - merge_distribution_mode: Option, - #[key(TableProperties::PROPERTY_UPSERT_ENABLED)] - #[default(TableProperties::PROPERTY_UPSERT_ENABLED_DEFAULT)] - upsert_enabled: bool, - /// Whether content-defined chunking is enabled. - /// `true` only when `write.parquet.content-defined-chunking.enabled = "true"`. - #[key(TableProperties::PROPERTY_PARQUET_CDC_ENABLED)] - #[default(TableProperties::PROPERTY_PARQUET_CDC_ENABLED_DEFAULT)] - cdc_enabled: bool, - /// Content-defined chunking minimum chunk size in bytes. - #[key(TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE)] - #[default(TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT)] - cdc_min_chunk_size: usize, - /// Content-defined chunking maximum chunk size in bytes. - #[key(TableProperties::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE)] - #[default(TableProperties::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE_DEFAULT)] - cdc_max_chunk_size: usize, - /// Content-defined chunking normalization level (gearhash bit adjustment). - #[key(TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL)] - #[default(TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL_DEFAULT)] - cdc_norm_level: i32, - /// The master key id used to encrypt this table's manifest list and data - /// files. `None` if `encryption.key-id` is not set. - #[key(TableProperties::PROPERTY_ENCRYPTION_KEY_ID)] - #[default(None)] - encryption_key_id: Option, - /// The encryption data encryption key length in bytes. - #[key(TableProperties::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH)] - #[default(TableProperties::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH_DEFAULT)] - encryption_data_key_length: usize, -} - /// TableProperties that contains the properties of a table. #[derive(Debug)] pub struct TableProperties { @@ -733,487 +325,101 @@ impl TableProperties { pub const PROPERTY_ENCRYPTION_DATA_KEY_LENGTH: &str = "encryption.data-key-length"; /// Default value for the encryption DEK length (16 bytes = AES-128). pub const PROPERTY_ENCRYPTION_DATA_KEY_LENGTH_DEFAULT: usize = 16; - - /// Table property documenting the business meaning and usage context of the table. - pub const PROPERTY_COMMENT: &str = "comment"; - - /// Property key for commit status-check retries. - pub const PROPERTY_COMMIT_NUM_STATUS_CHECKS: &str = "commit.status-check.num-retries"; - /// Default number of commit status-check retries. - pub const PROPERTY_COMMIT_NUM_STATUS_CHECKS_DEFAULT: usize = 3; - /// Property key for minimum wait between commit status checks. - pub const PROPERTY_COMMIT_STATUS_CHECKS_MIN_WAIT_MS: &str = "commit.status-check.min-wait-ms"; - /// Default minimum wait between commit status checks. - pub const PROPERTY_COMMIT_STATUS_CHECKS_MIN_WAIT_MS_DEFAULT: u64 = 1000; - /// Property key for maximum wait between commit status checks. - pub const PROPERTY_COMMIT_STATUS_CHECKS_MAX_WAIT_MS: &str = "commit.status-check.max-wait-ms"; - /// Default maximum wait between commit status checks. - pub const PROPERTY_COMMIT_STATUS_CHECKS_MAX_WAIT_MS_DEFAULT: u64 = 60 * 1000; - /// Property key for total commit status-check timeout. - pub const PROPERTY_COMMIT_STATUS_CHECKS_TOTAL_WAIT_MS: &str = - "commit.status-check.total-timeout-ms"; - /// Default total commit status-check timeout. - pub const PROPERTY_COMMIT_STATUS_CHECKS_TOTAL_WAIT_MS_DEFAULT: u64 = 30 * 60 * 1000; - - /// Property key for the target manifest file size. - pub const PROPERTY_MANIFEST_TARGET_SIZE_BYTES: &str = "commit.manifest.target-size-bytes"; - /// Default target manifest file size. - pub const PROPERTY_MANIFEST_TARGET_SIZE_BYTES_DEFAULT: usize = 8 * 1024 * 1024; - /// Property key for the minimum manifest count before merging. - pub const PROPERTY_MANIFEST_MIN_MERGE_COUNT: &str = "commit.manifest.min-count-to-merge"; - /// Default minimum manifest count before merging. - pub const PROPERTY_MANIFEST_MIN_MERGE_COUNT_DEFAULT: usize = 100; - /// Property key controlling automatic manifest merging. - pub const PROPERTY_MANIFEST_MERGE_ENABLED: &str = "commit.manifest-merge.enabled"; - /// Default automatic manifest merging setting. - pub const PROPERTY_MANIFEST_MERGE_ENABLED_DEFAULT: bool = true; - - /// Property key for Parquet row group size. - pub const PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES: &str = "write.parquet.row-group-size-bytes"; - /// Property key for delete-file Parquet row group size. - pub const PROPERTY_DELETE_PARQUET_ROW_GROUP_SIZE_BYTES: &str = - "write.delete.parquet.row-group-size-bytes"; - /// Default Parquet row group size. - pub const PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT: usize = 128 * 1024 * 1024; - /// Property key for Parquet page size. - pub const PROPERTY_PARQUET_PAGE_SIZE_BYTES: &str = "write.parquet.page-size-bytes"; - /// Property key for delete-file Parquet page size. - pub const PROPERTY_DELETE_PARQUET_PAGE_SIZE_BYTES: &str = - "write.delete.parquet.page-size-bytes"; - /// Default Parquet page size. - pub const PROPERTY_PARQUET_PAGE_SIZE_BYTES_DEFAULT: usize = 1024 * 1024; - /// Property key for Parquet page version. - pub const PROPERTY_PARQUET_PAGE_VERSION: &str = "write.parquet.page-version"; - /// Property key for delete-file Parquet page version. - pub const PROPERTY_DELETE_PARQUET_PAGE_VERSION: &str = "write.delete.parquet.page-version"; - /// Default Parquet page version. - pub const PROPERTY_PARQUET_PAGE_VERSION_DEFAULT: &str = "v1"; - /// Property key for the Parquet page row limit. - pub const PROPERTY_PARQUET_PAGE_ROW_LIMIT: &str = "write.parquet.page-row-limit"; - /// Property key for the delete-file Parquet page row limit. - pub const PROPERTY_DELETE_PARQUET_PAGE_ROW_LIMIT: &str = "write.delete.parquet.page-row-limit"; - /// Default Parquet page row limit. - pub const PROPERTY_PARQUET_PAGE_ROW_LIMIT_DEFAULT: usize = 20_000; - /// Property key for Parquet dictionary size. - pub const PROPERTY_PARQUET_DICT_SIZE_BYTES: &str = "write.parquet.dict-size-bytes"; - /// Property key for delete-file Parquet dictionary size. - pub const PROPERTY_DELETE_PARQUET_DICT_SIZE_BYTES: &str = - "write.delete.parquet.dict-size-bytes"; - /// Default Parquet dictionary size. - pub const PROPERTY_PARQUET_DICT_SIZE_BYTES_DEFAULT: usize = 2 * 1024 * 1024; - /// Property key for Parquet compression codec. - pub const PROPERTY_PARQUET_COMPRESSION: &str = "write.parquet.compression-codec"; - /// Property key for delete-file Parquet compression codec. - pub const PROPERTY_DELETE_PARQUET_COMPRESSION: &str = "write.delete.parquet.compression-codec"; - /// Original default Parquet compression codec. - pub const PROPERTY_PARQUET_COMPRESSION_DEFAULT: &str = "gzip"; - /// Default Parquet compression codec since Iceberg 1.4.0. - pub const PROPERTY_PARQUET_COMPRESSION_DEFAULT_SINCE_1_4_0: &str = "zstd"; - /// Property key for Parquet compression level. - pub const PROPERTY_PARQUET_COMPRESSION_LEVEL: &str = "write.parquet.compression-level"; - /// Property key for delete-file Parquet compression level. - pub const PROPERTY_DELETE_PARQUET_COMPRESSION_LEVEL: &str = - "write.delete.parquet.compression-level"; - /// Default Parquet compression level. - pub const PROPERTY_PARQUET_COMPRESSION_LEVEL_DEFAULT: Option<&str> = None; - /// Property key controlling Parquet variant shredding. - pub const PROPERTY_PARQUET_SHRED_VARIANTS: &str = "write.parquet.shred-variants"; - /// Default Parquet variant shredding setting. - pub const PROPERTY_PARQUET_SHRED_VARIANTS_DEFAULT: bool = false; - /// Property key for Parquet variant inference buffer size. - pub const PROPERTY_PARQUET_VARIANT_BUFFER_SIZE: &str = - "write.parquet.variant-inference-buffer-size"; - /// Default Parquet variant inference buffer size. - pub const PROPERTY_PARQUET_VARIANT_BUFFER_SIZE_DEFAULT: usize = 100; - /// Property key for minimum record-count row group checks. - pub const PROPERTY_PARQUET_ROW_GROUP_CHECK_MIN_RECORD_COUNT: &str = - "write.parquet.row-group-check-min-record-count"; - /// Delete-file property key for minimum record-count row group checks. - pub const PROPERTY_DELETE_PARQUET_ROW_GROUP_CHECK_MIN_RECORD_COUNT: &str = - "write.delete.parquet.row-group-check-min-record-count"; - /// Default minimum record-count row group check. - pub const PROPERTY_PARQUET_ROW_GROUP_CHECK_MIN_RECORD_COUNT_DEFAULT: usize = 100; - /// Property key for maximum record-count row group checks. - pub const PROPERTY_PARQUET_ROW_GROUP_CHECK_MAX_RECORD_COUNT: &str = - "write.parquet.row-group-check-max-record-count"; - /// Delete-file property key for maximum record-count row group checks. - pub const PROPERTY_DELETE_PARQUET_ROW_GROUP_CHECK_MAX_RECORD_COUNT: &str = - "write.delete.parquet.row-group-check-max-record-count"; - /// Default maximum record-count row group check. - pub const PROPERTY_PARQUET_ROW_GROUP_CHECK_MAX_RECORD_COUNT_DEFAULT: usize = 10_000; - /// Property key for tracking uncompressed Parquet row group size. - pub const PROPERTY_PARQUET_ROW_GROUP_SIZE_TRACK_UNCOMPRESSED: &str = - "write.parquet.row-group-size-track-uncompressed"; - /// Default uncompressed row group size tracking setting. - pub const PROPERTY_PARQUET_ROW_GROUP_SIZE_TRACK_UNCOMPRESSED_DEFAULT: bool = false; - /// Property key for maximum Parquet bloom filter bytes. - pub const PROPERTY_PARQUET_BLOOM_FILTER_MAX_BYTES: &str = - "write.parquet.bloom-filter-max-bytes"; - /// Default maximum Parquet bloom filter bytes. - pub const PROPERTY_PARQUET_BLOOM_FILTER_MAX_BYTES_DEFAULT: usize = 1024 * 1024; - /// Property key for adaptive Parquet bloom filters. - pub const PROPERTY_PARQUET_BLOOM_FILTER_ADAPTIVE_ENABLED: &str = - "write.parquet.bloom-filter-adaptive-enabled"; - /// Default adaptive Parquet bloom filter setting. - pub const PROPERTY_PARQUET_BLOOM_FILTER_ADAPTIVE_ENABLED_DEFAULT: bool = false; - /// Prefix for per-column Parquet bloom filter false-positive probability. - pub const PROPERTY_PARQUET_BLOOM_FILTER_COLUMN_FPP_PREFIX: &str = - "write.parquet.bloom-filter-fpp.column."; - /// Default per-column Parquet bloom filter false-positive probability. - pub const PROPERTY_PARQUET_BLOOM_FILTER_COLUMN_FPP_DEFAULT: f64 = 0.01; - /// Prefix for per-column Parquet bloom filter distinct-value counts. - pub const PROPERTY_PARQUET_BLOOM_FILTER_COLUMN_NDV_PREFIX: &str = - "write.parquet.bloom-filter-ndv.column."; - /// Prefix for enabling Parquet bloom filters by column. - pub const PROPERTY_PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX: &str = - "write.parquet.bloom-filter-enabled.column."; - /// Prefix for enabling Parquet column statistics. - pub const PROPERTY_PARQUET_COLUMN_STATS_ENABLED_PREFIX: &str = - "write.parquet.stats-enabled.column."; - /// Prefix for enabling Parquet dictionary encoding by column. - pub const PROPERTY_PARQUET_DICT_ENCODING_ENABLED_COLUMN_PREFIX: &str = - "write.parquet.dict-encoding-enabled.column."; - - /// Property key for Avro compression codec. - pub const PROPERTY_AVRO_COMPRESSION: &str = "write.avro.compression-codec"; - /// Property key for delete-file Avro compression codec. - pub const PROPERTY_DELETE_AVRO_COMPRESSION: &str = "write.delete.avro.compression-codec"; - /// Default Avro compression codec. - pub const PROPERTY_AVRO_COMPRESSION_DEFAULT: &str = "gzip"; - /// Property key for Avro compression level. - pub const PROPERTY_AVRO_COMPRESSION_LEVEL: &str = "write.avro.compression-level"; - /// Property key for delete-file Avro compression level. - pub const PROPERTY_DELETE_AVRO_COMPRESSION_LEVEL: &str = "write.delete.avro.compression-level"; - /// Default Avro compression level. - pub const PROPERTY_AVRO_COMPRESSION_LEVEL_DEFAULT: Option<&str> = None; - /// Property key for manifest compression codec. - pub const PROPERTY_MANIFEST_COMPRESSION: &str = "write.manifest.compression-codec"; - /// Default manifest compression codec. - pub const PROPERTY_MANIFEST_COMPRESSION_DEFAULT: &str = "gzip"; - /// Property key for manifest compression level. - pub const PROPERTY_MANIFEST_COMPRESSION_LEVEL: &str = "write.manifest.compression-level"; - /// Default manifest compression level. - pub const PROPERTY_MANIFEST_COMPRESSION_LEVEL_DEFAULT: Option<&str> = None; - - /// Property key for ORC stripe size. - pub const PROPERTY_ORC_STRIPE_SIZE_BYTES: &str = "write.orc.stripe-size-bytes"; - /// Property key for delete-file ORC stripe size. - pub const PROPERTY_DELETE_ORC_STRIPE_SIZE_BYTES: &str = "write.delete.orc.stripe-size-bytes"; - /// Default ORC stripe size. - pub const PROPERTY_ORC_STRIPE_SIZE_BYTES_DEFAULT: u64 = 64 * 1024 * 1024; - /// Property key for ORC bloom filter columns. - pub const PROPERTY_ORC_BLOOM_FILTER_COLUMNS: &str = "write.orc.bloom.filter.columns"; - /// Default ORC bloom filter columns. - pub const PROPERTY_ORC_BLOOM_FILTER_COLUMNS_DEFAULT: &str = ""; - /// Property key for ORC bloom filter false-positive probability. - pub const PROPERTY_ORC_BLOOM_FILTER_FPP: &str = "write.orc.bloom.filter.fpp"; - /// Default ORC bloom filter false-positive probability. - pub const PROPERTY_ORC_BLOOM_FILTER_FPP_DEFAULT: f64 = 0.05; - /// Property key for ORC block size. - pub const PROPERTY_ORC_BLOCK_SIZE_BYTES: &str = "write.orc.block-size-bytes"; - /// Property key for delete-file ORC block size. - pub const PROPERTY_DELETE_ORC_BLOCK_SIZE_BYTES: &str = "write.delete.orc.block-size-bytes"; - /// Default ORC block size. - pub const PROPERTY_ORC_BLOCK_SIZE_BYTES_DEFAULT: u64 = 256 * 1024 * 1024; - /// Property key for ORC vectorized write batch size. - pub const PROPERTY_ORC_WRITE_BATCH_SIZE: &str = "write.orc.vectorized.batch-size"; - /// Property key for delete-file ORC vectorized write batch size. - pub const PROPERTY_DELETE_ORC_WRITE_BATCH_SIZE: &str = "write.delete.orc.vectorized.batch-size"; - /// Default ORC vectorized write batch size. - pub const PROPERTY_ORC_WRITE_BATCH_SIZE_DEFAULT: usize = 1024; - /// Property key for ORC compression codec. - pub const PROPERTY_ORC_COMPRESSION: &str = "write.orc.compression-codec"; - /// Property key for delete-file ORC compression codec. - pub const PROPERTY_DELETE_ORC_COMPRESSION: &str = "write.delete.orc.compression-codec"; - /// Default ORC compression codec. - pub const PROPERTY_ORC_COMPRESSION_DEFAULT: &str = "zlib"; - /// Property key for ORC compression strategy. - pub const PROPERTY_ORC_COMPRESSION_STRATEGY: &str = "write.orc.compression-strategy"; - /// Property key for delete-file ORC compression strategy. - pub const PROPERTY_DELETE_ORC_COMPRESSION_STRATEGY: &str = - "write.delete.orc.compression-strategy"; - /// Default ORC compression strategy. - pub const PROPERTY_ORC_COMPRESSION_STRATEGY_DEFAULT: &str = "speed"; - - /// Property key for read split target size. - pub const PROPERTY_SPLIT_SIZE: &str = "read.split.target-size"; - /// Default read split target size. - pub const PROPERTY_SPLIT_SIZE_DEFAULT: u64 = 128 * 1024 * 1024; - /// Property key for metadata split target size. - pub const PROPERTY_METADATA_SPLIT_SIZE: &str = "read.split.metadata-target-size"; - /// Default metadata split target size. - pub const PROPERTY_METADATA_SPLIT_SIZE_DEFAULT: u64 = 32 * 1024 * 1024; - /// Property key for split planning lookback. - pub const PROPERTY_SPLIT_LOOKBACK: &str = "read.split.planning-lookback"; - /// Default split planning lookback. - pub const PROPERTY_SPLIT_LOOKBACK_DEFAULT: usize = 10; - /// Property key for split open-file cost. - pub const PROPERTY_SPLIT_OPEN_FILE_COST: &str = "read.split.open-file-cost"; - /// Default split open-file cost. - pub const PROPERTY_SPLIT_OPEN_FILE_COST_DEFAULT: u64 = 4 * 1024 * 1024; - /// Property key controlling adaptive split sizing. - pub const PROPERTY_ADAPTIVE_SPLIT_SIZE_ENABLED: &str = "read.split.adaptive-size.enabled"; - /// Default adaptive split sizing setting. - pub const PROPERTY_ADAPTIVE_SPLIT_SIZE_ENABLED_DEFAULT: bool = true; - /// Property key controlling Parquet vectorized reads. - pub const PROPERTY_PARQUET_VECTORIZATION_ENABLED: &str = "read.parquet.vectorization.enabled"; - /// Default Parquet vectorized read setting. - pub const PROPERTY_PARQUET_VECTORIZATION_ENABLED_DEFAULT: bool = true; - /// Property key for Parquet vectorized read batch size. - pub const PROPERTY_PARQUET_BATCH_SIZE: &str = "read.parquet.vectorization.batch-size"; - /// Default Parquet vectorized read batch size. - pub const PROPERTY_PARQUET_BATCH_SIZE_DEFAULT: usize = 5000; - /// Property key controlling ORC vectorized reads. - pub const PROPERTY_ORC_VECTORIZATION_ENABLED: &str = "read.orc.vectorization.enabled"; - /// Default ORC vectorized read setting. - pub const PROPERTY_ORC_VECTORIZATION_ENABLED_DEFAULT: bool = false; - /// Property key for ORC vectorized read batch size. - pub const PROPERTY_ORC_BATCH_SIZE: &str = "read.orc.vectorization.batch-size"; - /// Default ORC vectorized read batch size. - pub const PROPERTY_ORC_BATCH_SIZE_DEFAULT: usize = 5000; - /// Property key for data planning mode. - pub const PROPERTY_DATA_PLANNING_MODE: &str = "read.data-planning-mode"; - /// Property key for delete planning mode. - pub const PROPERTY_DELETE_PLANNING_MODE: &str = "read.delete-planning-mode"; - /// Default planning mode. - pub const PROPERTY_PLANNING_MODE_DEFAULT: &str = "auto"; - /// Property key declaring identifier fields reliable. - pub const PROPERTY_IDENTIFIER_FIELDS_RELY: &str = "identifier-fields.rely"; - /// Default identifier-field reliability setting. - pub const PROPERTY_IDENTIFIER_FIELDS_RELY_DEFAULT: bool = false; - - /// Property key controlling object-store locations. - pub const PROPERTY_OBJECT_STORE_ENABLED: &str = "write.object-storage.enabled"; - /// Default object-store location setting. - pub const PROPERTY_OBJECT_STORE_ENABLED_DEFAULT: bool = false; - /// Property key controlling partitioned paths for object storage. - pub const PROPERTY_WRITE_OBJECT_STORE_PARTITIONED_PATHS: &str = - "write.object-storage.partitioned-paths"; - /// Default partitioned-path setting for object storage. - pub const PROPERTY_WRITE_OBJECT_STORE_PARTITIONED_PATHS_DEFAULT: bool = true; - /// Deprecated object-store path property. - pub const PROPERTY_OBJECT_STORE_PATH: &str = "write.object-storage.path"; - /// Property key for a custom location provider implementation. - pub const PROPERTY_WRITE_LOCATION_PROVIDER_IMPL: &str = "write.location-provider.impl"; - /// Deprecated folder-storage location property. - pub const PROPERTY_WRITE_FOLDER_STORAGE_LOCATION: &str = "write.folder-storage.path"; - /// Property key for the table data location. - pub const PROPERTY_WRITE_DATA_LOCATION: &str = "write.data.path"; - /// Alias matching Java's metadata location property name. - pub const PROPERTY_WRITE_METADATA_LOCATION: &str = Self::PROPERTY_WRITE_METADATA_PATH; - /// Deprecated property controlling manifest-list writes. - pub const PROPERTY_MANIFEST_LISTS_ENABLED: &str = "write.manifest-lists.enabled"; - /// Default manifest-list write setting. - pub const PROPERTY_MANIFEST_LISTS_ENABLED_DEFAULT: bool = true; - /// Alias matching Java's metadata compression property name. - pub const PROPERTY_METADATA_COMPRESSION: &str = Self::PROPERTY_METADATA_COMPRESSION_CODEC; - /// Alias matching Java's metadata compression default name. - pub const PROPERTY_METADATA_COMPRESSION_DEFAULT: &str = - Self::PROPERTY_METADATA_COMPRESSION_CODEC_DEFAULT; - /// Property key controlling deletion of old metadata after commit. - pub const PROPERTY_METADATA_DELETE_AFTER_COMMIT_ENABLED: &str = - "write.metadata.delete-after-commit.enabled"; - /// Default old-metadata deletion setting. - pub const PROPERTY_METADATA_DELETE_AFTER_COMMIT_ENABLED_DEFAULT: bool = false; - /// Property key for the maximum inferred column metric defaults. - pub const PROPERTY_METRICS_MAX_INFERRED_COLUMN_DEFAULTS: &str = - "write.metadata.metrics.max-inferred-column-defaults"; - /// Default maximum inferred column metric defaults. - pub const PROPERTY_METRICS_MAX_INFERRED_COLUMN_DEFAULTS_DEFAULT: usize = 100; - /// Prefix for per-column metrics modes. - pub const PROPERTY_METRICS_MODE_COLUMN_CONF_PREFIX: &str = "write.metadata.metrics.column."; - /// Property key for the default write metrics mode. - pub const PROPERTY_DEFAULT_WRITE_METRICS_MODE: &str = "write.metadata.metrics.default"; - /// Default write metrics mode. - pub const PROPERTY_DEFAULT_WRITE_METRICS_MODE_DEFAULT: &str = "truncate(16)"; - /// Property key for the default schema name mapping. - pub const PROPERTY_DEFAULT_NAME_MAPPING: &str = "schema.name-mapping.default"; - /// Property key enabling write-audit-publish behavior. - pub const PROPERTY_WRITE_AUDIT_PUBLISH_ENABLED: &str = "write.wap.enabled"; - /// Default write-audit-publish setting. - pub const PROPERTY_WRITE_AUDIT_PUBLISH_ENABLED_DEFAULT: bool = false; - /// Property key for delete-file target size. - pub const PROPERTY_DELETE_TARGET_FILE_SIZE_BYTES: &str = "write.delete.target-file-size-bytes"; - /// Default delete-file target size. - pub const PROPERTY_DELETE_TARGET_FILE_SIZE_BYTES_DEFAULT: usize = 64 * 1024 * 1024; - - /// Deprecated Spark fanout writer property. - pub const PROPERTY_SPARK_WRITE_PARTITIONED_FANOUT_ENABLED: &str = "write.spark.fanout.enabled"; - /// Default deprecated Spark fanout writer setting. - pub const PROPERTY_SPARK_WRITE_PARTITIONED_FANOUT_ENABLED_DEFAULT: bool = false; - /// Deprecated Spark accept-any-schema property. - pub const PROPERTY_SPARK_WRITE_ACCEPT_ANY_SCHEMA: &str = "write.spark.accept-any-schema"; - /// Default deprecated Spark accept-any-schema setting. - pub const PROPERTY_SPARK_WRITE_ACCEPT_ANY_SCHEMA_DEFAULT: bool = false; - /// Deprecated Spark automatic schema evolution property. - pub const PROPERTY_SPARK_WRITE_AUTO_SCHEMA_EVOLUTION: &str = - "write.spark.auto-schema-evolution.enabled"; - /// Default deprecated Spark automatic schema evolution setting. - pub const PROPERTY_SPARK_WRITE_AUTO_SCHEMA_EVOLUTION_DEFAULT: bool = true; - /// Deprecated Spark advisory partition size property. - pub const PROPERTY_SPARK_WRITE_ADVISORY_PARTITION_SIZE_BYTES: &str = - "write.spark.advisory-partition-size-bytes"; - /// Property key for snapshot ID inheritance compatibility. - pub const PROPERTY_SNAPSHOT_ID_INHERITANCE_ENABLED: &str = - "compatibility.snapshot-id-inheritance.enabled"; - /// Default snapshot ID inheritance setting. - pub const PROPERTY_SNAPSHOT_ID_INHERITANCE_ENABLED_DEFAULT: bool = false; - /// Property key enabling Hive engine behavior. - pub const PROPERTY_ENGINE_HIVE_ENABLED: &str = "engine.hive.enabled"; - /// Default Hive engine setting. - pub const PROPERTY_ENGINE_HIVE_ENABLED_DEFAULT: bool = false; - /// Property key enabling Hive locking. - pub const PROPERTY_HIVE_LOCK_ENABLED: &str = "engine.hive.lock-enabled"; - /// Default Hive lock setting. - pub const PROPERTY_HIVE_LOCK_ENABLED_DEFAULT: bool = true; - /// Property key for write distribution mode. - pub const PROPERTY_WRITE_DISTRIBUTION_MODE: &str = "write.distribution-mode"; - /// No-distribution mode value. - pub const PROPERTY_WRITE_DISTRIBUTION_MODE_NONE: &str = "none"; - /// Hash-distribution mode value. - pub const PROPERTY_WRITE_DISTRIBUTION_MODE_HASH: &str = "hash"; - /// Range-distribution mode value. - pub const PROPERTY_WRITE_DISTRIBUTION_MODE_RANGE: &str = "range"; - - /// Property key for delete granularity. - pub const PROPERTY_DELETE_GRANULARITY: &str = "write.delete.granularity"; - /// Default delete granularity. - pub const PROPERTY_DELETE_GRANULARITY_DEFAULT: &str = "partition"; - /// Property key for delete isolation level. - pub const PROPERTY_DELETE_ISOLATION_LEVEL: &str = "write.delete.isolation-level"; - /// Default delete isolation level. - pub const PROPERTY_DELETE_ISOLATION_LEVEL_DEFAULT: &str = "serializable"; - /// Property key for delete operation mode. - pub const PROPERTY_DELETE_MODE: &str = "write.delete.mode"; - /// Default delete operation mode. - pub const PROPERTY_DELETE_MODE_DEFAULT: &str = "copy-on-write"; - /// Property key for delete distribution mode. - pub const PROPERTY_DELETE_DISTRIBUTION_MODE: &str = "write.delete.distribution-mode"; - /// Property key for update isolation level. - pub const PROPERTY_UPDATE_ISOLATION_LEVEL: &str = "write.update.isolation-level"; - /// Default update isolation level. - pub const PROPERTY_UPDATE_ISOLATION_LEVEL_DEFAULT: &str = "serializable"; - /// Property key for update operation mode. - pub const PROPERTY_UPDATE_MODE: &str = "write.update.mode"; - /// Default update operation mode. - pub const PROPERTY_UPDATE_MODE_DEFAULT: &str = "copy-on-write"; - /// Property key for update distribution mode. - pub const PROPERTY_UPDATE_DISTRIBUTION_MODE: &str = "write.update.distribution-mode"; - /// Property key for merge isolation level. - pub const PROPERTY_MERGE_ISOLATION_LEVEL: &str = "write.merge.isolation-level"; - /// Default merge isolation level. - pub const PROPERTY_MERGE_ISOLATION_LEVEL_DEFAULT: &str = "serializable"; - /// Property key for merge operation mode. - pub const PROPERTY_MERGE_MODE: &str = "write.merge.mode"; - /// Default merge operation mode. - pub const PROPERTY_MERGE_MODE_DEFAULT: &str = "copy-on-write"; - /// Property key for merge distribution mode. - pub const PROPERTY_MERGE_DISTRIBUTION_MODE: &str = "write.merge.distribution-mode"; - /// Property key enabling upserts. - pub const PROPERTY_UPSERT_ENABLED: &str = "write.upsert.enabled"; - /// Default upsert setting. - pub const PROPERTY_UPSERT_ENABLED_DEFAULT: bool = false; - /// Alias matching Java's encryption table key property name. - pub const PROPERTY_ENCRYPTION_TABLE_KEY: &str = Self::PROPERTY_ENCRYPTION_KEY_ID; - /// Alias matching Java's encryption DEK length property name. - pub const PROPERTY_ENCRYPTION_DEK_LENGTH: &str = Self::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH; - /// Alias matching Java's encryption DEK length default name. - pub const PROPERTY_ENCRYPTION_DEK_LENGTH_DEFAULT: usize = - Self::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH_DEFAULT; - /// Default encryption AAD length. - pub const PROPERTY_ENCRYPTION_AAD_LENGTH_DEFAULT: usize = 16; } impl TryFrom<&HashMap> for TableProperties { + // parse by entry key or use default value type Error = Error; fn try_from(props: &HashMap) -> Result { - Ok(Self { + Ok(TableProperties { commit_num_retries: parse_property( props, - Self::PROPERTY_COMMIT_NUM_RETRIES, - Self::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT, + TableProperties::PROPERTY_COMMIT_NUM_RETRIES, + TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT, )?, commit_min_retry_wait_ms: parse_property( props, - Self::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS, - Self::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT, + TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS, + TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT, )?, commit_max_retry_wait_ms: parse_property( props, - Self::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS, - Self::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT, + TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS, + TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT, )?, commit_total_retry_timeout_ms: parse_property( props, - Self::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS, - Self::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT, + TableProperties::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS, + TableProperties::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT, )?, write_format_default: parse_property( props, - Self::PROPERTY_DEFAULT_FILE_FORMAT, - Self::PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT.to_string(), + TableProperties::PROPERTY_DEFAULT_FILE_FORMAT, + TableProperties::PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT.to_string(), )?, write_target_file_size_bytes: parse_property( props, - Self::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES, - Self::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT, + TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES, + TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT, )?, write_metadata_path: parse_location_property( props, - Self::PROPERTY_WRITE_METADATA_PATH, + TableProperties::PROPERTY_WRITE_METADATA_PATH, )?, metadata_compression_codec: parse_metadata_file_compression(props)?, write_datafusion_fanout_enabled: parse_property( props, - Self::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED, - Self::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT, + TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED, + TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT, )?, gc_enabled: parse_property( props, - Self::PROPERTY_GC_ENABLED, - Self::PROPERTY_GC_ENABLED_DEFAULT, + TableProperties::PROPERTY_GC_ENABLED, + TableProperties::PROPERTY_GC_ENABLED_DEFAULT, )?, max_snapshot_age_ms: parse_property( props, - Self::PROPERTY_MAX_SNAPSHOT_AGE_MS, - Self::PROPERTY_MAX_SNAPSHOT_AGE_MS_DEFAULT, + TableProperties::PROPERTY_MAX_SNAPSHOT_AGE_MS, + TableProperties::PROPERTY_MAX_SNAPSHOT_AGE_MS_DEFAULT, )?, min_snapshots_to_keep: parse_property( props, - Self::PROPERTY_MIN_SNAPSHOTS_TO_KEEP, - Self::PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT, + TableProperties::PROPERTY_MIN_SNAPSHOTS_TO_KEEP, + TableProperties::PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT, )?, max_ref_age_ms: parse_property( props, - Self::PROPERTY_MAX_REF_AGE_MS, - Self::PROPERTY_MAX_REF_AGE_MS_DEFAULT, + TableProperties::PROPERTY_MAX_REF_AGE_MS, + TableProperties::PROPERTY_MAX_REF_AGE_MS_DEFAULT, )?, cdc_enabled: parse_property( props, - Self::PROPERTY_PARQUET_CDC_ENABLED, - Self::PROPERTY_PARQUET_CDC_ENABLED_DEFAULT, + TableProperties::PROPERTY_PARQUET_CDC_ENABLED, + TableProperties::PROPERTY_PARQUET_CDC_ENABLED_DEFAULT, )?, cdc_min_chunk_size: parse_property( props, - Self::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE, - Self::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT, + TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE, + TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT, )?, cdc_max_chunk_size: parse_property( props, - Self::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE, - Self::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE_DEFAULT, + TableProperties::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE, + TableProperties::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE_DEFAULT, )?, cdc_norm_level: parse_property( props, - Self::PROPERTY_PARQUET_CDC_NORM_LEVEL, - Self::PROPERTY_PARQUET_CDC_NORM_LEVEL_DEFAULT, + TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL, + TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL_DEFAULT, )?, - encryption_key_id: props.get(Self::PROPERTY_ENCRYPTION_KEY_ID).cloned(), + encryption_key_id: props + .get(TableProperties::PROPERTY_ENCRYPTION_KEY_ID) + .cloned(), encryption_data_key_length: parse_property( props, - Self::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH, - Self::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH_DEFAULT, + TableProperties::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH, + TableProperties::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH_DEFAULT, )?, }) } @@ -1221,313 +427,58 @@ impl TryFrom<&HashMap> for TableProperties { #[cfg(test)] mod tests { - use std::collections::HashMap; - use super::*; use crate::compression::CompressionCodec; - fn parse(properties: &HashMap) -> Result { - serde_json::to_value(properties) - .and_then(serde_json::from_value) - .map_err(|error| Error::new(ErrorKind::DataInvalid, error.to_string())) - } - #[test] - fn test_parsed_table_properties_default() { - let table_properties = ParsedTableProperties::default(); + fn test_table_properties_default() { + let props = HashMap::new(); + let table_properties = TableProperties::try_from(&props).unwrap(); assert_eq!( - table_properties.commit_num_retries(), + table_properties.commit_num_retries, TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT ); assert_eq!( - table_properties.commit_min_retry_wait_ms(), + table_properties.commit_min_retry_wait_ms, TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT ); assert_eq!( - table_properties.commit_max_retry_wait_ms(), + table_properties.commit_max_retry_wait_ms, TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT ); assert_eq!( - table_properties.write_format_default(), - DataFileFormat::Parquet + table_properties.write_format_default, + TableProperties::PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT.to_string() ); assert_eq!( - table_properties.write_target_file_size_bytes(), + table_properties.write_target_file_size_bytes, TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT ); // Test compression defaults (none means CompressionCodec::None) assert_eq!( - table_properties.metadata_compression_codec(), + table_properties.metadata_compression_codec, CompressionCodec::None ); assert_eq!( - table_properties.gc_enabled(), + table_properties.gc_enabled, TableProperties::PROPERTY_GC_ENABLED_DEFAULT ); assert_eq!( - table_properties.max_snapshot_age_ms(), + table_properties.max_snapshot_age_ms, TableProperties::PROPERTY_MAX_SNAPSHOT_AGE_MS_DEFAULT ); assert_eq!( - table_properties.min_snapshots_to_keep(), + table_properties.min_snapshots_to_keep, TableProperties::PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT ); assert_eq!( - table_properties.max_ref_age_ms(), + table_properties.max_ref_age_ms, TableProperties::PROPERTY_MAX_REF_AGE_MS_DEFAULT ); } #[test] - fn test_existing_table_properties_api() { - let properties = HashMap::from([ - ( - TableProperties::PROPERTY_COMMIT_NUM_RETRIES.to_string(), - "8".to_string(), - ), - ( - TableProperties::PROPERTY_DEFAULT_FILE_FORMAT.to_string(), - "orc".to_string(), - ), - ( - TableProperties::PROPERTY_WRITE_METADATA_PATH.to_string(), - "s3://warehouse/table/metadata/".to_string(), - ), - ]); - - let parsed = TableProperties::try_from(&properties).unwrap(); - assert_eq!(parsed.commit_num_retries, 8); - assert_eq!(parsed.write_format_default, "orc"); - assert_eq!( - parsed.write_metadata_path.as_deref(), - Some("s3://warehouse/table/metadata") - ); - } - - #[test] - fn test_parsed_table_properties_modifiers() { - let table_properties = ParsedTableProperties::default() - .with_commit_num_retries(9) - .with_comment(Some("orders table".to_string())) - .with_write_format_default(DataFileFormat::Avro); - - assert_eq!(table_properties.commit_num_retries(), 9); - assert_eq!(table_properties.comment(), Some("orders table".to_string())); - assert_eq!( - table_properties.write_format_default(), - DataFileFormat::Avro - ); - } - - #[test] - fn test_properties_from_java_table_properties() { - let properties = parse(&HashMap::from([ - ( - TableProperties::PROPERTY_COMMIT_NUM_STATUS_CHECKS.to_string(), - "7".to_string(), - ), - ( - TableProperties::PROPERTY_DELETE_DEFAULT_FILE_FORMAT.to_string(), - "orc".to_string(), - ), - ( - TableProperties::PROPERTY_PARQUET_PAGE_ROW_LIMIT.to_string(), - "1000".to_string(), - ), - ( - format!( - "{}customer_id", - TableProperties::PROPERTY_PARQUET_BLOOM_FILTER_COLUMN_FPP_PREFIX - ), - "0.02".to_string(), - ), - ( - TableProperties::PROPERTY_DELETE_AVRO_COMPRESSION.to_string(), - "snappy".to_string(), - ), - ( - TableProperties::PROPERTY_ORC_BLOOM_FILTER_FPP.to_string(), - "0.1".to_string(), - ), - ( - TableProperties::PROPERTY_SPLIT_LOOKBACK.to_string(), - "25".to_string(), - ), - ( - TableProperties::PROPERTY_WRITE_OBJECT_STORE_PARTITIONED_PATHS.to_string(), - "false".to_string(), - ), - ( - TableProperties::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX.to_string(), - "20".to_string(), - ), - ( - format!( - "{}customer_id", - TableProperties::PROPERTY_METRICS_MODE_COLUMN_CONF_PREFIX - ), - "full".to_string(), - ), - ( - TableProperties::PROPERTY_SPARK_WRITE_ADVISORY_PARTITION_SIZE_BYTES.to_string(), - "4096".to_string(), - ), - ( - TableProperties::PROPERTY_DELETE_MODE.to_string(), - "merge-on-read".to_string(), - ), - ( - TableProperties::PROPERTY_UPSERT_ENABLED.to_string(), - "true".to_string(), - ), - ])) - .unwrap(); - - assert_eq!(properties.commit_num_status_checks(), 7); - assert_eq!(properties.delete_format_default(), DataFileFormat::Orc); - assert_eq!(properties.parquet_page_row_limit(), 1000); - assert_eq!( - properties.parquet_bloom_filter_column_fpp()["customer_id"], - 0.02 - ); - assert_eq!(properties.delete_avro_compression(), "snappy"); - assert_eq!(properties.orc_bloom_filter_fpp(), 0.1); - assert_eq!(properties.split_lookback(), 25); - assert!(!properties.write_object_store_partitioned_paths()); - assert_eq!(properties.metadata_previous_versions_max(), 20); - assert_eq!( - properties.metrics_mode_column_config()["customer_id"], - "full" - ); - assert_eq!( - properties.spark_write_advisory_partition_size_bytes(), - Some(4096) - ); - assert_eq!(properties.delete_mode(), "merge-on-read"); - assert!(properties.upsert_enabled()); - } - - #[test] - fn test_empty_properties_match_default() { - let parsed = parse(&HashMap::new()).unwrap(); - let defaults = ParsedTableProperties::default(); - - assert_eq!(parsed.commit_num_retries(), defaults.commit_num_retries()); - assert_eq!( - parsed.commit_min_retry_wait_ms(), - defaults.commit_min_retry_wait_ms() - ); - assert_eq!( - parsed.commit_max_retry_wait_ms(), - defaults.commit_max_retry_wait_ms() - ); - assert_eq!( - parsed.commit_total_retry_timeout_ms(), - defaults.commit_total_retry_timeout_ms() - ); - assert_eq!( - parsed.write_format_default(), - defaults.write_format_default() - ); - assert_eq!( - parsed.write_target_file_size_bytes(), - defaults.write_target_file_size_bytes() - ); - assert_eq!(parsed.write_metadata_path(), defaults.write_metadata_path()); - assert_eq!( - parsed.metadata_compression_codec(), - defaults.metadata_compression_codec() - ); - assert_eq!( - parsed.write_datafusion_fanout_enabled(), - defaults.write_datafusion_fanout_enabled() - ); - assert_eq!(parsed.gc_enabled(), defaults.gc_enabled()); - assert_eq!(parsed.max_snapshot_age_ms(), defaults.max_snapshot_age_ms()); - assert_eq!( - parsed.min_snapshots_to_keep(), - defaults.min_snapshots_to_keep() - ); - assert_eq!(parsed.max_ref_age_ms(), defaults.max_ref_age_ms()); - assert_eq!(parsed.cdc_enabled(), defaults.cdc_enabled()); - assert_eq!(parsed.cdc_min_chunk_size(), defaults.cdc_min_chunk_size()); - assert_eq!(parsed.cdc_max_chunk_size(), defaults.cdc_max_chunk_size()); - assert_eq!(parsed.cdc_norm_level(), defaults.cdc_norm_level()); - assert_eq!(parsed.encryption_key_id(), defaults.encryption_key_id()); - assert_eq!( - parsed.encryption_data_key_length(), - defaults.encryption_data_key_length() - ); - } - - #[test] - fn test_parsed_table_properties_json_round_trip() { - let properties = parse(&HashMap::from([ - ( - TableProperties::PROPERTY_DEFAULT_FILE_FORMAT.to_string(), - "ORC".to_string(), - ), - ( - TableProperties::PROPERTY_WRITE_METADATA_PATH.to_string(), - "s3://warehouse/table/metadata/".to_string(), - ), - ( - TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), - "GZIP".to_string(), - ), - ( - TableProperties::PROPERTY_ENCRYPTION_KEY_ID.to_string(), - "table-key".to_string(), - ), - ])) - .unwrap(); - - let json = serde_json::to_value(&properties).unwrap(); - assert_eq!(json[TableProperties::PROPERTY_DEFAULT_FILE_FORMAT], "orc"); - assert_eq!( - json[TableProperties::PROPERTY_WRITE_METADATA_PATH], - "s3://warehouse/table/metadata" - ); - assert_eq!( - json[TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC], - "gzip" - ); - - let decoded: ParsedTableProperties = serde_json::from_value(json).unwrap(); - assert_eq!(decoded.write_format_default(), DataFileFormat::Orc); - assert_eq!( - decoded.write_metadata_path(), - Some("s3://warehouse/table/metadata".to_string()) - ); - assert_eq!( - decoded.metadata_compression_codec(), - CompressionCodec::gzip_default() - ); - assert_eq!(decoded.encryption_key_id(), Some("table-key".to_string())); - } - - #[test] - fn test_default_parsed_table_properties_json_round_trip() { - let defaults = ParsedTableProperties::default(); - let json = serde_json::to_value(&defaults).unwrap(); - - assert!( - json.get(TableProperties::PROPERTY_WRITE_METADATA_PATH) - .is_none() - ); - assert!( - json.get(TableProperties::PROPERTY_ENCRYPTION_KEY_ID) - .is_none() - ); - - let decoded: ParsedTableProperties = serde_json::from_value(json).unwrap(); - assert_eq!(decoded.write_format_default(), DataFileFormat::Parquet); - assert_eq!(decoded.write_metadata_path(), None); - assert_eq!(decoded.encryption_key_id(), None); - } - - #[test] - fn test_parsed_table_properties_history_expire_overrides() { + fn test_table_properties_history_expire_overrides() { let props = HashMap::from([ ( TableProperties::PROPERTY_MAX_SNAPSHOT_AGE_MS.to_string(), @@ -1542,24 +493,24 @@ mod tests { "5678".to_string(), ), ]); - let table_properties = parse(&props).unwrap(); - assert_eq!(table_properties.max_snapshot_age_ms(), 1234); - assert_eq!(table_properties.min_snapshots_to_keep(), 7); - assert_eq!(table_properties.max_ref_age_ms(), 5678); + let table_properties = TableProperties::try_from(&props).unwrap(); + assert_eq!(table_properties.max_snapshot_age_ms, 1234); + assert_eq!(table_properties.min_snapshots_to_keep, 7); + assert_eq!(table_properties.max_ref_age_ms, 5678); } #[test] - fn test_parsed_table_properties_write_metadata_path() { + fn test_table_properties_write_metadata_path() { // Test unset - let table_properties = parse(&HashMap::new()).unwrap(); - assert_eq!(table_properties.write_metadata_path(), None); + let table_properties = TableProperties::try_from(&HashMap::new()).unwrap(); + assert_eq!(table_properties.write_metadata_path, None); // Test empty path is invalid let props = HashMap::from([( TableProperties::PROPERTY_WRITE_METADATA_PATH.to_string(), String::new(), )]); - let error = parse(&props).unwrap_err(); + let error = TableProperties::try_from(&props).unwrap_err(); assert_eq!(error.kind(), ErrorKind::DataInvalid); assert!( error @@ -1571,10 +522,10 @@ mod tests { TableProperties::PROPERTY_WRITE_METADATA_PATH.to_string(), "s3://other-bucket/custom-meta/".to_string(), )]); - let table_properties = parse(&props).unwrap(); + let table_properties = TableProperties::try_from(&props).unwrap(); assert_eq!( - table_properties.write_metadata_path(), - Some("s3://other-bucket/custom-meta".to_string()) + table_properties.write_metadata_path.as_deref(), + Some("s3://other-bucket/custom-meta") ); } @@ -1595,41 +546,41 @@ mod tests { } #[test] - fn test_parsed_table_properties_compression() { + fn test_table_properties_compression() { let props = HashMap::from([( TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), "gzip".to_string(), )]); - let table_properties = parse(&props).unwrap(); + let table_properties = TableProperties::try_from(&props).unwrap(); assert_eq!( - table_properties.metadata_compression_codec(), + table_properties.metadata_compression_codec, CompressionCodec::gzip_default() ); } #[test] - fn test_parsed_table_properties_compression_none() { + fn test_table_properties_compression_none() { let props = HashMap::from([( TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), "none".to_string(), )]); - let table_properties = parse(&props).unwrap(); + let table_properties = TableProperties::try_from(&props).unwrap(); assert_eq!( - table_properties.metadata_compression_codec(), + table_properties.metadata_compression_codec, CompressionCodec::None ); } #[test] - fn test_parsed_table_properties_compression_case_insensitive() { + fn test_table_properties_compression_case_insensitive() { // Test uppercase let props_upper = HashMap::from([( TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), "GZIP".to_string(), )]); - let table_properties = parse(&props_upper).unwrap(); + let table_properties = TableProperties::try_from(&props_upper).unwrap(); assert_eq!( - table_properties.metadata_compression_codec(), + table_properties.metadata_compression_codec, CompressionCodec::gzip_default() ); @@ -1638,9 +589,9 @@ mod tests { TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), "GzIp".to_string(), )]); - let table_properties = parse(&props_mixed).unwrap(); + let table_properties = TableProperties::try_from(&props_mixed).unwrap(); assert_eq!( - table_properties.metadata_compression_codec(), + table_properties.metadata_compression_codec, CompressionCodec::gzip_default() ); @@ -1649,15 +600,15 @@ mod tests { TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), "NONE".to_string(), )]); - let table_properties = parse(&props_none_upper).unwrap(); + let table_properties = TableProperties::try_from(&props_none_upper).unwrap(); assert_eq!( - table_properties.metadata_compression_codec(), + table_properties.metadata_compression_codec, CompressionCodec::None ); } #[test] - fn test_parsed_table_properties_valid() { + fn test_table_properties_valid() { let props = HashMap::from([ ( TableProperties::PROPERTY_COMMIT_NUM_RETRIES.to_string(), @@ -1680,25 +631,22 @@ mod tests { "false".to_string(), ), ]); - let table_properties = parse(&props).unwrap(); - assert_eq!(table_properties.commit_num_retries(), 10); - assert_eq!(table_properties.commit_max_retry_wait_ms(), 20); - assert_eq!( - table_properties.write_format_default(), - DataFileFormat::Avro - ); - assert_eq!(table_properties.write_target_file_size_bytes(), 512); - assert!(!table_properties.gc_enabled()); + let table_properties = TableProperties::try_from(&props).unwrap(); + assert_eq!(table_properties.commit_num_retries, 10); + assert_eq!(table_properties.commit_max_retry_wait_ms, 20); + assert_eq!(table_properties.write_format_default, "avro".to_string()); + assert_eq!(table_properties.write_target_file_size_bytes, 512); + assert!(!table_properties.gc_enabled); } #[test] - fn test_parsed_table_properties_invalid() { + fn test_table_properties_invalid() { let invalid_retries = HashMap::from([( TableProperties::PROPERTY_COMMIT_NUM_RETRIES.to_string(), "abc".to_string(), )]); - let table_properties = parse(&invalid_retries).unwrap_err(); + let table_properties = TableProperties::try_from(&invalid_retries).unwrap_err(); assert!( table_properties.to_string().contains( "Invalid value for commit.retry.num-retries: invalid digit found in string" @@ -1709,7 +657,7 @@ mod tests { TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS.to_string(), "abc".to_string(), )]); - let table_properties = parse(&invalid_min_wait).unwrap_err(); + let table_properties = TableProperties::try_from(&invalid_min_wait).unwrap_err(); assert!( table_properties.to_string().contains( "Invalid value for commit.retry.min-wait-ms: invalid digit found in string" @@ -1720,7 +668,7 @@ mod tests { TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS.to_string(), "abc".to_string(), )]); - let table_properties = parse(&invalid_max_wait).unwrap_err(); + let table_properties = TableProperties::try_from(&invalid_max_wait).unwrap_err(); assert!( table_properties.to_string().contains( "Invalid value for commit.retry.max-wait-ms: invalid digit found in string" @@ -1731,7 +679,7 @@ mod tests { TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES.to_string(), "abc".to_string(), )]); - let table_properties = parse(&invalid_target_size).unwrap_err(); + let table_properties = TableProperties::try_from(&invalid_target_size).unwrap_err(); assert!(table_properties.to_string().contains( "Invalid value for write.target-file-size-bytes: invalid digit found in string" )); @@ -1740,7 +688,7 @@ mod tests { TableProperties::PROPERTY_GC_ENABLED.to_string(), "notabool".to_string(), )]); - let table_properties = parse(&invalid_gc_enabled).unwrap_err(); + let table_properties = TableProperties::try_from(&invalid_gc_enabled).unwrap_err(); assert!( table_properties .to_string() @@ -1749,7 +697,7 @@ mod tests { } #[test] - fn test_parsed_table_properties_compression_invalid_rejected() { + fn test_table_properties_compression_invalid_rejected() { let invalid_codecs = ["lz4", "zstd", "snappy"]; for codec in invalid_codecs { @@ -1757,7 +705,7 @@ mod tests { TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), codec.to_string(), )]); - let err = parse(&props).unwrap_err(); + let err = TableProperties::try_from(&props).unwrap_err(); let err_msg = err.to_string(); assert!( err_msg.contains(&format!("Invalid metadata compression codec: {codec}")), @@ -1778,7 +726,7 @@ mod tests { "none".to_string(), )]); assert_eq!( - parse(&props).unwrap().metadata_compression_codec(), + parse_metadata_file_compression(&props).unwrap(), CompressionCodec::None ); @@ -1788,7 +736,7 @@ mod tests { "".to_string(), )]); assert_eq!( - parse(&props).unwrap().metadata_compression_codec(), + parse_metadata_file_compression(&props).unwrap(), CompressionCodec::None ); @@ -1798,7 +746,7 @@ mod tests { "gzip".to_string(), )]); assert_eq!( - parse(&props).unwrap().metadata_compression_codec(), + parse_metadata_file_compression(&props).unwrap(), CompressionCodec::gzip_default() ); @@ -1808,7 +756,7 @@ mod tests { "NONE".to_string(), )]); assert_eq!( - parse(&props).unwrap().metadata_compression_codec(), + parse_metadata_file_compression(&props).unwrap(), CompressionCodec::None ); @@ -1818,7 +766,7 @@ mod tests { "GZIP".to_string(), )]); assert_eq!( - parse(&props).unwrap().metadata_compression_codec(), + parse_metadata_file_compression(&props).unwrap(), CompressionCodec::gzip_default() ); @@ -1828,14 +776,14 @@ mod tests { "GzIp".to_string(), )]); assert_eq!( - parse(&props).unwrap().metadata_compression_codec(), + parse_metadata_file_compression(&props).unwrap(), CompressionCodec::gzip_default() ); // Test default when property is missing let props = HashMap::new(); assert_eq!( - parse(&props).unwrap().metadata_compression_codec(), + parse_metadata_file_compression(&props).unwrap(), CompressionCodec::None ); } @@ -1849,7 +797,7 @@ mod tests { TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), codec.to_string(), )]); - let err = parse(&props).unwrap_err(); + let err = parse_metadata_file_compression(&props).unwrap_err(); let err_msg = err.to_string(); assert!( err_msg.contains("Invalid metadata compression codec"), @@ -1865,8 +813,8 @@ mod tests { #[test] fn test_cdc_disabled_by_default() { let props = HashMap::new(); - let tp = parse(&props).unwrap(); - assert!(!tp.cdc_enabled()); + let tp = TableProperties::try_from(&props).unwrap(); + assert!(!tp.cdc_enabled); } #[test] @@ -1875,11 +823,11 @@ mod tests { TableProperties::PROPERTY_PARQUET_CDC_ENABLED.to_string(), "true".to_string(), )]); - let tp = parse(&props).unwrap(); - assert!(tp.cdc_enabled()); - assert_eq!(tp.cdc_min_chunk_size(), 256 * 1024); - assert_eq!(tp.cdc_max_chunk_size(), 1024 * 1024); - assert_eq!(tp.cdc_norm_level(), 0); + let tp = TableProperties::try_from(&props).unwrap(); + assert!(tp.cdc_enabled); + assert_eq!(tp.cdc_min_chunk_size, 256 * 1024); + assert_eq!(tp.cdc_max_chunk_size, 1024 * 1024); + assert_eq!(tp.cdc_norm_level, 0); } #[test] @@ -1888,8 +836,8 @@ mod tests { TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE.to_string(), "262144".to_string(), )]); - let tp = parse(&props).unwrap(); - assert!(!tp.cdc_enabled()); + let tp = TableProperties::try_from(&props).unwrap(); + assert!(!tp.cdc_enabled); } #[test] @@ -1912,11 +860,11 @@ mod tests { "1".to_string(), ), ]); - let tp = parse(&props).unwrap(); - assert!(tp.cdc_enabled()); - assert_eq!(tp.cdc_min_chunk_size(), 200000); - assert_eq!(tp.cdc_max_chunk_size(), 900000); - assert_eq!(tp.cdc_norm_level(), 1); + let tp = TableProperties::try_from(&props).unwrap(); + assert!(tp.cdc_enabled); + assert_eq!(tp.cdc_min_chunk_size, 200000); + assert_eq!(tp.cdc_max_chunk_size, 900000); + assert_eq!(tp.cdc_norm_level, 1); } #[test] @@ -1931,11 +879,11 @@ mod tests { "2".to_string(), ), ]); - let tp = parse(&props).unwrap(); - assert!(tp.cdc_enabled()); - assert_eq!(tp.cdc_min_chunk_size(), 256 * 1024); - assert_eq!(tp.cdc_max_chunk_size(), 1024 * 1024); - assert_eq!(tp.cdc_norm_level(), 2); + let tp = TableProperties::try_from(&props).unwrap(); + assert!(tp.cdc_enabled); + assert_eq!(tp.cdc_min_chunk_size, 256 * 1024); + assert_eq!(tp.cdc_max_chunk_size, 1024 * 1024); + assert_eq!(tp.cdc_norm_level, 2); } #[test] @@ -1950,8 +898,8 @@ mod tests { "-2".to_string(), ), ]); - let tp = parse(&props).unwrap(); - assert_eq!(tp.cdc_norm_level(), -2); + let tp = TableProperties::try_from(&props).unwrap(); + assert_eq!(tp.cdc_norm_level, -2); } #[test] @@ -1966,7 +914,7 @@ mod tests { "not_a_number".to_string(), ), ]); - let err = parse(&props).unwrap_err(); + let err = TableProperties::try_from(&props).unwrap_err(); assert!( err.to_string().contains( "Invalid value for write.parquet.content-defined-chunking.min-chunk-size" @@ -1986,7 +934,7 @@ mod tests { "not_a_number".to_string(), ), ]); - let err = parse(&props).unwrap_err(); + let err = TableProperties::try_from(&props).unwrap_err(); assert!( err.to_string() .contains("Invalid value for write.parquet.content-defined-chunking.norm-level") @@ -1996,7 +944,7 @@ mod tests { #[test] fn test_cdc_no_properties() { let props = HashMap::from([("some.other.property".to_string(), "value".to_string())]); - let tp = parse(&props).unwrap(); - assert!(!tp.cdc_enabled()); + let tp = TableProperties::try_from(&props).unwrap(); + assert!(!tp.cdc_enabled); } } diff --git a/crates/iceberg/src/transaction/expire_snapshots.rs b/crates/iceberg/src/transaction/expire_snapshots.rs index 5178cb6bc8..0e7cdce10c 100644 --- a/crates/iceberg/src/transaction/expire_snapshots.rs +++ b/crates/iceberg/src/transaction/expire_snapshots.rs @@ -100,6 +100,7 @@ impl ExpireSnapshotsAction { /// Resolves the snapshots and refs to remove, following Java `RemoveSnapshots.internalApply`. fn plan(&self, table: &Table, properties: &ParsedTableProperties) -> Result { + let history = properties.history(); // Matches Java `RemoveSnapshots.retainLast`, which requires at least one snapshot. if self.retain_last == Some(0) { return Err(Error::new( @@ -115,10 +116,8 @@ impl ExpireSnapshotsAction { // days) the age path always runs, so even an explicit-id-only call applies the default cutoff. let default_cutoff = self .older_than_ms - .unwrap_or_else(|| now.saturating_sub(properties.max_snapshot_age_ms())); - let default_min_to_keep = self - .retain_last - .unwrap_or(properties.min_snapshots_to_keep()); + .unwrap_or_else(|| now.saturating_sub(history.max_snapshot_age_ms())); + let default_min_to_keep = self.retain_last.unwrap_or(history.min_snapshots_to_keep()); // Ref aging: `main` is always kept; any other ref whose head is older than its // `max_ref_age_ms` (defaulting to `history.expire.max-ref-age-ms`) is dropped, like Java's @@ -127,7 +126,7 @@ impl ExpireSnapshotsAction { let mut retained_refs: Vec<&SnapshotReference> = vec![]; for (ref_name, snapshot_ref) in &metadata.refs { if ref_name == MAIN_BRANCH - || !Self::ref_aged_out(metadata, snapshot_ref, now, properties.max_ref_age_ms()) + || !Self::ref_aged_out(metadata, snapshot_ref, now, history.max_ref_age_ms()) { retained_refs.push(snapshot_ref); } else { @@ -304,7 +303,7 @@ impl TransactionAction for ExpireSnapshotsAction { let properties = metadata.parsed_table_properties()?; // Expiring metadata defeats a user's explicit decision to disable GC (Java refuses too). - if !properties.gc_enabled() { + if !properties.history().gc_enabled() { return Err(Error::new( ErrorKind::DataInvalid, "Cannot expire snapshots: gc.enabled is false", diff --git a/crates/iceberg/src/transaction/mod.rs b/crates/iceberg/src/transaction/mod.rs index 69015d4002..19cc9f159f 100644 --- a/crates/iceberg/src/transaction/mod.rs +++ b/crates/iceberg/src/transaction/mod.rs @@ -181,7 +181,7 @@ impl Transaction { let table_props = self.table.metadata().parsed_table_properties()?; // TODO(https://github.com/apache/iceberg-rust/issues/2034): remove once encrypted writes are supported - if table_props.encryption_key_id().is_some() { + if table_props.encryption().key_id().is_some() { return Err(Error::new( ErrorKind::FeatureUnsupported, "Cannot commit to an encrypted table: encrypted writes are not yet supported", @@ -204,13 +204,12 @@ impl Transaction { } fn build_backoff(props: ParsedTableProperties) -> Result { + let commit = props.commit(); Ok(ExponentialBuilder::new() - .with_min_delay(Duration::from_millis(props.commit_min_retry_wait_ms())) - .with_max_delay(Duration::from_millis(props.commit_max_retry_wait_ms())) - .with_total_delay(Some(Duration::from_millis( - props.commit_total_retry_timeout_ms(), - ))) - .with_max_times(props.commit_num_retries()) + .with_min_delay(Duration::from_millis(commit.min_retry_wait_ms())) + .with_max_delay(Duration::from_millis(commit.max_retry_wait_ms())) + .with_total_delay(Some(Duration::from_millis(commit.total_retry_timeout_ms()))) + .with_max_times(commit.num_retries()) .with_factor(2.0) .build()) } diff --git a/crates/iceberg/src/writer/file_writer/parquet_writer.rs b/crates/iceberg/src/writer/file_writer/parquet_writer.rs index 53f531910a..a70ce5ac71 100644 --- a/crates/iceberg/src/writer/file_writer/parquet_writer.rs +++ b/crates/iceberg/src/writer/file_writer/parquet_writer.rs @@ -100,11 +100,14 @@ impl ParquetWriterBuilder { table_props: &ParsedTableProperties, schema: SchemaRef, ) -> Self { - let cdc = table_props.cdc_enabled().then_some(CdcOptions { - min_chunk_size: table_props.cdc_min_chunk_size(), - max_chunk_size: table_props.cdc_max_chunk_size(), - norm_level: table_props.cdc_norm_level(), - }); + let parquet = table_props.parquet(); + let cdc = parquet + .content_defined_chunking_enabled() + .then_some(CdcOptions { + min_chunk_size: parquet.content_defined_chunking_min_chunk_size(), + max_chunk_size: parquet.content_defined_chunking_max_chunk_size(), + norm_level: parquet.content_defined_chunking_norm_level(), + }); Self::from_cdc_options(cdc, schema) } diff --git a/crates/integrations/datafusion/src/physical_plan/write.rs b/crates/integrations/datafusion/src/physical_plan/write.rs index 312f525256..4e7dffb2bc 100644 --- a/crates/integrations/datafusion/src/physical_plan/write.rs +++ b/crates/integrations/datafusion/src/physical_plan/write.rs @@ -209,7 +209,8 @@ impl ExecutionPlan for IcebergWriteExec { .map_err(to_datafusion_error)?; // Check data file format - let file_format = table_props.write_format_default(); + let write_properties = table_props.write(); + let file_format = write_properties.format_default(); if file_format != DataFileFormat::Parquet { return Err(to_datafusion_error(Error::new( ErrorKind::FeatureUnsupported, @@ -225,7 +226,7 @@ impl ExecutionPlan for IcebergWriteExec { self.table.metadata().current_schema().clone(), ) .with_match_mode(FieldMatchMode::Name); - let target_file_size = table_props.write_target_file_size_bytes(); + let target_file_size = write_properties.target_file_size_bytes(); let file_io = self.table.file_io().clone(); // todo location_gen and file_name_gen should be configurable @@ -244,7 +245,7 @@ impl ExecutionPlan for IcebergWriteExec { let data_file_writer_builder = DataFileWriterBuilder::new(rolling_writer_builder); // Create TaskWriter - let fanout_enabled = table_props.write_datafusion_fanout_enabled(); + let fanout_enabled = write_properties.datafusion_fanout_enabled(); let schema = self.table.metadata().current_schema().clone(); let partition_spec = self.table.metadata().default_partition_spec().clone(); let task_writer = TaskWriter::try_new( diff --git a/crates/property-macro/src/lib.rs b/crates/property-macro/src/lib.rs index 5672d6450f..56b419f9de 100644 --- a/crates/property-macro/src/lib.rs +++ b/crates/property-macro/src/lib.rs @@ -27,27 +27,28 @@ use syn::{ /// Derive parsing, defaults, JSON serialization, and getters for a typed property map. /// -/// Each field must declare the table-property key and its default: +/// Leaf fields must declare the table-property key and its default: /// /// ```ignore /// #[derive(Properties)] /// struct Properties { -/// #[key(TableProperties::DEFAULT_FILE_FORMAT)] -/// #[default(DataFileFormat::Parquet)] +/// #[key = "write.format.default"] +/// #[default = DataFileFormat::Parquet] /// #[doc = "Default file format"] /// write_format_default: DataFileFormat, /// } /// ``` /// /// `prefix` captures a family of properties in a `HashMap`, keyed by the suffix after -/// the declared prefix. `parse_with` may be used for exact-key property types that do not implement +/// the declared prefix. `nested` embeds another `Properties` struct while keeping its serialized +/// property map flat. `parse_with` may be used for exact-key property types that do not implement /// `FromStr` or need validation. `serialize_with` supplies their string representation in JSON. /// Optional fields are omitted from JSON when they are `None`. Fields must implement `Clone`; they /// also need `FromStr` and `ToString` unless the relevant custom parsing or serialization attribute /// is supplied. #[proc_macro_derive( Properties, - attributes(key, prefix, default, parse_with, serialize_with) + attributes(key, prefix, nested, default, parse_with, serialize_with) )] pub fn derive_properties(input: TokenStream) -> TokenStream { let input = parse_macro_input!(input as DeriveInput); @@ -64,7 +65,8 @@ struct PropertyField { docs: Vec, key: Option, prefix: Option, - default: Expr, + nested: bool, + default: Option, parse_with: Option, serialize_with: Option, option_inner_type: Option, @@ -98,8 +100,12 @@ fn expand_properties(input: DeriveInput) -> syn::Result { let defaults = fields.iter().map(|field| { let ident = &field.ident; - let default = &field.default; - quote!(#ident: #default) + if field.nested { + quote!(#ident: ::std::default::Default::default()) + } else { + let default = field.default.as_ref().expect("leaf fields have defaults"); + quote!(#ident: #default) + } }); let accessors = fields.iter().map(|field| { @@ -130,7 +136,7 @@ fn expand_properties(input: DeriveInput) -> syn::Result { let parses = fields.iter().map(parse_field); - let serializes = fields.iter().map(serialize_field); + let property_writes = fields.iter().map(write_field); Ok(quote! { impl ::std::default::Default for #struct_name { @@ -150,6 +156,27 @@ fn expand_properties(input: DeriveInput) -> syn::Result { }) } + pub(crate) fn write_properties( + &self, + properties: &mut ::std::collections::HashMap< + ::std::string::String, + ::std::string::String, + >, + ) { + #(#property_writes)* + } + + fn to_properties( + &self, + ) -> ::std::collections::HashMap< + ::std::string::String, + ::std::string::String, + > { + let mut properties = ::std::collections::HashMap::new(); + self.write_properties(&mut properties); + properties + } + #(#accessors)* } @@ -158,11 +185,7 @@ fn expand_properties(input: DeriveInput) -> syn::Result { where S: ::serde::Serializer, { - use ::serde::ser::SerializeMap as _; - - let mut map = serializer.serialize_map(None)?; - #(#serializes)* - map.end() + ::serde::Serialize::serialize(&self.to_properties(), serializer) } } @@ -185,15 +208,26 @@ fn parse_property_field(field: &Field) -> syn::Result { .ok_or_else(|| Error::new_spanned(field, "Properties fields must be named"))?; let key = attribute_expression_value(&field.attrs, "key")?; let prefix = attribute_expression_value(&field.attrs, "prefix")?; - if key.is_some() == prefix.is_some() { + let nested = marker_attribute(&field.attrs, "nested")?; + if usize::from(key.is_some()) + usize::from(prefix.is_some()) + usize::from(nested) != 1 { + return Err(Error::new_spanned( + field, + "Properties fields must declare exactly one of #[key(...)], #[prefix(...)], or #[nested]", + )); + } + let default = attribute_expression_value(&field.attrs, "default")?; + if nested && default.is_some() { + return Err(Error::new_spanned( + field, + "#[nested] fields use the nested type's Default implementation and cannot declare #[default(...)]", + )); + } + if !nested && default.is_none() { return Err(Error::new_spanned( field, - "Properties fields must declare exactly one of #[key(...)] or #[prefix(...)]", + "Properties leaf fields must declare #[default(...)]", )); } - let default = attribute_expression_value(&field.attrs, "default")?.ok_or_else(|| { - Error::new_spanned(field, "Properties fields must declare #[default(...)]") - })?; let map_value_type = map_value_type(&field.ty); if prefix.is_some() && map_value_type.is_none() { @@ -202,13 +236,13 @@ fn parse_property_field(field: &Field) -> syn::Result { "#[prefix(...)] fields must have type HashMap", )); } - if prefix.is_some() + if (prefix.is_some() || nested) && (attribute_path_value(&field.attrs, "parse_with")?.is_some() || attribute_path_value(&field.attrs, "serialize_with")?.is_some()) { return Err(Error::new_spanned( field, - "#[prefix(...)] fields do not support parse_with or serialize_with", + "#[prefix(...)] and #[nested] fields do not support parse_with or serialize_with", )); } @@ -223,6 +257,7 @@ fn parse_property_field(field: &Field) -> syn::Result { .collect(), key, prefix, + nested, default, parse_with: attribute_path_value(&field.attrs, "parse_with")?, serialize_with: attribute_path_value(&field.attrs, "serialize_with")?, @@ -231,6 +266,20 @@ fn parse_property_field(field: &Field) -> syn::Result { }) } +fn marker_attribute(attributes: &[Attribute], name: &str) -> syn::Result { + let Some(attribute) = find_attribute(attributes, name)? else { + return Ok(false); + }; + + match &attribute.meta { + Meta::Path(_) => Ok(true), + _ => Err(Error::new_spanned( + attribute, + format!("{name} must use the form #[{name}]"), + )), + } +} + fn attribute_expression_value(attributes: &[Attribute], name: &str) -> syn::Result> { let Some(attribute) = find_attribute(attributes, name)? else { return Ok(None); @@ -279,7 +328,12 @@ fn find_attribute<'a>( fn parse_field(field: &PropertyField) -> TokenStream2 { let ident = &field.ident; - let default = &field.default; + if field.nested { + let ty = &field.ty; + return quote!(#ident: <#ty>::from_properties(properties)?); + } + + let default = field.default.as_ref().expect("leaf fields have defaults"); if let Some(prefix) = &field.prefix { let value_type = field @@ -378,13 +432,19 @@ fn map_value_type(ty: &Type) -> Option { Some(value_type.clone()) } -fn serialize_field(field: &PropertyField) -> TokenStream2 { +fn write_field(field: &PropertyField) -> TokenStream2 { let ident = &field.ident; + if field.nested { + return quote! { + self.#ident.write_properties(properties); + }; + } + if let Some(prefix) = &field.prefix { return quote! { for (suffix, value) in &self.#ident { let key = format!("{}{}", #prefix, suffix); - map.serialize_entry(&key, &::std::string::ToString::to_string(value))?; + properties.insert(key, ::std::string::ToString::to_string(value)); } }; } @@ -399,7 +459,7 @@ fn serialize_field(field: &PropertyField) -> TokenStream2 { }; quote! { if self.#ident.is_some() { - map.serialize_entry(#key, &#value)?; + properties.insert((#key).to_string(), #value); } } } else { @@ -408,7 +468,7 @@ fn serialize_field(field: &PropertyField) -> TokenStream2 { None => quote!(::std::string::ToString::to_string(&self.#ident)), }; quote! { - map.serialize_entry(#key, &#value)?; + properties.insert((#key).to_string(), #value); } } } diff --git a/crates/property-macro/tests/properties.rs b/crates/property-macro/tests/properties.rs index 7f4fa14663..22aea51a56 100644 --- a/crates/property-macro/tests/properties.rs +++ b/crates/property-macro/tests/properties.rs @@ -58,3 +58,30 @@ fn generates_defaults_getters_modifiers_and_serde() { assert_eq!(decoded.owner(), Some("iceberg".to_string())); assert_eq!(decoded.column_fpp()["id"], 0.01); } + +#[derive(Clone, Debug, Properties)] +struct CommitProperties { + #[key = "commit.retry.num-retries"] + #[default = 4] + #[doc = "Number of times to retry a commit before failing."] + num_retries: u64, +} + +#[derive(Debug, Properties)] +struct NestedProperties { + #[nested] + #[doc = "Commit behavior properties."] + commit: CommitProperties, +} + +#[test] +fn nested_properties_use_a_flat_property_map() { + let properties = + NestedProperties::default().with_commit(CommitProperties::default().with_num_retries(9)); + + let json = serde_json::to_value(&properties).unwrap(); + assert_eq!(json["commit.retry.num-retries"], "9"); + + let decoded: NestedProperties = serde_json::from_value(json).unwrap(); + assert_eq!(decoded.commit().num_retries(), 9); +} From 80d19aa502acdca72c16fa65216b443e9db5f9e0 Mon Sep 17 00:00:00 2001 From: Renjie Liu Date: Wed, 5 Aug 2026 10:27:16 +0800 Subject: [PATCH 04/14] PUb fields --- crates/iceberg/public-api.txt | 416 ++++++----------- crates/iceberg/src/catalog/utils.rs | 2 +- crates/iceberg/src/encryption/manager.rs | 6 +- crates/iceberg/src/spec/parsed_table_prop.rs | 422 ++++++++++-------- crates/iceberg/src/spec/table_metadata.rs | 8 +- .../src/transaction/expire_snapshots.rs | 10 +- crates/iceberg/src/transaction/mod.rs | 12 +- .../src/writer/file_writer/parquet_writer.rs | 10 +- .../datafusion/src/physical_plan/write.rs | 8 +- crates/property-macro/src/lib.rs | 46 +- crates/property-macro/tests/properties.rs | 39 +- 11 files changed, 422 insertions(+), 557 deletions(-) diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index 4f9fe5df5f..aef65ac0c2 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -2245,33 +2245,19 @@ pub fn iceberg::spec::NestedField::serialize<__S>(&self, __serializer: __S) -> c impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::NestedField pub fn iceberg::spec::NestedField::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> pub struct iceberg::spec::ParsedTableProperties -impl iceberg::spec::ParsedTableProperties -pub fn iceberg::spec::ParsedTableProperties::avro(&self) -> iceberg::spec::TableAvroProperties -pub fn iceberg::spec::ParsedTableProperties::commit(&self) -> iceberg::spec::TableCommitProperties -pub fn iceberg::spec::ParsedTableProperties::compatibility(&self) -> iceberg::spec::TableCompatibilityProperties -pub fn iceberg::spec::ParsedTableProperties::encryption(&self) -> iceberg::spec::TableEncryptionProperties -pub fn iceberg::spec::ParsedTableProperties::general(&self) -> iceberg::spec::TableGeneralProperties -pub fn iceberg::spec::ParsedTableProperties::history(&self) -> iceberg::spec::TableHistoryProperties -pub fn iceberg::spec::ParsedTableProperties::manifest(&self) -> iceberg::spec::TableManifestProperties -pub fn iceberg::spec::ParsedTableProperties::metadata(&self) -> iceberg::spec::TableMetadataProperties -pub fn iceberg::spec::ParsedTableProperties::orc(&self) -> iceberg::spec::TableOrcProperties -pub fn iceberg::spec::ParsedTableProperties::parquet(&self) -> iceberg::spec::TableParquetProperties -pub fn iceberg::spec::ParsedTableProperties::read(&self) -> iceberg::spec::TableReadProperties -pub fn iceberg::spec::ParsedTableProperties::row_level(&self) -> iceberg::spec::TableRowLevelProperties -pub fn iceberg::spec::ParsedTableProperties::with_avro(self, value: iceberg::spec::TableAvroProperties) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_commit(self, value: iceberg::spec::TableCommitProperties) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_compatibility(self, value: iceberg::spec::TableCompatibilityProperties) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_encryption(self, value: iceberg::spec::TableEncryptionProperties) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_general(self, value: iceberg::spec::TableGeneralProperties) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_history(self, value: iceberg::spec::TableHistoryProperties) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_manifest(self, value: iceberg::spec::TableManifestProperties) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_metadata(self, value: iceberg::spec::TableMetadataProperties) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_orc(self, value: iceberg::spec::TableOrcProperties) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_parquet(self, value: iceberg::spec::TableParquetProperties) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_read(self, value: iceberg::spec::TableReadProperties) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_row_level(self, value: iceberg::spec::TableRowLevelProperties) -> Self -pub fn iceberg::spec::ParsedTableProperties::with_write(self, value: iceberg::spec::TableWriteProperties) -> Self -pub fn iceberg::spec::ParsedTableProperties::write(&self) -> iceberg::spec::TableWriteProperties +pub iceberg::spec::ParsedTableProperties::avro: iceberg::spec::TableAvroProperties +pub iceberg::spec::ParsedTableProperties::commit: iceberg::spec::TableCommitProperties +pub iceberg::spec::ParsedTableProperties::compatibility: iceberg::spec::TableCompatibilityProperties +pub iceberg::spec::ParsedTableProperties::encryption: iceberg::spec::TableEncryptionProperties +pub iceberg::spec::ParsedTableProperties::general: iceberg::spec::TableGeneralProperties +pub iceberg::spec::ParsedTableProperties::history: iceberg::spec::TableHistoryProperties +pub iceberg::spec::ParsedTableProperties::manifest: iceberg::spec::TableManifestProperties +pub iceberg::spec::ParsedTableProperties::metadata: iceberg::spec::TableMetadataProperties +pub iceberg::spec::ParsedTableProperties::orc: iceberg::spec::TableOrcProperties +pub iceberg::spec::ParsedTableProperties::parquet: iceberg::spec::TableParquetProperties +pub iceberg::spec::ParsedTableProperties::read: iceberg::spec::TableReadProperties +pub iceberg::spec::ParsedTableProperties::row_level: iceberg::spec::TableRowLevelProperties +pub iceberg::spec::ParsedTableProperties::write: iceberg::spec::TableWriteProperties impl core::clone::Clone for iceberg::spec::ParsedTableProperties pub fn iceberg::spec::ParsedTableProperties::clone(&self) -> iceberg::spec::ParsedTableProperties impl core::default::Default for iceberg::spec::ParsedTableProperties @@ -2664,15 +2650,10 @@ pub fn iceberg::spec::Summary::serialize<__S>(&self, __serializer: __S) -> core: impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::Summary pub fn iceberg::spec::Summary::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> pub struct iceberg::spec::TableAvroProperties -impl iceberg::spec::TableAvroProperties -pub fn iceberg::spec::TableAvroProperties::compression(&self) -> alloc::string::String -pub fn iceberg::spec::TableAvroProperties::compression_level(&self) -> core::option::Option -pub fn iceberg::spec::TableAvroProperties::delete_compression(&self) -> alloc::string::String -pub fn iceberg::spec::TableAvroProperties::delete_compression_level(&self) -> core::option::Option -pub fn iceberg::spec::TableAvroProperties::with_compression(self, value: alloc::string::String) -> Self -pub fn iceberg::spec::TableAvroProperties::with_compression_level(self, value: core::option::Option) -> Self -pub fn iceberg::spec::TableAvroProperties::with_delete_compression(self, value: alloc::string::String) -> Self -pub fn iceberg::spec::TableAvroProperties::with_delete_compression_level(self, value: core::option::Option) -> Self +pub iceberg::spec::TableAvroProperties::compression: alloc::string::String +pub iceberg::spec::TableAvroProperties::compression_level: core::option::Option +pub iceberg::spec::TableAvroProperties::delete_compression: alloc::string::String +pub iceberg::spec::TableAvroProperties::delete_compression_level: core::option::Option impl core::clone::Clone for iceberg::spec::TableAvroProperties pub fn iceberg::spec::TableAvroProperties::clone(&self) -> iceberg::spec::TableAvroProperties impl core::default::Default for iceberg::spec::TableAvroProperties @@ -2684,23 +2665,14 @@ pub fn iceberg::spec::TableAvroProperties::serialize(&self, serializer: S) -> impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::TableAvroProperties pub fn iceberg::spec::TableAvroProperties::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> pub struct iceberg::spec::TableCommitProperties -impl iceberg::spec::TableCommitProperties -pub fn iceberg::spec::TableCommitProperties::max_retry_wait_ms(&self) -> u64 -pub fn iceberg::spec::TableCommitProperties::min_retry_wait_ms(&self) -> u64 -pub fn iceberg::spec::TableCommitProperties::num_retries(&self) -> usize -pub fn iceberg::spec::TableCommitProperties::num_status_checks(&self) -> usize -pub fn iceberg::spec::TableCommitProperties::status_checks_max_wait_ms(&self) -> u64 -pub fn iceberg::spec::TableCommitProperties::status_checks_min_wait_ms(&self) -> u64 -pub fn iceberg::spec::TableCommitProperties::status_checks_total_wait_ms(&self) -> u64 -pub fn iceberg::spec::TableCommitProperties::total_retry_timeout_ms(&self) -> u64 -pub fn iceberg::spec::TableCommitProperties::with_max_retry_wait_ms(self, value: u64) -> Self -pub fn iceberg::spec::TableCommitProperties::with_min_retry_wait_ms(self, value: u64) -> Self -pub fn iceberg::spec::TableCommitProperties::with_num_retries(self, value: usize) -> Self -pub fn iceberg::spec::TableCommitProperties::with_num_status_checks(self, value: usize) -> Self -pub fn iceberg::spec::TableCommitProperties::with_status_checks_max_wait_ms(self, value: u64) -> Self -pub fn iceberg::spec::TableCommitProperties::with_status_checks_min_wait_ms(self, value: u64) -> Self -pub fn iceberg::spec::TableCommitProperties::with_status_checks_total_wait_ms(self, value: u64) -> Self -pub fn iceberg::spec::TableCommitProperties::with_total_retry_timeout_ms(self, value: u64) -> Self +pub iceberg::spec::TableCommitProperties::max_retry_wait_ms: u64 +pub iceberg::spec::TableCommitProperties::min_retry_wait_ms: u64 +pub iceberg::spec::TableCommitProperties::num_retries: usize +pub iceberg::spec::TableCommitProperties::num_status_checks: usize +pub iceberg::spec::TableCommitProperties::status_checks_max_wait_ms: u64 +pub iceberg::spec::TableCommitProperties::status_checks_min_wait_ms: u64 +pub iceberg::spec::TableCommitProperties::status_checks_total_wait_ms: u64 +pub iceberg::spec::TableCommitProperties::total_retry_timeout_ms: u64 impl core::clone::Clone for iceberg::spec::TableCommitProperties pub fn iceberg::spec::TableCommitProperties::clone(&self) -> iceberg::spec::TableCommitProperties impl core::default::Default for iceberg::spec::TableCommitProperties @@ -2712,21 +2684,13 @@ pub fn iceberg::spec::TableCommitProperties::serialize(&self, serializer: S) impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::TableCommitProperties pub fn iceberg::spec::TableCommitProperties::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> pub struct iceberg::spec::TableCompatibilityProperties -impl iceberg::spec::TableCompatibilityProperties -pub fn iceberg::spec::TableCompatibilityProperties::engine_hive_enabled(&self) -> bool -pub fn iceberg::spec::TableCompatibilityProperties::hive_lock_enabled(&self) -> bool -pub fn iceberg::spec::TableCompatibilityProperties::snapshot_id_inheritance_enabled(&self) -> bool -pub fn iceberg::spec::TableCompatibilityProperties::spark_write_accept_any_schema(&self) -> bool -pub fn iceberg::spec::TableCompatibilityProperties::spark_write_advisory_partition_size_bytes(&self) -> core::option::Option -pub fn iceberg::spec::TableCompatibilityProperties::spark_write_auto_schema_evolution(&self) -> bool -pub fn iceberg::spec::TableCompatibilityProperties::spark_write_partitioned_fanout_enabled(&self) -> bool -pub fn iceberg::spec::TableCompatibilityProperties::with_engine_hive_enabled(self, value: bool) -> Self -pub fn iceberg::spec::TableCompatibilityProperties::with_hive_lock_enabled(self, value: bool) -> Self -pub fn iceberg::spec::TableCompatibilityProperties::with_snapshot_id_inheritance_enabled(self, value: bool) -> Self -pub fn iceberg::spec::TableCompatibilityProperties::with_spark_write_accept_any_schema(self, value: bool) -> Self -pub fn iceberg::spec::TableCompatibilityProperties::with_spark_write_advisory_partition_size_bytes(self, value: core::option::Option) -> Self -pub fn iceberg::spec::TableCompatibilityProperties::with_spark_write_auto_schema_evolution(self, value: bool) -> Self -pub fn iceberg::spec::TableCompatibilityProperties::with_spark_write_partitioned_fanout_enabled(self, value: bool) -> Self +pub iceberg::spec::TableCompatibilityProperties::engine_hive_enabled: bool +pub iceberg::spec::TableCompatibilityProperties::hive_lock_enabled: bool +pub iceberg::spec::TableCompatibilityProperties::snapshot_id_inheritance_enabled: bool +pub iceberg::spec::TableCompatibilityProperties::spark_write_accept_any_schema: bool +pub iceberg::spec::TableCompatibilityProperties::spark_write_advisory_partition_size_bytes: core::option::Option +pub iceberg::spec::TableCompatibilityProperties::spark_write_auto_schema_evolution: bool +pub iceberg::spec::TableCompatibilityProperties::spark_write_partitioned_fanout_enabled: bool impl core::clone::Clone for iceberg::spec::TableCompatibilityProperties pub fn iceberg::spec::TableCompatibilityProperties::clone(&self) -> iceberg::spec::TableCompatibilityProperties impl core::default::Default for iceberg::spec::TableCompatibilityProperties @@ -2738,11 +2702,8 @@ pub fn iceberg::spec::TableCompatibilityProperties::serialize(&self, serializ impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::TableCompatibilityProperties pub fn iceberg::spec::TableCompatibilityProperties::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> pub struct iceberg::spec::TableEncryptionProperties -impl iceberg::spec::TableEncryptionProperties -pub fn iceberg::spec::TableEncryptionProperties::data_key_length(&self) -> usize -pub fn iceberg::spec::TableEncryptionProperties::key_id(&self) -> core::option::Option -pub fn iceberg::spec::TableEncryptionProperties::with_data_key_length(self, value: usize) -> Self -pub fn iceberg::spec::TableEncryptionProperties::with_key_id(self, value: core::option::Option) -> Self +pub iceberg::spec::TableEncryptionProperties::data_key_length: usize +pub iceberg::spec::TableEncryptionProperties::key_id: core::option::Option impl core::clone::Clone for iceberg::spec::TableEncryptionProperties pub fn iceberg::spec::TableEncryptionProperties::clone(&self) -> iceberg::spec::TableEncryptionProperties impl core::default::Default for iceberg::spec::TableEncryptionProperties @@ -2754,11 +2715,8 @@ pub fn iceberg::spec::TableEncryptionProperties::serialize(&self, serializer: impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::TableEncryptionProperties pub fn iceberg::spec::TableEncryptionProperties::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> pub struct iceberg::spec::TableGeneralProperties -impl iceberg::spec::TableGeneralProperties -pub fn iceberg::spec::TableGeneralProperties::comment(&self) -> core::option::Option -pub fn iceberg::spec::TableGeneralProperties::identifier_fields_rely(&self) -> bool -pub fn iceberg::spec::TableGeneralProperties::with_comment(self, value: core::option::Option) -> Self -pub fn iceberg::spec::TableGeneralProperties::with_identifier_fields_rely(self, value: bool) -> Self +pub iceberg::spec::TableGeneralProperties::comment: core::option::Option +pub iceberg::spec::TableGeneralProperties::identifier_fields_rely: bool impl core::clone::Clone for iceberg::spec::TableGeneralProperties pub fn iceberg::spec::TableGeneralProperties::clone(&self) -> iceberg::spec::TableGeneralProperties impl core::default::Default for iceberg::spec::TableGeneralProperties @@ -2770,15 +2728,10 @@ pub fn iceberg::spec::TableGeneralProperties::serialize(&self, serializer: S) impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::TableGeneralProperties pub fn iceberg::spec::TableGeneralProperties::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> pub struct iceberg::spec::TableHistoryProperties -impl iceberg::spec::TableHistoryProperties -pub fn iceberg::spec::TableHistoryProperties::gc_enabled(&self) -> bool -pub fn iceberg::spec::TableHistoryProperties::max_ref_age_ms(&self) -> i64 -pub fn iceberg::spec::TableHistoryProperties::max_snapshot_age_ms(&self) -> i64 -pub fn iceberg::spec::TableHistoryProperties::min_snapshots_to_keep(&self) -> usize -pub fn iceberg::spec::TableHistoryProperties::with_gc_enabled(self, value: bool) -> Self -pub fn iceberg::spec::TableHistoryProperties::with_max_ref_age_ms(self, value: i64) -> Self -pub fn iceberg::spec::TableHistoryProperties::with_max_snapshot_age_ms(self, value: i64) -> Self -pub fn iceberg::spec::TableHistoryProperties::with_min_snapshots_to_keep(self, value: usize) -> Self +pub iceberg::spec::TableHistoryProperties::gc_enabled: bool +pub iceberg::spec::TableHistoryProperties::max_ref_age_ms: i64 +pub iceberg::spec::TableHistoryProperties::max_snapshot_age_ms: i64 +pub iceberg::spec::TableHistoryProperties::min_snapshots_to_keep: usize impl core::clone::Clone for iceberg::spec::TableHistoryProperties pub fn iceberg::spec::TableHistoryProperties::clone(&self) -> iceberg::spec::TableHistoryProperties impl core::default::Default for iceberg::spec::TableHistoryProperties @@ -2790,19 +2743,12 @@ pub fn iceberg::spec::TableHistoryProperties::serialize(&self, serializer: S) impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::TableHistoryProperties pub fn iceberg::spec::TableHistoryProperties::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> pub struct iceberg::spec::TableManifestProperties -impl iceberg::spec::TableManifestProperties -pub fn iceberg::spec::TableManifestProperties::compression(&self) -> alloc::string::String -pub fn iceberg::spec::TableManifestProperties::compression_level(&self) -> core::option::Option -pub fn iceberg::spec::TableManifestProperties::lists_enabled(&self) -> bool -pub fn iceberg::spec::TableManifestProperties::merge_enabled(&self) -> bool -pub fn iceberg::spec::TableManifestProperties::min_merge_count(&self) -> usize -pub fn iceberg::spec::TableManifestProperties::target_size_bytes(&self) -> usize -pub fn iceberg::spec::TableManifestProperties::with_compression(self, value: alloc::string::String) -> Self -pub fn iceberg::spec::TableManifestProperties::with_compression_level(self, value: core::option::Option) -> Self -pub fn iceberg::spec::TableManifestProperties::with_lists_enabled(self, value: bool) -> Self -pub fn iceberg::spec::TableManifestProperties::with_merge_enabled(self, value: bool) -> Self -pub fn iceberg::spec::TableManifestProperties::with_min_merge_count(self, value: usize) -> Self -pub fn iceberg::spec::TableManifestProperties::with_target_size_bytes(self, value: usize) -> Self +pub iceberg::spec::TableManifestProperties::compression: alloc::string::String +pub iceberg::spec::TableManifestProperties::compression_level: core::option::Option +pub iceberg::spec::TableManifestProperties::lists_enabled: bool +pub iceberg::spec::TableManifestProperties::merge_enabled: bool +pub iceberg::spec::TableManifestProperties::min_merge_count: usize +pub iceberg::spec::TableManifestProperties::target_size_bytes: usize impl core::clone::Clone for iceberg::spec::TableManifestProperties pub fn iceberg::spec::TableManifestProperties::clone(&self) -> iceberg::spec::TableManifestProperties impl core::default::Default for iceberg::spec::TableManifestProperties @@ -2924,25 +2870,15 @@ pub fn iceberg::spec::TableMetadataBuilder::clone(&self) -> iceberg::spec::Table impl core::fmt::Debug for iceberg::spec::TableMetadataBuilder pub fn iceberg::spec::TableMetadataBuilder::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result pub struct iceberg::spec::TableMetadataProperties -impl iceberg::spec::TableMetadataProperties -pub fn iceberg::spec::TableMetadataProperties::compression_codec(&self) -> iceberg::compression::CompressionCodec -pub fn iceberg::spec::TableMetadataProperties::default_metrics_mode(&self) -> alloc::string::String -pub fn iceberg::spec::TableMetadataProperties::default_name_mapping(&self) -> core::option::Option -pub fn iceberg::spec::TableMetadataProperties::delete_after_commit_enabled(&self) -> bool -pub fn iceberg::spec::TableMetadataProperties::metrics_max_inferred_column_defaults(&self) -> usize -pub fn iceberg::spec::TableMetadataProperties::metrics_mode_by_column(&self) -> std::collections::hash::map::HashMap -pub fn iceberg::spec::TableMetadataProperties::partition_summary_limit(&self) -> u64 -pub fn iceberg::spec::TableMetadataProperties::path(&self) -> core::option::Option -pub fn iceberg::spec::TableMetadataProperties::previous_versions_max(&self) -> usize -pub fn iceberg::spec::TableMetadataProperties::with_compression_codec(self, value: iceberg::compression::CompressionCodec) -> Self -pub fn iceberg::spec::TableMetadataProperties::with_default_metrics_mode(self, value: alloc::string::String) -> Self -pub fn iceberg::spec::TableMetadataProperties::with_default_name_mapping(self, value: core::option::Option) -> Self -pub fn iceberg::spec::TableMetadataProperties::with_delete_after_commit_enabled(self, value: bool) -> Self -pub fn iceberg::spec::TableMetadataProperties::with_metrics_max_inferred_column_defaults(self, value: usize) -> Self -pub fn iceberg::spec::TableMetadataProperties::with_metrics_mode_by_column(self, value: std::collections::hash::map::HashMap) -> Self -pub fn iceberg::spec::TableMetadataProperties::with_partition_summary_limit(self, value: u64) -> Self -pub fn iceberg::spec::TableMetadataProperties::with_path(self, value: core::option::Option) -> Self -pub fn iceberg::spec::TableMetadataProperties::with_previous_versions_max(self, value: usize) -> Self +pub iceberg::spec::TableMetadataProperties::compression_codec: iceberg::compression::CompressionCodec +pub iceberg::spec::TableMetadataProperties::default_metrics_mode: alloc::string::String +pub iceberg::spec::TableMetadataProperties::default_name_mapping: core::option::Option +pub iceberg::spec::TableMetadataProperties::delete_after_commit_enabled: bool +pub iceberg::spec::TableMetadataProperties::metrics_max_inferred_column_defaults: usize +pub iceberg::spec::TableMetadataProperties::metrics_mode_by_column: std::collections::hash::map::HashMap +pub iceberg::spec::TableMetadataProperties::partition_summary_limit: u64 +pub iceberg::spec::TableMetadataProperties::path: core::option::Option +pub iceberg::spec::TableMetadataProperties::previous_versions_max: usize impl core::clone::Clone for iceberg::spec::TableMetadataProperties pub fn iceberg::spec::TableMetadataProperties::clone(&self) -> iceberg::spec::TableMetadataProperties impl core::default::Default for iceberg::spec::TableMetadataProperties @@ -2954,31 +2890,18 @@ pub fn iceberg::spec::TableMetadataProperties::serialize(&self, serializer: S impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::TableMetadataProperties pub fn iceberg::spec::TableMetadataProperties::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> pub struct iceberg::spec::TableOrcProperties -impl iceberg::spec::TableOrcProperties -pub fn iceberg::spec::TableOrcProperties::block_size_bytes(&self) -> u64 -pub fn iceberg::spec::TableOrcProperties::bloom_filter_columns(&self) -> alloc::string::String -pub fn iceberg::spec::TableOrcProperties::bloom_filter_fpp(&self) -> f64 -pub fn iceberg::spec::TableOrcProperties::compression(&self) -> alloc::string::String -pub fn iceberg::spec::TableOrcProperties::compression_strategy(&self) -> alloc::string::String -pub fn iceberg::spec::TableOrcProperties::delete_block_size_bytes(&self) -> u64 -pub fn iceberg::spec::TableOrcProperties::delete_compression(&self) -> alloc::string::String -pub fn iceberg::spec::TableOrcProperties::delete_compression_strategy(&self) -> alloc::string::String -pub fn iceberg::spec::TableOrcProperties::delete_stripe_size_bytes(&self) -> u64 -pub fn iceberg::spec::TableOrcProperties::delete_write_batch_size(&self) -> usize -pub fn iceberg::spec::TableOrcProperties::stripe_size_bytes(&self) -> u64 -pub fn iceberg::spec::TableOrcProperties::with_block_size_bytes(self, value: u64) -> Self -pub fn iceberg::spec::TableOrcProperties::with_bloom_filter_columns(self, value: alloc::string::String) -> Self -pub fn iceberg::spec::TableOrcProperties::with_bloom_filter_fpp(self, value: f64) -> Self -pub fn iceberg::spec::TableOrcProperties::with_compression(self, value: alloc::string::String) -> Self -pub fn iceberg::spec::TableOrcProperties::with_compression_strategy(self, value: alloc::string::String) -> Self -pub fn iceberg::spec::TableOrcProperties::with_delete_block_size_bytes(self, value: u64) -> Self -pub fn iceberg::spec::TableOrcProperties::with_delete_compression(self, value: alloc::string::String) -> Self -pub fn iceberg::spec::TableOrcProperties::with_delete_compression_strategy(self, value: alloc::string::String) -> Self -pub fn iceberg::spec::TableOrcProperties::with_delete_stripe_size_bytes(self, value: u64) -> Self -pub fn iceberg::spec::TableOrcProperties::with_delete_write_batch_size(self, value: usize) -> Self -pub fn iceberg::spec::TableOrcProperties::with_stripe_size_bytes(self, value: u64) -> Self -pub fn iceberg::spec::TableOrcProperties::with_write_batch_size(self, value: usize) -> Self -pub fn iceberg::spec::TableOrcProperties::write_batch_size(&self) -> usize +pub iceberg::spec::TableOrcProperties::block_size_bytes: u64 +pub iceberg::spec::TableOrcProperties::bloom_filter_columns: alloc::string::String +pub iceberg::spec::TableOrcProperties::bloom_filter_fpp: f64 +pub iceberg::spec::TableOrcProperties::compression: alloc::string::String +pub iceberg::spec::TableOrcProperties::compression_strategy: alloc::string::String +pub iceberg::spec::TableOrcProperties::delete_block_size_bytes: u64 +pub iceberg::spec::TableOrcProperties::delete_compression: alloc::string::String +pub iceberg::spec::TableOrcProperties::delete_compression_strategy: alloc::string::String +pub iceberg::spec::TableOrcProperties::delete_stripe_size_bytes: u64 +pub iceberg::spec::TableOrcProperties::delete_write_batch_size: usize +pub iceberg::spec::TableOrcProperties::stripe_size_bytes: u64 +pub iceberg::spec::TableOrcProperties::write_batch_size: usize impl core::clone::Clone for iceberg::spec::TableOrcProperties pub fn iceberg::spec::TableOrcProperties::clone(&self) -> iceberg::spec::TableOrcProperties impl core::default::Default for iceberg::spec::TableOrcProperties @@ -2990,71 +2913,38 @@ pub fn iceberg::spec::TableOrcProperties::serialize(&self, serializer: S) -> impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::TableOrcProperties pub fn iceberg::spec::TableOrcProperties::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> pub struct iceberg::spec::TableParquetProperties -impl iceberg::spec::TableParquetProperties -pub fn iceberg::spec::TableParquetProperties::bloom_filter_adaptive_enabled(&self) -> bool -pub fn iceberg::spec::TableParquetProperties::bloom_filter_column_enabled(&self) -> std::collections::hash::map::HashMap -pub fn iceberg::spec::TableParquetProperties::bloom_filter_column_fpp(&self) -> std::collections::hash::map::HashMap -pub fn iceberg::spec::TableParquetProperties::bloom_filter_column_ndv(&self) -> std::collections::hash::map::HashMap -pub fn iceberg::spec::TableParquetProperties::bloom_filter_max_bytes(&self) -> usize -pub fn iceberg::spec::TableParquetProperties::column_stats_enabled(&self) -> std::collections::hash::map::HashMap -pub fn iceberg::spec::TableParquetProperties::compression(&self) -> alloc::string::String -pub fn iceberg::spec::TableParquetProperties::compression_level(&self) -> core::option::Option -pub fn iceberg::spec::TableParquetProperties::content_defined_chunking_enabled(&self) -> bool -pub fn iceberg::spec::TableParquetProperties::content_defined_chunking_max_chunk_size(&self) -> usize -pub fn iceberg::spec::TableParquetProperties::content_defined_chunking_min_chunk_size(&self) -> usize -pub fn iceberg::spec::TableParquetProperties::content_defined_chunking_norm_level(&self) -> i32 -pub fn iceberg::spec::TableParquetProperties::delete_compression(&self) -> alloc::string::String -pub fn iceberg::spec::TableParquetProperties::delete_compression_level(&self) -> core::option::Option -pub fn iceberg::spec::TableParquetProperties::delete_dict_size_bytes(&self) -> usize -pub fn iceberg::spec::TableParquetProperties::delete_page_row_limit(&self) -> usize -pub fn iceberg::spec::TableParquetProperties::delete_page_size_bytes(&self) -> usize -pub fn iceberg::spec::TableParquetProperties::delete_page_version(&self) -> alloc::string::String -pub fn iceberg::spec::TableParquetProperties::delete_row_group_check_max_record_count(&self) -> usize -pub fn iceberg::spec::TableParquetProperties::delete_row_group_check_min_record_count(&self) -> usize -pub fn iceberg::spec::TableParquetProperties::delete_row_group_size_bytes(&self) -> usize -pub fn iceberg::spec::TableParquetProperties::dict_encoding_enabled_by_column(&self) -> std::collections::hash::map::HashMap -pub fn iceberg::spec::TableParquetProperties::dict_size_bytes(&self) -> usize -pub fn iceberg::spec::TableParquetProperties::page_row_limit(&self) -> usize -pub fn iceberg::spec::TableParquetProperties::page_size_bytes(&self) -> usize -pub fn iceberg::spec::TableParquetProperties::page_version(&self) -> alloc::string::String -pub fn iceberg::spec::TableParquetProperties::row_group_check_max_record_count(&self) -> usize -pub fn iceberg::spec::TableParquetProperties::row_group_check_min_record_count(&self) -> usize -pub fn iceberg::spec::TableParquetProperties::row_group_size_bytes(&self) -> usize -pub fn iceberg::spec::TableParquetProperties::row_group_size_track_uncompressed(&self) -> bool -pub fn iceberg::spec::TableParquetProperties::shred_variants(&self) -> bool -pub fn iceberg::spec::TableParquetProperties::variant_inference_buffer_size(&self) -> usize -pub fn iceberg::spec::TableParquetProperties::with_bloom_filter_adaptive_enabled(self, value: bool) -> Self -pub fn iceberg::spec::TableParquetProperties::with_bloom_filter_column_enabled(self, value: std::collections::hash::map::HashMap) -> Self -pub fn iceberg::spec::TableParquetProperties::with_bloom_filter_column_fpp(self, value: std::collections::hash::map::HashMap) -> Self -pub fn iceberg::spec::TableParquetProperties::with_bloom_filter_column_ndv(self, value: std::collections::hash::map::HashMap) -> Self -pub fn iceberg::spec::TableParquetProperties::with_bloom_filter_max_bytes(self, value: usize) -> Self -pub fn iceberg::spec::TableParquetProperties::with_column_stats_enabled(self, value: std::collections::hash::map::HashMap) -> Self -pub fn iceberg::spec::TableParquetProperties::with_compression(self, value: alloc::string::String) -> Self -pub fn iceberg::spec::TableParquetProperties::with_compression_level(self, value: core::option::Option) -> Self -pub fn iceberg::spec::TableParquetProperties::with_content_defined_chunking_enabled(self, value: bool) -> Self -pub fn iceberg::spec::TableParquetProperties::with_content_defined_chunking_max_chunk_size(self, value: usize) -> Self -pub fn iceberg::spec::TableParquetProperties::with_content_defined_chunking_min_chunk_size(self, value: usize) -> Self -pub fn iceberg::spec::TableParquetProperties::with_content_defined_chunking_norm_level(self, value: i32) -> Self -pub fn iceberg::spec::TableParquetProperties::with_delete_compression(self, value: alloc::string::String) -> Self -pub fn iceberg::spec::TableParquetProperties::with_delete_compression_level(self, value: core::option::Option) -> Self -pub fn iceberg::spec::TableParquetProperties::with_delete_dict_size_bytes(self, value: usize) -> Self -pub fn iceberg::spec::TableParquetProperties::with_delete_page_row_limit(self, value: usize) -> Self -pub fn iceberg::spec::TableParquetProperties::with_delete_page_size_bytes(self, value: usize) -> Self -pub fn iceberg::spec::TableParquetProperties::with_delete_page_version(self, value: alloc::string::String) -> Self -pub fn iceberg::spec::TableParquetProperties::with_delete_row_group_check_max_record_count(self, value: usize) -> Self -pub fn iceberg::spec::TableParquetProperties::with_delete_row_group_check_min_record_count(self, value: usize) -> Self -pub fn iceberg::spec::TableParquetProperties::with_delete_row_group_size_bytes(self, value: usize) -> Self -pub fn iceberg::spec::TableParquetProperties::with_dict_encoding_enabled_by_column(self, value: std::collections::hash::map::HashMap) -> Self -pub fn iceberg::spec::TableParquetProperties::with_dict_size_bytes(self, value: usize) -> Self -pub fn iceberg::spec::TableParquetProperties::with_page_row_limit(self, value: usize) -> Self -pub fn iceberg::spec::TableParquetProperties::with_page_size_bytes(self, value: usize) -> Self -pub fn iceberg::spec::TableParquetProperties::with_page_version(self, value: alloc::string::String) -> Self -pub fn iceberg::spec::TableParquetProperties::with_row_group_check_max_record_count(self, value: usize) -> Self -pub fn iceberg::spec::TableParquetProperties::with_row_group_check_min_record_count(self, value: usize) -> Self -pub fn iceberg::spec::TableParquetProperties::with_row_group_size_bytes(self, value: usize) -> Self -pub fn iceberg::spec::TableParquetProperties::with_row_group_size_track_uncompressed(self, value: bool) -> Self -pub fn iceberg::spec::TableParquetProperties::with_shred_variants(self, value: bool) -> Self -pub fn iceberg::spec::TableParquetProperties::with_variant_inference_buffer_size(self, value: usize) -> Self +pub iceberg::spec::TableParquetProperties::bloom_filter_adaptive_enabled: bool +pub iceberg::spec::TableParquetProperties::bloom_filter_column_enabled: std::collections::hash::map::HashMap +pub iceberg::spec::TableParquetProperties::bloom_filter_column_fpp: std::collections::hash::map::HashMap +pub iceberg::spec::TableParquetProperties::bloom_filter_column_ndv: std::collections::hash::map::HashMap +pub iceberg::spec::TableParquetProperties::bloom_filter_max_bytes: usize +pub iceberg::spec::TableParquetProperties::column_stats_enabled: std::collections::hash::map::HashMap +pub iceberg::spec::TableParquetProperties::compression: alloc::string::String +pub iceberg::spec::TableParquetProperties::compression_level: core::option::Option +pub iceberg::spec::TableParquetProperties::content_defined_chunking_enabled: bool +pub iceberg::spec::TableParquetProperties::content_defined_chunking_max_chunk_size: usize +pub iceberg::spec::TableParquetProperties::content_defined_chunking_min_chunk_size: usize +pub iceberg::spec::TableParquetProperties::content_defined_chunking_norm_level: i32 +pub iceberg::spec::TableParquetProperties::delete_compression: alloc::string::String +pub iceberg::spec::TableParquetProperties::delete_compression_level: core::option::Option +pub iceberg::spec::TableParquetProperties::delete_dict_size_bytes: usize +pub iceberg::spec::TableParquetProperties::delete_page_row_limit: usize +pub iceberg::spec::TableParquetProperties::delete_page_size_bytes: usize +pub iceberg::spec::TableParquetProperties::delete_page_version: alloc::string::String +pub iceberg::spec::TableParquetProperties::delete_row_group_check_max_record_count: usize +pub iceberg::spec::TableParquetProperties::delete_row_group_check_min_record_count: usize +pub iceberg::spec::TableParquetProperties::delete_row_group_size_bytes: usize +pub iceberg::spec::TableParquetProperties::dict_encoding_enabled_by_column: std::collections::hash::map::HashMap +pub iceberg::spec::TableParquetProperties::dict_size_bytes: usize +pub iceberg::spec::TableParquetProperties::page_row_limit: usize +pub iceberg::spec::TableParquetProperties::page_size_bytes: usize +pub iceberg::spec::TableParquetProperties::page_version: alloc::string::String +pub iceberg::spec::TableParquetProperties::row_group_check_max_record_count: usize +pub iceberg::spec::TableParquetProperties::row_group_check_min_record_count: usize +pub iceberg::spec::TableParquetProperties::row_group_size_bytes: usize +pub iceberg::spec::TableParquetProperties::row_group_size_track_uncompressed: bool +pub iceberg::spec::TableParquetProperties::shred_variants: bool +pub iceberg::spec::TableParquetProperties::variant_inference_buffer_size: usize impl core::clone::Clone for iceberg::spec::TableParquetProperties pub fn iceberg::spec::TableParquetProperties::clone(&self) -> iceberg::spec::TableParquetProperties impl core::default::Default for iceberg::spec::TableParquetProperties @@ -3143,29 +3033,17 @@ pub fn iceberg::spec::TableProperties::try_from(props: &std::collections::hash:: impl core::fmt::Debug for iceberg::spec::TableProperties pub fn iceberg::spec::TableProperties::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result pub struct iceberg::spec::TableReadProperties -impl iceberg::spec::TableReadProperties -pub fn iceberg::spec::TableReadProperties::adaptive_split_size_enabled(&self) -> bool -pub fn iceberg::spec::TableReadProperties::data_planning_mode(&self) -> alloc::string::String -pub fn iceberg::spec::TableReadProperties::delete_planning_mode(&self) -> alloc::string::String -pub fn iceberg::spec::TableReadProperties::metadata_split_target_size(&self) -> u64 -pub fn iceberg::spec::TableReadProperties::orc_batch_size(&self) -> usize -pub fn iceberg::spec::TableReadProperties::orc_vectorization_enabled(&self) -> bool -pub fn iceberg::spec::TableReadProperties::parquet_batch_size(&self) -> usize -pub fn iceberg::spec::TableReadProperties::parquet_vectorization_enabled(&self) -> bool -pub fn iceberg::spec::TableReadProperties::split_open_file_cost(&self) -> u64 -pub fn iceberg::spec::TableReadProperties::split_planning_lookback(&self) -> usize -pub fn iceberg::spec::TableReadProperties::split_target_size(&self) -> u64 -pub fn iceberg::spec::TableReadProperties::with_adaptive_split_size_enabled(self, value: bool) -> Self -pub fn iceberg::spec::TableReadProperties::with_data_planning_mode(self, value: alloc::string::String) -> Self -pub fn iceberg::spec::TableReadProperties::with_delete_planning_mode(self, value: alloc::string::String) -> Self -pub fn iceberg::spec::TableReadProperties::with_metadata_split_target_size(self, value: u64) -> Self -pub fn iceberg::spec::TableReadProperties::with_orc_batch_size(self, value: usize) -> Self -pub fn iceberg::spec::TableReadProperties::with_orc_vectorization_enabled(self, value: bool) -> Self -pub fn iceberg::spec::TableReadProperties::with_parquet_batch_size(self, value: usize) -> Self -pub fn iceberg::spec::TableReadProperties::with_parquet_vectorization_enabled(self, value: bool) -> Self -pub fn iceberg::spec::TableReadProperties::with_split_open_file_cost(self, value: u64) -> Self -pub fn iceberg::spec::TableReadProperties::with_split_planning_lookback(self, value: usize) -> Self -pub fn iceberg::spec::TableReadProperties::with_split_target_size(self, value: u64) -> Self +pub iceberg::spec::TableReadProperties::adaptive_split_size_enabled: bool +pub iceberg::spec::TableReadProperties::data_planning_mode: alloc::string::String +pub iceberg::spec::TableReadProperties::delete_planning_mode: alloc::string::String +pub iceberg::spec::TableReadProperties::metadata_split_target_size: u64 +pub iceberg::spec::TableReadProperties::orc_batch_size: usize +pub iceberg::spec::TableReadProperties::orc_vectorization_enabled: bool +pub iceberg::spec::TableReadProperties::parquet_batch_size: usize +pub iceberg::spec::TableReadProperties::parquet_vectorization_enabled: bool +pub iceberg::spec::TableReadProperties::split_open_file_cost: u64 +pub iceberg::spec::TableReadProperties::split_planning_lookback: usize +pub iceberg::spec::TableReadProperties::split_target_size: u64 impl core::clone::Clone for iceberg::spec::TableReadProperties pub fn iceberg::spec::TableReadProperties::clone(&self) -> iceberg::spec::TableReadProperties impl core::default::Default for iceberg::spec::TableReadProperties @@ -3177,29 +3055,17 @@ pub fn iceberg::spec::TableReadProperties::serialize(&self, serializer: S) -> impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::TableReadProperties pub fn iceberg::spec::TableReadProperties::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> pub struct iceberg::spec::TableRowLevelProperties -impl iceberg::spec::TableRowLevelProperties -pub fn iceberg::spec::TableRowLevelProperties::delete_distribution_mode(&self) -> core::option::Option -pub fn iceberg::spec::TableRowLevelProperties::delete_granularity(&self) -> alloc::string::String -pub fn iceberg::spec::TableRowLevelProperties::delete_isolation_level(&self) -> alloc::string::String -pub fn iceberg::spec::TableRowLevelProperties::delete_mode(&self) -> alloc::string::String -pub fn iceberg::spec::TableRowLevelProperties::merge_distribution_mode(&self) -> core::option::Option -pub fn iceberg::spec::TableRowLevelProperties::merge_isolation_level(&self) -> alloc::string::String -pub fn iceberg::spec::TableRowLevelProperties::merge_mode(&self) -> alloc::string::String -pub fn iceberg::spec::TableRowLevelProperties::update_distribution_mode(&self) -> core::option::Option -pub fn iceberg::spec::TableRowLevelProperties::update_isolation_level(&self) -> alloc::string::String -pub fn iceberg::spec::TableRowLevelProperties::update_mode(&self) -> alloc::string::String -pub fn iceberg::spec::TableRowLevelProperties::upsert_enabled(&self) -> bool -pub fn iceberg::spec::TableRowLevelProperties::with_delete_distribution_mode(self, value: core::option::Option) -> Self -pub fn iceberg::spec::TableRowLevelProperties::with_delete_granularity(self, value: alloc::string::String) -> Self -pub fn iceberg::spec::TableRowLevelProperties::with_delete_isolation_level(self, value: alloc::string::String) -> Self -pub fn iceberg::spec::TableRowLevelProperties::with_delete_mode(self, value: alloc::string::String) -> Self -pub fn iceberg::spec::TableRowLevelProperties::with_merge_distribution_mode(self, value: core::option::Option) -> Self -pub fn iceberg::spec::TableRowLevelProperties::with_merge_isolation_level(self, value: alloc::string::String) -> Self -pub fn iceberg::spec::TableRowLevelProperties::with_merge_mode(self, value: alloc::string::String) -> Self -pub fn iceberg::spec::TableRowLevelProperties::with_update_distribution_mode(self, value: core::option::Option) -> Self -pub fn iceberg::spec::TableRowLevelProperties::with_update_isolation_level(self, value: alloc::string::String) -> Self -pub fn iceberg::spec::TableRowLevelProperties::with_update_mode(self, value: alloc::string::String) -> Self -pub fn iceberg::spec::TableRowLevelProperties::with_upsert_enabled(self, value: bool) -> Self +pub iceberg::spec::TableRowLevelProperties::delete_distribution_mode: core::option::Option +pub iceberg::spec::TableRowLevelProperties::delete_granularity: alloc::string::String +pub iceberg::spec::TableRowLevelProperties::delete_isolation_level: alloc::string::String +pub iceberg::spec::TableRowLevelProperties::delete_mode: alloc::string::String +pub iceberg::spec::TableRowLevelProperties::merge_distribution_mode: core::option::Option +pub iceberg::spec::TableRowLevelProperties::merge_isolation_level: alloc::string::String +pub iceberg::spec::TableRowLevelProperties::merge_mode: alloc::string::String +pub iceberg::spec::TableRowLevelProperties::update_distribution_mode: core::option::Option +pub iceberg::spec::TableRowLevelProperties::update_isolation_level: alloc::string::String +pub iceberg::spec::TableRowLevelProperties::update_mode: alloc::string::String +pub iceberg::spec::TableRowLevelProperties::upsert_enabled: bool impl core::clone::Clone for iceberg::spec::TableRowLevelProperties pub fn iceberg::spec::TableRowLevelProperties::clone(&self) -> iceberg::spec::TableRowLevelProperties impl core::default::Default for iceberg::spec::TableRowLevelProperties @@ -3211,33 +3077,19 @@ pub fn iceberg::spec::TableRowLevelProperties::serialize(&self, serializer: S impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::TableRowLevelProperties pub fn iceberg::spec::TableRowLevelProperties::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> pub struct iceberg::spec::TableWriteProperties -impl iceberg::spec::TableWriteProperties -pub fn iceberg::spec::TableWriteProperties::audit_publish_enabled(&self) -> bool -pub fn iceberg::spec::TableWriteProperties::data_location(&self) -> core::option::Option -pub fn iceberg::spec::TableWriteProperties::datafusion_fanout_enabled(&self) -> bool -pub fn iceberg::spec::TableWriteProperties::delete_format_default(&self) -> iceberg::spec::DataFileFormat -pub fn iceberg::spec::TableWriteProperties::delete_target_file_size_bytes(&self) -> usize -pub fn iceberg::spec::TableWriteProperties::distribution_mode(&self) -> core::option::Option -pub fn iceberg::spec::TableWriteProperties::folder_storage_location(&self) -> core::option::Option -pub fn iceberg::spec::TableWriteProperties::format_default(&self) -> iceberg::spec::DataFileFormat -pub fn iceberg::spec::TableWriteProperties::location_provider_impl(&self) -> core::option::Option -pub fn iceberg::spec::TableWriteProperties::object_store_enabled(&self) -> bool -pub fn iceberg::spec::TableWriteProperties::object_store_partitioned_paths(&self) -> bool -pub fn iceberg::spec::TableWriteProperties::object_store_path(&self) -> core::option::Option -pub fn iceberg::spec::TableWriteProperties::target_file_size_bytes(&self) -> usize -pub fn iceberg::spec::TableWriteProperties::with_audit_publish_enabled(self, value: bool) -> Self -pub fn iceberg::spec::TableWriteProperties::with_data_location(self, value: core::option::Option) -> Self -pub fn iceberg::spec::TableWriteProperties::with_datafusion_fanout_enabled(self, value: bool) -> Self -pub fn iceberg::spec::TableWriteProperties::with_delete_format_default(self, value: iceberg::spec::DataFileFormat) -> Self -pub fn iceberg::spec::TableWriteProperties::with_delete_target_file_size_bytes(self, value: usize) -> Self -pub fn iceberg::spec::TableWriteProperties::with_distribution_mode(self, value: core::option::Option) -> Self -pub fn iceberg::spec::TableWriteProperties::with_folder_storage_location(self, value: core::option::Option) -> Self -pub fn iceberg::spec::TableWriteProperties::with_format_default(self, value: iceberg::spec::DataFileFormat) -> Self -pub fn iceberg::spec::TableWriteProperties::with_location_provider_impl(self, value: core::option::Option) -> Self -pub fn iceberg::spec::TableWriteProperties::with_object_store_enabled(self, value: bool) -> Self -pub fn iceberg::spec::TableWriteProperties::with_object_store_partitioned_paths(self, value: bool) -> Self -pub fn iceberg::spec::TableWriteProperties::with_object_store_path(self, value: core::option::Option) -> Self -pub fn iceberg::spec::TableWriteProperties::with_target_file_size_bytes(self, value: usize) -> Self +pub iceberg::spec::TableWriteProperties::audit_publish_enabled: bool +pub iceberg::spec::TableWriteProperties::data_location: core::option::Option +pub iceberg::spec::TableWriteProperties::datafusion_fanout_enabled: bool +pub iceberg::spec::TableWriteProperties::delete_format_default: iceberg::spec::DataFileFormat +pub iceberg::spec::TableWriteProperties::delete_target_file_size_bytes: usize +pub iceberg::spec::TableWriteProperties::distribution_mode: core::option::Option +pub iceberg::spec::TableWriteProperties::folder_storage_location: core::option::Option +pub iceberg::spec::TableWriteProperties::format_default: iceberg::spec::DataFileFormat +pub iceberg::spec::TableWriteProperties::location_provider_impl: core::option::Option +pub iceberg::spec::TableWriteProperties::object_store_enabled: bool +pub iceberg::spec::TableWriteProperties::object_store_partitioned_paths: bool +pub iceberg::spec::TableWriteProperties::object_store_path: core::option::Option +pub iceberg::spec::TableWriteProperties::target_file_size_bytes: usize impl core::clone::Clone for iceberg::spec::TableWriteProperties pub fn iceberg::spec::TableWriteProperties::clone(&self) -> iceberg::spec::TableWriteProperties impl core::default::Default for iceberg::spec::TableWriteProperties diff --git a/crates/iceberg/src/catalog/utils.rs b/crates/iceberg/src/catalog/utils.rs index 8ed4c51678..a754b14ab5 100644 --- a/crates/iceberg/src/catalog/utils.rs +++ b/crates/iceberg/src/catalog/utils.rs @@ -60,7 +60,7 @@ pub async fn drop_table_data(table_info: &Table) -> Result<()> { } // Delete data files only if gc.enabled is true, to avoid corrupting shared tables - if metadata.parsed_table_properties()?.history().gc_enabled() { + if metadata.parsed_table_properties()?.history.gc_enabled { delete_data_files(io, &manifests_to_delete).await?; } diff --git a/crates/iceberg/src/encryption/manager.rs b/crates/iceberg/src/encryption/manager.rs index 59ef661d95..cd9db8cb42 100644 --- a/crates/iceberg/src/encryption/manager.rs +++ b/crates/iceberg/src/encryption/manager.rs @@ -119,8 +119,8 @@ impl EncryptionManager { } let table_properties = metadata.parsed_table_properties()?; - let encryption = table_properties.encryption(); - let Some(table_key_id) = encryption.key_id() else { + let encryption = table_properties.encryption; + let Some(table_key_id) = encryption.key_id else { if kms_client.is_some() { tracing::warn!( "KeyManagementClient provided but table does not have encryption.key-id set" @@ -140,7 +140,7 @@ impl EncryptionManager { .kms_client(Arc::clone(kms_client)) .table_key_id(table_key_id) .encryption_keys(metadata.encryption_keys.clone()) - .key_size(AesKeySize::from_key_length(encryption.data_key_length())?) + .key_size(AesKeySize::from_key_length(encryption.data_key_length)?) .build(); Ok(Some(Arc::new(em))) } diff --git a/crates/iceberg/src/spec/parsed_table_prop.rs b/crates/iceberg/src/spec/parsed_table_prop.rs index 1e482b15ab..fd56ae70fd 100644 --- a/crates/iceberg/src/spec/parsed_table_prop.rs +++ b/crates/iceberg/src/spec/parsed_table_prop.rs @@ -15,6 +15,64 @@ // specific language governing permissions and limitations // under the License. +//! Typed access to Iceberg table properties. +//! +//! [`ParsedTableProperties`] organizes Iceberg's string-keyed table properties into typed groups +//! such as [`TableWriteProperties`] and [`TableCommitProperties`]. The Rust representation is +//! nested, while its JSON representation remains a flat object whose keys and values are strings. +//! +//! # Create from defaults +//! +//! Start with Iceberg's defaults and modify public fields directly: +//! +//! ``` +//! use iceberg::spec::{DataFileFormat, ParsedTableProperties}; +//! +//! let mut properties = ParsedTableProperties::default(); +//! properties.write.format_default = DataFileFormat::Orc; +//! properties.write.data_location = Some("s3://warehouse/table/data".to_string()); +//! +//! assert_eq!(properties.write.format_default, DataFileFormat::Orc); +//! ``` +//! +//! # Deserialize from JSON +//! +//! JSON property values must be strings, matching Iceberg's table property map: +//! +//! ``` +//! use iceberg::spec::{DataFileFormat, ParsedTableProperties}; +//! +//! # fn main() -> Result<(), serde_json::Error> { +//! let properties: ParsedTableProperties = serde_json::from_value(serde_json::json!({ +//! "commit.retry.num-retries": "8", +//! "write.format.default": "orc" +//! }))?; +//! +//! assert_eq!(properties.commit.num_retries, 8); +//! assert_eq!(properties.write.format_default, DataFileFormat::Orc); +//! # Ok(()) +//! # } +//! ``` +//! +//! # Serialize to JSON +//! +//! Serialization flattens the nested groups back into Iceberg property keys: +//! +//! ``` +//! use iceberg::spec::ParsedTableProperties; +//! +//! # fn main() -> Result<(), serde_json::Error> { +//! let mut properties = ParsedTableProperties::default(); +//! properties.commit.num_retries = 8; +//! properties.write.data_location = Some("s3://warehouse/table/data".to_string()); +//! +//! let json = serde_json::to_value(&properties)?; +//! assert_eq!(json["commit.retry.num-retries"], "8"); +//! assert_eq!(json["write.data.path"], "s3://warehouse/table/data"); +//! # Ok(()) +//! # } +//! ``` + use std::collections::HashMap; use iceberg_property_macro::Properties; @@ -91,43 +149,43 @@ fn serialize_compression_codec(codec: &CompressionCodec) -> String { pub struct ParsedTableProperties { #[nested] #[doc = "Informational properties and identifier-field semantics."] - general: TableGeneralProperties, + pub general: TableGeneralProperties, #[nested] #[doc = "Commit retry and commit status-check behavior."] - commit: TableCommitProperties, + pub commit: TableCommitProperties, #[nested] #[doc = "Manifest creation, merging, and compression behavior."] - manifest: TableManifestProperties, + pub manifest: TableManifestProperties, #[nested] #[doc = "General table write behavior and output locations."] - write: TableWriteProperties, + pub write: TableWriteProperties, #[nested] #[doc = "Parquet data and delete file behavior."] - parquet: TableParquetProperties, + pub parquet: TableParquetProperties, #[nested] #[doc = "Avro data and delete file behavior."] - avro: TableAvroProperties, + pub avro: TableAvroProperties, #[nested] #[doc = "ORC data and delete file behavior."] - orc: TableOrcProperties, + pub orc: TableOrcProperties, #[nested] #[doc = "Read split planning and vectorization behavior."] - read: TableReadProperties, + pub read: TableReadProperties, #[nested] #[doc = "Table metadata retention, compression, and metrics behavior."] - metadata: TableMetadataProperties, + pub metadata: TableMetadataProperties, #[nested] #[doc = "Engine-specific and compatibility behavior."] - compatibility: TableCompatibilityProperties, + pub compatibility: TableCompatibilityProperties, #[nested] #[doc = "Garbage collection and snapshot retention behavior."] - history: TableHistoryProperties, + pub history: TableHistoryProperties, #[nested] #[doc = "Delete, update, merge, and upsert behavior."] - row_level: TableRowLevelProperties, + pub row_level: TableRowLevelProperties, #[nested] #[doc = "Table encryption behavior."] - encryption: TableEncryptionProperties, + pub encryption: TableEncryptionProperties, } /// Informational properties and identifier-field semantics. @@ -136,11 +194,11 @@ pub struct TableGeneralProperties { #[key = "comment"] #[default(None)] #[doc = "Table-level description of the table's business meaning and usage context."] - comment: Option, + pub comment: Option, #[key = "identifier-fields.rely"] #[default(false)] #[doc = "Whether query engines may rely on identifier fields as a primary key for optimization; this is not enforced on writes."] - identifier_fields_rely: bool, + pub identifier_fields_rely: bool, } /// Commit retries and post-failure status checks. @@ -149,35 +207,35 @@ pub struct TableCommitProperties { #[key = "commit.retry.num-retries"] #[default(4)] #[doc = "Number of times to retry a commit before failing."] - num_retries: usize, + pub num_retries: usize, #[key = "commit.retry.min-wait-ms"] #[default(100)] #[doc = "Minimum time in milliseconds to wait before retrying a commit."] - min_retry_wait_ms: u64, + pub min_retry_wait_ms: u64, #[key = "commit.retry.max-wait-ms"] #[default(60 * 1000)] #[doc = "Maximum time in milliseconds to wait before retrying a commit."] - max_retry_wait_ms: u64, + pub max_retry_wait_ms: u64, #[key = "commit.retry.total-timeout-ms"] #[default(30 * 60 * 1000)] #[doc = "Total commit retry timeout in milliseconds."] - total_retry_timeout_ms: u64, + pub total_retry_timeout_ms: u64, #[key = "commit.status-check.num-retries"] #[default(3)] #[doc = "Number of times to check whether a commit succeeded after connectivity is lost."] - num_status_checks: usize, + pub num_status_checks: usize, #[key = "commit.status-check.min-wait-ms"] #[default(1000)] #[doc = "Minimum time in milliseconds to wait before retrying a commit status check."] - status_checks_min_wait_ms: u64, + pub status_checks_min_wait_ms: u64, #[key = "commit.status-check.max-wait-ms"] #[default(60 * 1000)] #[doc = "Maximum time in milliseconds to wait before retrying a commit status check."] - status_checks_max_wait_ms: u64, + pub status_checks_max_wait_ms: u64, #[key = "commit.status-check.total-timeout-ms"] #[default(30 * 60 * 1000)] #[doc = "Total timeout in milliseconds in which commit status checking must succeed."] - status_checks_total_wait_ms: u64, + pub status_checks_total_wait_ms: u64, } /// Manifest creation, merging, and compression properties. @@ -186,27 +244,27 @@ pub struct TableManifestProperties { #[key = "commit.manifest.target-size-bytes"] #[default(8 * 1024 * 1024)] #[doc = "Target size in bytes when merging manifest files."] - target_size_bytes: usize, + pub target_size_bytes: usize, #[key = "commit.manifest.min-count-to-merge"] #[default(100)] #[doc = "Minimum number of manifests to accumulate before merging."] - min_merge_count: usize, + pub min_merge_count: usize, #[key = "commit.manifest-merge.enabled"] #[default(true)] #[doc = "Whether manifests are automatically merged during writes."] - merge_enabled: bool, + pub merge_enabled: bool, #[key = "write.manifest.compression-codec"] #[default("gzip".to_string())] #[doc = "Compression codec used for manifest files."] - compression: String, + pub compression: String, #[key = "write.manifest.compression-level"] #[default(None)] #[doc = "Optional compression level used for manifest files."] - compression_level: Option, + pub compression_level: Option, #[key = "write.manifest-lists.enabled"] #[default(true)] #[doc = "Deprecated flag for writing manifest lists; manifest lists are always enabled."] - lists_enabled: bool, + pub lists_enabled: bool, } /// General write properties and output locations. @@ -215,55 +273,55 @@ pub struct TableWriteProperties { #[key = "write.format.default"] #[default(DataFileFormat::Parquet)] #[doc = "Default data file format: Parquet, Avro, or ORC."] - format_default: DataFileFormat, + pub format_default: DataFileFormat, #[key = "write.delete.format.default"] #[default(DataFileFormat::Parquet)] #[doc = "Default delete file format: Parquet, Avro, or ORC."] - delete_format_default: DataFileFormat, + pub delete_format_default: DataFileFormat, #[key = "write.target-file-size-bytes"] #[default(512 * 1024 * 1024)] #[doc = "Target size in bytes for generated data files."] - target_file_size_bytes: usize, + pub target_file_size_bytes: usize, #[key = "write.delete.target-file-size-bytes"] #[default(64 * 1024 * 1024)] #[doc = "Target size in bytes for generated delete files."] - delete_target_file_size_bytes: usize, + pub delete_target_file_size_bytes: usize, #[key = "write.object-storage.enabled"] #[default(false)] #[doc = "Whether the object-storage location provider adds a hash component to file paths."] - object_store_enabled: bool, + pub object_store_enabled: bool, #[key = "write.object-storage.partitioned-paths"] #[default(true)] #[doc = "Whether object-storage file paths include partition values."] - object_store_partitioned_paths: bool, + pub object_store_partitioned_paths: bool, #[key = "write.object-storage.path"] #[default(None)] #[doc = "Deprecated base object-storage path; use write.data.path instead."] - object_store_path: Option, + pub object_store_path: Option, #[key = "write.location-provider.impl"] #[default(None)] #[doc = "Optional custom location provider implementation."] - location_provider_impl: Option, + pub location_provider_impl: Option, #[key = "write.folder-storage.path"] #[default(None)] #[doc = "Deprecated base folder-storage path; use write.data.path instead."] - folder_storage_location: Option, + pub folder_storage_location: Option, #[key = "write.data.path"] #[default(None)] #[doc = "Base location for data files written after this property is set."] - data_location: Option, + pub data_location: Option, #[key = "write.wap.enabled"] #[default(false)] #[doc = "Whether write-audit-publish writes are enabled."] - audit_publish_enabled: bool, + pub audit_publish_enabled: bool, #[key = "write.distribution-mode"] #[default(None)] #[doc = "Optional write distribution mode: none, hash, or range."] - distribution_mode: Option, + pub distribution_mode: Option, #[key = "write.datafusion.fanout.enabled"] #[default(true)] #[doc = "Whether DataFusion uses a fanout writer for partitioned tables."] - datafusion_fanout_enabled: bool, + pub datafusion_fanout_enabled: bool, } /// Parquet data and delete file properties. @@ -272,131 +330,131 @@ pub struct TableParquetProperties { #[key = "write.parquet.row-group-size-bytes"] #[default(128 * 1024 * 1024)] #[doc = "Parquet row group size in bytes for data files."] - row_group_size_bytes: usize, + pub row_group_size_bytes: usize, #[key = "write.delete.parquet.row-group-size-bytes"] #[default(128 * 1024 * 1024)] #[doc = "Parquet row group size in bytes for delete files."] - delete_row_group_size_bytes: usize, + pub delete_row_group_size_bytes: usize, #[key = "write.parquet.page-size-bytes"] #[default(1024 * 1024)] #[doc = "Parquet page size in bytes for data files."] - page_size_bytes: usize, + pub page_size_bytes: usize, #[key = "write.delete.parquet.page-size-bytes"] #[default(1024 * 1024)] #[doc = "Parquet page size in bytes for delete files."] - delete_page_size_bytes: usize, + pub delete_page_size_bytes: usize, #[key = "write.parquet.page-version"] #[default("v1".to_string())] #[doc = "Parquet data page version for data files: v1 or v2."] - page_version: String, + pub page_version: String, #[key = "write.delete.parquet.page-version"] #[default("v1".to_string())] #[doc = "Parquet data page version for delete files: v1 or v2."] - delete_page_version: String, + pub delete_page_version: String, #[key = "write.parquet.page-row-limit"] #[default(20_000)] #[doc = "Maximum number of rows per Parquet page in data files."] - page_row_limit: usize, + pub page_row_limit: usize, #[key = "write.delete.parquet.page-row-limit"] #[default(20_000)] #[doc = "Maximum number of rows per Parquet page in delete files."] - delete_page_row_limit: usize, + pub delete_page_row_limit: usize, #[key = "write.parquet.dict-size-bytes"] #[default(2 * 1024 * 1024)] #[doc = "Parquet dictionary page size in bytes for data files."] - dict_size_bytes: usize, + pub dict_size_bytes: usize, #[key = "write.delete.parquet.dict-size-bytes"] #[default(2 * 1024 * 1024)] #[doc = "Parquet dictionary page size in bytes for delete files."] - delete_dict_size_bytes: usize, + pub delete_dict_size_bytes: usize, #[key = "write.parquet.compression-codec"] #[default("zstd".to_string())] #[doc = "Parquet compression codec used for data files."] - compression: String, + pub compression: String, #[key = "write.delete.parquet.compression-codec"] #[default("zstd".to_string())] #[doc = "Parquet compression codec used for delete files."] - delete_compression: String, + pub delete_compression: String, #[key = "write.parquet.compression-level"] #[default(None)] #[doc = "Optional Parquet compression level for data files."] - compression_level: Option, + pub compression_level: Option, #[key = "write.delete.parquet.compression-level"] #[default(None)] #[doc = "Optional Parquet compression level for delete files."] - delete_compression_level: Option, + pub delete_compression_level: Option, #[key = "write.parquet.shred-variants"] #[default(false)] #[doc = "Whether variant columns use shredded Parquet encoding for improved query performance."] - shred_variants: bool, + pub shred_variants: bool, #[key = "write.parquet.variant-inference-buffer-size"] #[default(100)] #[doc = "Number of rows buffered for schema inference when variant shredding is enabled."] - variant_inference_buffer_size: usize, + pub variant_inference_buffer_size: usize, #[key = "write.parquet.row-group-check-min-record-count"] #[default(100)] #[doc = "Minimum record count between Parquet data-file row group size checks."] - row_group_check_min_record_count: usize, + pub row_group_check_min_record_count: usize, #[key = "write.delete.parquet.row-group-check-min-record-count"] #[default(100)] #[doc = "Minimum record count between Parquet delete-file row group size checks."] - delete_row_group_check_min_record_count: usize, + pub delete_row_group_check_min_record_count: usize, #[key = "write.parquet.row-group-check-max-record-count"] #[default(10_000)] #[doc = "Maximum record count between Parquet data-file row group size checks."] - row_group_check_max_record_count: usize, + pub row_group_check_max_record_count: usize, #[key = "write.delete.parquet.row-group-check-max-record-count"] #[default(10_000)] #[doc = "Maximum record count between Parquet delete-file row group size checks."] - delete_row_group_check_max_record_count: usize, + pub delete_row_group_check_max_record_count: usize, #[key = "write.parquet.row-group-size-track-uncompressed"] #[default(false)] #[doc = "Whether uncompressed data size is tracked to enforce the Parquet row group target."] - row_group_size_track_uncompressed: bool, + pub row_group_size_track_uncompressed: bool, #[key = "write.parquet.bloom-filter-max-bytes"] #[default(1024 * 1024)] #[doc = "Maximum number of bytes for a Parquet bloom filter bitset."] - bloom_filter_max_bytes: usize, + pub bloom_filter_max_bytes: usize, #[key = "write.parquet.bloom-filter-adaptive-enabled"] #[default(false)] #[doc = "Whether adaptive Parquet bloom filter sizing selects the smallest suitable filter."] - bloom_filter_adaptive_enabled: bool, + pub bloom_filter_adaptive_enabled: bool, #[prefix = "write.parquet.bloom-filter-fpp.column."] #[default(HashMap::new())] #[doc = "Per-column Parquet bloom filter false-positive probabilities, keyed by column name."] - bloom_filter_column_fpp: HashMap, + pub bloom_filter_column_fpp: HashMap, #[prefix = "write.parquet.bloom-filter-ndv.column."] #[default(HashMap::new())] #[doc = "Per-column expected distinct-value counts for Parquet bloom filters."] - bloom_filter_column_ndv: HashMap, + pub bloom_filter_column_ndv: HashMap, #[prefix = "write.parquet.bloom-filter-enabled.column."] #[default(HashMap::new())] #[doc = "Per-column flags controlling whether Parquet bloom filters are written."] - bloom_filter_column_enabled: HashMap, + pub bloom_filter_column_enabled: HashMap, #[prefix = "write.parquet.stats-enabled.column."] #[default(HashMap::new())] #[doc = "Per-column flags controlling whether Parquet column statistics are collected."] - column_stats_enabled: HashMap, + pub column_stats_enabled: HashMap, #[prefix = "write.parquet.dict-encoding-enabled.column."] #[default(HashMap::new())] #[doc = "Per-column flags controlling whether Parquet dictionary encoding is used."] - dict_encoding_enabled_by_column: HashMap, + pub dict_encoding_enabled_by_column: HashMap, #[key = "write.parquet.content-defined-chunking.enabled"] #[default(false)] #[doc = "Whether Parquet content-defined chunking is enabled."] - content_defined_chunking_enabled: bool, + pub content_defined_chunking_enabled: bool, #[key = "write.parquet.content-defined-chunking.min-chunk-size"] #[default(256 * 1024)] #[doc = "Minimum Parquet content-defined chunk size in bytes."] - content_defined_chunking_min_chunk_size: usize, + pub content_defined_chunking_min_chunk_size: usize, #[key = "write.parquet.content-defined-chunking.max-chunk-size"] #[default(1024 * 1024)] #[doc = "Maximum Parquet content-defined chunk size in bytes."] - content_defined_chunking_max_chunk_size: usize, + pub content_defined_chunking_max_chunk_size: usize, #[key = "write.parquet.content-defined-chunking.norm-level"] #[default(0)] #[doc = "Gearhash normalization level used by Parquet content-defined chunking."] - content_defined_chunking_norm_level: i32, + pub content_defined_chunking_norm_level: i32, } /// Avro data and delete file properties. @@ -405,19 +463,19 @@ pub struct TableAvroProperties { #[key = "write.avro.compression-codec"] #[default("gzip".to_string())] #[doc = "Avro compression codec used for data files."] - compression: String, + pub compression: String, #[key = "write.delete.avro.compression-codec"] #[default("gzip".to_string())] #[doc = "Avro compression codec used for delete files."] - delete_compression: String, + pub delete_compression: String, #[key = "write.avro.compression-level"] #[default(None)] #[doc = "Optional Avro compression level for data files."] - compression_level: Option, + pub compression_level: Option, #[key = "write.delete.avro.compression-level"] #[default(None)] #[doc = "Optional Avro compression level for delete files."] - delete_compression_level: Option, + pub delete_compression_level: Option, } /// ORC data and delete file properties. @@ -426,51 +484,51 @@ pub struct TableOrcProperties { #[key = "write.orc.stripe-size-bytes"] #[default(64 * 1024 * 1024)] #[doc = "Default ORC stripe size in bytes for data files."] - stripe_size_bytes: u64, + pub stripe_size_bytes: u64, #[key = "write.delete.orc.stripe-size-bytes"] #[default(64 * 1024 * 1024)] #[doc = "Default ORC stripe size in bytes for delete files."] - delete_stripe_size_bytes: u64, + pub delete_stripe_size_bytes: u64, #[key = "write.orc.bloom.filter.columns"] #[default(String::new())] #[doc = "Comma-separated column names for which ORC bloom filters are created."] - bloom_filter_columns: String, + pub bloom_filter_columns: String, #[key = "write.orc.bloom.filter.fpp"] #[default(0.05)] #[doc = "False-positive probability for ORC bloom filters."] - bloom_filter_fpp: f64, + pub bloom_filter_fpp: f64, #[key = "write.orc.block-size-bytes"] #[default(256 * 1024 * 1024)] #[doc = "Default file-system block size in bytes for ORC data files."] - block_size_bytes: u64, + pub block_size_bytes: u64, #[key = "write.delete.orc.block-size-bytes"] #[default(256 * 1024 * 1024)] #[doc = "Default file-system block size in bytes for ORC delete files."] - delete_block_size_bytes: u64, + pub delete_block_size_bytes: u64, #[key = "write.orc.vectorized.batch-size"] #[default(1024)] #[doc = "ORC vectorized write batch size for data files."] - write_batch_size: usize, + pub write_batch_size: usize, #[key = "write.delete.orc.vectorized.batch-size"] #[default(1024)] #[doc = "ORC vectorized write batch size for delete files."] - delete_write_batch_size: usize, + pub delete_write_batch_size: usize, #[key = "write.orc.compression-codec"] #[default("zlib".to_string())] #[doc = "ORC compression codec used for data files."] - compression: String, + pub compression: String, #[key = "write.delete.orc.compression-codec"] #[default("zlib".to_string())] #[doc = "ORC compression codec used for delete files."] - delete_compression: String, + pub delete_compression: String, #[key = "write.orc.compression-strategy"] #[default("speed".to_string())] #[doc = "ORC compression strategy for data files: speed or compression."] - compression_strategy: String, + pub compression_strategy: String, #[key = "write.delete.orc.compression-strategy"] #[default("speed".to_string())] #[doc = "ORC compression strategy for delete files: speed or compression."] - delete_compression_strategy: String, + pub delete_compression_strategy: String, } /// Read split planning and vectorization properties. @@ -479,47 +537,47 @@ pub struct TableReadProperties { #[key = "read.split.target-size"] #[default(128 * 1024 * 1024)] #[doc = "Target size in bytes when combining data input splits."] - split_target_size: u64, + pub split_target_size: u64, #[key = "read.split.metadata-target-size"] #[default(32 * 1024 * 1024)] #[doc = "Target size in bytes when combining metadata input splits."] - metadata_split_target_size: u64, + pub metadata_split_target_size: u64, #[key = "read.split.planning-lookback"] #[default(10)] #[doc = "Number of bins considered when combining input splits."] - split_planning_lookback: usize, + pub split_planning_lookback: usize, #[key = "read.split.open-file-cost"] #[default(4 * 1024 * 1024)] #[doc = "Estimated file-open cost used as a minimum weight when combining splits."] - split_open_file_cost: u64, + pub split_open_file_cost: u64, #[key = "read.split.adaptive-size.enabled"] #[default(true)] #[doc = "Whether split size is adapted to the scan size."] - adaptive_split_size_enabled: bool, + pub adaptive_split_size_enabled: bool, #[key = "read.parquet.vectorization.enabled"] #[default(true)] #[doc = "Whether Parquet vectorized reads are enabled."] - parquet_vectorization_enabled: bool, + pub parquet_vectorization_enabled: bool, #[key = "read.parquet.vectorization.batch-size"] #[default(5000)] #[doc = "Batch size for Parquet vectorized reads."] - parquet_batch_size: usize, + pub parquet_batch_size: usize, #[key = "read.orc.vectorization.enabled"] #[default(false)] #[doc = "Whether ORC vectorized reads are enabled."] - orc_vectorization_enabled: bool, + pub orc_vectorization_enabled: bool, #[key = "read.orc.vectorization.batch-size"] #[default(5000)] #[doc = "Batch size for ORC vectorized reads."] - orc_batch_size: usize, + pub orc_batch_size: usize, #[key = "read.data-planning-mode"] #[default("auto".to_string())] #[doc = "Planning mode used for data files."] - data_planning_mode: String, + pub data_planning_mode: String, #[key = "read.delete-planning-mode"] #[default("auto".to_string())] #[doc = "Planning mode used for delete files."] - delete_planning_mode: String, + pub delete_planning_mode: String, } /// Table metadata retention, compression, and metrics properties. @@ -529,41 +587,41 @@ pub struct TableMetadataProperties { #[default(None)] #[parse_with(parse_metadata_location)] #[doc = "Base location for metadata files written after this property is set, with trailing slashes removed."] - path: Option, + pub path: Option, #[key = "write.summary.partition-limit"] #[default(0)] #[doc = "Maximum changed-partition count for including partition-level statistics in snapshot summaries."] - partition_summary_limit: u64, + pub partition_summary_limit: u64, #[key = "write.metadata.compression-codec"] #[default(CompressionCodec::None)] #[parse_with(parse_metadata_file_compression)] #[serialize_with(serialize_compression_codec)] #[doc = "Compression codec for metadata JSON files: none or gzip."] - compression_codec: CompressionCodec, + pub compression_codec: CompressionCodec, #[key = "write.metadata.previous-versions-max"] #[default(100)] #[doc = "Maximum number of previous metadata file versions to track."] - previous_versions_max: usize, + pub previous_versions_max: usize, #[key = "write.metadata.delete-after-commit.enabled"] #[default(false)] #[doc = "Whether the oldest tracked metadata file is deleted after each commit."] - delete_after_commit_enabled: bool, + pub delete_after_commit_enabled: bool, #[key = "write.metadata.metrics.max-inferred-column-defaults"] #[default(100)] #[doc = "Maximum number of columns that receive inferred metrics defaults."] - metrics_max_inferred_column_defaults: usize, + pub metrics_max_inferred_column_defaults: usize, #[prefix = "write.metadata.metrics.column."] #[default(HashMap::new())] #[doc = "Per-column metrics modes keyed by column name."] - metrics_mode_by_column: HashMap, + pub metrics_mode_by_column: HashMap, #[key = "write.metadata.metrics.default"] #[default("truncate(16)".to_string())] #[doc = "Default metrics mode for table columns."] - default_metrics_mode: String, + pub default_metrics_mode: String, #[key = "schema.name-mapping.default"] #[default(None)] #[doc = "Default JSON name mapping used to resolve columns in files without field IDs."] - default_name_mapping: Option, + pub default_name_mapping: Option, } /// Engine-specific and compatibility properties. @@ -572,31 +630,31 @@ pub struct TableCompatibilityProperties { #[key = "write.spark.fanout.enabled"] #[default(false)] #[doc = "Deprecated Spark fanout-writer flag; the fanout writer accepts unclustered data but uses more memory."] - spark_write_partitioned_fanout_enabled: bool, + pub spark_write_partitioned_fanout_enabled: bool, #[key = "write.spark.accept-any-schema"] #[default(false)] #[doc = "Deprecated Spark flag allowing writes with any compatible schema."] - spark_write_accept_any_schema: bool, + pub spark_write_accept_any_schema: bool, #[key = "write.spark.auto-schema-evolution.enabled"] #[default(true)] #[doc = "Deprecated Spark flag enabling automatic schema evolution during writes."] - spark_write_auto_schema_evolution: bool, + pub spark_write_auto_schema_evolution: bool, #[key = "write.spark.advisory-partition-size-bytes"] #[default(None)] #[doc = "Deprecated Spark advisory partition size in bytes."] - spark_write_advisory_partition_size_bytes: Option, + pub spark_write_advisory_partition_size_bytes: Option, #[key = "compatibility.snapshot-id-inheritance.enabled"] #[default(false)] #[doc = "Whether snapshots may be committed without explicit snapshot IDs; format version 2 and later always allow this."] - snapshot_id_inheritance_enabled: bool, + pub snapshot_id_inheritance_enabled: bool, #[key = "engine.hive.enabled"] #[default(false)] #[doc = "Whether Hive engine integration behavior is enabled."] - engine_hive_enabled: bool, + pub engine_hive_enabled: bool, #[key = "engine.hive.lock-enabled"] #[default(true)] #[doc = "Whether Hive locking is enabled."] - hive_lock_enabled: bool, + pub hive_lock_enabled: bool, } /// Garbage collection and snapshot retention properties. @@ -605,19 +663,19 @@ pub struct TableHistoryProperties { #[key = "gc.enabled"] #[default(true)] #[doc = "Whether garbage collection operations such as snapshot expiration and orphan-file removal are allowed."] - gc_enabled: bool, + pub gc_enabled: bool, #[key = "history.expire.max-snapshot-age-ms"] #[default(5 * 24 * 60 * 60 * 1000)] #[doc = "Default maximum snapshot age in milliseconds while expiring snapshots."] - max_snapshot_age_ms: i64, + pub max_snapshot_age_ms: i64, #[key = "history.expire.min-snapshots-to-keep"] #[default(1)] #[doc = "Default minimum number of snapshots retained per branch while expiring snapshots."] - min_snapshots_to_keep: usize, + pub min_snapshots_to_keep: usize, #[key = "history.expire.max-ref-age-ms"] #[default(i64::MAX)] #[doc = "Default maximum age in milliseconds for snapshot references other than the main branch."] - max_ref_age_ms: i64, + pub max_ref_age_ms: i64, } /// Delete, update, merge, and upsert properties. @@ -626,47 +684,47 @@ pub struct TableRowLevelProperties { #[key = "write.delete.granularity"] #[default("partition".to_string())] #[doc = "Granularity of generated delete files: partition or file."] - delete_granularity: String, + pub delete_granularity: String, #[key = "write.delete.isolation-level"] #[default("serializable".to_string())] #[doc = "Isolation level for delete commands: serializable or snapshot."] - delete_isolation_level: String, + pub delete_isolation_level: String, #[key = "write.delete.mode"] #[default("copy-on-write".to_string())] #[doc = "Execution mode for delete commands: copy-on-write or merge-on-read."] - delete_mode: String, + pub delete_mode: String, #[key = "write.delete.distribution-mode"] #[default(None)] #[doc = "Optional distribution mode for delete command data."] - delete_distribution_mode: Option, + pub delete_distribution_mode: Option, #[key = "write.update.isolation-level"] #[default("serializable".to_string())] #[doc = "Isolation level for update commands: serializable or snapshot."] - update_isolation_level: String, + pub update_isolation_level: String, #[key = "write.update.mode"] #[default("copy-on-write".to_string())] #[doc = "Execution mode for update commands: copy-on-write or merge-on-read."] - update_mode: String, + pub update_mode: String, #[key = "write.update.distribution-mode"] #[default(None)] #[doc = "Optional distribution mode for update command data."] - update_distribution_mode: Option, + pub update_distribution_mode: Option, #[key = "write.merge.isolation-level"] #[default("serializable".to_string())] #[doc = "Isolation level for merge commands: serializable or snapshot."] - merge_isolation_level: String, + pub merge_isolation_level: String, #[key = "write.merge.mode"] #[default("copy-on-write".to_string())] #[doc = "Execution mode for merge commands: copy-on-write or merge-on-read."] - merge_mode: String, + pub merge_mode: String, #[key = "write.merge.distribution-mode"] #[default(None)] #[doc = "Optional distribution mode for merge command data."] - merge_distribution_mode: Option, + pub merge_distribution_mode: Option, #[key = "write.upsert.enabled"] #[default(false)] #[doc = "Whether upsert behavior is enabled."] - upsert_enabled: bool, + pub upsert_enabled: bool, } /// Table encryption properties. @@ -675,11 +733,11 @@ pub struct TableEncryptionProperties { #[key = "encryption.key-id"] #[default(None)] #[doc = "Identifier of the table's master encryption key."] - key_id: Option, + pub key_id: Option, #[key = "encryption.data-key-length"] #[default(16)] #[doc = "Length in bytes of data-encryption keys; valid AES lengths are 16, 24, and 32 bytes."] - data_key_length: usize, + pub data_key_length: usize, } #[cfg(test)] @@ -692,35 +750,25 @@ mod tests { } #[test] - fn defaults_are_organized_by_property_group() { + fn creates_properties_from_defaults() { let properties = ParsedTableProperties::default(); - assert_eq!(properties.commit().num_retries(), 4); - assert_eq!(properties.write().format_default(), DataFileFormat::Parquet); - assert_eq!( - properties.parquet().row_group_size_bytes(), - 128 * 1024 * 1024 - ); - assert_eq!(properties.read().split_target_size(), 128 * 1024 * 1024); - assert!(properties.history().gc_enabled()); - assert_eq!(properties.encryption().data_key_length(), 16); + assert_eq!(properties.commit.num_retries, 4); + assert_eq!(properties.write.format_default, DataFileFormat::Parquet); + assert_eq!(properties.parquet.row_group_size_bytes, 128 * 1024 * 1024); + assert_eq!(properties.read.split_target_size, 128 * 1024 * 1024); + assert!(properties.history.gc_enabled); + assert_eq!(properties.encryption.data_key_length, 16); } #[test] - fn nested_properties_serialize_to_a_flat_json_map() { - let properties = ParsedTableProperties::default() - .with_commit(TableCommitProperties::default().with_num_retries(9)) - .with_write( - TableWriteProperties::default() - .with_format_default(DataFileFormat::Orc) - .with_data_location(Some("s3://warehouse/table/data".to_string())), - ) - .with_parquet( - TableParquetProperties::default().with_bloom_filter_column_fpp(HashMap::from([( - "customer_id".to_string(), - 0.02, - )])), - ); + fn serializes_to_flat_json_object() { + let mut properties = ParsedTableProperties::default(); + properties.commit.num_retries = 9; + properties.write.format_default = DataFileFormat::Orc; + properties.write.data_location = Some("s3://warehouse/table/data".to_string()); + properties.parquet.bloom_filter_column_fpp = + HashMap::from([("customer_id".to_string(), 0.02)]); let json = serde_json::to_value(&properties).unwrap(); assert_eq!(json["commit.retry.num-retries"], "9"); @@ -732,13 +780,22 @@ mod tests { ); assert!(json.get("commit").is_none()); assert!(json.get("write").is_none()); + } + + #[test] + fn deserializes_from_flat_json_object() { + let properties: ParsedTableProperties = serde_json::from_value(serde_json::json!({ + "commit.retry.num-retries": "8", + "write.format.default": "orc", + "write.data.path": "s3://warehouse/table/data" + })) + .unwrap(); - let decoded: ParsedTableProperties = serde_json::from_value(json).unwrap(); - assert_eq!(decoded.commit().num_retries(), 9); - assert_eq!(decoded.write().format_default(), DataFileFormat::Orc); + assert_eq!(properties.commit.num_retries, 8); + assert_eq!(properties.write.format_default, DataFileFormat::Orc); assert_eq!( - decoded.parquet().bloom_filter_column_fpp()["customer_id"], - 0.02 + properties.write.data_location, + Some("s3://warehouse/table/data".to_string()) ); } @@ -748,22 +805,16 @@ mod tests { let decoded: ParsedTableProperties = serde_json::from_value(serde_json::to_value(&defaults).unwrap()).unwrap(); + assert_eq!(decoded.commit.num_retries, defaults.commit.num_retries); assert_eq!( - decoded.commit().num_retries(), - defaults.commit().num_retries() + decoded.metadata.compression_codec, + defaults.metadata.compression_codec ); assert_eq!( - decoded.metadata().compression_codec(), - defaults.metadata().compression_codec() - ); - assert_eq!( - decoded.parquet().content_defined_chunking_max_chunk_size(), - defaults.parquet().content_defined_chunking_max_chunk_size() - ); - assert_eq!( - decoded.row_level().merge_mode(), - defaults.row_level().merge_mode() + decoded.parquet.content_defined_chunking_max_chunk_size, + defaults.parquet.content_defined_chunking_max_chunk_size ); + assert_eq!(decoded.row_level.merge_mode, defaults.row_level.merge_mode); } #[test] @@ -788,16 +839,13 @@ mod tests { ])) .unwrap(); - assert_eq!( - properties.general().comment(), - Some("orders table".to_string()) - ); - assert_eq!(properties.commit().num_status_checks(), 7); - assert_eq!(properties.avro().delete_compression(), "snappy"); - assert_eq!(properties.read().split_planning_lookback(), 25); - assert_eq!(properties.history().min_snapshots_to_keep(), 4); - assert_eq!(properties.row_level().delete_mode(), "merge-on-read"); - assert_eq!(properties.encryption().data_key_length(), 32); + assert_eq!(properties.general.comment, Some("orders table".to_string())); + assert_eq!(properties.commit.num_status_checks, 7); + assert_eq!(properties.avro.delete_compression, "snappy"); + assert_eq!(properties.read.split_planning_lookback, 25); + assert_eq!(properties.history.min_snapshots_to_keep, 4); + assert_eq!(properties.row_level.delete_mode, "merge-on-read"); + assert_eq!(properties.encryption.data_key_length, 32); } #[test] @@ -815,11 +863,11 @@ mod tests { .unwrap(); assert_eq!( - properties.metadata().path(), + properties.metadata.path, Some("s3://warehouse/table/metadata".to_string()) ); assert_eq!( - properties.metadata().compression_codec(), + properties.metadata.compression_codec, CompressionCodec::gzip_default() ); diff --git a/crates/iceberg/src/spec/table_metadata.rs b/crates/iceberg/src/spec/table_metadata.rs index 97da227616..38fca30f76 100644 --- a/crates/iceberg/src/spec/table_metadata.rs +++ b/crates/iceberg/src/spec/table_metadata.rs @@ -4052,11 +4052,11 @@ mod tests { let props = metadata.parsed_table_properties().unwrap(); assert_eq!( - props.commit().num_retries(), + props.commit.num_retries, TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT ); assert_eq!( - props.write().target_file_size_bytes(), + props.write.target_file_size_bytes, TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT ); @@ -4104,8 +4104,8 @@ mod tests { let props = metadata.parsed_table_properties().unwrap(); - assert_eq!(props.commit().num_retries(), 10); - assert_eq!(props.write().target_file_size_bytes(), 1024); + assert_eq!(props.commit.num_retries, 10); + assert_eq!(props.write.target_file_size_bytes, 1024); } #[test] diff --git a/crates/iceberg/src/transaction/expire_snapshots.rs b/crates/iceberg/src/transaction/expire_snapshots.rs index 0e7cdce10c..b308e64da6 100644 --- a/crates/iceberg/src/transaction/expire_snapshots.rs +++ b/crates/iceberg/src/transaction/expire_snapshots.rs @@ -100,7 +100,7 @@ impl ExpireSnapshotsAction { /// Resolves the snapshots and refs to remove, following Java `RemoveSnapshots.internalApply`. fn plan(&self, table: &Table, properties: &ParsedTableProperties) -> Result { - let history = properties.history(); + let history = &properties.history; // Matches Java `RemoveSnapshots.retainLast`, which requires at least one snapshot. if self.retain_last == Some(0) { return Err(Error::new( @@ -116,8 +116,8 @@ impl ExpireSnapshotsAction { // days) the age path always runs, so even an explicit-id-only call applies the default cutoff. let default_cutoff = self .older_than_ms - .unwrap_or_else(|| now.saturating_sub(history.max_snapshot_age_ms())); - let default_min_to_keep = self.retain_last.unwrap_or(history.min_snapshots_to_keep()); + .unwrap_or_else(|| now.saturating_sub(history.max_snapshot_age_ms)); + let default_min_to_keep = self.retain_last.unwrap_or(history.min_snapshots_to_keep); // Ref aging: `main` is always kept; any other ref whose head is older than its // `max_ref_age_ms` (defaulting to `history.expire.max-ref-age-ms`) is dropped, like Java's @@ -126,7 +126,7 @@ impl ExpireSnapshotsAction { let mut retained_refs: Vec<&SnapshotReference> = vec![]; for (ref_name, snapshot_ref) in &metadata.refs { if ref_name == MAIN_BRANCH - || !Self::ref_aged_out(metadata, snapshot_ref, now, history.max_ref_age_ms()) + || !Self::ref_aged_out(metadata, snapshot_ref, now, history.max_ref_age_ms) { retained_refs.push(snapshot_ref); } else { @@ -303,7 +303,7 @@ impl TransactionAction for ExpireSnapshotsAction { let properties = metadata.parsed_table_properties()?; // Expiring metadata defeats a user's explicit decision to disable GC (Java refuses too). - if !properties.history().gc_enabled() { + if !properties.history.gc_enabled { return Err(Error::new( ErrorKind::DataInvalid, "Cannot expire snapshots: gc.enabled is false", diff --git a/crates/iceberg/src/transaction/mod.rs b/crates/iceberg/src/transaction/mod.rs index 19cc9f159f..f5ab4247d9 100644 --- a/crates/iceberg/src/transaction/mod.rs +++ b/crates/iceberg/src/transaction/mod.rs @@ -181,7 +181,7 @@ impl Transaction { let table_props = self.table.metadata().parsed_table_properties()?; // TODO(https://github.com/apache/iceberg-rust/issues/2034): remove once encrypted writes are supported - if table_props.encryption().key_id().is_some() { + if table_props.encryption.key_id.is_some() { return Err(Error::new( ErrorKind::FeatureUnsupported, "Cannot commit to an encrypted table: encrypted writes are not yet supported", @@ -204,12 +204,12 @@ impl Transaction { } fn build_backoff(props: ParsedTableProperties) -> Result { - let commit = props.commit(); + let commit = props.commit; Ok(ExponentialBuilder::new() - .with_min_delay(Duration::from_millis(commit.min_retry_wait_ms())) - .with_max_delay(Duration::from_millis(commit.max_retry_wait_ms())) - .with_total_delay(Some(Duration::from_millis(commit.total_retry_timeout_ms()))) - .with_max_times(commit.num_retries()) + .with_min_delay(Duration::from_millis(commit.min_retry_wait_ms)) + .with_max_delay(Duration::from_millis(commit.max_retry_wait_ms)) + .with_total_delay(Some(Duration::from_millis(commit.total_retry_timeout_ms))) + .with_max_times(commit.num_retries) .with_factor(2.0) .build()) } diff --git a/crates/iceberg/src/writer/file_writer/parquet_writer.rs b/crates/iceberg/src/writer/file_writer/parquet_writer.rs index a70ce5ac71..58de75a8b1 100644 --- a/crates/iceberg/src/writer/file_writer/parquet_writer.rs +++ b/crates/iceberg/src/writer/file_writer/parquet_writer.rs @@ -100,13 +100,13 @@ impl ParquetWriterBuilder { table_props: &ParsedTableProperties, schema: SchemaRef, ) -> Self { - let parquet = table_props.parquet(); + let parquet = &table_props.parquet; let cdc = parquet - .content_defined_chunking_enabled() + .content_defined_chunking_enabled .then_some(CdcOptions { - min_chunk_size: parquet.content_defined_chunking_min_chunk_size(), - max_chunk_size: parquet.content_defined_chunking_max_chunk_size(), - norm_level: parquet.content_defined_chunking_norm_level(), + min_chunk_size: parquet.content_defined_chunking_min_chunk_size, + max_chunk_size: parquet.content_defined_chunking_max_chunk_size, + norm_level: parquet.content_defined_chunking_norm_level, }); Self::from_cdc_options(cdc, schema) } diff --git a/crates/integrations/datafusion/src/physical_plan/write.rs b/crates/integrations/datafusion/src/physical_plan/write.rs index 4e7dffb2bc..6fc9a061b8 100644 --- a/crates/integrations/datafusion/src/physical_plan/write.rs +++ b/crates/integrations/datafusion/src/physical_plan/write.rs @@ -209,8 +209,8 @@ impl ExecutionPlan for IcebergWriteExec { .map_err(to_datafusion_error)?; // Check data file format - let write_properties = table_props.write(); - let file_format = write_properties.format_default(); + let write_properties = &table_props.write; + let file_format = write_properties.format_default; if file_format != DataFileFormat::Parquet { return Err(to_datafusion_error(Error::new( ErrorKind::FeatureUnsupported, @@ -226,7 +226,7 @@ impl ExecutionPlan for IcebergWriteExec { self.table.metadata().current_schema().clone(), ) .with_match_mode(FieldMatchMode::Name); - let target_file_size = write_properties.target_file_size_bytes(); + let target_file_size = write_properties.target_file_size_bytes; let file_io = self.table.file_io().clone(); // todo location_gen and file_name_gen should be configurable @@ -245,7 +245,7 @@ impl ExecutionPlan for IcebergWriteExec { let data_file_writer_builder = DataFileWriterBuilder::new(rolling_writer_builder); // Create TaskWriter - let fanout_enabled = write_properties.datafusion_fanout_enabled(); + let fanout_enabled = write_properties.datafusion_fanout_enabled; let schema = self.table.metadata().current_schema().clone(); let partition_spec = self.table.metadata().default_partition_spec().clone(); let task_writer = TaskWriter::try_new( diff --git a/crates/property-macro/src/lib.rs b/crates/property-macro/src/lib.rs index 56b419f9de..598ec7cc10 100644 --- a/crates/property-macro/src/lib.rs +++ b/crates/property-macro/src/lib.rs @@ -19,13 +19,13 @@ use proc_macro::TokenStream; use proc_macro2::TokenStream as TokenStream2; -use quote::{format_ident, quote}; +use quote::quote; use syn::{ Attribute, Data, DeriveInput, Error, Expr, ExprPath, Field, Fields, Ident, Meta, Path, Type, parse_macro_input, }; -/// Derive parsing, defaults, JSON serialization, and getters for a typed property map. +/// Derive parsing, defaults, and JSON serialization for a typed property map. /// /// Leaf fields must declare the table-property key and its default: /// @@ -35,7 +35,7 @@ use syn::{ /// #[key = "write.format.default"] /// #[default = DataFileFormat::Parquet] /// #[doc = "Default file format"] -/// write_format_default: DataFileFormat, +/// pub write_format_default: DataFileFormat, /// } /// ``` /// @@ -43,9 +43,8 @@ use syn::{ /// the declared prefix. `nested` embeds another `Properties` struct while keeping its serialized /// property map flat. `parse_with` may be used for exact-key property types that do not implement /// `FromStr` or need validation. `serialize_with` supplies their string representation in JSON. -/// Optional fields are omitted from JSON when they are `None`. Fields must implement `Clone`; they -/// also need `FromStr` and `ToString` unless the relevant custom parsing or serialization attribute -/// is supplied. +/// Optional fields are omitted from JSON when they are `None`. Fields need `FromStr` and `ToString` +/// unless the relevant custom parsing or serialization attribute is supplied. #[proc_macro_derive( Properties, attributes(key, prefix, nested, default, parse_with, serialize_with) @@ -62,7 +61,6 @@ pub fn derive_properties(input: TokenStream) -> TokenStream { struct PropertyField { ident: Ident, ty: Type, - docs: Vec, key: Option, prefix: Option, nested: bool, @@ -108,32 +106,6 @@ fn expand_properties(input: DeriveInput) -> syn::Result { } }); - let accessors = fields.iter().map(|field| { - let ident = &field.ident; - let ty = &field.ty; - let docs = &field.docs; - let with_ident = format_ident!("with_{ident}"); - let getter_doc = format!("Returns the `{ident}` property."); - let with_doc = format!("Sets the `{ident}` property."); - let getter_docs = if docs.is_empty() { - quote!(#[doc = #getter_doc]) - } else { - quote!(#(#docs)*) - }; - quote! { - #getter_docs - pub fn #ident(&self) -> #ty { - self.#ident.clone() - } - - #[doc = #with_doc] - pub fn #with_ident(mut self, value: #ty) -> Self { - self.#ident = value; - self - } - } - }); - let parses = fields.iter().map(parse_field); let property_writes = fields.iter().map(write_field); @@ -176,8 +148,6 @@ fn expand_properties(input: DeriveInput) -> syn::Result { self.write_properties(&mut properties); properties } - - #(#accessors)* } impl ::serde::Serialize for #struct_name { @@ -249,12 +219,6 @@ fn parse_property_field(field: &Field) -> syn::Result { Ok(PropertyField { ident, ty: field.ty.clone(), - docs: field - .attrs - .iter() - .filter(|attribute| attribute.path().is_ident("doc")) - .cloned() - .collect(), key, prefix, nested, diff --git a/crates/property-macro/tests/properties.rs b/crates/property-macro/tests/properties.rs index 22aea51a56..1863499999 100644 --- a/crates/property-macro/tests/properties.rs +++ b/crates/property-macro/tests/properties.rs @@ -28,25 +28,26 @@ struct TestProperties { #[key(RETRIES)] #[default(4)] #[doc = "Number of retries."] - retries: u64, + pub retries: u64, #[key(OWNER)] #[default(None)] - owner: Option, + pub owner: Option, #[prefix(COLUMN_FPP_PREFIX)] #[default(HashMap::new())] - column_fpp: HashMap, + pub column_fpp: HashMap, } #[test] -fn generates_defaults_getters_modifiers_and_serde() { - let properties = TestProperties::default() - .with_retries(8) - .with_owner(Some("iceberg".to_string())) - .with_column_fpp(HashMap::from([("id".to_string(), 0.01)])); +fn generates_defaults_and_serde_for_public_fields() { + let properties = TestProperties { + retries: 8, + owner: Some("iceberg".to_string()), + column_fpp: HashMap::from([("id".to_string(), 0.01)]), + }; - assert_eq!(properties.retries(), 8); - assert_eq!(properties.owner(), Some("iceberg".to_string())); - assert_eq!(properties.column_fpp()["id"], 0.01); + assert_eq!(properties.retries, 8); + assert_eq!(properties.owner, Some("iceberg".to_string())); + assert_eq!(properties.column_fpp["id"], 0.01); let json = serde_json::to_value(&properties).unwrap(); assert_eq!(json[RETRIES], "8"); @@ -54,9 +55,9 @@ fn generates_defaults_getters_modifiers_and_serde() { assert_eq!(json[format!("{COLUMN_FPP_PREFIX}id")], "0.01"); let decoded: TestProperties = serde_json::from_value(json).unwrap(); - assert_eq!(decoded.retries(), 8); - assert_eq!(decoded.owner(), Some("iceberg".to_string())); - assert_eq!(decoded.column_fpp()["id"], 0.01); + assert_eq!(decoded.retries, 8); + assert_eq!(decoded.owner, Some("iceberg".to_string())); + assert_eq!(decoded.column_fpp["id"], 0.01); } #[derive(Clone, Debug, Properties)] @@ -64,24 +65,24 @@ struct CommitProperties { #[key = "commit.retry.num-retries"] #[default = 4] #[doc = "Number of times to retry a commit before failing."] - num_retries: u64, + pub num_retries: u64, } #[derive(Debug, Properties)] struct NestedProperties { #[nested] #[doc = "Commit behavior properties."] - commit: CommitProperties, + pub commit: CommitProperties, } #[test] fn nested_properties_use_a_flat_property_map() { - let properties = - NestedProperties::default().with_commit(CommitProperties::default().with_num_retries(9)); + let mut properties = NestedProperties::default(); + properties.commit.num_retries = 9; let json = serde_json::to_value(&properties).unwrap(); assert_eq!(json["commit.retry.num-retries"], "9"); let decoded: NestedProperties = serde_json::from_value(json).unwrap(); - assert_eq!(decoded.commit().num_retries(), 9); + assert_eq!(decoded.commit.num_retries, 9); } From 389c8ef7b0d885c18c8dd3d576b300b8458669e4 Mon Sep 17 00:00:00 2001 From: Renjie Liu Date: Wed, 5 Aug 2026 11:03:31 +0800 Subject: [PATCH 05/14] No nested --- crates/iceberg/public-api.txt | 398 ++++--------- crates/iceberg/src/catalog/utils.rs | 2 +- crates/iceberg/src/encryption/manager.rs | 7 +- crates/iceberg/src/spec/parsed_table_prop.rs | 542 ++++++++++-------- crates/iceberg/src/spec/table_metadata.rs | 8 +- .../src/transaction/expire_snapshots.rs | 16 +- crates/iceberg/src/transaction/mod.rs | 13 +- .../src/writer/file_writer/parquet_writer.rs | 11 +- .../datafusion/src/physical_plan/write.rs | 7 +- crates/property-macro/tests/properties.rs | 2 + 10 files changed, 448 insertions(+), 558 deletions(-) diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index aef65ac0c2..f90c664fa2 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -2245,19 +2245,127 @@ pub fn iceberg::spec::NestedField::serialize<__S>(&self, __serializer: __S) -> c impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::NestedField pub fn iceberg::spec::NestedField::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> pub struct iceberg::spec::ParsedTableProperties -pub iceberg::spec::ParsedTableProperties::avro: iceberg::spec::TableAvroProperties -pub iceberg::spec::ParsedTableProperties::commit: iceberg::spec::TableCommitProperties -pub iceberg::spec::ParsedTableProperties::compatibility: iceberg::spec::TableCompatibilityProperties -pub iceberg::spec::ParsedTableProperties::encryption: iceberg::spec::TableEncryptionProperties -pub iceberg::spec::ParsedTableProperties::general: iceberg::spec::TableGeneralProperties -pub iceberg::spec::ParsedTableProperties::history: iceberg::spec::TableHistoryProperties -pub iceberg::spec::ParsedTableProperties::manifest: iceberg::spec::TableManifestProperties -pub iceberg::spec::ParsedTableProperties::metadata: iceberg::spec::TableMetadataProperties -pub iceberg::spec::ParsedTableProperties::orc: iceberg::spec::TableOrcProperties -pub iceberg::spec::ParsedTableProperties::parquet: iceberg::spec::TableParquetProperties -pub iceberg::spec::ParsedTableProperties::read: iceberg::spec::TableReadProperties -pub iceberg::spec::ParsedTableProperties::row_level: iceberg::spec::TableRowLevelProperties -pub iceberg::spec::ParsedTableProperties::write: iceberg::spec::TableWriteProperties +pub iceberg::spec::ParsedTableProperties::comment: core::option::Option +pub iceberg::spec::ParsedTableProperties::commit_manifest_merge_enabled: bool +pub iceberg::spec::ParsedTableProperties::commit_manifest_min_count_to_merge: usize +pub iceberg::spec::ParsedTableProperties::commit_manifest_target_size_bytes: usize +pub iceberg::spec::ParsedTableProperties::commit_retry_max_wait_ms: u64 +pub iceberg::spec::ParsedTableProperties::commit_retry_min_wait_ms: u64 +pub iceberg::spec::ParsedTableProperties::commit_retry_num_retries: usize +pub iceberg::spec::ParsedTableProperties::commit_retry_total_timeout_ms: u64 +pub iceberg::spec::ParsedTableProperties::commit_status_check_max_wait_ms: u64 +pub iceberg::spec::ParsedTableProperties::commit_status_check_min_wait_ms: u64 +pub iceberg::spec::ParsedTableProperties::commit_status_check_num_retries: usize +pub iceberg::spec::ParsedTableProperties::commit_status_check_total_timeout_ms: u64 +pub iceberg::spec::ParsedTableProperties::compatibility_snapshot_id_inheritance_enabled: bool +pub iceberg::spec::ParsedTableProperties::encryption_data_key_length: usize +pub iceberg::spec::ParsedTableProperties::encryption_key_id: core::option::Option +pub iceberg::spec::ParsedTableProperties::engine_hive_enabled: bool +pub iceberg::spec::ParsedTableProperties::engine_hive_lock_enabled: bool +pub iceberg::spec::ParsedTableProperties::gc_enabled: bool +pub iceberg::spec::ParsedTableProperties::history_expire_max_ref_age_ms: i64 +pub iceberg::spec::ParsedTableProperties::history_expire_max_snapshot_age_ms: i64 +pub iceberg::spec::ParsedTableProperties::history_expire_min_snapshots_to_keep: usize +pub iceberg::spec::ParsedTableProperties::identifier_fields_rely: bool +pub iceberg::spec::ParsedTableProperties::read_data_planning_mode: alloc::string::String +pub iceberg::spec::ParsedTableProperties::read_delete_planning_mode: alloc::string::String +pub iceberg::spec::ParsedTableProperties::read_orc_vectorization_batch_size: usize +pub iceberg::spec::ParsedTableProperties::read_orc_vectorization_enabled: bool +pub iceberg::spec::ParsedTableProperties::read_parquet_vectorization_batch_size: usize +pub iceberg::spec::ParsedTableProperties::read_parquet_vectorization_enabled: bool +pub iceberg::spec::ParsedTableProperties::read_split_adaptive_size_enabled: bool +pub iceberg::spec::ParsedTableProperties::read_split_metadata_target_size: u64 +pub iceberg::spec::ParsedTableProperties::read_split_open_file_cost: u64 +pub iceberg::spec::ParsedTableProperties::read_split_planning_lookback: usize +pub iceberg::spec::ParsedTableProperties::read_split_target_size: u64 +pub iceberg::spec::ParsedTableProperties::schema_name_mapping_default: core::option::Option +pub iceberg::spec::ParsedTableProperties::write_avro_compression_codec: alloc::string::String +pub iceberg::spec::ParsedTableProperties::write_avro_compression_level: core::option::Option +pub iceberg::spec::ParsedTableProperties::write_data_path: core::option::Option +pub iceberg::spec::ParsedTableProperties::write_datafusion_fanout_enabled: bool +pub iceberg::spec::ParsedTableProperties::write_delete_avro_compression_codec: alloc::string::String +pub iceberg::spec::ParsedTableProperties::write_delete_avro_compression_level: core::option::Option +pub iceberg::spec::ParsedTableProperties::write_delete_distribution_mode: core::option::Option +pub iceberg::spec::ParsedTableProperties::write_delete_format_default: iceberg::spec::DataFileFormat +pub iceberg::spec::ParsedTableProperties::write_delete_granularity: alloc::string::String +pub iceberg::spec::ParsedTableProperties::write_delete_isolation_level: alloc::string::String +pub iceberg::spec::ParsedTableProperties::write_delete_mode: alloc::string::String +pub iceberg::spec::ParsedTableProperties::write_delete_orc_block_size_bytes: u64 +pub iceberg::spec::ParsedTableProperties::write_delete_orc_compression_codec: alloc::string::String +pub iceberg::spec::ParsedTableProperties::write_delete_orc_compression_strategy: alloc::string::String +pub iceberg::spec::ParsedTableProperties::write_delete_orc_stripe_size_bytes: u64 +pub iceberg::spec::ParsedTableProperties::write_delete_orc_vectorized_batch_size: usize +pub iceberg::spec::ParsedTableProperties::write_delete_parquet_compression_codec: alloc::string::String +pub iceberg::spec::ParsedTableProperties::write_delete_parquet_compression_level: core::option::Option +pub iceberg::spec::ParsedTableProperties::write_delete_parquet_dict_size_bytes: usize +pub iceberg::spec::ParsedTableProperties::write_delete_parquet_page_row_limit: usize +pub iceberg::spec::ParsedTableProperties::write_delete_parquet_page_size_bytes: usize +pub iceberg::spec::ParsedTableProperties::write_delete_parquet_page_version: alloc::string::String +pub iceberg::spec::ParsedTableProperties::write_delete_parquet_row_group_check_max_record_count: usize +pub iceberg::spec::ParsedTableProperties::write_delete_parquet_row_group_check_min_record_count: usize +pub iceberg::spec::ParsedTableProperties::write_delete_parquet_row_group_size_bytes: usize +pub iceberg::spec::ParsedTableProperties::write_delete_target_file_size_bytes: usize +pub iceberg::spec::ParsedTableProperties::write_distribution_mode: core::option::Option +pub iceberg::spec::ParsedTableProperties::write_folder_storage_path: core::option::Option +pub iceberg::spec::ParsedTableProperties::write_format_default: iceberg::spec::DataFileFormat +pub iceberg::spec::ParsedTableProperties::write_location_provider_impl: core::option::Option +pub iceberg::spec::ParsedTableProperties::write_manifest_compression_codec: alloc::string::String +pub iceberg::spec::ParsedTableProperties::write_manifest_compression_level: core::option::Option +pub iceberg::spec::ParsedTableProperties::write_manifest_lists_enabled: bool +pub iceberg::spec::ParsedTableProperties::write_merge_distribution_mode: core::option::Option +pub iceberg::spec::ParsedTableProperties::write_merge_isolation_level: alloc::string::String +pub iceberg::spec::ParsedTableProperties::write_merge_mode: alloc::string::String +pub iceberg::spec::ParsedTableProperties::write_metadata_compression_codec: iceberg::compression::CompressionCodec +pub iceberg::spec::ParsedTableProperties::write_metadata_delete_after_commit_enabled: bool +pub iceberg::spec::ParsedTableProperties::write_metadata_metrics_column: std::collections::hash::map::HashMap +pub iceberg::spec::ParsedTableProperties::write_metadata_metrics_default: alloc::string::String +pub iceberg::spec::ParsedTableProperties::write_metadata_metrics_max_inferred_column_defaults: usize +pub iceberg::spec::ParsedTableProperties::write_metadata_path: core::option::Option +pub iceberg::spec::ParsedTableProperties::write_metadata_previous_versions_max: usize +pub iceberg::spec::ParsedTableProperties::write_object_storage_enabled: bool +pub iceberg::spec::ParsedTableProperties::write_object_storage_partitioned_paths: bool +pub iceberg::spec::ParsedTableProperties::write_object_storage_path: core::option::Option +pub iceberg::spec::ParsedTableProperties::write_orc_block_size_bytes: u64 +pub iceberg::spec::ParsedTableProperties::write_orc_bloom_filter_columns: alloc::string::String +pub iceberg::spec::ParsedTableProperties::write_orc_bloom_filter_fpp: f64 +pub iceberg::spec::ParsedTableProperties::write_orc_compression_codec: alloc::string::String +pub iceberg::spec::ParsedTableProperties::write_orc_compression_strategy: alloc::string::String +pub iceberg::spec::ParsedTableProperties::write_orc_stripe_size_bytes: u64 +pub iceberg::spec::ParsedTableProperties::write_orc_vectorized_batch_size: usize +pub iceberg::spec::ParsedTableProperties::write_parquet_bloom_filter_adaptive_enabled: bool +pub iceberg::spec::ParsedTableProperties::write_parquet_bloom_filter_enabled_column: std::collections::hash::map::HashMap +pub iceberg::spec::ParsedTableProperties::write_parquet_bloom_filter_fpp_column: std::collections::hash::map::HashMap +pub iceberg::spec::ParsedTableProperties::write_parquet_bloom_filter_max_bytes: usize +pub iceberg::spec::ParsedTableProperties::write_parquet_bloom_filter_ndv_column: std::collections::hash::map::HashMap +pub iceberg::spec::ParsedTableProperties::write_parquet_compression_codec: alloc::string::String +pub iceberg::spec::ParsedTableProperties::write_parquet_compression_level: core::option::Option +pub iceberg::spec::ParsedTableProperties::write_parquet_content_defined_chunking_enabled: bool +pub iceberg::spec::ParsedTableProperties::write_parquet_content_defined_chunking_max_chunk_size: usize +pub iceberg::spec::ParsedTableProperties::write_parquet_content_defined_chunking_min_chunk_size: usize +pub iceberg::spec::ParsedTableProperties::write_parquet_content_defined_chunking_norm_level: i32 +pub iceberg::spec::ParsedTableProperties::write_parquet_dict_encoding_enabled_column: std::collections::hash::map::HashMap +pub iceberg::spec::ParsedTableProperties::write_parquet_dict_size_bytes: usize +pub iceberg::spec::ParsedTableProperties::write_parquet_page_row_limit: usize +pub iceberg::spec::ParsedTableProperties::write_parquet_page_size_bytes: usize +pub iceberg::spec::ParsedTableProperties::write_parquet_page_version: alloc::string::String +pub iceberg::spec::ParsedTableProperties::write_parquet_row_group_check_max_record_count: usize +pub iceberg::spec::ParsedTableProperties::write_parquet_row_group_check_min_record_count: usize +pub iceberg::spec::ParsedTableProperties::write_parquet_row_group_size_bytes: usize +pub iceberg::spec::ParsedTableProperties::write_parquet_row_group_size_track_uncompressed: bool +pub iceberg::spec::ParsedTableProperties::write_parquet_shred_variants: bool +pub iceberg::spec::ParsedTableProperties::write_parquet_stats_enabled_column: std::collections::hash::map::HashMap +pub iceberg::spec::ParsedTableProperties::write_parquet_variant_inference_buffer_size: usize +pub iceberg::spec::ParsedTableProperties::write_spark_accept_any_schema: bool +pub iceberg::spec::ParsedTableProperties::write_spark_advisory_partition_size_bytes: core::option::Option +pub iceberg::spec::ParsedTableProperties::write_spark_auto_schema_evolution_enabled: bool +pub iceberg::spec::ParsedTableProperties::write_spark_fanout_enabled: bool +pub iceberg::spec::ParsedTableProperties::write_summary_partition_limit: u64 +pub iceberg::spec::ParsedTableProperties::write_target_file_size_bytes: usize +pub iceberg::spec::ParsedTableProperties::write_update_distribution_mode: core::option::Option +pub iceberg::spec::ParsedTableProperties::write_update_isolation_level: alloc::string::String +pub iceberg::spec::ParsedTableProperties::write_update_mode: alloc::string::String +pub iceberg::spec::ParsedTableProperties::write_upsert_enabled: bool +pub iceberg::spec::ParsedTableProperties::write_wap_enabled: bool impl core::clone::Clone for iceberg::spec::ParsedTableProperties pub fn iceberg::spec::ParsedTableProperties::clone(&self) -> iceberg::spec::ParsedTableProperties impl core::default::Default for iceberg::spec::ParsedTableProperties @@ -2649,116 +2757,6 @@ impl serde_core::ser::Serialize for iceberg::spec::Summary pub fn iceberg::spec::Summary::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::Summary pub fn iceberg::spec::Summary::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> -pub struct iceberg::spec::TableAvroProperties -pub iceberg::spec::TableAvroProperties::compression: alloc::string::String -pub iceberg::spec::TableAvroProperties::compression_level: core::option::Option -pub iceberg::spec::TableAvroProperties::delete_compression: alloc::string::String -pub iceberg::spec::TableAvroProperties::delete_compression_level: core::option::Option -impl core::clone::Clone for iceberg::spec::TableAvroProperties -pub fn iceberg::spec::TableAvroProperties::clone(&self) -> iceberg::spec::TableAvroProperties -impl core::default::Default for iceberg::spec::TableAvroProperties -pub fn iceberg::spec::TableAvroProperties::default() -> Self -impl core::fmt::Debug for iceberg::spec::TableAvroProperties -pub fn iceberg::spec::TableAvroProperties::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl serde_core::ser::Serialize for iceberg::spec::TableAvroProperties -pub fn iceberg::spec::TableAvroProperties::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> where S: serde_core::ser::Serializer -impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::TableAvroProperties -pub fn iceberg::spec::TableAvroProperties::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> -pub struct iceberg::spec::TableCommitProperties -pub iceberg::spec::TableCommitProperties::max_retry_wait_ms: u64 -pub iceberg::spec::TableCommitProperties::min_retry_wait_ms: u64 -pub iceberg::spec::TableCommitProperties::num_retries: usize -pub iceberg::spec::TableCommitProperties::num_status_checks: usize -pub iceberg::spec::TableCommitProperties::status_checks_max_wait_ms: u64 -pub iceberg::spec::TableCommitProperties::status_checks_min_wait_ms: u64 -pub iceberg::spec::TableCommitProperties::status_checks_total_wait_ms: u64 -pub iceberg::spec::TableCommitProperties::total_retry_timeout_ms: u64 -impl core::clone::Clone for iceberg::spec::TableCommitProperties -pub fn iceberg::spec::TableCommitProperties::clone(&self) -> iceberg::spec::TableCommitProperties -impl core::default::Default for iceberg::spec::TableCommitProperties -pub fn iceberg::spec::TableCommitProperties::default() -> Self -impl core::fmt::Debug for iceberg::spec::TableCommitProperties -pub fn iceberg::spec::TableCommitProperties::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl serde_core::ser::Serialize for iceberg::spec::TableCommitProperties -pub fn iceberg::spec::TableCommitProperties::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> where S: serde_core::ser::Serializer -impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::TableCommitProperties -pub fn iceberg::spec::TableCommitProperties::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> -pub struct iceberg::spec::TableCompatibilityProperties -pub iceberg::spec::TableCompatibilityProperties::engine_hive_enabled: bool -pub iceberg::spec::TableCompatibilityProperties::hive_lock_enabled: bool -pub iceberg::spec::TableCompatibilityProperties::snapshot_id_inheritance_enabled: bool -pub iceberg::spec::TableCompatibilityProperties::spark_write_accept_any_schema: bool -pub iceberg::spec::TableCompatibilityProperties::spark_write_advisory_partition_size_bytes: core::option::Option -pub iceberg::spec::TableCompatibilityProperties::spark_write_auto_schema_evolution: bool -pub iceberg::spec::TableCompatibilityProperties::spark_write_partitioned_fanout_enabled: bool -impl core::clone::Clone for iceberg::spec::TableCompatibilityProperties -pub fn iceberg::spec::TableCompatibilityProperties::clone(&self) -> iceberg::spec::TableCompatibilityProperties -impl core::default::Default for iceberg::spec::TableCompatibilityProperties -pub fn iceberg::spec::TableCompatibilityProperties::default() -> Self -impl core::fmt::Debug for iceberg::spec::TableCompatibilityProperties -pub fn iceberg::spec::TableCompatibilityProperties::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl serde_core::ser::Serialize for iceberg::spec::TableCompatibilityProperties -pub fn iceberg::spec::TableCompatibilityProperties::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> where S: serde_core::ser::Serializer -impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::TableCompatibilityProperties -pub fn iceberg::spec::TableCompatibilityProperties::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> -pub struct iceberg::spec::TableEncryptionProperties -pub iceberg::spec::TableEncryptionProperties::data_key_length: usize -pub iceberg::spec::TableEncryptionProperties::key_id: core::option::Option -impl core::clone::Clone for iceberg::spec::TableEncryptionProperties -pub fn iceberg::spec::TableEncryptionProperties::clone(&self) -> iceberg::spec::TableEncryptionProperties -impl core::default::Default for iceberg::spec::TableEncryptionProperties -pub fn iceberg::spec::TableEncryptionProperties::default() -> Self -impl core::fmt::Debug for iceberg::spec::TableEncryptionProperties -pub fn iceberg::spec::TableEncryptionProperties::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl serde_core::ser::Serialize for iceberg::spec::TableEncryptionProperties -pub fn iceberg::spec::TableEncryptionProperties::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> where S: serde_core::ser::Serializer -impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::TableEncryptionProperties -pub fn iceberg::spec::TableEncryptionProperties::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> -pub struct iceberg::spec::TableGeneralProperties -pub iceberg::spec::TableGeneralProperties::comment: core::option::Option -pub iceberg::spec::TableGeneralProperties::identifier_fields_rely: bool -impl core::clone::Clone for iceberg::spec::TableGeneralProperties -pub fn iceberg::spec::TableGeneralProperties::clone(&self) -> iceberg::spec::TableGeneralProperties -impl core::default::Default for iceberg::spec::TableGeneralProperties -pub fn iceberg::spec::TableGeneralProperties::default() -> Self -impl core::fmt::Debug for iceberg::spec::TableGeneralProperties -pub fn iceberg::spec::TableGeneralProperties::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl serde_core::ser::Serialize for iceberg::spec::TableGeneralProperties -pub fn iceberg::spec::TableGeneralProperties::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> where S: serde_core::ser::Serializer -impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::TableGeneralProperties -pub fn iceberg::spec::TableGeneralProperties::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> -pub struct iceberg::spec::TableHistoryProperties -pub iceberg::spec::TableHistoryProperties::gc_enabled: bool -pub iceberg::spec::TableHistoryProperties::max_ref_age_ms: i64 -pub iceberg::spec::TableHistoryProperties::max_snapshot_age_ms: i64 -pub iceberg::spec::TableHistoryProperties::min_snapshots_to_keep: usize -impl core::clone::Clone for iceberg::spec::TableHistoryProperties -pub fn iceberg::spec::TableHistoryProperties::clone(&self) -> iceberg::spec::TableHistoryProperties -impl core::default::Default for iceberg::spec::TableHistoryProperties -pub fn iceberg::spec::TableHistoryProperties::default() -> Self -impl core::fmt::Debug for iceberg::spec::TableHistoryProperties -pub fn iceberg::spec::TableHistoryProperties::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl serde_core::ser::Serialize for iceberg::spec::TableHistoryProperties -pub fn iceberg::spec::TableHistoryProperties::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> where S: serde_core::ser::Serializer -impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::TableHistoryProperties -pub fn iceberg::spec::TableHistoryProperties::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> -pub struct iceberg::spec::TableManifestProperties -pub iceberg::spec::TableManifestProperties::compression: alloc::string::String -pub iceberg::spec::TableManifestProperties::compression_level: core::option::Option -pub iceberg::spec::TableManifestProperties::lists_enabled: bool -pub iceberg::spec::TableManifestProperties::merge_enabled: bool -pub iceberg::spec::TableManifestProperties::min_merge_count: usize -pub iceberg::spec::TableManifestProperties::target_size_bytes: usize -impl core::clone::Clone for iceberg::spec::TableManifestProperties -pub fn iceberg::spec::TableManifestProperties::clone(&self) -> iceberg::spec::TableManifestProperties -impl core::default::Default for iceberg::spec::TableManifestProperties -pub fn iceberg::spec::TableManifestProperties::default() -> Self -impl core::fmt::Debug for iceberg::spec::TableManifestProperties -pub fn iceberg::spec::TableManifestProperties::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl serde_core::ser::Serialize for iceberg::spec::TableManifestProperties -pub fn iceberg::spec::TableManifestProperties::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> where S: serde_core::ser::Serializer -impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::TableManifestProperties -pub fn iceberg::spec::TableManifestProperties::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> pub struct iceberg::spec::TableMetadata impl iceberg::spec::TableMetadata pub fn iceberg::spec::TableMetadata::current_schema(&self) -> &iceberg::spec::SchemaRef @@ -2869,92 +2867,6 @@ impl core::clone::Clone for iceberg::spec::TableMetadataBuilder pub fn iceberg::spec::TableMetadataBuilder::clone(&self) -> iceberg::spec::TableMetadataBuilder impl core::fmt::Debug for iceberg::spec::TableMetadataBuilder pub fn iceberg::spec::TableMetadataBuilder::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub struct iceberg::spec::TableMetadataProperties -pub iceberg::spec::TableMetadataProperties::compression_codec: iceberg::compression::CompressionCodec -pub iceberg::spec::TableMetadataProperties::default_metrics_mode: alloc::string::String -pub iceberg::spec::TableMetadataProperties::default_name_mapping: core::option::Option -pub iceberg::spec::TableMetadataProperties::delete_after_commit_enabled: bool -pub iceberg::spec::TableMetadataProperties::metrics_max_inferred_column_defaults: usize -pub iceberg::spec::TableMetadataProperties::metrics_mode_by_column: std::collections::hash::map::HashMap -pub iceberg::spec::TableMetadataProperties::partition_summary_limit: u64 -pub iceberg::spec::TableMetadataProperties::path: core::option::Option -pub iceberg::spec::TableMetadataProperties::previous_versions_max: usize -impl core::clone::Clone for iceberg::spec::TableMetadataProperties -pub fn iceberg::spec::TableMetadataProperties::clone(&self) -> iceberg::spec::TableMetadataProperties -impl core::default::Default for iceberg::spec::TableMetadataProperties -pub fn iceberg::spec::TableMetadataProperties::default() -> Self -impl core::fmt::Debug for iceberg::spec::TableMetadataProperties -pub fn iceberg::spec::TableMetadataProperties::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl serde_core::ser::Serialize for iceberg::spec::TableMetadataProperties -pub fn iceberg::spec::TableMetadataProperties::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> where S: serde_core::ser::Serializer -impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::TableMetadataProperties -pub fn iceberg::spec::TableMetadataProperties::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> -pub struct iceberg::spec::TableOrcProperties -pub iceberg::spec::TableOrcProperties::block_size_bytes: u64 -pub iceberg::spec::TableOrcProperties::bloom_filter_columns: alloc::string::String -pub iceberg::spec::TableOrcProperties::bloom_filter_fpp: f64 -pub iceberg::spec::TableOrcProperties::compression: alloc::string::String -pub iceberg::spec::TableOrcProperties::compression_strategy: alloc::string::String -pub iceberg::spec::TableOrcProperties::delete_block_size_bytes: u64 -pub iceberg::spec::TableOrcProperties::delete_compression: alloc::string::String -pub iceberg::spec::TableOrcProperties::delete_compression_strategy: alloc::string::String -pub iceberg::spec::TableOrcProperties::delete_stripe_size_bytes: u64 -pub iceberg::spec::TableOrcProperties::delete_write_batch_size: usize -pub iceberg::spec::TableOrcProperties::stripe_size_bytes: u64 -pub iceberg::spec::TableOrcProperties::write_batch_size: usize -impl core::clone::Clone for iceberg::spec::TableOrcProperties -pub fn iceberg::spec::TableOrcProperties::clone(&self) -> iceberg::spec::TableOrcProperties -impl core::default::Default for iceberg::spec::TableOrcProperties -pub fn iceberg::spec::TableOrcProperties::default() -> Self -impl core::fmt::Debug for iceberg::spec::TableOrcProperties -pub fn iceberg::spec::TableOrcProperties::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl serde_core::ser::Serialize for iceberg::spec::TableOrcProperties -pub fn iceberg::spec::TableOrcProperties::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> where S: serde_core::ser::Serializer -impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::TableOrcProperties -pub fn iceberg::spec::TableOrcProperties::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> -pub struct iceberg::spec::TableParquetProperties -pub iceberg::spec::TableParquetProperties::bloom_filter_adaptive_enabled: bool -pub iceberg::spec::TableParquetProperties::bloom_filter_column_enabled: std::collections::hash::map::HashMap -pub iceberg::spec::TableParquetProperties::bloom_filter_column_fpp: std::collections::hash::map::HashMap -pub iceberg::spec::TableParquetProperties::bloom_filter_column_ndv: std::collections::hash::map::HashMap -pub iceberg::spec::TableParquetProperties::bloom_filter_max_bytes: usize -pub iceberg::spec::TableParquetProperties::column_stats_enabled: std::collections::hash::map::HashMap -pub iceberg::spec::TableParquetProperties::compression: alloc::string::String -pub iceberg::spec::TableParquetProperties::compression_level: core::option::Option -pub iceberg::spec::TableParquetProperties::content_defined_chunking_enabled: bool -pub iceberg::spec::TableParquetProperties::content_defined_chunking_max_chunk_size: usize -pub iceberg::spec::TableParquetProperties::content_defined_chunking_min_chunk_size: usize -pub iceberg::spec::TableParquetProperties::content_defined_chunking_norm_level: i32 -pub iceberg::spec::TableParquetProperties::delete_compression: alloc::string::String -pub iceberg::spec::TableParquetProperties::delete_compression_level: core::option::Option -pub iceberg::spec::TableParquetProperties::delete_dict_size_bytes: usize -pub iceberg::spec::TableParquetProperties::delete_page_row_limit: usize -pub iceberg::spec::TableParquetProperties::delete_page_size_bytes: usize -pub iceberg::spec::TableParquetProperties::delete_page_version: alloc::string::String -pub iceberg::spec::TableParquetProperties::delete_row_group_check_max_record_count: usize -pub iceberg::spec::TableParquetProperties::delete_row_group_check_min_record_count: usize -pub iceberg::spec::TableParquetProperties::delete_row_group_size_bytes: usize -pub iceberg::spec::TableParquetProperties::dict_encoding_enabled_by_column: std::collections::hash::map::HashMap -pub iceberg::spec::TableParquetProperties::dict_size_bytes: usize -pub iceberg::spec::TableParquetProperties::page_row_limit: usize -pub iceberg::spec::TableParquetProperties::page_size_bytes: usize -pub iceberg::spec::TableParquetProperties::page_version: alloc::string::String -pub iceberg::spec::TableParquetProperties::row_group_check_max_record_count: usize -pub iceberg::spec::TableParquetProperties::row_group_check_min_record_count: usize -pub iceberg::spec::TableParquetProperties::row_group_size_bytes: usize -pub iceberg::spec::TableParquetProperties::row_group_size_track_uncompressed: bool -pub iceberg::spec::TableParquetProperties::shred_variants: bool -pub iceberg::spec::TableParquetProperties::variant_inference_buffer_size: usize -impl core::clone::Clone for iceberg::spec::TableParquetProperties -pub fn iceberg::spec::TableParquetProperties::clone(&self) -> iceberg::spec::TableParquetProperties -impl core::default::Default for iceberg::spec::TableParquetProperties -pub fn iceberg::spec::TableParquetProperties::default() -> Self -impl core::fmt::Debug for iceberg::spec::TableParquetProperties -pub fn iceberg::spec::TableParquetProperties::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl serde_core::ser::Serialize for iceberg::spec::TableParquetProperties -pub fn iceberg::spec::TableParquetProperties::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> where S: serde_core::ser::Serializer -impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::TableParquetProperties -pub fn iceberg::spec::TableParquetProperties::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> pub struct iceberg::spec::TableProperties pub iceberg::spec::TableProperties::cdc_enabled: bool pub iceberg::spec::TableProperties::cdc_max_chunk_size: usize @@ -3032,74 +2944,6 @@ pub type iceberg::spec::TableProperties::Error = iceberg::Error pub fn iceberg::spec::TableProperties::try_from(props: &std::collections::hash::map::HashMap) -> iceberg::Result impl core::fmt::Debug for iceberg::spec::TableProperties pub fn iceberg::spec::TableProperties::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub struct iceberg::spec::TableReadProperties -pub iceberg::spec::TableReadProperties::adaptive_split_size_enabled: bool -pub iceberg::spec::TableReadProperties::data_planning_mode: alloc::string::String -pub iceberg::spec::TableReadProperties::delete_planning_mode: alloc::string::String -pub iceberg::spec::TableReadProperties::metadata_split_target_size: u64 -pub iceberg::spec::TableReadProperties::orc_batch_size: usize -pub iceberg::spec::TableReadProperties::orc_vectorization_enabled: bool -pub iceberg::spec::TableReadProperties::parquet_batch_size: usize -pub iceberg::spec::TableReadProperties::parquet_vectorization_enabled: bool -pub iceberg::spec::TableReadProperties::split_open_file_cost: u64 -pub iceberg::spec::TableReadProperties::split_planning_lookback: usize -pub iceberg::spec::TableReadProperties::split_target_size: u64 -impl core::clone::Clone for iceberg::spec::TableReadProperties -pub fn iceberg::spec::TableReadProperties::clone(&self) -> iceberg::spec::TableReadProperties -impl core::default::Default for iceberg::spec::TableReadProperties -pub fn iceberg::spec::TableReadProperties::default() -> Self -impl core::fmt::Debug for iceberg::spec::TableReadProperties -pub fn iceberg::spec::TableReadProperties::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl serde_core::ser::Serialize for iceberg::spec::TableReadProperties -pub fn iceberg::spec::TableReadProperties::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> where S: serde_core::ser::Serializer -impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::TableReadProperties -pub fn iceberg::spec::TableReadProperties::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> -pub struct iceberg::spec::TableRowLevelProperties -pub iceberg::spec::TableRowLevelProperties::delete_distribution_mode: core::option::Option -pub iceberg::spec::TableRowLevelProperties::delete_granularity: alloc::string::String -pub iceberg::spec::TableRowLevelProperties::delete_isolation_level: alloc::string::String -pub iceberg::spec::TableRowLevelProperties::delete_mode: alloc::string::String -pub iceberg::spec::TableRowLevelProperties::merge_distribution_mode: core::option::Option -pub iceberg::spec::TableRowLevelProperties::merge_isolation_level: alloc::string::String -pub iceberg::spec::TableRowLevelProperties::merge_mode: alloc::string::String -pub iceberg::spec::TableRowLevelProperties::update_distribution_mode: core::option::Option -pub iceberg::spec::TableRowLevelProperties::update_isolation_level: alloc::string::String -pub iceberg::spec::TableRowLevelProperties::update_mode: alloc::string::String -pub iceberg::spec::TableRowLevelProperties::upsert_enabled: bool -impl core::clone::Clone for iceberg::spec::TableRowLevelProperties -pub fn iceberg::spec::TableRowLevelProperties::clone(&self) -> iceberg::spec::TableRowLevelProperties -impl core::default::Default for iceberg::spec::TableRowLevelProperties -pub fn iceberg::spec::TableRowLevelProperties::default() -> Self -impl core::fmt::Debug for iceberg::spec::TableRowLevelProperties -pub fn iceberg::spec::TableRowLevelProperties::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl serde_core::ser::Serialize for iceberg::spec::TableRowLevelProperties -pub fn iceberg::spec::TableRowLevelProperties::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> where S: serde_core::ser::Serializer -impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::TableRowLevelProperties -pub fn iceberg::spec::TableRowLevelProperties::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> -pub struct iceberg::spec::TableWriteProperties -pub iceberg::spec::TableWriteProperties::audit_publish_enabled: bool -pub iceberg::spec::TableWriteProperties::data_location: core::option::Option -pub iceberg::spec::TableWriteProperties::datafusion_fanout_enabled: bool -pub iceberg::spec::TableWriteProperties::delete_format_default: iceberg::spec::DataFileFormat -pub iceberg::spec::TableWriteProperties::delete_target_file_size_bytes: usize -pub iceberg::spec::TableWriteProperties::distribution_mode: core::option::Option -pub iceberg::spec::TableWriteProperties::folder_storage_location: core::option::Option -pub iceberg::spec::TableWriteProperties::format_default: iceberg::spec::DataFileFormat -pub iceberg::spec::TableWriteProperties::location_provider_impl: core::option::Option -pub iceberg::spec::TableWriteProperties::object_store_enabled: bool -pub iceberg::spec::TableWriteProperties::object_store_partitioned_paths: bool -pub iceberg::spec::TableWriteProperties::object_store_path: core::option::Option -pub iceberg::spec::TableWriteProperties::target_file_size_bytes: usize -impl core::clone::Clone for iceberg::spec::TableWriteProperties -pub fn iceberg::spec::TableWriteProperties::clone(&self) -> iceberg::spec::TableWriteProperties -impl core::default::Default for iceberg::spec::TableWriteProperties -pub fn iceberg::spec::TableWriteProperties::default() -> Self -impl core::fmt::Debug for iceberg::spec::TableWriteProperties -pub fn iceberg::spec::TableWriteProperties::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl serde_core::ser::Serialize for iceberg::spec::TableWriteProperties -pub fn iceberg::spec::TableWriteProperties::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> where S: serde_core::ser::Serializer -impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::TableWriteProperties -pub fn iceberg::spec::TableWriteProperties::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> pub struct iceberg::spec::UnboundPartitionField pub iceberg::spec::UnboundPartitionField::field_id: core::option::Option pub iceberg::spec::UnboundPartitionField::name: alloc::string::String diff --git a/crates/iceberg/src/catalog/utils.rs b/crates/iceberg/src/catalog/utils.rs index a754b14ab5..f6117f71ed 100644 --- a/crates/iceberg/src/catalog/utils.rs +++ b/crates/iceberg/src/catalog/utils.rs @@ -60,7 +60,7 @@ pub async fn drop_table_data(table_info: &Table) -> Result<()> { } // Delete data files only if gc.enabled is true, to avoid corrupting shared tables - if metadata.parsed_table_properties()?.history.gc_enabled { + if metadata.parsed_table_properties()?.gc_enabled { delete_data_files(io, &manifests_to_delete).await?; } diff --git a/crates/iceberg/src/encryption/manager.rs b/crates/iceberg/src/encryption/manager.rs index cd9db8cb42..26e316af19 100644 --- a/crates/iceberg/src/encryption/manager.rs +++ b/crates/iceberg/src/encryption/manager.rs @@ -119,8 +119,7 @@ impl EncryptionManager { } let table_properties = metadata.parsed_table_properties()?; - let encryption = table_properties.encryption; - let Some(table_key_id) = encryption.key_id else { + let Some(table_key_id) = table_properties.encryption_key_id else { if kms_client.is_some() { tracing::warn!( "KeyManagementClient provided but table does not have encryption.key-id set" @@ -140,7 +139,9 @@ impl EncryptionManager { .kms_client(Arc::clone(kms_client)) .table_key_id(table_key_id) .encryption_keys(metadata.encryption_keys.clone()) - .key_size(AesKeySize::from_key_length(encryption.data_key_length)?) + .key_size(AesKeySize::from_key_length( + table_properties.encryption_data_key_length, + )?) .build(); Ok(Some(Arc::new(em))) } diff --git a/crates/iceberg/src/spec/parsed_table_prop.rs b/crates/iceberg/src/spec/parsed_table_prop.rs index fd56ae70fd..da7586e3c7 100644 --- a/crates/iceberg/src/spec/parsed_table_prop.rs +++ b/crates/iceberg/src/spec/parsed_table_prop.rs @@ -17,9 +17,8 @@ //! Typed access to Iceberg table properties. //! -//! [`ParsedTableProperties`] organizes Iceberg's string-keyed table properties into typed groups -//! such as [`TableWriteProperties`] and [`TableCommitProperties`]. The Rust representation is -//! nested, while its JSON representation remains a flat object whose keys and values are strings. +//! [`ParsedTableProperties`] exposes Iceberg's string-keyed table properties as typed public +//! fields. Its JSON representation is a flat object whose keys and values are strings. //! //! # Create from defaults //! @@ -29,10 +28,10 @@ //! use iceberg::spec::{DataFileFormat, ParsedTableProperties}; //! //! let mut properties = ParsedTableProperties::default(); -//! properties.write.format_default = DataFileFormat::Orc; -//! properties.write.data_location = Some("s3://warehouse/table/data".to_string()); +//! properties.write_format_default = DataFileFormat::Orc; +//! properties.write_data_path = Some("s3://warehouse/table/data".to_string()); //! -//! assert_eq!(properties.write.format_default, DataFileFormat::Orc); +//! assert_eq!(properties.write_format_default, DataFileFormat::Orc); //! ``` //! //! # Deserialize from JSON @@ -48,23 +47,23 @@ //! "write.format.default": "orc" //! }))?; //! -//! assert_eq!(properties.commit.num_retries, 8); -//! assert_eq!(properties.write.format_default, DataFileFormat::Orc); +//! assert_eq!(properties.commit_retry_num_retries, 8); +//! assert_eq!(properties.write_format_default, DataFileFormat::Orc); //! # Ok(()) //! # } //! ``` //! //! # Serialize to JSON //! -//! Serialization flattens the nested groups back into Iceberg property keys: +//! Serialization converts the typed fields back into Iceberg property keys: //! //! ``` //! use iceberg::spec::ParsedTableProperties; //! //! # fn main() -> Result<(), serde_json::Error> { //! let mut properties = ParsedTableProperties::default(); -//! properties.commit.num_retries = 8; -//! properties.write.data_location = Some("s3://warehouse/table/data".to_string()); +//! properties.commit_retry_num_retries = 8; +//! properties.write_data_path = Some("s3://warehouse/table/data".to_string()); //! //! let json = serde_json::to_value(&properties)?; //! assert_eq!(json["commit.retry.num-retries"], "8"); @@ -137,9 +136,9 @@ fn serialize_compression_codec(codec: &CompressionCodec) -> String { codec.name().to_string() } -/// Typed Iceberg table properties organized by behavior and file format. +/// Typed Iceberg table properties organized into documented sections. /// -/// Serde represents this hierarchy as Iceberg's flat string-to-string property map. Property +/// Serde represents this struct as Iceberg's flat string-to-string property map. Property /// definitions and descriptions are based on the pinned [Java TableProperties implementation] /// and [Apache Iceberg configuration documentation]. /// @@ -147,597 +146,626 @@ fn serialize_compression_codec(codec: &CompressionCodec) -> String { /// [Apache Iceberg configuration documentation]: https://github.com/apache/iceberg/blob/d8c10a1608170f0ba83be740d6ab0b6a3757cb3e/docs/docs/configuration.md #[derive(Clone, Debug, Properties)] pub struct ParsedTableProperties { - #[nested] - #[doc = "Informational properties and identifier-field semantics."] - pub general: TableGeneralProperties, - #[nested] - #[doc = "Commit retry and commit status-check behavior."] - pub commit: TableCommitProperties, - #[nested] - #[doc = "Manifest creation, merging, and compression behavior."] - pub manifest: TableManifestProperties, - #[nested] - #[doc = "General table write behavior and output locations."] - pub write: TableWriteProperties, - #[nested] - #[doc = "Parquet data and delete file behavior."] - pub parquet: TableParquetProperties, - #[nested] - #[doc = "Avro data and delete file behavior."] - pub avro: TableAvroProperties, - #[nested] - #[doc = "ORC data and delete file behavior."] - pub orc: TableOrcProperties, - #[nested] - #[doc = "Read split planning and vectorization behavior."] - pub read: TableReadProperties, - #[nested] - #[doc = "Table metadata retention, compression, and metrics behavior."] - pub metadata: TableMetadataProperties, - #[nested] - #[doc = "Engine-specific and compatibility behavior."] - pub compatibility: TableCompatibilityProperties, - #[nested] - #[doc = "Garbage collection and snapshot retention behavior."] - pub history: TableHistoryProperties, - #[nested] - #[doc = "Delete, update, merge, and upsert behavior."] - pub row_level: TableRowLevelProperties, - #[nested] - #[doc = "Table encryption behavior."] - pub encryption: TableEncryptionProperties, -} - -/// Informational properties and identifier-field semantics. -#[derive(Clone, Debug, Properties)] -pub struct TableGeneralProperties { + // General properties. #[key = "comment"] #[default(None)] #[doc = "Table-level description of the table's business meaning and usage context."] pub comment: Option, + #[key = "identifier-fields.rely"] #[default(false)] #[doc = "Whether query engines may rely on identifier fields as a primary key for optimization; this is not enforced on writes."] pub identifier_fields_rely: bool, -} -/// Commit retries and post-failure status checks. -#[derive(Clone, Debug, Properties)] -pub struct TableCommitProperties { + // Commit properties. #[key = "commit.retry.num-retries"] #[default(4)] #[doc = "Number of times to retry a commit before failing."] - pub num_retries: usize, + pub commit_retry_num_retries: usize, + #[key = "commit.retry.min-wait-ms"] #[default(100)] #[doc = "Minimum time in milliseconds to wait before retrying a commit."] - pub min_retry_wait_ms: u64, + pub commit_retry_min_wait_ms: u64, + #[key = "commit.retry.max-wait-ms"] #[default(60 * 1000)] #[doc = "Maximum time in milliseconds to wait before retrying a commit."] - pub max_retry_wait_ms: u64, + pub commit_retry_max_wait_ms: u64, + #[key = "commit.retry.total-timeout-ms"] #[default(30 * 60 * 1000)] #[doc = "Total commit retry timeout in milliseconds."] - pub total_retry_timeout_ms: u64, + pub commit_retry_total_timeout_ms: u64, + #[key = "commit.status-check.num-retries"] #[default(3)] #[doc = "Number of times to check whether a commit succeeded after connectivity is lost."] - pub num_status_checks: usize, + pub commit_status_check_num_retries: usize, + #[key = "commit.status-check.min-wait-ms"] #[default(1000)] #[doc = "Minimum time in milliseconds to wait before retrying a commit status check."] - pub status_checks_min_wait_ms: u64, + pub commit_status_check_min_wait_ms: u64, + #[key = "commit.status-check.max-wait-ms"] #[default(60 * 1000)] #[doc = "Maximum time in milliseconds to wait before retrying a commit status check."] - pub status_checks_max_wait_ms: u64, + pub commit_status_check_max_wait_ms: u64, + #[key = "commit.status-check.total-timeout-ms"] #[default(30 * 60 * 1000)] #[doc = "Total timeout in milliseconds in which commit status checking must succeed."] - pub status_checks_total_wait_ms: u64, -} + pub commit_status_check_total_timeout_ms: u64, -/// Manifest creation, merging, and compression properties. -#[derive(Clone, Debug, Properties)] -pub struct TableManifestProperties { + // Manifest properties. #[key = "commit.manifest.target-size-bytes"] #[default(8 * 1024 * 1024)] #[doc = "Target size in bytes when merging manifest files."] - pub target_size_bytes: usize, + pub commit_manifest_target_size_bytes: usize, + #[key = "commit.manifest.min-count-to-merge"] #[default(100)] #[doc = "Minimum number of manifests to accumulate before merging."] - pub min_merge_count: usize, + pub commit_manifest_min_count_to_merge: usize, + #[key = "commit.manifest-merge.enabled"] #[default(true)] #[doc = "Whether manifests are automatically merged during writes."] - pub merge_enabled: bool, + pub commit_manifest_merge_enabled: bool, + #[key = "write.manifest.compression-codec"] #[default("gzip".to_string())] #[doc = "Compression codec used for manifest files."] - pub compression: String, + pub write_manifest_compression_codec: String, + #[key = "write.manifest.compression-level"] #[default(None)] #[doc = "Optional compression level used for manifest files."] - pub compression_level: Option, + pub write_manifest_compression_level: Option, + #[key = "write.manifest-lists.enabled"] #[default(true)] #[doc = "Deprecated flag for writing manifest lists; manifest lists are always enabled."] - pub lists_enabled: bool, -} + pub write_manifest_lists_enabled: bool, -/// General write properties and output locations. -#[derive(Clone, Debug, Properties)] -pub struct TableWriteProperties { + // Write properties. #[key = "write.format.default"] #[default(DataFileFormat::Parquet)] #[doc = "Default data file format: Parquet, Avro, or ORC."] - pub format_default: DataFileFormat, + pub write_format_default: DataFileFormat, + #[key = "write.delete.format.default"] #[default(DataFileFormat::Parquet)] #[doc = "Default delete file format: Parquet, Avro, or ORC."] - pub delete_format_default: DataFileFormat, + pub write_delete_format_default: DataFileFormat, + #[key = "write.target-file-size-bytes"] #[default(512 * 1024 * 1024)] #[doc = "Target size in bytes for generated data files."] - pub target_file_size_bytes: usize, + pub write_target_file_size_bytes: usize, + #[key = "write.delete.target-file-size-bytes"] #[default(64 * 1024 * 1024)] #[doc = "Target size in bytes for generated delete files."] - pub delete_target_file_size_bytes: usize, + pub write_delete_target_file_size_bytes: usize, + #[key = "write.object-storage.enabled"] #[default(false)] #[doc = "Whether the object-storage location provider adds a hash component to file paths."] - pub object_store_enabled: bool, + pub write_object_storage_enabled: bool, + #[key = "write.object-storage.partitioned-paths"] #[default(true)] #[doc = "Whether object-storage file paths include partition values."] - pub object_store_partitioned_paths: bool, + pub write_object_storage_partitioned_paths: bool, + #[key = "write.object-storage.path"] #[default(None)] #[doc = "Deprecated base object-storage path; use write.data.path instead."] - pub object_store_path: Option, + pub write_object_storage_path: Option, + #[key = "write.location-provider.impl"] #[default(None)] #[doc = "Optional custom location provider implementation."] - pub location_provider_impl: Option, + pub write_location_provider_impl: Option, + #[key = "write.folder-storage.path"] #[default(None)] #[doc = "Deprecated base folder-storage path; use write.data.path instead."] - pub folder_storage_location: Option, + pub write_folder_storage_path: Option, + #[key = "write.data.path"] #[default(None)] #[doc = "Base location for data files written after this property is set."] - pub data_location: Option, + pub write_data_path: Option, + #[key = "write.wap.enabled"] #[default(false)] #[doc = "Whether write-audit-publish writes are enabled."] - pub audit_publish_enabled: bool, + pub write_wap_enabled: bool, + #[key = "write.distribution-mode"] #[default(None)] #[doc = "Optional write distribution mode: none, hash, or range."] - pub distribution_mode: Option, + pub write_distribution_mode: Option, + #[key = "write.datafusion.fanout.enabled"] #[default(true)] #[doc = "Whether DataFusion uses a fanout writer for partitioned tables."] - pub datafusion_fanout_enabled: bool, -} + pub write_datafusion_fanout_enabled: bool, -/// Parquet data and delete file properties. -#[derive(Clone, Debug, Properties)] -pub struct TableParquetProperties { + // Parquet properties. #[key = "write.parquet.row-group-size-bytes"] #[default(128 * 1024 * 1024)] #[doc = "Parquet row group size in bytes for data files."] - pub row_group_size_bytes: usize, + pub write_parquet_row_group_size_bytes: usize, + #[key = "write.delete.parquet.row-group-size-bytes"] #[default(128 * 1024 * 1024)] #[doc = "Parquet row group size in bytes for delete files."] - pub delete_row_group_size_bytes: usize, + pub write_delete_parquet_row_group_size_bytes: usize, + #[key = "write.parquet.page-size-bytes"] #[default(1024 * 1024)] #[doc = "Parquet page size in bytes for data files."] - pub page_size_bytes: usize, + pub write_parquet_page_size_bytes: usize, + #[key = "write.delete.parquet.page-size-bytes"] #[default(1024 * 1024)] #[doc = "Parquet page size in bytes for delete files."] - pub delete_page_size_bytes: usize, + pub write_delete_parquet_page_size_bytes: usize, + #[key = "write.parquet.page-version"] #[default("v1".to_string())] #[doc = "Parquet data page version for data files: v1 or v2."] - pub page_version: String, + pub write_parquet_page_version: String, + #[key = "write.delete.parquet.page-version"] #[default("v1".to_string())] #[doc = "Parquet data page version for delete files: v1 or v2."] - pub delete_page_version: String, + pub write_delete_parquet_page_version: String, + #[key = "write.parquet.page-row-limit"] #[default(20_000)] #[doc = "Maximum number of rows per Parquet page in data files."] - pub page_row_limit: usize, + pub write_parquet_page_row_limit: usize, + #[key = "write.delete.parquet.page-row-limit"] #[default(20_000)] #[doc = "Maximum number of rows per Parquet page in delete files."] - pub delete_page_row_limit: usize, + pub write_delete_parquet_page_row_limit: usize, + #[key = "write.parquet.dict-size-bytes"] #[default(2 * 1024 * 1024)] #[doc = "Parquet dictionary page size in bytes for data files."] - pub dict_size_bytes: usize, + pub write_parquet_dict_size_bytes: usize, + #[key = "write.delete.parquet.dict-size-bytes"] #[default(2 * 1024 * 1024)] #[doc = "Parquet dictionary page size in bytes for delete files."] - pub delete_dict_size_bytes: usize, + pub write_delete_parquet_dict_size_bytes: usize, + #[key = "write.parquet.compression-codec"] #[default("zstd".to_string())] #[doc = "Parquet compression codec used for data files."] - pub compression: String, + pub write_parquet_compression_codec: String, + #[key = "write.delete.parquet.compression-codec"] #[default("zstd".to_string())] #[doc = "Parquet compression codec used for delete files."] - pub delete_compression: String, + pub write_delete_parquet_compression_codec: String, + #[key = "write.parquet.compression-level"] #[default(None)] #[doc = "Optional Parquet compression level for data files."] - pub compression_level: Option, + pub write_parquet_compression_level: Option, + #[key = "write.delete.parquet.compression-level"] #[default(None)] #[doc = "Optional Parquet compression level for delete files."] - pub delete_compression_level: Option, + pub write_delete_parquet_compression_level: Option, + #[key = "write.parquet.shred-variants"] #[default(false)] #[doc = "Whether variant columns use shredded Parquet encoding for improved query performance."] - pub shred_variants: bool, + pub write_parquet_shred_variants: bool, + #[key = "write.parquet.variant-inference-buffer-size"] #[default(100)] #[doc = "Number of rows buffered for schema inference when variant shredding is enabled."] - pub variant_inference_buffer_size: usize, + pub write_parquet_variant_inference_buffer_size: usize, + #[key = "write.parquet.row-group-check-min-record-count"] #[default(100)] #[doc = "Minimum record count between Parquet data-file row group size checks."] - pub row_group_check_min_record_count: usize, + pub write_parquet_row_group_check_min_record_count: usize, + #[key = "write.delete.parquet.row-group-check-min-record-count"] #[default(100)] #[doc = "Minimum record count between Parquet delete-file row group size checks."] - pub delete_row_group_check_min_record_count: usize, + pub write_delete_parquet_row_group_check_min_record_count: usize, + #[key = "write.parquet.row-group-check-max-record-count"] #[default(10_000)] #[doc = "Maximum record count between Parquet data-file row group size checks."] - pub row_group_check_max_record_count: usize, + pub write_parquet_row_group_check_max_record_count: usize, + #[key = "write.delete.parquet.row-group-check-max-record-count"] #[default(10_000)] #[doc = "Maximum record count between Parquet delete-file row group size checks."] - pub delete_row_group_check_max_record_count: usize, + pub write_delete_parquet_row_group_check_max_record_count: usize, + #[key = "write.parquet.row-group-size-track-uncompressed"] #[default(false)] #[doc = "Whether uncompressed data size is tracked to enforce the Parquet row group target."] - pub row_group_size_track_uncompressed: bool, + pub write_parquet_row_group_size_track_uncompressed: bool, + #[key = "write.parquet.bloom-filter-max-bytes"] #[default(1024 * 1024)] #[doc = "Maximum number of bytes for a Parquet bloom filter bitset."] - pub bloom_filter_max_bytes: usize, + pub write_parquet_bloom_filter_max_bytes: usize, + #[key = "write.parquet.bloom-filter-adaptive-enabled"] #[default(false)] #[doc = "Whether adaptive Parquet bloom filter sizing selects the smallest suitable filter."] - pub bloom_filter_adaptive_enabled: bool, + pub write_parquet_bloom_filter_adaptive_enabled: bool, + #[prefix = "write.parquet.bloom-filter-fpp.column."] #[default(HashMap::new())] #[doc = "Per-column Parquet bloom filter false-positive probabilities, keyed by column name."] - pub bloom_filter_column_fpp: HashMap, + pub write_parquet_bloom_filter_fpp_column: HashMap, + #[prefix = "write.parquet.bloom-filter-ndv.column."] #[default(HashMap::new())] #[doc = "Per-column expected distinct-value counts for Parquet bloom filters."] - pub bloom_filter_column_ndv: HashMap, + pub write_parquet_bloom_filter_ndv_column: HashMap, + #[prefix = "write.parquet.bloom-filter-enabled.column."] #[default(HashMap::new())] #[doc = "Per-column flags controlling whether Parquet bloom filters are written."] - pub bloom_filter_column_enabled: HashMap, + pub write_parquet_bloom_filter_enabled_column: HashMap, + #[prefix = "write.parquet.stats-enabled.column."] #[default(HashMap::new())] #[doc = "Per-column flags controlling whether Parquet column statistics are collected."] - pub column_stats_enabled: HashMap, + pub write_parquet_stats_enabled_column: HashMap, + #[prefix = "write.parquet.dict-encoding-enabled.column."] #[default(HashMap::new())] #[doc = "Per-column flags controlling whether Parquet dictionary encoding is used."] - pub dict_encoding_enabled_by_column: HashMap, + pub write_parquet_dict_encoding_enabled_column: HashMap, + #[key = "write.parquet.content-defined-chunking.enabled"] #[default(false)] #[doc = "Whether Parquet content-defined chunking is enabled."] - pub content_defined_chunking_enabled: bool, + pub write_parquet_content_defined_chunking_enabled: bool, + #[key = "write.parquet.content-defined-chunking.min-chunk-size"] #[default(256 * 1024)] #[doc = "Minimum Parquet content-defined chunk size in bytes."] - pub content_defined_chunking_min_chunk_size: usize, + pub write_parquet_content_defined_chunking_min_chunk_size: usize, + #[key = "write.parquet.content-defined-chunking.max-chunk-size"] #[default(1024 * 1024)] #[doc = "Maximum Parquet content-defined chunk size in bytes."] - pub content_defined_chunking_max_chunk_size: usize, + pub write_parquet_content_defined_chunking_max_chunk_size: usize, + #[key = "write.parquet.content-defined-chunking.norm-level"] #[default(0)] #[doc = "Gearhash normalization level used by Parquet content-defined chunking."] - pub content_defined_chunking_norm_level: i32, -} + pub write_parquet_content_defined_chunking_norm_level: i32, -/// Avro data and delete file properties. -#[derive(Clone, Debug, Properties)] -pub struct TableAvroProperties { + // Avro properties. #[key = "write.avro.compression-codec"] #[default("gzip".to_string())] #[doc = "Avro compression codec used for data files."] - pub compression: String, + pub write_avro_compression_codec: String, + #[key = "write.delete.avro.compression-codec"] #[default("gzip".to_string())] #[doc = "Avro compression codec used for delete files."] - pub delete_compression: String, + pub write_delete_avro_compression_codec: String, + #[key = "write.avro.compression-level"] #[default(None)] #[doc = "Optional Avro compression level for data files."] - pub compression_level: Option, + pub write_avro_compression_level: Option, + #[key = "write.delete.avro.compression-level"] #[default(None)] #[doc = "Optional Avro compression level for delete files."] - pub delete_compression_level: Option, -} + pub write_delete_avro_compression_level: Option, -/// ORC data and delete file properties. -#[derive(Clone, Debug, Properties)] -pub struct TableOrcProperties { + // ORC properties. #[key = "write.orc.stripe-size-bytes"] #[default(64 * 1024 * 1024)] #[doc = "Default ORC stripe size in bytes for data files."] - pub stripe_size_bytes: u64, + pub write_orc_stripe_size_bytes: u64, + #[key = "write.delete.orc.stripe-size-bytes"] #[default(64 * 1024 * 1024)] #[doc = "Default ORC stripe size in bytes for delete files."] - pub delete_stripe_size_bytes: u64, + pub write_delete_orc_stripe_size_bytes: u64, + #[key = "write.orc.bloom.filter.columns"] #[default(String::new())] #[doc = "Comma-separated column names for which ORC bloom filters are created."] - pub bloom_filter_columns: String, + pub write_orc_bloom_filter_columns: String, + #[key = "write.orc.bloom.filter.fpp"] #[default(0.05)] #[doc = "False-positive probability for ORC bloom filters."] - pub bloom_filter_fpp: f64, + pub write_orc_bloom_filter_fpp: f64, + #[key = "write.orc.block-size-bytes"] #[default(256 * 1024 * 1024)] #[doc = "Default file-system block size in bytes for ORC data files."] - pub block_size_bytes: u64, + pub write_orc_block_size_bytes: u64, + #[key = "write.delete.orc.block-size-bytes"] #[default(256 * 1024 * 1024)] #[doc = "Default file-system block size in bytes for ORC delete files."] - pub delete_block_size_bytes: u64, + pub write_delete_orc_block_size_bytes: u64, + #[key = "write.orc.vectorized.batch-size"] #[default(1024)] #[doc = "ORC vectorized write batch size for data files."] - pub write_batch_size: usize, + pub write_orc_vectorized_batch_size: usize, + #[key = "write.delete.orc.vectorized.batch-size"] #[default(1024)] #[doc = "ORC vectorized write batch size for delete files."] - pub delete_write_batch_size: usize, + pub write_delete_orc_vectorized_batch_size: usize, + #[key = "write.orc.compression-codec"] #[default("zlib".to_string())] #[doc = "ORC compression codec used for data files."] - pub compression: String, + pub write_orc_compression_codec: String, + #[key = "write.delete.orc.compression-codec"] #[default("zlib".to_string())] #[doc = "ORC compression codec used for delete files."] - pub delete_compression: String, + pub write_delete_orc_compression_codec: String, + #[key = "write.orc.compression-strategy"] #[default("speed".to_string())] #[doc = "ORC compression strategy for data files: speed or compression."] - pub compression_strategy: String, + pub write_orc_compression_strategy: String, + #[key = "write.delete.orc.compression-strategy"] #[default("speed".to_string())] #[doc = "ORC compression strategy for delete files: speed or compression."] - pub delete_compression_strategy: String, -} + pub write_delete_orc_compression_strategy: String, -/// Read split planning and vectorization properties. -#[derive(Clone, Debug, Properties)] -pub struct TableReadProperties { + // Read properties. #[key = "read.split.target-size"] #[default(128 * 1024 * 1024)] #[doc = "Target size in bytes when combining data input splits."] - pub split_target_size: u64, + pub read_split_target_size: u64, + #[key = "read.split.metadata-target-size"] #[default(32 * 1024 * 1024)] #[doc = "Target size in bytes when combining metadata input splits."] - pub metadata_split_target_size: u64, + pub read_split_metadata_target_size: u64, + #[key = "read.split.planning-lookback"] #[default(10)] #[doc = "Number of bins considered when combining input splits."] - pub split_planning_lookback: usize, + pub read_split_planning_lookback: usize, + #[key = "read.split.open-file-cost"] #[default(4 * 1024 * 1024)] #[doc = "Estimated file-open cost used as a minimum weight when combining splits."] - pub split_open_file_cost: u64, + pub read_split_open_file_cost: u64, + #[key = "read.split.adaptive-size.enabled"] #[default(true)] #[doc = "Whether split size is adapted to the scan size."] - pub adaptive_split_size_enabled: bool, + pub read_split_adaptive_size_enabled: bool, + #[key = "read.parquet.vectorization.enabled"] #[default(true)] #[doc = "Whether Parquet vectorized reads are enabled."] - pub parquet_vectorization_enabled: bool, + pub read_parquet_vectorization_enabled: bool, + #[key = "read.parquet.vectorization.batch-size"] #[default(5000)] #[doc = "Batch size for Parquet vectorized reads."] - pub parquet_batch_size: usize, + pub read_parquet_vectorization_batch_size: usize, + #[key = "read.orc.vectorization.enabled"] #[default(false)] #[doc = "Whether ORC vectorized reads are enabled."] - pub orc_vectorization_enabled: bool, + pub read_orc_vectorization_enabled: bool, + #[key = "read.orc.vectorization.batch-size"] #[default(5000)] #[doc = "Batch size for ORC vectorized reads."] - pub orc_batch_size: usize, + pub read_orc_vectorization_batch_size: usize, + #[key = "read.data-planning-mode"] #[default("auto".to_string())] #[doc = "Planning mode used for data files."] - pub data_planning_mode: String, + pub read_data_planning_mode: String, + #[key = "read.delete-planning-mode"] #[default("auto".to_string())] #[doc = "Planning mode used for delete files."] - pub delete_planning_mode: String, -} + pub read_delete_planning_mode: String, -/// Table metadata retention, compression, and metrics properties. -#[derive(Clone, Debug, Properties)] -pub struct TableMetadataProperties { + // Metadata properties. #[key = "write.metadata.path"] #[default(None)] #[parse_with(parse_metadata_location)] #[doc = "Base location for metadata files written after this property is set, with trailing slashes removed."] - pub path: Option, + pub write_metadata_path: Option, + #[key = "write.summary.partition-limit"] #[default(0)] #[doc = "Maximum changed-partition count for including partition-level statistics in snapshot summaries."] - pub partition_summary_limit: u64, + pub write_summary_partition_limit: u64, + #[key = "write.metadata.compression-codec"] #[default(CompressionCodec::None)] #[parse_with(parse_metadata_file_compression)] #[serialize_with(serialize_compression_codec)] #[doc = "Compression codec for metadata JSON files: none or gzip."] - pub compression_codec: CompressionCodec, + pub write_metadata_compression_codec: CompressionCodec, + #[key = "write.metadata.previous-versions-max"] #[default(100)] #[doc = "Maximum number of previous metadata file versions to track."] - pub previous_versions_max: usize, + pub write_metadata_previous_versions_max: usize, + #[key = "write.metadata.delete-after-commit.enabled"] #[default(false)] #[doc = "Whether the oldest tracked metadata file is deleted after each commit."] - pub delete_after_commit_enabled: bool, + pub write_metadata_delete_after_commit_enabled: bool, + #[key = "write.metadata.metrics.max-inferred-column-defaults"] #[default(100)] #[doc = "Maximum number of columns that receive inferred metrics defaults."] - pub metrics_max_inferred_column_defaults: usize, + pub write_metadata_metrics_max_inferred_column_defaults: usize, + #[prefix = "write.metadata.metrics.column."] #[default(HashMap::new())] #[doc = "Per-column metrics modes keyed by column name."] - pub metrics_mode_by_column: HashMap, + pub write_metadata_metrics_column: HashMap, + #[key = "write.metadata.metrics.default"] #[default("truncate(16)".to_string())] #[doc = "Default metrics mode for table columns."] - pub default_metrics_mode: String, + pub write_metadata_metrics_default: String, + #[key = "schema.name-mapping.default"] #[default(None)] #[doc = "Default JSON name mapping used to resolve columns in files without field IDs."] - pub default_name_mapping: Option, -} + pub schema_name_mapping_default: Option, -/// Engine-specific and compatibility properties. -#[derive(Clone, Debug, Properties)] -pub struct TableCompatibilityProperties { + // Compatibility properties. #[key = "write.spark.fanout.enabled"] #[default(false)] #[doc = "Deprecated Spark fanout-writer flag; the fanout writer accepts unclustered data but uses more memory."] - pub spark_write_partitioned_fanout_enabled: bool, + pub write_spark_fanout_enabled: bool, + #[key = "write.spark.accept-any-schema"] #[default(false)] #[doc = "Deprecated Spark flag allowing writes with any compatible schema."] - pub spark_write_accept_any_schema: bool, + pub write_spark_accept_any_schema: bool, + #[key = "write.spark.auto-schema-evolution.enabled"] #[default(true)] #[doc = "Deprecated Spark flag enabling automatic schema evolution during writes."] - pub spark_write_auto_schema_evolution: bool, + pub write_spark_auto_schema_evolution_enabled: bool, + #[key = "write.spark.advisory-partition-size-bytes"] #[default(None)] #[doc = "Deprecated Spark advisory partition size in bytes."] - pub spark_write_advisory_partition_size_bytes: Option, + pub write_spark_advisory_partition_size_bytes: Option, + #[key = "compatibility.snapshot-id-inheritance.enabled"] #[default(false)] #[doc = "Whether snapshots may be committed without explicit snapshot IDs; format version 2 and later always allow this."] - pub snapshot_id_inheritance_enabled: bool, + pub compatibility_snapshot_id_inheritance_enabled: bool, + #[key = "engine.hive.enabled"] #[default(false)] #[doc = "Whether Hive engine integration behavior is enabled."] pub engine_hive_enabled: bool, + #[key = "engine.hive.lock-enabled"] #[default(true)] #[doc = "Whether Hive locking is enabled."] - pub hive_lock_enabled: bool, -} + pub engine_hive_lock_enabled: bool, -/// Garbage collection and snapshot retention properties. -#[derive(Clone, Debug, Properties)] -pub struct TableHistoryProperties { + // History properties. #[key = "gc.enabled"] #[default(true)] #[doc = "Whether garbage collection operations such as snapshot expiration and orphan-file removal are allowed."] pub gc_enabled: bool, + #[key = "history.expire.max-snapshot-age-ms"] #[default(5 * 24 * 60 * 60 * 1000)] #[doc = "Default maximum snapshot age in milliseconds while expiring snapshots."] - pub max_snapshot_age_ms: i64, + pub history_expire_max_snapshot_age_ms: i64, + #[key = "history.expire.min-snapshots-to-keep"] #[default(1)] #[doc = "Default minimum number of snapshots retained per branch while expiring snapshots."] - pub min_snapshots_to_keep: usize, + pub history_expire_min_snapshots_to_keep: usize, + #[key = "history.expire.max-ref-age-ms"] #[default(i64::MAX)] #[doc = "Default maximum age in milliseconds for snapshot references other than the main branch."] - pub max_ref_age_ms: i64, -} + pub history_expire_max_ref_age_ms: i64, -/// Delete, update, merge, and upsert properties. -#[derive(Clone, Debug, Properties)] -pub struct TableRowLevelProperties { + // Row-level operation properties. #[key = "write.delete.granularity"] #[default("partition".to_string())] #[doc = "Granularity of generated delete files: partition or file."] - pub delete_granularity: String, + pub write_delete_granularity: String, + #[key = "write.delete.isolation-level"] #[default("serializable".to_string())] #[doc = "Isolation level for delete commands: serializable or snapshot."] - pub delete_isolation_level: String, + pub write_delete_isolation_level: String, + #[key = "write.delete.mode"] #[default("copy-on-write".to_string())] #[doc = "Execution mode for delete commands: copy-on-write or merge-on-read."] - pub delete_mode: String, + pub write_delete_mode: String, + #[key = "write.delete.distribution-mode"] #[default(None)] #[doc = "Optional distribution mode for delete command data."] - pub delete_distribution_mode: Option, + pub write_delete_distribution_mode: Option, + #[key = "write.update.isolation-level"] #[default("serializable".to_string())] #[doc = "Isolation level for update commands: serializable or snapshot."] - pub update_isolation_level: String, + pub write_update_isolation_level: String, + #[key = "write.update.mode"] #[default("copy-on-write".to_string())] #[doc = "Execution mode for update commands: copy-on-write or merge-on-read."] - pub update_mode: String, + pub write_update_mode: String, + #[key = "write.update.distribution-mode"] #[default(None)] #[doc = "Optional distribution mode for update command data."] - pub update_distribution_mode: Option, + pub write_update_distribution_mode: Option, + #[key = "write.merge.isolation-level"] #[default("serializable".to_string())] #[doc = "Isolation level for merge commands: serializable or snapshot."] - pub merge_isolation_level: String, + pub write_merge_isolation_level: String, + #[key = "write.merge.mode"] #[default("copy-on-write".to_string())] #[doc = "Execution mode for merge commands: copy-on-write or merge-on-read."] - pub merge_mode: String, + pub write_merge_mode: String, + #[key = "write.merge.distribution-mode"] #[default(None)] #[doc = "Optional distribution mode for merge command data."] - pub merge_distribution_mode: Option, + pub write_merge_distribution_mode: Option, + #[key = "write.upsert.enabled"] #[default(false)] #[doc = "Whether upsert behavior is enabled."] - pub upsert_enabled: bool, -} + pub write_upsert_enabled: bool, -/// Table encryption properties. -#[derive(Clone, Debug, Properties)] -pub struct TableEncryptionProperties { + // Encryption properties. #[key = "encryption.key-id"] #[default(None)] #[doc = "Identifier of the table's master encryption key."] - pub key_id: Option, + pub encryption_key_id: Option, + #[key = "encryption.data-key-length"] #[default(16)] #[doc = "Length in bytes of data-encryption keys; valid AES lengths are 16, 24, and 32 bytes."] - pub data_key_length: usize, + pub encryption_data_key_length: usize, } #[cfg(test)] @@ -753,22 +781,29 @@ mod tests { fn creates_properties_from_defaults() { let properties = ParsedTableProperties::default(); - assert_eq!(properties.commit.num_retries, 4); - assert_eq!(properties.write.format_default, DataFileFormat::Parquet); - assert_eq!(properties.parquet.row_group_size_bytes, 128 * 1024 * 1024); - assert_eq!(properties.read.split_target_size, 128 * 1024 * 1024); - assert!(properties.history.gc_enabled); - assert_eq!(properties.encryption.data_key_length, 16); + assert_eq!(properties.commit_retry_num_retries, 4); + assert_eq!(properties.write_format_default, DataFileFormat::Parquet); + assert_eq!( + properties.write_parquet_row_group_size_bytes, + 128 * 1024 * 1024 + ); + assert_eq!(properties.read_split_target_size, 128 * 1024 * 1024); + assert!(properties.gc_enabled); + assert_eq!(properties.encryption_data_key_length, 16); } #[test] fn serializes_to_flat_json_object() { - let mut properties = ParsedTableProperties::default(); - properties.commit.num_retries = 9; - properties.write.format_default = DataFileFormat::Orc; - properties.write.data_location = Some("s3://warehouse/table/data".to_string()); - properties.parquet.bloom_filter_column_fpp = - HashMap::from([("customer_id".to_string(), 0.02)]); + let properties = ParsedTableProperties { + commit_retry_num_retries: 9, + write_format_default: DataFileFormat::Orc, + write_data_path: Some("s3://warehouse/table/data".to_string()), + write_parquet_bloom_filter_fpp_column: HashMap::from([( + "customer_id".to_string(), + 0.02, + )]), + ..Default::default() + }; let json = serde_json::to_value(&properties).unwrap(); assert_eq!(json["commit.retry.num-retries"], "9"); @@ -791,10 +826,10 @@ mod tests { })) .unwrap(); - assert_eq!(properties.commit.num_retries, 8); - assert_eq!(properties.write.format_default, DataFileFormat::Orc); + assert_eq!(properties.commit_retry_num_retries, 8); + assert_eq!(properties.write_format_default, DataFileFormat::Orc); assert_eq!( - properties.write.data_location, + properties.write_data_path, Some("s3://warehouse/table/data".to_string()) ); } @@ -805,16 +840,19 @@ mod tests { let decoded: ParsedTableProperties = serde_json::from_value(serde_json::to_value(&defaults).unwrap()).unwrap(); - assert_eq!(decoded.commit.num_retries, defaults.commit.num_retries); assert_eq!( - decoded.metadata.compression_codec, - defaults.metadata.compression_codec + decoded.commit_retry_num_retries, + defaults.commit_retry_num_retries + ); + assert_eq!( + decoded.write_metadata_compression_codec, + defaults.write_metadata_compression_codec ); assert_eq!( - decoded.parquet.content_defined_chunking_max_chunk_size, - defaults.parquet.content_defined_chunking_max_chunk_size + decoded.write_parquet_content_defined_chunking_max_chunk_size, + defaults.write_parquet_content_defined_chunking_max_chunk_size ); - assert_eq!(decoded.row_level.merge_mode, defaults.row_level.merge_mode); + assert_eq!(decoded.write_merge_mode, defaults.write_merge_mode); } #[test] @@ -839,13 +877,13 @@ mod tests { ])) .unwrap(); - assert_eq!(properties.general.comment, Some("orders table".to_string())); - assert_eq!(properties.commit.num_status_checks, 7); - assert_eq!(properties.avro.delete_compression, "snappy"); - assert_eq!(properties.read.split_planning_lookback, 25); - assert_eq!(properties.history.min_snapshots_to_keep, 4); - assert_eq!(properties.row_level.delete_mode, "merge-on-read"); - assert_eq!(properties.encryption.data_key_length, 32); + assert_eq!(properties.comment, Some("orders table".to_string())); + assert_eq!(properties.commit_status_check_num_retries, 7); + assert_eq!(properties.write_delete_avro_compression_codec, "snappy"); + assert_eq!(properties.read_split_planning_lookback, 25); + assert_eq!(properties.history_expire_min_snapshots_to_keep, 4); + assert_eq!(properties.write_delete_mode, "merge-on-read"); + assert_eq!(properties.encryption_data_key_length, 32); } #[test] @@ -863,11 +901,11 @@ mod tests { .unwrap(); assert_eq!( - properties.metadata.path, + properties.write_metadata_path, Some("s3://warehouse/table/metadata".to_string()) ); assert_eq!( - properties.metadata.compression_codec, + properties.write_metadata_compression_codec, CompressionCodec::gzip_default() ); diff --git a/crates/iceberg/src/spec/table_metadata.rs b/crates/iceberg/src/spec/table_metadata.rs index 38fca30f76..5c872eeaf6 100644 --- a/crates/iceberg/src/spec/table_metadata.rs +++ b/crates/iceberg/src/spec/table_metadata.rs @@ -4052,11 +4052,11 @@ mod tests { let props = metadata.parsed_table_properties().unwrap(); assert_eq!( - props.commit.num_retries, + props.commit_retry_num_retries, TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT ); assert_eq!( - props.write.target_file_size_bytes, + props.write_target_file_size_bytes, TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT ); @@ -4104,8 +4104,8 @@ mod tests { let props = metadata.parsed_table_properties().unwrap(); - assert_eq!(props.commit.num_retries, 10); - assert_eq!(props.write.target_file_size_bytes, 1024); + assert_eq!(props.commit_retry_num_retries, 10); + assert_eq!(props.write_target_file_size_bytes, 1024); } #[test] diff --git a/crates/iceberg/src/transaction/expire_snapshots.rs b/crates/iceberg/src/transaction/expire_snapshots.rs index b308e64da6..a673b11683 100644 --- a/crates/iceberg/src/transaction/expire_snapshots.rs +++ b/crates/iceberg/src/transaction/expire_snapshots.rs @@ -100,7 +100,6 @@ impl ExpireSnapshotsAction { /// Resolves the snapshots and refs to remove, following Java `RemoveSnapshots.internalApply`. fn plan(&self, table: &Table, properties: &ParsedTableProperties) -> Result { - let history = &properties.history; // Matches Java `RemoveSnapshots.retainLast`, which requires at least one snapshot. if self.retain_last == Some(0) { return Err(Error::new( @@ -116,8 +115,10 @@ impl ExpireSnapshotsAction { // days) the age path always runs, so even an explicit-id-only call applies the default cutoff. let default_cutoff = self .older_than_ms - .unwrap_or_else(|| now.saturating_sub(history.max_snapshot_age_ms)); - let default_min_to_keep = self.retain_last.unwrap_or(history.min_snapshots_to_keep); + .unwrap_or_else(|| now.saturating_sub(properties.history_expire_max_snapshot_age_ms)); + let default_min_to_keep = self + .retain_last + .unwrap_or(properties.history_expire_min_snapshots_to_keep); // Ref aging: `main` is always kept; any other ref whose head is older than its // `max_ref_age_ms` (defaulting to `history.expire.max-ref-age-ms`) is dropped, like Java's @@ -126,7 +127,12 @@ impl ExpireSnapshotsAction { let mut retained_refs: Vec<&SnapshotReference> = vec![]; for (ref_name, snapshot_ref) in &metadata.refs { if ref_name == MAIN_BRANCH - || !Self::ref_aged_out(metadata, snapshot_ref, now, history.max_ref_age_ms) + || !Self::ref_aged_out( + metadata, + snapshot_ref, + now, + properties.history_expire_max_ref_age_ms, + ) { retained_refs.push(snapshot_ref); } else { @@ -303,7 +309,7 @@ impl TransactionAction for ExpireSnapshotsAction { let properties = metadata.parsed_table_properties()?; // Expiring metadata defeats a user's explicit decision to disable GC (Java refuses too). - if !properties.history.gc_enabled { + if !properties.gc_enabled { return Err(Error::new( ErrorKind::DataInvalid, "Cannot expire snapshots: gc.enabled is false", diff --git a/crates/iceberg/src/transaction/mod.rs b/crates/iceberg/src/transaction/mod.rs index f5ab4247d9..79afd34233 100644 --- a/crates/iceberg/src/transaction/mod.rs +++ b/crates/iceberg/src/transaction/mod.rs @@ -181,7 +181,7 @@ impl Transaction { let table_props = self.table.metadata().parsed_table_properties()?; // TODO(https://github.com/apache/iceberg-rust/issues/2034): remove once encrypted writes are supported - if table_props.encryption.key_id.is_some() { + if table_props.encryption_key_id.is_some() { return Err(Error::new( ErrorKind::FeatureUnsupported, "Cannot commit to an encrypted table: encrypted writes are not yet supported", @@ -204,12 +204,13 @@ impl Transaction { } fn build_backoff(props: ParsedTableProperties) -> Result { - let commit = props.commit; Ok(ExponentialBuilder::new() - .with_min_delay(Duration::from_millis(commit.min_retry_wait_ms)) - .with_max_delay(Duration::from_millis(commit.max_retry_wait_ms)) - .with_total_delay(Some(Duration::from_millis(commit.total_retry_timeout_ms))) - .with_max_times(commit.num_retries) + .with_min_delay(Duration::from_millis(props.commit_retry_min_wait_ms)) + .with_max_delay(Duration::from_millis(props.commit_retry_max_wait_ms)) + .with_total_delay(Some(Duration::from_millis( + props.commit_retry_total_timeout_ms, + ))) + .with_max_times(props.commit_retry_num_retries) .with_factor(2.0) .build()) } diff --git a/crates/iceberg/src/writer/file_writer/parquet_writer.rs b/crates/iceberg/src/writer/file_writer/parquet_writer.rs index 58de75a8b1..444052bde2 100644 --- a/crates/iceberg/src/writer/file_writer/parquet_writer.rs +++ b/crates/iceberg/src/writer/file_writer/parquet_writer.rs @@ -100,13 +100,12 @@ impl ParquetWriterBuilder { table_props: &ParsedTableProperties, schema: SchemaRef, ) -> Self { - let parquet = &table_props.parquet; - let cdc = parquet - .content_defined_chunking_enabled + let cdc = table_props + .write_parquet_content_defined_chunking_enabled .then_some(CdcOptions { - min_chunk_size: parquet.content_defined_chunking_min_chunk_size, - max_chunk_size: parquet.content_defined_chunking_max_chunk_size, - norm_level: parquet.content_defined_chunking_norm_level, + min_chunk_size: table_props.write_parquet_content_defined_chunking_min_chunk_size, + max_chunk_size: table_props.write_parquet_content_defined_chunking_max_chunk_size, + norm_level: table_props.write_parquet_content_defined_chunking_norm_level, }); Self::from_cdc_options(cdc, schema) } diff --git a/crates/integrations/datafusion/src/physical_plan/write.rs b/crates/integrations/datafusion/src/physical_plan/write.rs index 6fc9a061b8..a3d4de3cc3 100644 --- a/crates/integrations/datafusion/src/physical_plan/write.rs +++ b/crates/integrations/datafusion/src/physical_plan/write.rs @@ -209,8 +209,7 @@ impl ExecutionPlan for IcebergWriteExec { .map_err(to_datafusion_error)?; // Check data file format - let write_properties = &table_props.write; - let file_format = write_properties.format_default; + let file_format = table_props.write_format_default; if file_format != DataFileFormat::Parquet { return Err(to_datafusion_error(Error::new( ErrorKind::FeatureUnsupported, @@ -226,7 +225,7 @@ impl ExecutionPlan for IcebergWriteExec { self.table.metadata().current_schema().clone(), ) .with_match_mode(FieldMatchMode::Name); - let target_file_size = write_properties.target_file_size_bytes; + let target_file_size = table_props.write_target_file_size_bytes; let file_io = self.table.file_io().clone(); // todo location_gen and file_name_gen should be configurable @@ -245,7 +244,7 @@ impl ExecutionPlan for IcebergWriteExec { let data_file_writer_builder = DataFileWriterBuilder::new(rolling_writer_builder); // Create TaskWriter - let fanout_enabled = write_properties.datafusion_fanout_enabled; + let fanout_enabled = table_props.write_datafusion_fanout_enabled; let schema = self.table.metadata().current_schema().clone(); let partition_spec = self.table.metadata().default_partition_spec().clone(); let task_writer = TaskWriter::try_new( diff --git a/crates/property-macro/tests/properties.rs b/crates/property-macro/tests/properties.rs index 1863499999..a24f05b224 100644 --- a/crates/property-macro/tests/properties.rs +++ b/crates/property-macro/tests/properties.rs @@ -29,9 +29,11 @@ struct TestProperties { #[default(4)] #[doc = "Number of retries."] pub retries: u64, + #[key(OWNER)] #[default(None)] pub owner: Option, + #[prefix(COLUMN_FPP_PREFIX)] #[default(HashMap::new())] pub column_fpp: HashMap, From 0824c7bd102bcf79590c91815d1a380b39307165 Mon Sep 17 00:00:00 2001 From: Renjie Liu Date: Wed, 5 Aug 2026 11:35:43 +0800 Subject: [PATCH 06/14] Address comments --- crates/iceberg/public-api.txt | 32 +++-- crates/iceberg/src/compression.rs | 115 +++++++++++++---- crates/iceberg/src/spec/parsed_table_prop.rs | 124 +++++++++++++------ crates/property-macro/Cargo.toml | 2 +- crates/property-macro/src/lib.rs | 39 ++++-- crates/property-macro/tests/properties.rs | 11 ++ 6 files changed, 247 insertions(+), 76 deletions(-) diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index f90c664fa2..0075d44e73 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -142,10 +142,14 @@ pub fn iceberg::cache::ObjectCacheProvide::manifest_list_cache(&self) -> &dyn ic pub type iceberg::cache::ObjectCacheProvider = alloc::sync::Arc pub mod iceberg::compression pub enum iceberg::compression::CompressionCodec +pub iceberg::compression::CompressionCodec::Brotli pub iceberg::compression::CompressionCodec::Gzip(u8) pub iceberg::compression::CompressionCodec::Lz4 +pub iceberg::compression::CompressionCodec::Lzo pub iceberg::compression::CompressionCodec::None pub iceberg::compression::CompressionCodec::Snappy +pub iceberg::compression::CompressionCodec::Uncompressed +pub iceberg::compression::CompressionCodec::Zlib pub iceberg::compression::CompressionCodec::Zstd(u8) impl iceberg::compression::CompressionCodec pub const fn iceberg::compression::CompressionCodec::gzip_default() -> Self @@ -1153,10 +1157,14 @@ pub fn iceberg::metadata_columns::row_id_field() -> &'static iceberg::spec::Nest pub fn iceberg::metadata_columns::spec_id_field() -> &'static iceberg::spec::NestedFieldRef pub mod iceberg::puffin pub enum iceberg::puffin::CompressionCodec +pub iceberg::puffin::CompressionCodec::Brotli pub iceberg::puffin::CompressionCodec::Gzip(u8) pub iceberg::puffin::CompressionCodec::Lz4 +pub iceberg::puffin::CompressionCodec::Lzo pub iceberg::puffin::CompressionCodec::None pub iceberg::puffin::CompressionCodec::Snappy +pub iceberg::puffin::CompressionCodec::Uncompressed +pub iceberg::puffin::CompressionCodec::Zlib pub iceberg::puffin::CompressionCodec::Zstd(u8) impl iceberg::compression::CompressionCodec pub const fn iceberg::compression::CompressionCodec::gzip_default() -> Self @@ -2279,24 +2287,24 @@ pub iceberg::spec::ParsedTableProperties::read_split_open_file_cost: u64 pub iceberg::spec::ParsedTableProperties::read_split_planning_lookback: usize pub iceberg::spec::ParsedTableProperties::read_split_target_size: u64 pub iceberg::spec::ParsedTableProperties::schema_name_mapping_default: core::option::Option -pub iceberg::spec::ParsedTableProperties::write_avro_compression_codec: alloc::string::String -pub iceberg::spec::ParsedTableProperties::write_avro_compression_level: core::option::Option +pub iceberg::spec::ParsedTableProperties::write_avro_compression_codec: iceberg::compression::CompressionCodec +pub iceberg::spec::ParsedTableProperties::write_avro_compression_level: core::option::Option pub iceberg::spec::ParsedTableProperties::write_data_path: core::option::Option pub iceberg::spec::ParsedTableProperties::write_datafusion_fanout_enabled: bool -pub iceberg::spec::ParsedTableProperties::write_delete_avro_compression_codec: alloc::string::String -pub iceberg::spec::ParsedTableProperties::write_delete_avro_compression_level: core::option::Option +pub iceberg::spec::ParsedTableProperties::write_delete_avro_compression_codec: iceberg::compression::CompressionCodec +pub iceberg::spec::ParsedTableProperties::write_delete_avro_compression_level: core::option::Option pub iceberg::spec::ParsedTableProperties::write_delete_distribution_mode: core::option::Option pub iceberg::spec::ParsedTableProperties::write_delete_format_default: iceberg::spec::DataFileFormat pub iceberg::spec::ParsedTableProperties::write_delete_granularity: alloc::string::String pub iceberg::spec::ParsedTableProperties::write_delete_isolation_level: alloc::string::String pub iceberg::spec::ParsedTableProperties::write_delete_mode: alloc::string::String pub iceberg::spec::ParsedTableProperties::write_delete_orc_block_size_bytes: u64 -pub iceberg::spec::ParsedTableProperties::write_delete_orc_compression_codec: alloc::string::String +pub iceberg::spec::ParsedTableProperties::write_delete_orc_compression_codec: iceberg::compression::CompressionCodec pub iceberg::spec::ParsedTableProperties::write_delete_orc_compression_strategy: alloc::string::String pub iceberg::spec::ParsedTableProperties::write_delete_orc_stripe_size_bytes: u64 pub iceberg::spec::ParsedTableProperties::write_delete_orc_vectorized_batch_size: usize -pub iceberg::spec::ParsedTableProperties::write_delete_parquet_compression_codec: alloc::string::String -pub iceberg::spec::ParsedTableProperties::write_delete_parquet_compression_level: core::option::Option +pub iceberg::spec::ParsedTableProperties::write_delete_parquet_compression_codec: iceberg::compression::CompressionCodec +pub iceberg::spec::ParsedTableProperties::write_delete_parquet_compression_level: core::option::Option pub iceberg::spec::ParsedTableProperties::write_delete_parquet_dict_size_bytes: usize pub iceberg::spec::ParsedTableProperties::write_delete_parquet_page_row_limit: usize pub iceberg::spec::ParsedTableProperties::write_delete_parquet_page_size_bytes: usize @@ -2309,8 +2317,8 @@ pub iceberg::spec::ParsedTableProperties::write_distribution_mode: core::option: pub iceberg::spec::ParsedTableProperties::write_folder_storage_path: core::option::Option pub iceberg::spec::ParsedTableProperties::write_format_default: iceberg::spec::DataFileFormat pub iceberg::spec::ParsedTableProperties::write_location_provider_impl: core::option::Option -pub iceberg::spec::ParsedTableProperties::write_manifest_compression_codec: alloc::string::String -pub iceberg::spec::ParsedTableProperties::write_manifest_compression_level: core::option::Option +pub iceberg::spec::ParsedTableProperties::write_manifest_compression_codec: iceberg::compression::CompressionCodec +pub iceberg::spec::ParsedTableProperties::write_manifest_compression_level: core::option::Option pub iceberg::spec::ParsedTableProperties::write_manifest_lists_enabled: bool pub iceberg::spec::ParsedTableProperties::write_merge_distribution_mode: core::option::Option pub iceberg::spec::ParsedTableProperties::write_merge_isolation_level: alloc::string::String @@ -2328,7 +2336,7 @@ pub iceberg::spec::ParsedTableProperties::write_object_storage_path: core::optio pub iceberg::spec::ParsedTableProperties::write_orc_block_size_bytes: u64 pub iceberg::spec::ParsedTableProperties::write_orc_bloom_filter_columns: alloc::string::String pub iceberg::spec::ParsedTableProperties::write_orc_bloom_filter_fpp: f64 -pub iceberg::spec::ParsedTableProperties::write_orc_compression_codec: alloc::string::String +pub iceberg::spec::ParsedTableProperties::write_orc_compression_codec: iceberg::compression::CompressionCodec pub iceberg::spec::ParsedTableProperties::write_orc_compression_strategy: alloc::string::String pub iceberg::spec::ParsedTableProperties::write_orc_stripe_size_bytes: u64 pub iceberg::spec::ParsedTableProperties::write_orc_vectorized_batch_size: usize @@ -2337,8 +2345,8 @@ pub iceberg::spec::ParsedTableProperties::write_parquet_bloom_filter_enabled_col pub iceberg::spec::ParsedTableProperties::write_parquet_bloom_filter_fpp_column: std::collections::hash::map::HashMap pub iceberg::spec::ParsedTableProperties::write_parquet_bloom_filter_max_bytes: usize pub iceberg::spec::ParsedTableProperties::write_parquet_bloom_filter_ndv_column: std::collections::hash::map::HashMap -pub iceberg::spec::ParsedTableProperties::write_parquet_compression_codec: alloc::string::String -pub iceberg::spec::ParsedTableProperties::write_parquet_compression_level: core::option::Option +pub iceberg::spec::ParsedTableProperties::write_parquet_compression_codec: iceberg::compression::CompressionCodec +pub iceberg::spec::ParsedTableProperties::write_parquet_compression_level: core::option::Option pub iceberg::spec::ParsedTableProperties::write_parquet_content_defined_chunking_enabled: bool pub iceberg::spec::ParsedTableProperties::write_parquet_content_defined_chunking_max_chunk_size: usize pub iceberg::spec::ParsedTableProperties::write_parquet_content_defined_chunking_min_chunk_size: usize diff --git a/crates/iceberg/src/compression.rs b/crates/iceberg/src/compression.rs index 929d9226e7..49cd20712e 100644 --- a/crates/iceberg/src/compression.rs +++ b/crates/iceberg/src/compression.rs @@ -40,8 +40,14 @@ pub enum CompressionCodec { #[default] /// No compression None, + /// Uncompressed file-format encoding + Uncompressed, + /// Brotli compression + Brotli, /// LZ4 single compression frame with content size present Lz4, + /// LZO compression + Lzo, /// Zstandard single compression frame with content size present. /// Level range is 0–22, where 0 means default compression level (not no compression). /// Use [`CompressionCodec::zstd_default`] to construct with the default level. @@ -51,6 +57,8 @@ pub enum CompressionCodec { Gzip(u8), /// Snappy compression Snappy, + /// Zlib compression + Zlib, } impl CompressionCodec { @@ -68,10 +76,14 @@ impl CompressionCodec { pub fn name(&self) -> &'static str { match self { CompressionCodec::None => "none", + CompressionCodec::Uncompressed => "uncompressed", + CompressionCodec::Brotli => "brotli", CompressionCodec::Lz4 => "lz4", + CompressionCodec::Lzo => "lzo", CompressionCodec::Zstd(_) => "zstd", CompressionCodec::Gzip(_) => "gzip", CompressionCodec::Snappy => "snappy", + CompressionCodec::Zlib => "zlib", } } } @@ -91,12 +103,24 @@ impl<'de> Deserialize<'de> for CompressionCodec { let s = String::deserialize(deserializer)?; match s.to_lowercase().as_str() { "none" => Ok(CompressionCodec::None), + "uncompressed" => Ok(CompressionCodec::Uncompressed), + "brotli" => Ok(CompressionCodec::Brotli), "lz4" => Ok(CompressionCodec::Lz4), + "lzo" => Ok(CompressionCodec::Lzo), "zstd" => Ok(CompressionCodec::zstd_default()), "gzip" => Ok(CompressionCodec::gzip_default()), "snappy" => Ok(CompressionCodec::Snappy), + "zlib" => Ok(CompressionCodec::Zlib), other => Err(serde::de::Error::unknown_variant(other, &[ - "none", "lz4", "zstd", "gzip", "snappy", + "none", + "uncompressed", + "brotli", + "lz4", + "lzo", + "zstd", + "gzip", + "snappy", + "zlib", ])), } } @@ -106,10 +130,14 @@ impl fmt::Display for CompressionCodec { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { CompressionCodec::None => write!(f, "None"), + CompressionCodec::Uncompressed => write!(f, "Uncompressed"), + CompressionCodec::Brotli => write!(f, "Brotli"), CompressionCodec::Lz4 => write!(f, "Lz4"), + CompressionCodec::Lzo => write!(f, "Lzo"), CompressionCodec::Zstd(level) => write!(f, "Zstd(level={level})"), CompressionCodec::Gzip(level) => write!(f, "Gzip(level={level})"), CompressionCodec::Snappy => write!(f, "Snappy"), + CompressionCodec::Zlib => write!(f, "Zlib"), } } } @@ -117,11 +145,21 @@ impl fmt::Display for CompressionCodec { impl CompressionCodec { pub(crate) fn decompress(&self, bytes: Vec) -> Result> { match self { - CompressionCodec::None => Ok(bytes), + CompressionCodec::None | CompressionCodec::Uncompressed => Ok(bytes), CompressionCodec::Lz4 => Err(Error::new( ErrorKind::FeatureUnsupported, "LZ4 decompression is not supported currently", )), + CompressionCodec::Snappy => Err(Error::new( + ErrorKind::FeatureUnsupported, + "Snappy decompression is not supported currently", + )), + codec @ (CompressionCodec::Brotli | CompressionCodec::Lzo | CompressionCodec::Zlib) => { + Err(Error::new( + ErrorKind::FeatureUnsupported, + format!("{codec} decompression is not supported currently"), + )) + } CompressionCodec::Zstd(_) => Ok(zstd::stream::decode_all(&bytes[..])?), CompressionCodec::Gzip(_) => { let mut decoder = GzDecoder::new(&bytes[..]); @@ -129,20 +167,26 @@ impl CompressionCodec { decoder.read_to_end(&mut decompressed)?; Ok(decompressed) } - CompressionCodec::Snappy => Err(Error::new( - ErrorKind::FeatureUnsupported, - "Snappy decompression is not supported currently", - )), } } pub(crate) fn compress(&self, bytes: Vec) -> Result> { match self { - CompressionCodec::None => Ok(bytes), + CompressionCodec::None | CompressionCodec::Uncompressed => Ok(bytes), CompressionCodec::Lz4 => Err(Error::new( ErrorKind::FeatureUnsupported, "LZ4 compression is not supported currently", )), + CompressionCodec::Snappy => Err(Error::new( + ErrorKind::FeatureUnsupported, + "Snappy compression is not supported currently", + )), + codec @ (CompressionCodec::Brotli | CompressionCodec::Lzo | CompressionCodec::Zlib) => { + Err(Error::new( + ErrorKind::FeatureUnsupported, + format!("{codec} compression is not supported currently"), + )) + } CompressionCodec::Zstd(level) => { let writer = Vec::::new(); let mut encoder = zstd::stream::Encoder::new(writer, *level as i32)?; @@ -157,15 +201,14 @@ impl CompressionCodec { encoder.write_all(&bytes)?; Ok(encoder.finish()?) } - CompressionCodec::Snappy => Err(Error::new( - ErrorKind::FeatureUnsupported, - "Snappy compression is not supported currently", - )), } } pub(crate) fn is_none(&self) -> bool { - matches!(self, CompressionCodec::None) + matches!( + self, + CompressionCodec::None | CompressionCodec::Uncompressed + ) } /// Returns the file extension suffix for this compression codec. @@ -173,14 +216,17 @@ impl CompressionCodec { /// /// # Errors /// - /// Returns an error for Lz4 and Zstd as they are not fully supported. + /// Returns an error when the codec does not have a defined suffix. pub fn suffix(&self) -> Result<&'static str> { match self { - CompressionCodec::None => Ok(""), + CompressionCodec::None | CompressionCodec::Uncompressed => Ok(""), CompressionCodec::Gzip(_) => Ok(".gz"), - codec @ (CompressionCodec::Lz4 + codec @ (CompressionCodec::Brotli + | CompressionCodec::Lz4 + | CompressionCodec::Lzo | CompressionCodec::Zstd(_) - | CompressionCodec::Snappy) => Err(Error::new( + | CompressionCodec::Snappy + | CompressionCodec::Zlib) => Err(Error::new( ErrorKind::FeatureUnsupported, format!("suffix not defined for {codec:?}"), )), @@ -196,11 +242,12 @@ mod tests { async fn test_compression_codec_none() { let bytes_vec = [0_u8; 100].to_vec(); - let codec = CompressionCodec::None; - let compressed = codec.compress(bytes_vec.clone()).unwrap(); - assert_eq!(bytes_vec, compressed); - let decompressed = codec.decompress(compressed).unwrap(); - assert_eq!(bytes_vec, decompressed); + for codec in [CompressionCodec::None, CompressionCodec::Uncompressed] { + let compressed = codec.compress(bytes_vec.clone()).unwrap(); + assert_eq!(bytes_vec, compressed); + let decompressed = codec.decompress(compressed).unwrap(); + assert_eq!(bytes_vec, decompressed); + } } #[tokio::test] @@ -225,6 +272,9 @@ mod tests { let unsupported_codecs = [ (CompressionCodec::Lz4, "LZ4"), (CompressionCodec::Snappy, "Snappy"), + (CompressionCodec::Brotli, "Brotli"), + (CompressionCodec::Lzo, "Lzo"), + (CompressionCodec::Zlib, "Zlib"), ]; let bytes_vec = [0_u8; 100].to_vec(); @@ -273,4 +323,27 @@ mod tests { assert_eq!(CompressionCodec::Gzip(9).to_string(), "Gzip(level=9)"); assert_eq!(CompressionCodec::Snappy.to_string(), "Snappy"); } + + #[test] + fn test_serde_names() { + let codecs = [ + ("none", CompressionCodec::None), + ("uncompressed", CompressionCodec::Uncompressed), + ("brotli", CompressionCodec::Brotli), + ("lz4", CompressionCodec::Lz4), + ("lzo", CompressionCodec::Lzo), + ("zstd", CompressionCodec::zstd_default()), + ("gzip", CompressionCodec::gzip_default()), + ("snappy", CompressionCodec::Snappy), + ("zlib", CompressionCodec::Zlib), + ]; + + for (name, codec) in codecs { + assert_eq!(serde_json::to_value(codec).unwrap(), name); + assert_eq!( + serde_json::from_value::(serde_json::json!(name)).unwrap(), + codec + ); + } + } } diff --git a/crates/iceberg/src/spec/parsed_table_prop.rs b/crates/iceberg/src/spec/parsed_table_prop.rs index da7586e3c7..f6b9f1233e 100644 --- a/crates/iceberg/src/spec/parsed_table_prop.rs +++ b/crates/iceberg/src/spec/parsed_table_prop.rs @@ -100,6 +100,15 @@ fn parse_metadata_location(value: &str) -> Result> { Ok(Some(strip_trailing_slash(value).to_string())) } +fn parse_compression_codec(value: &str) -> Result { + serde_json::from_value(serde_json::Value::String(value.to_lowercase())).map_err(|_| { + Error::new( + ErrorKind::DataInvalid, + format!("Invalid compression codec: {value}"), + ) + }) +} + fn parse_metadata_file_compression(value: &str) -> Result { if value.is_empty() { return Ok(CompressionCodec::None); @@ -215,14 +224,16 @@ pub struct ParsedTableProperties { pub commit_manifest_merge_enabled: bool, #[key = "write.manifest.compression-codec"] - #[default("gzip".to_string())] + #[default(CompressionCodec::gzip_default())] + #[parse_with(parse_compression_codec)] + #[serialize_with(serialize_compression_codec)] #[doc = "Compression codec used for manifest files."] - pub write_manifest_compression_codec: String, + pub write_manifest_compression_codec: CompressionCodec, #[key = "write.manifest.compression-level"] #[default(None)] #[doc = "Optional compression level used for manifest files."] - pub write_manifest_compression_level: Option, + pub write_manifest_compression_level: Option, #[key = "write.manifest-lists.enabled"] #[default(true)] @@ -317,12 +328,12 @@ pub struct ParsedTableProperties { pub write_delete_parquet_page_size_bytes: usize, #[key = "write.parquet.page-version"] - #[default("v1".to_string())] + #[default("v1")] #[doc = "Parquet data page version for data files: v1 or v2."] pub write_parquet_page_version: String, #[key = "write.delete.parquet.page-version"] - #[default("v1".to_string())] + #[default("v1")] #[doc = "Parquet data page version for delete files: v1 or v2."] pub write_delete_parquet_page_version: String, @@ -347,24 +358,28 @@ pub struct ParsedTableProperties { pub write_delete_parquet_dict_size_bytes: usize, #[key = "write.parquet.compression-codec"] - #[default("zstd".to_string())] + #[default(CompressionCodec::zstd_default())] + #[parse_with(parse_compression_codec)] + #[serialize_with(serialize_compression_codec)] #[doc = "Parquet compression codec used for data files."] - pub write_parquet_compression_codec: String, + pub write_parquet_compression_codec: CompressionCodec, #[key = "write.delete.parquet.compression-codec"] - #[default("zstd".to_string())] + #[default(CompressionCodec::zstd_default())] + #[parse_with(parse_compression_codec)] + #[serialize_with(serialize_compression_codec)] #[doc = "Parquet compression codec used for delete files."] - pub write_delete_parquet_compression_codec: String, + pub write_delete_parquet_compression_codec: CompressionCodec, #[key = "write.parquet.compression-level"] #[default(None)] #[doc = "Optional Parquet compression level for data files."] - pub write_parquet_compression_level: Option, + pub write_parquet_compression_level: Option, #[key = "write.delete.parquet.compression-level"] #[default(None)] #[doc = "Optional Parquet compression level for delete files."] - pub write_delete_parquet_compression_level: Option, + pub write_delete_parquet_compression_level: Option, #[key = "write.parquet.shred-variants"] #[default(false)] @@ -458,24 +473,28 @@ pub struct ParsedTableProperties { // Avro properties. #[key = "write.avro.compression-codec"] - #[default("gzip".to_string())] + #[default(CompressionCodec::gzip_default())] + #[parse_with(parse_compression_codec)] + #[serialize_with(serialize_compression_codec)] #[doc = "Avro compression codec used for data files."] - pub write_avro_compression_codec: String, + pub write_avro_compression_codec: CompressionCodec, #[key = "write.delete.avro.compression-codec"] - #[default("gzip".to_string())] + #[default(CompressionCodec::gzip_default())] + #[parse_with(parse_compression_codec)] + #[serialize_with(serialize_compression_codec)] #[doc = "Avro compression codec used for delete files."] - pub write_delete_avro_compression_codec: String, + pub write_delete_avro_compression_codec: CompressionCodec, #[key = "write.avro.compression-level"] #[default(None)] #[doc = "Optional Avro compression level for data files."] - pub write_avro_compression_level: Option, + pub write_avro_compression_level: Option, #[key = "write.delete.avro.compression-level"] #[default(None)] #[doc = "Optional Avro compression level for delete files."] - pub write_delete_avro_compression_level: Option, + pub write_delete_avro_compression_level: Option, // ORC properties. #[key = "write.orc.stripe-size-bytes"] @@ -489,7 +508,7 @@ pub struct ParsedTableProperties { pub write_delete_orc_stripe_size_bytes: u64, #[key = "write.orc.bloom.filter.columns"] - #[default(String::new())] + #[default("")] #[doc = "Comma-separated column names for which ORC bloom filters are created."] pub write_orc_bloom_filter_columns: String, @@ -519,22 +538,26 @@ pub struct ParsedTableProperties { pub write_delete_orc_vectorized_batch_size: usize, #[key = "write.orc.compression-codec"] - #[default("zlib".to_string())] + #[default(CompressionCodec::Zlib)] + #[parse_with(parse_compression_codec)] + #[serialize_with(serialize_compression_codec)] #[doc = "ORC compression codec used for data files."] - pub write_orc_compression_codec: String, + pub write_orc_compression_codec: CompressionCodec, #[key = "write.delete.orc.compression-codec"] - #[default("zlib".to_string())] + #[default(CompressionCodec::Zlib)] + #[parse_with(parse_compression_codec)] + #[serialize_with(serialize_compression_codec)] #[doc = "ORC compression codec used for delete files."] - pub write_delete_orc_compression_codec: String, + pub write_delete_orc_compression_codec: CompressionCodec, #[key = "write.orc.compression-strategy"] - #[default("speed".to_string())] + #[default("speed")] #[doc = "ORC compression strategy for data files: speed or compression."] pub write_orc_compression_strategy: String, #[key = "write.delete.orc.compression-strategy"] - #[default("speed".to_string())] + #[default("speed")] #[doc = "ORC compression strategy for delete files: speed or compression."] pub write_delete_orc_compression_strategy: String, @@ -585,12 +608,12 @@ pub struct ParsedTableProperties { pub read_orc_vectorization_batch_size: usize, #[key = "read.data-planning-mode"] - #[default("auto".to_string())] + #[default("auto")] #[doc = "Planning mode used for data files."] pub read_data_planning_mode: String, #[key = "read.delete-planning-mode"] - #[default("auto".to_string())] + #[default("auto")] #[doc = "Planning mode used for delete files."] pub read_delete_planning_mode: String, @@ -634,7 +657,7 @@ pub struct ParsedTableProperties { pub write_metadata_metrics_column: HashMap, #[key = "write.metadata.metrics.default"] - #[default("truncate(16)".to_string())] + #[default("truncate(16)")] #[doc = "Default metrics mode for table columns."] pub write_metadata_metrics_default: String, @@ -702,17 +725,17 @@ pub struct ParsedTableProperties { // Row-level operation properties. #[key = "write.delete.granularity"] - #[default("partition".to_string())] + #[default("partition")] #[doc = "Granularity of generated delete files: partition or file."] pub write_delete_granularity: String, #[key = "write.delete.isolation-level"] - #[default("serializable".to_string())] + #[default("serializable")] #[doc = "Isolation level for delete commands: serializable or snapshot."] pub write_delete_isolation_level: String, #[key = "write.delete.mode"] - #[default("copy-on-write".to_string())] + #[default("copy-on-write")] #[doc = "Execution mode for delete commands: copy-on-write or merge-on-read."] pub write_delete_mode: String, @@ -722,12 +745,12 @@ pub struct ParsedTableProperties { pub write_delete_distribution_mode: Option, #[key = "write.update.isolation-level"] - #[default("serializable".to_string())] + #[default("serializable")] #[doc = "Isolation level for update commands: serializable or snapshot."] pub write_update_isolation_level: String, #[key = "write.update.mode"] - #[default("copy-on-write".to_string())] + #[default("copy-on-write")] #[doc = "Execution mode for update commands: copy-on-write or merge-on-read."] pub write_update_mode: String, @@ -737,12 +760,12 @@ pub struct ParsedTableProperties { pub write_update_distribution_mode: Option, #[key = "write.merge.isolation-level"] - #[default("serializable".to_string())] + #[default("serializable")] #[doc = "Isolation level for merge commands: serializable or snapshot."] pub write_merge_isolation_level: String, #[key = "write.merge.mode"] - #[default("copy-on-write".to_string())] + #[default("copy-on-write")] #[doc = "Execution mode for merge commands: copy-on-write or merge-on-read."] pub write_merge_mode: String, @@ -783,6 +806,25 @@ mod tests { assert_eq!(properties.commit_retry_num_retries, 4); assert_eq!(properties.write_format_default, DataFileFormat::Parquet); + assert_eq!( + properties.write_manifest_compression_codec, + CompressionCodec::gzip_default() + ); + assert_eq!( + properties.write_parquet_compression_codec, + CompressionCodec::zstd_default() + ); + assert_eq!( + properties.write_avro_compression_codec, + CompressionCodec::gzip_default() + ); + assert_eq!( + properties.write_orc_compression_codec, + CompressionCodec::Zlib + ); + assert_eq!(properties.write_manifest_compression_level, None); + assert_eq!(properties.write_parquet_compression_level, None); + assert_eq!(properties.write_avro_compression_level, None); assert_eq!( properties.write_parquet_row_group_size_bytes, 128 * 1024 * 1024 @@ -798,6 +840,8 @@ mod tests { commit_retry_num_retries: 9, write_format_default: DataFileFormat::Orc, write_data_path: Some("s3://warehouse/table/data".to_string()), + write_orc_compression_codec: CompressionCodec::Lzo, + write_parquet_compression_level: Some(5), write_parquet_bloom_filter_fpp_column: HashMap::from([( "customer_id".to_string(), 0.02, @@ -809,6 +853,8 @@ mod tests { assert_eq!(json["commit.retry.num-retries"], "9"); assert_eq!(json["write.format.default"], "orc"); assert_eq!(json["write.data.path"], "s3://warehouse/table/data"); + assert_eq!(json["write.orc.compression-codec"], "lzo"); + assert_eq!(json["write.parquet.compression-level"], "5"); assert_eq!( json["write.parquet.bloom-filter-fpp.column.customer_id"], "0.02" @@ -867,6 +913,10 @@ mod tests { "write.delete.avro.compression-codec".to_string(), "snappy".to_string(), ), + ( + "write.delete.avro.compression-level".to_string(), + "7".to_string(), + ), ("read.split.planning-lookback".to_string(), "25".to_string()), ( "history.expire.min-snapshots-to-keep".to_string(), @@ -879,7 +929,11 @@ mod tests { assert_eq!(properties.comment, Some("orders table".to_string())); assert_eq!(properties.commit_status_check_num_retries, 7); - assert_eq!(properties.write_delete_avro_compression_codec, "snappy"); + assert_eq!( + properties.write_delete_avro_compression_codec, + CompressionCodec::Snappy + ); + assert_eq!(properties.write_delete_avro_compression_level, Some(7)); assert_eq!(properties.read_split_planning_lookback, 25); assert_eq!(properties.history_expire_min_snapshots_to_keep, 4); assert_eq!(properties.write_delete_mode, "merge-on-read"); diff --git a/crates/property-macro/Cargo.toml b/crates/property-macro/Cargo.toml index f67c8aabf5..1fc1c73693 100644 --- a/crates/property-macro/Cargo.toml +++ b/crates/property-macro/Cargo.toml @@ -19,7 +19,7 @@ edition = { workspace = true } homepage = { workspace = true } name = "iceberg-property-macro" -publish = true +publish = false rust-version = { workspace = true } version = { workspace = true } diff --git a/crates/property-macro/src/lib.rs b/crates/property-macro/src/lib.rs index 598ec7cc10..784b5b2125 100644 --- a/crates/property-macro/src/lib.rs +++ b/crates/property-macro/src/lib.rs @@ -21,22 +21,27 @@ use proc_macro::TokenStream; use proc_macro2::TokenStream as TokenStream2; use quote::quote; use syn::{ - Attribute, Data, DeriveInput, Error, Expr, ExprPath, Field, Fields, Ident, Meta, Path, Type, - parse_macro_input, + Attribute, Data, DeriveInput, Error, Expr, ExprLit, ExprPath, Field, Fields, Ident, Lit, Meta, + Path, Type, parse_macro_input, }; /// Derive parsing, defaults, and JSON serialization for a typed property map. /// /// Leaf fields must declare the table-property key and its default: /// -/// ```ignore +/// ``` +/// use iceberg_property_macro::Properties; +/// /// #[derive(Properties)] /// struct Properties { /// #[key = "write.format.default"] -/// #[default = DataFileFormat::Parquet] +/// #[default = "parquet"] /// #[doc = "Default file format"] -/// pub write_format_default: DataFileFormat, +/// pub write_format_default: String, /// } +/// +/// let properties = Properties::default(); +/// assert_eq!(properties.write_format_default, "parquet"); /// ``` /// /// `prefix` captures a family of properties in a `HashMap`, keyed by the suffix after @@ -44,7 +49,8 @@ use syn::{ /// property map flat. `parse_with` may be used for exact-key property types that do not implement /// `FromStr` or need validation. `serialize_with` supplies their string representation in JSON. /// Optional fields are omitted from JSON when they are `None`. Fields need `FromStr` and `ToString` -/// unless the relevant custom parsing or serialization attribute is supplied. +/// unless the relevant custom parsing or serialization attribute is supplied. String-literal +/// defaults are converted into their field type with `Into`. #[proc_macro_derive( Properties, attributes(key, prefix, nested, default, parse_with, serialize_with) @@ -102,6 +108,8 @@ fn expand_properties(input: DeriveInput) -> syn::Result { quote!(#ident: ::std::default::Default::default()) } else { let default = field.default.as_ref().expect("leaf fields have defaults"); + let ty = &field.ty; + let default = default_value(default, ty); quote!(#ident: #default) } }); @@ -297,7 +305,10 @@ fn parse_field(field: &PropertyField) -> TokenStream2 { return quote!(#ident: <#ty>::from_properties(properties)?); } - let default = field.default.as_ref().expect("leaf fields have defaults"); + let default = default_value( + field.default.as_ref().expect("leaf fields have defaults"), + &field.ty, + ); if let Some(prefix) = &field.prefix { let value_type = field @@ -356,6 +367,20 @@ fn parse_field(field: &PropertyField) -> TokenStream2 { } } +fn default_value(default: &Expr, ty: &Type) -> TokenStream2 { + if matches!( + default, + Expr::Lit(ExprLit { + lit: Lit::Str(_), + .. + }) + ) { + quote!(::std::convert::Into::<#ty>::into(#default)) + } else { + quote!(#default) + } +} + fn option_inner_type(ty: &Type) -> Option { let Type::Path(type_path) = ty else { return None; diff --git a/crates/property-macro/tests/properties.rs b/crates/property-macro/tests/properties.rs index a24f05b224..b360ac9890 100644 --- a/crates/property-macro/tests/properties.rs +++ b/crates/property-macro/tests/properties.rs @@ -21,6 +21,7 @@ use iceberg_property_macro::Properties; const RETRIES: &str = "commit.retry.num-retries"; const OWNER: &str = "owner"; +const FORMAT: &str = "write.format.default"; const COLUMN_FPP_PREFIX: &str = "write.parquet.bloom-filter-fpp.column."; #[derive(Debug, Properties)] @@ -34,6 +35,10 @@ struct TestProperties { #[default(None)] pub owner: Option, + #[key(FORMAT)] + #[default("parquet")] + pub format: String, + #[prefix(COLUMN_FPP_PREFIX)] #[default(HashMap::new())] pub column_fpp: HashMap, @@ -41,24 +46,30 @@ struct TestProperties { #[test] fn generates_defaults_and_serde_for_public_fields() { + assert_eq!(TestProperties::default().format, "parquet"); + let properties = TestProperties { retries: 8, owner: Some("iceberg".to_string()), + format: "orc".to_string(), column_fpp: HashMap::from([("id".to_string(), 0.01)]), }; assert_eq!(properties.retries, 8); assert_eq!(properties.owner, Some("iceberg".to_string())); + assert_eq!(properties.format, "orc"); assert_eq!(properties.column_fpp["id"], 0.01); let json = serde_json::to_value(&properties).unwrap(); assert_eq!(json[RETRIES], "8"); assert_eq!(json[OWNER], "iceberg"); + assert_eq!(json[FORMAT], "orc"); assert_eq!(json[format!("{COLUMN_FPP_PREFIX}id")], "0.01"); let decoded: TestProperties = serde_json::from_value(json).unwrap(); assert_eq!(decoded.retries, 8); assert_eq!(decoded.owner, Some("iceberg".to_string())); + assert_eq!(decoded.format, "orc"); assert_eq!(decoded.column_fpp["id"], 0.01); } From 0cc83b7632fce1037fac20973069d92c1f688830 Mon Sep 17 00:00:00 2001 From: Renjie Liu Date: Wed, 5 Aug 2026 14:20:18 +0800 Subject: [PATCH 07/14] Address comments --- crates/iceberg/public-api.txt | 130 ++++++-- crates/iceberg/src/spec/parsed_table_prop.rs | 315 +++++++++++++++---- crates/property-macro/src/lib.rs | 6 +- 3 files changed, 369 insertions(+), 82 deletions(-) diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index 0075d44e73..82858bcbf1 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -1409,6 +1409,55 @@ impl serde_core::ser::Serialize for iceberg::spec::DataFileFormat where Self: co pub fn iceberg::spec::DataFileFormat::serialize<__S>(&self, serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::DataFileFormat where Self: core::str::traits::FromStr, ::Err: core::fmt::Display pub fn iceberg::spec::DataFileFormat::deserialize<__D>(deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub enum iceberg::spec::DeleteGranularity +pub iceberg::spec::DeleteGranularity::File +pub iceberg::spec::DeleteGranularity::Partition +impl core::clone::Clone for iceberg::spec::DeleteGranularity +pub fn iceberg::spec::DeleteGranularity::clone(&self) -> iceberg::spec::DeleteGranularity +impl core::cmp::Eq for iceberg::spec::DeleteGranularity +impl core::cmp::PartialEq for iceberg::spec::DeleteGranularity +pub fn iceberg::spec::DeleteGranularity::eq(&self, other: &iceberg::spec::DeleteGranularity) -> bool +impl core::convert::TryFrom<&str> for iceberg::spec::DeleteGranularity +pub type iceberg::spec::DeleteGranularity::Error = strum::ParseError +pub fn iceberg::spec::DeleteGranularity::try_from(s: &str) -> core::result::Result>::Error> +impl core::fmt::Debug for iceberg::spec::DeleteGranularity +pub fn iceberg::spec::DeleteGranularity::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for iceberg::spec::DeleteGranularity +pub fn iceberg::spec::DeleteGranularity::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::result::Result<(), core::fmt::Error> +impl core::marker::Copy for iceberg::spec::DeleteGranularity +impl core::marker::StructuralPartialEq for iceberg::spec::DeleteGranularity +impl core::str::traits::FromStr for iceberg::spec::DeleteGranularity +pub type iceberg::spec::DeleteGranularity::Err = strum::ParseError +pub fn iceberg::spec::DeleteGranularity::from_str(s: &str) -> core::result::Result::Err> +impl serde_core::ser::Serialize for iceberg::spec::DeleteGranularity where Self: core::fmt::Display +pub fn iceberg::spec::DeleteGranularity::serialize<__S>(&self, serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::DeleteGranularity where Self: core::str::traits::FromStr, ::Err: core::fmt::Display +pub fn iceberg::spec::DeleteGranularity::deserialize<__D>(deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub enum iceberg::spec::DistributionMode +pub iceberg::spec::DistributionMode::Hash +pub iceberg::spec::DistributionMode::None +pub iceberg::spec::DistributionMode::Range +impl core::clone::Clone for iceberg::spec::DistributionMode +pub fn iceberg::spec::DistributionMode::clone(&self) -> iceberg::spec::DistributionMode +impl core::cmp::Eq for iceberg::spec::DistributionMode +impl core::cmp::PartialEq for iceberg::spec::DistributionMode +pub fn iceberg::spec::DistributionMode::eq(&self, other: &iceberg::spec::DistributionMode) -> bool +impl core::convert::TryFrom<&str> for iceberg::spec::DistributionMode +pub type iceberg::spec::DistributionMode::Error = strum::ParseError +pub fn iceberg::spec::DistributionMode::try_from(s: &str) -> core::result::Result>::Error> +impl core::fmt::Debug for iceberg::spec::DistributionMode +pub fn iceberg::spec::DistributionMode::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for iceberg::spec::DistributionMode +pub fn iceberg::spec::DistributionMode::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::result::Result<(), core::fmt::Error> +impl core::marker::Copy for iceberg::spec::DistributionMode +impl core::marker::StructuralPartialEq for iceberg::spec::DistributionMode +impl core::str::traits::FromStr for iceberg::spec::DistributionMode +pub type iceberg::spec::DistributionMode::Err = strum::ParseError +pub fn iceberg::spec::DistributionMode::from_str(s: &str) -> core::result::Result::Err> +impl serde_core::ser::Serialize for iceberg::spec::DistributionMode where Self: core::fmt::Display +pub fn iceberg::spec::DistributionMode::serialize<__S>(&self, serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::DistributionMode where Self: core::str::traits::FromStr, ::Err: core::fmt::Display +pub fn iceberg::spec::DistributionMode::deserialize<__D>(deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> #[repr(u8)] pub enum iceberg::spec::FormatVersion pub iceberg::spec::FormatVersion::V1 = 1 pub iceberg::spec::FormatVersion::V2 = 2 @@ -1434,6 +1483,30 @@ impl serde_core::ser::Serialize for iceberg::spec::FormatVersion pub fn iceberg::spec::FormatVersion::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> where S: serde_core::ser::Serializer impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::FormatVersion pub fn iceberg::spec::FormatVersion::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> +pub enum iceberg::spec::IsolationLevel +pub iceberg::spec::IsolationLevel::Serializable +pub iceberg::spec::IsolationLevel::Snapshot +impl core::clone::Clone for iceberg::spec::IsolationLevel +pub fn iceberg::spec::IsolationLevel::clone(&self) -> iceberg::spec::IsolationLevel +impl core::cmp::Eq for iceberg::spec::IsolationLevel +impl core::cmp::PartialEq for iceberg::spec::IsolationLevel +pub fn iceberg::spec::IsolationLevel::eq(&self, other: &iceberg::spec::IsolationLevel) -> bool +impl core::convert::TryFrom<&str> for iceberg::spec::IsolationLevel +pub type iceberg::spec::IsolationLevel::Error = strum::ParseError +pub fn iceberg::spec::IsolationLevel::try_from(s: &str) -> core::result::Result>::Error> +impl core::fmt::Debug for iceberg::spec::IsolationLevel +pub fn iceberg::spec::IsolationLevel::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for iceberg::spec::IsolationLevel +pub fn iceberg::spec::IsolationLevel::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::result::Result<(), core::fmt::Error> +impl core::marker::Copy for iceberg::spec::IsolationLevel +impl core::marker::StructuralPartialEq for iceberg::spec::IsolationLevel +impl core::str::traits::FromStr for iceberg::spec::IsolationLevel +pub type iceberg::spec::IsolationLevel::Err = strum::ParseError +pub fn iceberg::spec::IsolationLevel::from_str(s: &str) -> core::result::Result::Err> +impl serde_core::ser::Serialize for iceberg::spec::IsolationLevel where Self: core::fmt::Display +pub fn iceberg::spec::IsolationLevel::serialize<__S>(&self, serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::IsolationLevel where Self: core::str::traits::FromStr, ::Err: core::fmt::Display +pub fn iceberg::spec::IsolationLevel::deserialize<__D>(deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> pub enum iceberg::spec::Literal pub iceberg::spec::Literal::List(alloc::vec::Vec>) pub iceberg::spec::Literal::Map(iceberg::spec::Map) @@ -1631,6 +1704,30 @@ impl<'de> iceberg::spec::PrimitiveType pub fn iceberg::spec::PrimitiveType::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::PrimitiveType pub fn iceberg::spec::PrimitiveType::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> +pub enum iceberg::spec::RowLevelOperationMode +pub iceberg::spec::RowLevelOperationMode::CopyOnWrite +pub iceberg::spec::RowLevelOperationMode::MergeOnRead +impl core::clone::Clone for iceberg::spec::RowLevelOperationMode +pub fn iceberg::spec::RowLevelOperationMode::clone(&self) -> iceberg::spec::RowLevelOperationMode +impl core::cmp::Eq for iceberg::spec::RowLevelOperationMode +impl core::cmp::PartialEq for iceberg::spec::RowLevelOperationMode +pub fn iceberg::spec::RowLevelOperationMode::eq(&self, other: &iceberg::spec::RowLevelOperationMode) -> bool +impl core::convert::TryFrom<&str> for iceberg::spec::RowLevelOperationMode +pub type iceberg::spec::RowLevelOperationMode::Error = strum::ParseError +pub fn iceberg::spec::RowLevelOperationMode::try_from(s: &str) -> core::result::Result>::Error> +impl core::fmt::Debug for iceberg::spec::RowLevelOperationMode +pub fn iceberg::spec::RowLevelOperationMode::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for iceberg::spec::RowLevelOperationMode +pub fn iceberg::spec::RowLevelOperationMode::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::result::Result<(), core::fmt::Error> +impl core::marker::Copy for iceberg::spec::RowLevelOperationMode +impl core::marker::StructuralPartialEq for iceberg::spec::RowLevelOperationMode +impl core::str::traits::FromStr for iceberg::spec::RowLevelOperationMode +pub type iceberg::spec::RowLevelOperationMode::Err = strum::ParseError +pub fn iceberg::spec::RowLevelOperationMode::from_str(s: &str) -> core::result::Result::Err> +impl serde_core::ser::Serialize for iceberg::spec::RowLevelOperationMode where Self: core::fmt::Display +pub fn iceberg::spec::RowLevelOperationMode::serialize<__S>(&self, serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::RowLevelOperationMode where Self: core::str::traits::FromStr, ::Err: core::fmt::Display +pub fn iceberg::spec::RowLevelOperationMode::deserialize<__D>(deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> pub enum iceberg::spec::SnapshotRetention pub iceberg::spec::SnapshotRetention::Branch pub iceberg::spec::SnapshotRetention::Branch::max_ref_age_ms: core::option::Option @@ -2286,25 +2383,22 @@ pub iceberg::spec::ParsedTableProperties::read_split_metadata_target_size: u64 pub iceberg::spec::ParsedTableProperties::read_split_open_file_cost: u64 pub iceberg::spec::ParsedTableProperties::read_split_planning_lookback: usize pub iceberg::spec::ParsedTableProperties::read_split_target_size: u64 -pub iceberg::spec::ParsedTableProperties::schema_name_mapping_default: core::option::Option +pub iceberg::spec::ParsedTableProperties::schema_name_mapping_default: core::option::Option pub iceberg::spec::ParsedTableProperties::write_avro_compression_codec: iceberg::compression::CompressionCodec -pub iceberg::spec::ParsedTableProperties::write_avro_compression_level: core::option::Option pub iceberg::spec::ParsedTableProperties::write_data_path: core::option::Option pub iceberg::spec::ParsedTableProperties::write_datafusion_fanout_enabled: bool pub iceberg::spec::ParsedTableProperties::write_delete_avro_compression_codec: iceberg::compression::CompressionCodec -pub iceberg::spec::ParsedTableProperties::write_delete_avro_compression_level: core::option::Option -pub iceberg::spec::ParsedTableProperties::write_delete_distribution_mode: core::option::Option +pub iceberg::spec::ParsedTableProperties::write_delete_distribution_mode: core::option::Option pub iceberg::spec::ParsedTableProperties::write_delete_format_default: iceberg::spec::DataFileFormat -pub iceberg::spec::ParsedTableProperties::write_delete_granularity: alloc::string::String -pub iceberg::spec::ParsedTableProperties::write_delete_isolation_level: alloc::string::String -pub iceberg::spec::ParsedTableProperties::write_delete_mode: alloc::string::String +pub iceberg::spec::ParsedTableProperties::write_delete_granularity: iceberg::spec::DeleteGranularity +pub iceberg::spec::ParsedTableProperties::write_delete_isolation_level: iceberg::spec::IsolationLevel +pub iceberg::spec::ParsedTableProperties::write_delete_mode: iceberg::spec::RowLevelOperationMode pub iceberg::spec::ParsedTableProperties::write_delete_orc_block_size_bytes: u64 pub iceberg::spec::ParsedTableProperties::write_delete_orc_compression_codec: iceberg::compression::CompressionCodec pub iceberg::spec::ParsedTableProperties::write_delete_orc_compression_strategy: alloc::string::String pub iceberg::spec::ParsedTableProperties::write_delete_orc_stripe_size_bytes: u64 pub iceberg::spec::ParsedTableProperties::write_delete_orc_vectorized_batch_size: usize pub iceberg::spec::ParsedTableProperties::write_delete_parquet_compression_codec: iceberg::compression::CompressionCodec -pub iceberg::spec::ParsedTableProperties::write_delete_parquet_compression_level: core::option::Option pub iceberg::spec::ParsedTableProperties::write_delete_parquet_dict_size_bytes: usize pub iceberg::spec::ParsedTableProperties::write_delete_parquet_page_row_limit: usize pub iceberg::spec::ParsedTableProperties::write_delete_parquet_page_size_bytes: usize @@ -2313,16 +2407,15 @@ pub iceberg::spec::ParsedTableProperties::write_delete_parquet_row_group_check_m pub iceberg::spec::ParsedTableProperties::write_delete_parquet_row_group_check_min_record_count: usize pub iceberg::spec::ParsedTableProperties::write_delete_parquet_row_group_size_bytes: usize pub iceberg::spec::ParsedTableProperties::write_delete_target_file_size_bytes: usize -pub iceberg::spec::ParsedTableProperties::write_distribution_mode: core::option::Option +pub iceberg::spec::ParsedTableProperties::write_distribution_mode: core::option::Option pub iceberg::spec::ParsedTableProperties::write_folder_storage_path: core::option::Option pub iceberg::spec::ParsedTableProperties::write_format_default: iceberg::spec::DataFileFormat pub iceberg::spec::ParsedTableProperties::write_location_provider_impl: core::option::Option pub iceberg::spec::ParsedTableProperties::write_manifest_compression_codec: iceberg::compression::CompressionCodec -pub iceberg::spec::ParsedTableProperties::write_manifest_compression_level: core::option::Option pub iceberg::spec::ParsedTableProperties::write_manifest_lists_enabled: bool -pub iceberg::spec::ParsedTableProperties::write_merge_distribution_mode: core::option::Option -pub iceberg::spec::ParsedTableProperties::write_merge_isolation_level: alloc::string::String -pub iceberg::spec::ParsedTableProperties::write_merge_mode: alloc::string::String +pub iceberg::spec::ParsedTableProperties::write_merge_distribution_mode: core::option::Option +pub iceberg::spec::ParsedTableProperties::write_merge_isolation_level: iceberg::spec::IsolationLevel +pub iceberg::spec::ParsedTableProperties::write_merge_mode: iceberg::spec::RowLevelOperationMode pub iceberg::spec::ParsedTableProperties::write_metadata_compression_codec: iceberg::compression::CompressionCodec pub iceberg::spec::ParsedTableProperties::write_metadata_delete_after_commit_enabled: bool pub iceberg::spec::ParsedTableProperties::write_metadata_metrics_column: std::collections::hash::map::HashMap @@ -2334,7 +2427,7 @@ pub iceberg::spec::ParsedTableProperties::write_object_storage_enabled: bool pub iceberg::spec::ParsedTableProperties::write_object_storage_partitioned_paths: bool pub iceberg::spec::ParsedTableProperties::write_object_storage_path: core::option::Option pub iceberg::spec::ParsedTableProperties::write_orc_block_size_bytes: u64 -pub iceberg::spec::ParsedTableProperties::write_orc_bloom_filter_columns: alloc::string::String +pub iceberg::spec::ParsedTableProperties::write_orc_bloom_filter_columns: alloc::vec::Vec pub iceberg::spec::ParsedTableProperties::write_orc_bloom_filter_fpp: f64 pub iceberg::spec::ParsedTableProperties::write_orc_compression_codec: iceberg::compression::CompressionCodec pub iceberg::spec::ParsedTableProperties::write_orc_compression_strategy: alloc::string::String @@ -2346,7 +2439,6 @@ pub iceberg::spec::ParsedTableProperties::write_parquet_bloom_filter_fpp_column: pub iceberg::spec::ParsedTableProperties::write_parquet_bloom_filter_max_bytes: usize pub iceberg::spec::ParsedTableProperties::write_parquet_bloom_filter_ndv_column: std::collections::hash::map::HashMap pub iceberg::spec::ParsedTableProperties::write_parquet_compression_codec: iceberg::compression::CompressionCodec -pub iceberg::spec::ParsedTableProperties::write_parquet_compression_level: core::option::Option pub iceberg::spec::ParsedTableProperties::write_parquet_content_defined_chunking_enabled: bool pub iceberg::spec::ParsedTableProperties::write_parquet_content_defined_chunking_max_chunk_size: usize pub iceberg::spec::ParsedTableProperties::write_parquet_content_defined_chunking_min_chunk_size: usize @@ -2369,9 +2461,9 @@ pub iceberg::spec::ParsedTableProperties::write_spark_auto_schema_evolution_enab pub iceberg::spec::ParsedTableProperties::write_spark_fanout_enabled: bool pub iceberg::spec::ParsedTableProperties::write_summary_partition_limit: u64 pub iceberg::spec::ParsedTableProperties::write_target_file_size_bytes: usize -pub iceberg::spec::ParsedTableProperties::write_update_distribution_mode: core::option::Option -pub iceberg::spec::ParsedTableProperties::write_update_isolation_level: alloc::string::String -pub iceberg::spec::ParsedTableProperties::write_update_mode: alloc::string::String +pub iceberg::spec::ParsedTableProperties::write_update_distribution_mode: core::option::Option +pub iceberg::spec::ParsedTableProperties::write_update_isolation_level: iceberg::spec::IsolationLevel +pub iceberg::spec::ParsedTableProperties::write_update_mode: iceberg::spec::RowLevelOperationMode pub iceberg::spec::ParsedTableProperties::write_upsert_enabled: bool pub iceberg::spec::ParsedTableProperties::write_wap_enabled: bool impl core::clone::Clone for iceberg::spec::ParsedTableProperties @@ -3147,6 +3239,8 @@ pub const iceberg::spec::MAIN_BRANCH: &str pub const iceberg::spec::MAP_KEY_FIELD_NAME: &str pub const iceberg::spec::MAP_VALUE_FIELD_NAME: &str pub const iceberg::spec::MIN_FORMAT_VERSION_ROW_LINEAGE: iceberg::spec::FormatVersion +pub const iceberg::spec::PARQUET_PAGE_VERSION_V1: &str +pub const iceberg::spec::PARQUET_PAGE_VERSION_V2: &str pub const iceberg::spec::SCHEMA_NAME_DELIMITER: &str pub const iceberg::spec::UNASSIGNED_SEQUENCE_NUMBER: i64 pub const iceberg::spec::VIEW_PROPERTY_REPLACE_DROP_DIALECT_ALLOWED: &str diff --git a/crates/iceberg/src/spec/parsed_table_prop.rs b/crates/iceberg/src/spec/parsed_table_prop.rs index f6b9f1233e..adee395d94 100644 --- a/crates/iceberg/src/spec/parsed_table_prop.rs +++ b/crates/iceberg/src/spec/parsed_table_prop.rs @@ -75,10 +75,99 @@ use std::collections::HashMap; use iceberg_property_macro::Properties; +use serde_with::{DeserializeFromStr, SerializeDisplay}; use crate::compression::CompressionCodec; use crate::error::{Error, ErrorKind, Result}; -use crate::spec::DataFileFormat; +use crate::spec::{DataFileFormat, NameMapping}; + +/// Parquet data page version 1. +pub const PARQUET_PAGE_VERSION_V1: &str = "v1"; + +/// Parquet data page version 2. +pub const PARQUET_PAGE_VERSION_V2: &str = "v2"; + +/// Distribution applied to rows before writing files. +#[derive( + Debug, + PartialEq, + Eq, + Clone, + Copy, + SerializeDisplay, + DeserializeFromStr, + strum::Display, + strum::EnumString, +)] +#[strum(ascii_case_insensitive, serialize_all = "kebab-case")] +pub enum DistributionMode { + /// Do not redistribute rows. + None, + /// Hash-distribute rows by partition values. + Hash, + /// Range-distribute rows by partition or sort values. + Range, +} + +/// Granularity used when creating position delete files. +#[derive( + Debug, + PartialEq, + Eq, + Clone, + Copy, + SerializeDisplay, + DeserializeFromStr, + strum::Display, + strum::EnumString, +)] +#[strum(ascii_case_insensitive, serialize_all = "kebab-case")] +pub enum DeleteGranularity { + /// Group deletes for each referenced data file separately. + File, + /// Group deletes for different data files within a partition. + Partition, +} + +/// Isolation level used by row-level operations. +#[derive( + Debug, + PartialEq, + Eq, + Clone, + Copy, + SerializeDisplay, + DeserializeFromStr, + strum::Display, + strum::EnumString, +)] +#[strum(ascii_case_insensitive, serialize_all = "kebab-case")] +pub enum IsolationLevel { + /// Fail if concurrent changes may contain rows matching the operation. + Serializable, + /// Validate only against data visible in the operation's snapshot. + Snapshot, +} + +/// Strategy used to apply row-level changes. +#[derive( + Debug, + PartialEq, + Eq, + Clone, + Copy, + SerializeDisplay, + DeserializeFromStr, + strum::Display, + strum::EnumString, +)] +#[strum(ascii_case_insensitive, serialize_all = "kebab-case")] +pub enum RowLevelOperationMode { + /// Replace affected data files immediately. + CopyOnWrite, + /// Write delete files and merge changes while reading. + MergeOnRead, +} /// Strips trailing slashes from a location, preserving a bare URI scheme root. fn strip_trailing_slash(path: &str) -> &str { @@ -145,6 +234,30 @@ fn serialize_compression_codec(codec: &CompressionCodec) -> String { codec.name().to_string() } +fn parse_comma_separated_strings(value: &str) -> Result> { + Ok(value + .split(',') + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_string) + .collect()) +} + +fn serialize_comma_separated_strings(values: &[String]) -> String { + values.join(",") +} + +fn parse_name_mapping(value: &str) -> Result> { + serde_json::from_str(value).map(Some).map_err(|error| { + Error::new(ErrorKind::DataInvalid, "Invalid name mapping").with_source(error) + }) +} + +fn serialize_name_mapping(mapping: &Option) -> String { + serde_json::to_string(mapping.as_ref().expect("checked is_some above")) + .expect("serializing a NameMapping cannot fail") +} + /// Typed Iceberg table properties organized into documented sections. /// /// Serde represents this struct as Iceberg's flat string-to-string property map. Property @@ -230,11 +343,6 @@ pub struct ParsedTableProperties { #[doc = "Compression codec used for manifest files."] pub write_manifest_compression_codec: CompressionCodec, - #[key = "write.manifest.compression-level"] - #[default(None)] - #[doc = "Optional compression level used for manifest files."] - pub write_manifest_compression_level: Option, - #[key = "write.manifest-lists.enabled"] #[default(true)] #[doc = "Deprecated flag for writing manifest lists; manifest lists are always enabled."] @@ -299,7 +407,7 @@ pub struct ParsedTableProperties { #[key = "write.distribution-mode"] #[default(None)] #[doc = "Optional write distribution mode: none, hash, or range."] - pub write_distribution_mode: Option, + pub write_distribution_mode: Option, #[key = "write.datafusion.fanout.enabled"] #[default(true)] @@ -328,12 +436,12 @@ pub struct ParsedTableProperties { pub write_delete_parquet_page_size_bytes: usize, #[key = "write.parquet.page-version"] - #[default("v1")] + #[default(PARQUET_PAGE_VERSION_V1)] #[doc = "Parquet data page version for data files: v1 or v2."] pub write_parquet_page_version: String, #[key = "write.delete.parquet.page-version"] - #[default("v1")] + #[default(PARQUET_PAGE_VERSION_V1)] #[doc = "Parquet data page version for delete files: v1 or v2."] pub write_delete_parquet_page_version: String, @@ -371,16 +479,6 @@ pub struct ParsedTableProperties { #[doc = "Parquet compression codec used for delete files."] pub write_delete_parquet_compression_codec: CompressionCodec, - #[key = "write.parquet.compression-level"] - #[default(None)] - #[doc = "Optional Parquet compression level for data files."] - pub write_parquet_compression_level: Option, - - #[key = "write.delete.parquet.compression-level"] - #[default(None)] - #[doc = "Optional Parquet compression level for delete files."] - pub write_delete_parquet_compression_level: Option, - #[key = "write.parquet.shred-variants"] #[default(false)] #[doc = "Whether variant columns use shredded Parquet encoding for improved query performance."] @@ -486,16 +584,6 @@ pub struct ParsedTableProperties { #[doc = "Avro compression codec used for delete files."] pub write_delete_avro_compression_codec: CompressionCodec, - #[key = "write.avro.compression-level"] - #[default(None)] - #[doc = "Optional Avro compression level for data files."] - pub write_avro_compression_level: Option, - - #[key = "write.delete.avro.compression-level"] - #[default(None)] - #[doc = "Optional Avro compression level for delete files."] - pub write_delete_avro_compression_level: Option, - // ORC properties. #[key = "write.orc.stripe-size-bytes"] #[default(64 * 1024 * 1024)] @@ -508,9 +596,11 @@ pub struct ParsedTableProperties { pub write_delete_orc_stripe_size_bytes: u64, #[key = "write.orc.bloom.filter.columns"] - #[default("")] + #[default(Vec::new())] + #[parse_with(parse_comma_separated_strings)] + #[serialize_with(serialize_comma_separated_strings)] #[doc = "Comma-separated column names for which ORC bloom filters are created."] - pub write_orc_bloom_filter_columns: String, + pub write_orc_bloom_filter_columns: Vec, #[key = "write.orc.bloom.filter.fpp"] #[default(0.05)] @@ -663,8 +753,10 @@ pub struct ParsedTableProperties { #[key = "schema.name-mapping.default"] #[default(None)] + #[parse_with(parse_name_mapping)] + #[serialize_with(serialize_name_mapping)] #[doc = "Default JSON name mapping used to resolve columns in files without field IDs."] - pub schema_name_mapping_default: Option, + pub schema_name_mapping_default: Option, // Compatibility properties. #[key = "write.spark.fanout.enabled"] @@ -725,54 +817,54 @@ pub struct ParsedTableProperties { // Row-level operation properties. #[key = "write.delete.granularity"] - #[default("partition")] + #[default(DeleteGranularity::Partition)] #[doc = "Granularity of generated delete files: partition or file."] - pub write_delete_granularity: String, + pub write_delete_granularity: DeleteGranularity, #[key = "write.delete.isolation-level"] - #[default("serializable")] + #[default(IsolationLevel::Serializable)] #[doc = "Isolation level for delete commands: serializable or snapshot."] - pub write_delete_isolation_level: String, + pub write_delete_isolation_level: IsolationLevel, #[key = "write.delete.mode"] - #[default("copy-on-write")] + #[default(RowLevelOperationMode::CopyOnWrite)] #[doc = "Execution mode for delete commands: copy-on-write or merge-on-read."] - pub write_delete_mode: String, + pub write_delete_mode: RowLevelOperationMode, #[key = "write.delete.distribution-mode"] #[default(None)] #[doc = "Optional distribution mode for delete command data."] - pub write_delete_distribution_mode: Option, + pub write_delete_distribution_mode: Option, #[key = "write.update.isolation-level"] - #[default("serializable")] + #[default(IsolationLevel::Serializable)] #[doc = "Isolation level for update commands: serializable or snapshot."] - pub write_update_isolation_level: String, + pub write_update_isolation_level: IsolationLevel, #[key = "write.update.mode"] - #[default("copy-on-write")] + #[default(RowLevelOperationMode::CopyOnWrite)] #[doc = "Execution mode for update commands: copy-on-write or merge-on-read."] - pub write_update_mode: String, + pub write_update_mode: RowLevelOperationMode, #[key = "write.update.distribution-mode"] #[default(None)] #[doc = "Optional distribution mode for update command data."] - pub write_update_distribution_mode: Option, + pub write_update_distribution_mode: Option, #[key = "write.merge.isolation-level"] - #[default("serializable")] + #[default(IsolationLevel::Serializable)] #[doc = "Isolation level for merge commands: serializable or snapshot."] - pub write_merge_isolation_level: String, + pub write_merge_isolation_level: IsolationLevel, #[key = "write.merge.mode"] - #[default("copy-on-write")] + #[default(RowLevelOperationMode::CopyOnWrite)] #[doc = "Execution mode for merge commands: copy-on-write or merge-on-read."] - pub write_merge_mode: String, + pub write_merge_mode: RowLevelOperationMode, #[key = "write.merge.distribution-mode"] #[default(None)] #[doc = "Optional distribution mode for merge command data."] - pub write_merge_distribution_mode: Option, + pub write_merge_distribution_mode: Option, #[key = "write.upsert.enabled"] #[default(false)] @@ -794,6 +886,7 @@ pub struct ParsedTableProperties { #[cfg(test)] mod tests { use super::*; + use crate::spec::MappedField; fn parse(properties: HashMap) -> Result { serde_json::from_value(serde_json::to_value(properties).unwrap()) @@ -822,9 +915,30 @@ mod tests { properties.write_orc_compression_codec, CompressionCodec::Zlib ); - assert_eq!(properties.write_manifest_compression_level, None); - assert_eq!(properties.write_parquet_compression_level, None); - assert_eq!(properties.write_avro_compression_level, None); + assert_eq!(properties.write_distribution_mode, None); + assert_eq!( + properties.write_parquet_page_version, + PARQUET_PAGE_VERSION_V1 + ); + assert_eq!( + properties.write_delete_parquet_page_version, + PARQUET_PAGE_VERSION_V1 + ); + assert_eq!(PARQUET_PAGE_VERSION_V2, "v2"); + assert!(properties.write_orc_bloom_filter_columns.is_empty()); + assert_eq!(properties.schema_name_mapping_default, None); + assert_eq!( + properties.write_delete_granularity, + DeleteGranularity::Partition + ); + assert_eq!( + properties.write_delete_isolation_level, + IsolationLevel::Serializable + ); + assert_eq!( + properties.write_delete_mode, + RowLevelOperationMode::CopyOnWrite + ); assert_eq!( properties.write_parquet_row_group_size_bytes, 128 * 1024 * 1024 @@ -840,8 +954,18 @@ mod tests { commit_retry_num_retries: 9, write_format_default: DataFileFormat::Orc, write_data_path: Some("s3://warehouse/table/data".to_string()), + write_distribution_mode: Some(DistributionMode::Range), write_orc_compression_codec: CompressionCodec::Lzo, - write_parquet_compression_level: Some(5), + write_orc_bloom_filter_columns: vec!["id".to_string(), "category".to_string()], + schema_name_mapping_default: Some(NameMapping::new(vec![MappedField::new( + Some(1), + vec!["id".to_string()], + vec![], + )])), + write_delete_granularity: DeleteGranularity::File, + write_delete_isolation_level: IsolationLevel::Snapshot, + write_delete_mode: RowLevelOperationMode::MergeOnRead, + write_update_distribution_mode: Some(DistributionMode::Hash), write_parquet_bloom_filter_fpp_column: HashMap::from([( "customer_id".to_string(), 0.02, @@ -853,12 +977,33 @@ mod tests { assert_eq!(json["commit.retry.num-retries"], "9"); assert_eq!(json["write.format.default"], "orc"); assert_eq!(json["write.data.path"], "s3://warehouse/table/data"); + assert_eq!(json["write.distribution-mode"], "range"); assert_eq!(json["write.orc.compression-codec"], "lzo"); - assert_eq!(json["write.parquet.compression-level"], "5"); + assert_eq!(json["write.orc.bloom.filter.columns"], "id,category"); + assert_eq!(json["write.delete.granularity"], "file"); + assert_eq!(json["write.delete.isolation-level"], "snapshot"); + assert_eq!(json["write.delete.mode"], "merge-on-read"); + assert_eq!(json["write.update.distribution-mode"], "hash"); + assert_eq!( + serde_json::from_str::( + json["schema.name-mapping.default"].as_str().unwrap() + ) + .unwrap(), + serde_json::json!([{"field-id": 1, "names": ["id"]}]) + ); assert_eq!( json["write.parquet.bloom-filter-fpp.column.customer_id"], "0.02" ); + for key in [ + "write.manifest.compression-level", + "write.parquet.compression-level", + "write.delete.parquet.compression-level", + "write.avro.compression-level", + "write.delete.avro.compression-level", + ] { + assert!(json.get(key).is_none()); + } assert!(json.get("commit").is_none()); assert!(json.get("write").is_none()); } @@ -868,7 +1013,13 @@ mod tests { let properties: ParsedTableProperties = serde_json::from_value(serde_json::json!({ "commit.retry.num-retries": "8", "write.format.default": "orc", - "write.data.path": "s3://warehouse/table/data" + "write.data.path": "s3://warehouse/table/data", + "write.distribution-mode": "HASH", + "write.orc.bloom.filter.columns": "id, category", + "schema.name-mapping.default": r#"[{"field-id":1,"names":["id"]}]"#, + "write.delete.granularity": "FILE", + "write.update.isolation-level": "snapshot", + "write.merge.mode": "merge-on-read" })) .unwrap(); @@ -878,6 +1029,31 @@ mod tests { properties.write_data_path, Some("s3://warehouse/table/data".to_string()) ); + assert_eq!( + properties.write_distribution_mode, + Some(DistributionMode::Hash) + ); + assert_eq!(properties.write_orc_bloom_filter_columns, vec![ + "id".to_string(), + "category".to_string() + ]); + assert_eq!( + properties.schema_name_mapping_default, + Some(NameMapping::new(vec![MappedField::new( + Some(1), + vec!["id".to_string()], + vec![], + )])) + ); + assert_eq!(properties.write_delete_granularity, DeleteGranularity::File); + assert_eq!( + properties.write_update_isolation_level, + IsolationLevel::Snapshot + ); + assert_eq!( + properties.write_merge_mode, + RowLevelOperationMode::MergeOnRead + ); } #[test] @@ -913,10 +1089,6 @@ mod tests { "write.delete.avro.compression-codec".to_string(), "snappy".to_string(), ), - ( - "write.delete.avro.compression-level".to_string(), - "7".to_string(), - ), ("read.split.planning-lookback".to_string(), "25".to_string()), ( "history.expire.min-snapshots-to-keep".to_string(), @@ -933,10 +1105,12 @@ mod tests { properties.write_delete_avro_compression_codec, CompressionCodec::Snappy ); - assert_eq!(properties.write_delete_avro_compression_level, Some(7)); assert_eq!(properties.read_split_planning_lookback, 25); assert_eq!(properties.history_expire_min_snapshots_to_keep, 4); - assert_eq!(properties.write_delete_mode, "merge-on-read"); + assert_eq!( + properties.write_delete_mode, + RowLevelOperationMode::MergeOnRead + ); assert_eq!(properties.encryption_data_key_length, 32); } @@ -985,4 +1159,23 @@ mod tests { assert!(error.message().contains("commit.retry.num-retries")); } + + #[test] + fn invalid_typed_value_reports_its_property_key() { + let error = parse(HashMap::from([( + "write.distribution-mode".to_string(), + "random".to_string(), + )])) + .unwrap_err(); + + assert!(error.message().contains("write.distribution-mode")); + + let error = parse(HashMap::from([( + "schema.name-mapping.default".to_string(), + "not-json".to_string(), + )])) + .unwrap_err(); + + assert!(error.message().contains("schema.name-mapping.default")); + } } diff --git a/crates/property-macro/src/lib.rs b/crates/property-macro/src/lib.rs index 784b5b2125..bfeb30656d 100644 --- a/crates/property-macro/src/lib.rs +++ b/crates/property-macro/src/lib.rs @@ -49,8 +49,8 @@ use syn::{ /// property map flat. `parse_with` may be used for exact-key property types that do not implement /// `FromStr` or need validation. `serialize_with` supplies their string representation in JSON. /// Optional fields are omitted from JSON when they are `None`. Fields need `FromStr` and `ToString` -/// unless the relevant custom parsing or serialization attribute is supplied. String-literal -/// defaults are converted into their field type with `Into`. +/// unless the relevant custom parsing or serialization attribute is supplied. String-literal and +/// path defaults are converted into their field type with `Into`. #[proc_macro_derive( Properties, attributes(key, prefix, nested, default, parse_with, serialize_with) @@ -373,7 +373,7 @@ fn default_value(default: &Expr, ty: &Type) -> TokenStream2 { Expr::Lit(ExprLit { lit: Lit::Str(_), .. - }) + }) | Expr::Path(_) ) { quote!(::std::convert::Into::<#ty>::into(#default)) } else { From 7a69ad5821ab1c75bf9e693e327f926ea047d056 Mon Sep 17 00:00:00 2001 From: Renjie Liu Date: Wed, 5 Aug 2026 14:54:00 +0800 Subject: [PATCH 08/14] Address comments --- crates/iceberg/public-api.txt | 15 +- crates/iceberg/src/spec/name_mapping/mod.rs | 21 ++ crates/iceberg/src/spec/parsed_table_prop.rs | 208 ++++++++++++++----- crates/property-macro/src/lib.rs | 61 +++++- crates/property-macro/tests/properties.rs | 34 +++ 5 files changed, 278 insertions(+), 61 deletions(-) diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index 82858bcbf1..e6e2134f08 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -2312,7 +2312,12 @@ impl core::cmp::PartialEq for iceberg::spec::NameMapping pub fn iceberg::spec::NameMapping::eq(&self, other: &iceberg::spec::NameMapping) -> bool impl core::fmt::Debug for iceberg::spec::NameMapping pub fn iceberg::spec::NameMapping::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for iceberg::spec::NameMapping +pub fn iceberg::spec::NameMapping::fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result impl core::marker::StructuralPartialEq for iceberg::spec::NameMapping +impl core::str::traits::FromStr for iceberg::spec::NameMapping +pub type iceberg::spec::NameMapping::Err = iceberg::Error +pub fn iceberg::spec::NameMapping::from_str(value: &str) -> core::result::Result impl serde_core::ser::Serialize for iceberg::spec::NameMapping pub fn iceberg::spec::NameMapping::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::NameMapping @@ -2388,7 +2393,7 @@ pub iceberg::spec::ParsedTableProperties::write_avro_compression_codec: iceberg: pub iceberg::spec::ParsedTableProperties::write_data_path: core::option::Option pub iceberg::spec::ParsedTableProperties::write_datafusion_fanout_enabled: bool pub iceberg::spec::ParsedTableProperties::write_delete_avro_compression_codec: iceberg::compression::CompressionCodec -pub iceberg::spec::ParsedTableProperties::write_delete_distribution_mode: core::option::Option +pub iceberg::spec::ParsedTableProperties::write_delete_distribution_mode: iceberg::spec::DistributionMode pub iceberg::spec::ParsedTableProperties::write_delete_format_default: iceberg::spec::DataFileFormat pub iceberg::spec::ParsedTableProperties::write_delete_granularity: iceberg::spec::DeleteGranularity pub iceberg::spec::ParsedTableProperties::write_delete_isolation_level: iceberg::spec::IsolationLevel @@ -2407,13 +2412,13 @@ pub iceberg::spec::ParsedTableProperties::write_delete_parquet_row_group_check_m pub iceberg::spec::ParsedTableProperties::write_delete_parquet_row_group_check_min_record_count: usize pub iceberg::spec::ParsedTableProperties::write_delete_parquet_row_group_size_bytes: usize pub iceberg::spec::ParsedTableProperties::write_delete_target_file_size_bytes: usize -pub iceberg::spec::ParsedTableProperties::write_distribution_mode: core::option::Option +pub iceberg::spec::ParsedTableProperties::write_distribution_mode: iceberg::spec::DistributionMode pub iceberg::spec::ParsedTableProperties::write_folder_storage_path: core::option::Option pub iceberg::spec::ParsedTableProperties::write_format_default: iceberg::spec::DataFileFormat pub iceberg::spec::ParsedTableProperties::write_location_provider_impl: core::option::Option pub iceberg::spec::ParsedTableProperties::write_manifest_compression_codec: iceberg::compression::CompressionCodec pub iceberg::spec::ParsedTableProperties::write_manifest_lists_enabled: bool -pub iceberg::spec::ParsedTableProperties::write_merge_distribution_mode: core::option::Option +pub iceberg::spec::ParsedTableProperties::write_merge_distribution_mode: iceberg::spec::DistributionMode pub iceberg::spec::ParsedTableProperties::write_merge_isolation_level: iceberg::spec::IsolationLevel pub iceberg::spec::ParsedTableProperties::write_merge_mode: iceberg::spec::RowLevelOperationMode pub iceberg::spec::ParsedTableProperties::write_metadata_compression_codec: iceberg::compression::CompressionCodec @@ -2461,7 +2466,7 @@ pub iceberg::spec::ParsedTableProperties::write_spark_auto_schema_evolution_enab pub iceberg::spec::ParsedTableProperties::write_spark_fanout_enabled: bool pub iceberg::spec::ParsedTableProperties::write_summary_partition_limit: u64 pub iceberg::spec::ParsedTableProperties::write_target_file_size_bytes: usize -pub iceberg::spec::ParsedTableProperties::write_update_distribution_mode: core::option::Option +pub iceberg::spec::ParsedTableProperties::write_update_distribution_mode: iceberg::spec::DistributionMode pub iceberg::spec::ParsedTableProperties::write_update_isolation_level: iceberg::spec::IsolationLevel pub iceberg::spec::ParsedTableProperties::write_update_mode: iceberg::spec::RowLevelOperationMode pub iceberg::spec::ParsedTableProperties::write_upsert_enabled: bool @@ -3239,6 +3244,8 @@ pub const iceberg::spec::MAIN_BRANCH: &str pub const iceberg::spec::MAP_KEY_FIELD_NAME: &str pub const iceberg::spec::MAP_VALUE_FIELD_NAME: &str pub const iceberg::spec::MIN_FORMAT_VERSION_ROW_LINEAGE: iceberg::spec::FormatVersion +pub const iceberg::spec::ORC_COMPRESSION_STRATEGY_COMPRESSION: &str +pub const iceberg::spec::ORC_COMPRESSION_STRATEGY_SPEED: &str pub const iceberg::spec::PARQUET_PAGE_VERSION_V1: &str pub const iceberg::spec::PARQUET_PAGE_VERSION_V2: &str pub const iceberg::spec::SCHEMA_NAME_DELIMITER: &str diff --git a/crates/iceberg/src/spec/name_mapping/mod.rs b/crates/iceberg/src/spec/name_mapping/mod.rs index db9e44c290..3d11478e9d 100644 --- a/crates/iceberg/src/spec/name_mapping/mod.rs +++ b/crates/iceberg/src/spec/name_mapping/mod.rs @@ -17,11 +17,15 @@ //! Iceberg name mapping. +use std::fmt::{Display, Formatter}; +use std::str::FromStr; use std::sync::Arc; use serde::{Deserialize, Serialize}; use serde_with::{DefaultOnNull, serde_as}; +use crate::{Error, ErrorKind}; + /// Property name for name mapping. pub const DEFAULT_SCHEMA_NAME_MAPPING: &str = "schema.name-mapping.default"; @@ -44,6 +48,23 @@ impl NameMapping { } } +impl FromStr for NameMapping { + type Err = Error; + + fn from_str(value: &str) -> Result { + serde_json::from_str(value).map_err(|error| { + Error::new(ErrorKind::DataInvalid, "Invalid name mapping").with_source(error) + }) + } +} + +impl Display for NameMapping { + fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + let value = serde_json::to_string(self).map_err(|_| std::fmt::Error)?; + formatter.write_str(&value) + } +} + /// Maps field names to IDs. #[serde_as] #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)] diff --git a/crates/iceberg/src/spec/parsed_table_prop.rs b/crates/iceberg/src/spec/parsed_table_prop.rs index adee395d94..9cdb06cc79 100644 --- a/crates/iceberg/src/spec/parsed_table_prop.rs +++ b/crates/iceberg/src/spec/parsed_table_prop.rs @@ -87,6 +87,12 @@ pub const PARQUET_PAGE_VERSION_V1: &str = "v1"; /// Parquet data page version 2. pub const PARQUET_PAGE_VERSION_V2: &str = "v2"; +/// ORC compression strategy that prioritizes speed. +pub const ORC_COMPRESSION_STRATEGY_SPEED: &str = "speed"; + +/// ORC compression strategy that prioritizes compression ratio. +pub const ORC_COMPRESSION_STRATEGY_COMPRESSION: &str = "compression"; + /// Distribution applied to rows before writing files. #[derive( Debug, @@ -234,6 +240,68 @@ fn serialize_compression_codec(codec: &CompressionCodec) -> String { codec.name().to_string() } +fn compression_level_key(codec_key: &str) -> String { + format!( + "{}level", + codec_key + .strip_suffix("codec") + .expect("compression codec property keys end with 'codec'") + ) +} + +fn parse_compression_codec_properties( + properties: &HashMap, + codec_key: &str, + default: CompressionCodec, +) -> Result { + let codec = properties + .get(codec_key) + .map(|value| parse_compression_codec(value)) + .transpose()? + .unwrap_or(default); + let level_key = compression_level_key(codec_key); + let Some(level) = properties.get(&level_key) else { + return Ok(codec); + }; + let level = level.parse::().map_err(|error| { + Error::new( + ErrorKind::DataInvalid, + format!("Invalid compression level for {level_key}: {level}"), + ) + .with_source(error) + })?; + + match codec { + CompressionCodec::Gzip(_) => Ok(CompressionCodec::Gzip(level)), + CompressionCodec::Zstd(_) => Ok(CompressionCodec::Zstd(level)), + _ => Err(Error::new( + ErrorKind::DataInvalid, + format!( + "Compression level {level_key} is not supported for codec '{}'", + codec.name() + ), + )), + } +} + +fn write_compression_codec_properties( + codec: &CompressionCodec, + properties: &mut HashMap, + codec_key: &str, +) { + properties.insert(codec_key.to_string(), serialize_compression_codec(codec)); + + let level_key = compression_level_key(codec_key); + match codec { + CompressionCodec::Gzip(level) | CompressionCodec::Zstd(level) => { + properties.insert(level_key, level.to_string()); + } + _ => { + properties.remove(&level_key); + } + } +} + fn parse_comma_separated_strings(value: &str) -> Result> { Ok(value .split(',') @@ -247,17 +315,6 @@ fn serialize_comma_separated_strings(values: &[String]) -> String { values.join(",") } -fn parse_name_mapping(value: &str) -> Result> { - serde_json::from_str(value).map(Some).map_err(|error| { - Error::new(ErrorKind::DataInvalid, "Invalid name mapping").with_source(error) - }) -} - -fn serialize_name_mapping(mapping: &Option) -> String { - serde_json::to_string(mapping.as_ref().expect("checked is_some above")) - .expect("serializing a NameMapping cannot fail") -} - /// Typed Iceberg table properties organized into documented sections. /// /// Serde represents this struct as Iceberg's flat string-to-string property map. Property @@ -338,8 +395,8 @@ pub struct ParsedTableProperties { #[key = "write.manifest.compression-codec"] #[default(CompressionCodec::gzip_default())] - #[parse_with(parse_compression_codec)] - #[serialize_with(serialize_compression_codec)] + #[parse_properties_with(parse_compression_codec_properties)] + #[write_properties_with(write_compression_codec_properties)] #[doc = "Compression codec used for manifest files."] pub write_manifest_compression_codec: CompressionCodec, @@ -405,9 +462,9 @@ pub struct ParsedTableProperties { pub write_wap_enabled: bool, #[key = "write.distribution-mode"] - #[default(None)] - #[doc = "Optional write distribution mode: none, hash, or range."] - pub write_distribution_mode: Option, + #[default(DistributionMode::None)] + #[doc = "Write distribution mode: none, hash, or range."] + pub write_distribution_mode: DistributionMode, #[key = "write.datafusion.fanout.enabled"] #[default(true)] @@ -467,15 +524,15 @@ pub struct ParsedTableProperties { #[key = "write.parquet.compression-codec"] #[default(CompressionCodec::zstd_default())] - #[parse_with(parse_compression_codec)] - #[serialize_with(serialize_compression_codec)] + #[parse_properties_with(parse_compression_codec_properties)] + #[write_properties_with(write_compression_codec_properties)] #[doc = "Parquet compression codec used for data files."] pub write_parquet_compression_codec: CompressionCodec, #[key = "write.delete.parquet.compression-codec"] #[default(CompressionCodec::zstd_default())] - #[parse_with(parse_compression_codec)] - #[serialize_with(serialize_compression_codec)] + #[parse_properties_with(parse_compression_codec_properties)] + #[write_properties_with(write_compression_codec_properties)] #[doc = "Parquet compression codec used for delete files."] pub write_delete_parquet_compression_codec: CompressionCodec, @@ -572,15 +629,15 @@ pub struct ParsedTableProperties { // Avro properties. #[key = "write.avro.compression-codec"] #[default(CompressionCodec::gzip_default())] - #[parse_with(parse_compression_codec)] - #[serialize_with(serialize_compression_codec)] + #[parse_properties_with(parse_compression_codec_properties)] + #[write_properties_with(write_compression_codec_properties)] #[doc = "Avro compression codec used for data files."] pub write_avro_compression_codec: CompressionCodec, #[key = "write.delete.avro.compression-codec"] #[default(CompressionCodec::gzip_default())] - #[parse_with(parse_compression_codec)] - #[serialize_with(serialize_compression_codec)] + #[parse_properties_with(parse_compression_codec_properties)] + #[write_properties_with(write_compression_codec_properties)] #[doc = "Avro compression codec used for delete files."] pub write_delete_avro_compression_codec: CompressionCodec, @@ -642,12 +699,12 @@ pub struct ParsedTableProperties { pub write_delete_orc_compression_codec: CompressionCodec, #[key = "write.orc.compression-strategy"] - #[default("speed")] + #[default(ORC_COMPRESSION_STRATEGY_SPEED)] #[doc = "ORC compression strategy for data files: speed or compression."] pub write_orc_compression_strategy: String, #[key = "write.delete.orc.compression-strategy"] - #[default("speed")] + #[default(ORC_COMPRESSION_STRATEGY_SPEED)] #[doc = "ORC compression strategy for delete files: speed or compression."] pub write_delete_orc_compression_strategy: String, @@ -753,8 +810,6 @@ pub struct ParsedTableProperties { #[key = "schema.name-mapping.default"] #[default(None)] - #[parse_with(parse_name_mapping)] - #[serialize_with(serialize_name_mapping)] #[doc = "Default JSON name mapping used to resolve columns in files without field IDs."] pub schema_name_mapping_default: Option, @@ -832,9 +887,9 @@ pub struct ParsedTableProperties { pub write_delete_mode: RowLevelOperationMode, #[key = "write.delete.distribution-mode"] - #[default(None)] - #[doc = "Optional distribution mode for delete command data."] - pub write_delete_distribution_mode: Option, + #[default(DistributionMode::None)] + #[doc = "Distribution mode for delete command data."] + pub write_delete_distribution_mode: DistributionMode, #[key = "write.update.isolation-level"] #[default(IsolationLevel::Serializable)] @@ -847,9 +902,9 @@ pub struct ParsedTableProperties { pub write_update_mode: RowLevelOperationMode, #[key = "write.update.distribution-mode"] - #[default(None)] - #[doc = "Optional distribution mode for update command data."] - pub write_update_distribution_mode: Option, + #[default(DistributionMode::None)] + #[doc = "Distribution mode for update command data."] + pub write_update_distribution_mode: DistributionMode, #[key = "write.merge.isolation-level"] #[default(IsolationLevel::Serializable)] @@ -862,9 +917,9 @@ pub struct ParsedTableProperties { pub write_merge_mode: RowLevelOperationMode, #[key = "write.merge.distribution-mode"] - #[default(None)] - #[doc = "Optional distribution mode for merge command data."] - pub write_merge_distribution_mode: Option, + #[default(DistributionMode::None)] + #[doc = "Distribution mode for merge command data."] + pub write_merge_distribution_mode: DistributionMode, #[key = "write.upsert.enabled"] #[default(false)] @@ -915,7 +970,19 @@ mod tests { properties.write_orc_compression_codec, CompressionCodec::Zlib ); - assert_eq!(properties.write_distribution_mode, None); + assert_eq!(properties.write_distribution_mode, DistributionMode::None); + assert_eq!( + properties.write_delete_distribution_mode, + DistributionMode::None + ); + assert_eq!( + properties.write_update_distribution_mode, + DistributionMode::None + ); + assert_eq!( + properties.write_merge_distribution_mode, + DistributionMode::None + ); assert_eq!( properties.write_parquet_page_version, PARQUET_PAGE_VERSION_V1 @@ -925,6 +992,11 @@ mod tests { PARQUET_PAGE_VERSION_V1 ); assert_eq!(PARQUET_PAGE_VERSION_V2, "v2"); + assert_eq!( + properties.write_orc_compression_strategy, + ORC_COMPRESSION_STRATEGY_SPEED + ); + assert_eq!(ORC_COMPRESSION_STRATEGY_COMPRESSION, "compression"); assert!(properties.write_orc_bloom_filter_columns.is_empty()); assert_eq!(properties.schema_name_mapping_default, None); assert_eq!( @@ -954,7 +1026,10 @@ mod tests { commit_retry_num_retries: 9, write_format_default: DataFileFormat::Orc, write_data_path: Some("s3://warehouse/table/data".to_string()), - write_distribution_mode: Some(DistributionMode::Range), + write_manifest_compression_codec: CompressionCodec::Gzip(9), + write_parquet_compression_codec: CompressionCodec::Zstd(5), + write_delete_avro_compression_codec: CompressionCodec::Gzip(4), + write_distribution_mode: DistributionMode::Range, write_orc_compression_codec: CompressionCodec::Lzo, write_orc_bloom_filter_columns: vec!["id".to_string(), "category".to_string()], schema_name_mapping_default: Some(NameMapping::new(vec![MappedField::new( @@ -965,7 +1040,7 @@ mod tests { write_delete_granularity: DeleteGranularity::File, write_delete_isolation_level: IsolationLevel::Snapshot, write_delete_mode: RowLevelOperationMode::MergeOnRead, - write_update_distribution_mode: Some(DistributionMode::Hash), + write_update_distribution_mode: DistributionMode::Hash, write_parquet_bloom_filter_fpp_column: HashMap::from([( "customer_id".to_string(), 0.02, @@ -977,6 +1052,12 @@ mod tests { assert_eq!(json["commit.retry.num-retries"], "9"); assert_eq!(json["write.format.default"], "orc"); assert_eq!(json["write.data.path"], "s3://warehouse/table/data"); + assert_eq!(json["write.manifest.compression-codec"], "gzip"); + assert_eq!(json["write.manifest.compression-level"], "9"); + assert_eq!(json["write.parquet.compression-codec"], "zstd"); + assert_eq!(json["write.parquet.compression-level"], "5"); + assert_eq!(json["write.delete.avro.compression-codec"], "gzip"); + assert_eq!(json["write.delete.avro.compression-level"], "4"); assert_eq!(json["write.distribution-mode"], "range"); assert_eq!(json["write.orc.compression-codec"], "lzo"); assert_eq!(json["write.orc.bloom.filter.columns"], "id,category"); @@ -995,15 +1076,6 @@ mod tests { json["write.parquet.bloom-filter-fpp.column.customer_id"], "0.02" ); - for key in [ - "write.manifest.compression-level", - "write.parquet.compression-level", - "write.delete.parquet.compression-level", - "write.avro.compression-level", - "write.delete.avro.compression-level", - ] { - assert!(json.get(key).is_none()); - } assert!(json.get("commit").is_none()); assert!(json.get("write").is_none()); } @@ -1014,6 +1086,9 @@ mod tests { "commit.retry.num-retries": "8", "write.format.default": "orc", "write.data.path": "s3://warehouse/table/data", + "write.manifest.compression-codec": "gzip", + "write.manifest.compression-level": "8", + "write.parquet.compression-level": "5", "write.distribution-mode": "HASH", "write.orc.bloom.filter.columns": "id, category", "schema.name-mapping.default": r#"[{"field-id":1,"names":["id"]}]"#, @@ -1029,9 +1104,14 @@ mod tests { properties.write_data_path, Some("s3://warehouse/table/data".to_string()) ); + assert_eq!(properties.write_distribution_mode, DistributionMode::Hash); + assert_eq!( + properties.write_manifest_compression_codec, + CompressionCodec::Gzip(8) + ); assert_eq!( - properties.write_distribution_mode, - Some(DistributionMode::Hash) + properties.write_parquet_compression_codec, + CompressionCodec::Zstd(5) ); assert_eq!(properties.write_orc_bloom_filter_columns, vec![ "id".to_string(), @@ -1075,6 +1155,18 @@ mod tests { defaults.write_parquet_content_defined_chunking_max_chunk_size ); assert_eq!(decoded.write_merge_mode, defaults.write_merge_mode); + assert_eq!( + decoded.write_manifest_compression_codec, + defaults.write_manifest_compression_codec + ); + assert_eq!( + decoded.write_parquet_compression_codec, + defaults.write_parquet_compression_codec + ); + assert_eq!( + decoded.write_avro_compression_codec, + defaults.write_avro_compression_codec + ); } #[test] @@ -1177,5 +1269,19 @@ mod tests { .unwrap_err(); assert!(error.message().contains("schema.name-mapping.default")); + + let error = parse(HashMap::from([ + ( + "write.parquet.compression-codec".to_string(), + "snappy".to_string(), + ), + ( + "write.parquet.compression-level".to_string(), + "7".to_string(), + ), + ])) + .unwrap_err(); + + assert!(error.message().contains("write.parquet.compression-level")); } } diff --git a/crates/property-macro/src/lib.rs b/crates/property-macro/src/lib.rs index bfeb30656d..14133bd95e 100644 --- a/crates/property-macro/src/lib.rs +++ b/crates/property-macro/src/lib.rs @@ -48,12 +48,23 @@ use syn::{ /// the declared prefix. `nested` embeds another `Properties` struct while keeping its serialized /// property map flat. `parse_with` may be used for exact-key property types that do not implement /// `FromStr` or need validation. `serialize_with` supplies their string representation in JSON. +/// `parse_properties_with` and `write_properties_with` provide access to the complete property map +/// for fields represented by more than one key. /// Optional fields are omitted from JSON when they are `None`. Fields need `FromStr` and `ToString` /// unless the relevant custom parsing or serialization attribute is supplied. String-literal and /// path defaults are converted into their field type with `Into`. #[proc_macro_derive( Properties, - attributes(key, prefix, nested, default, parse_with, serialize_with) + attributes( + key, + prefix, + nested, + default, + parse_with, + serialize_with, + parse_properties_with, + write_properties_with + ) )] pub fn derive_properties(input: TokenStream) -> TokenStream { let input = parse_macro_input!(input as DeriveInput); @@ -73,6 +84,8 @@ struct PropertyField { default: Option, parse_with: Option, serialize_with: Option, + parse_properties_with: Option, + write_properties_with: Option, option_inner_type: Option, map_value_type: Option, } @@ -214,13 +227,31 @@ fn parse_property_field(field: &Field) -> syn::Result { "#[prefix(...)] fields must have type HashMap", )); } + let parse_with = attribute_path_value(&field.attrs, "parse_with")?; + let serialize_with = attribute_path_value(&field.attrs, "serialize_with")?; + let parse_properties_with = attribute_path_value(&field.attrs, "parse_properties_with")?; + let write_properties_with = attribute_path_value(&field.attrs, "write_properties_with")?; if (prefix.is_some() || nested) - && (attribute_path_value(&field.attrs, "parse_with")?.is_some() - || attribute_path_value(&field.attrs, "serialize_with")?.is_some()) + && (parse_with.is_some() + || serialize_with.is_some() + || parse_properties_with.is_some() + || write_properties_with.is_some()) { return Err(Error::new_spanned( field, - "#[prefix(...)] and #[nested] fields do not support parse_with or serialize_with", + "#[prefix(...)] and #[nested] fields do not support custom parse or write functions", + )); + } + if parse_with.is_some() && parse_properties_with.is_some() { + return Err(Error::new_spanned( + field, + "fields cannot declare both parse_with and parse_properties_with", + )); + } + if serialize_with.is_some() && write_properties_with.is_some() { + return Err(Error::new_spanned( + field, + "fields cannot declare both serialize_with and write_properties_with", )); } @@ -231,8 +262,10 @@ fn parse_property_field(field: &Field) -> syn::Result { prefix, nested, default, - parse_with: attribute_path_value(&field.attrs, "parse_with")?, - serialize_with: attribute_path_value(&field.attrs, "serialize_with")?, + parse_with, + serialize_with, + parse_properties_with, + write_properties_with, option_inner_type: option_inner_type(&field.ty), map_value_type, }) @@ -310,6 +343,15 @@ fn parse_field(field: &PropertyField) -> TokenStream2 { &field.ty, ); + if let Some(parse_properties_with) = &field.parse_properties_with { + let key = field.key.as_ref().expect("exact-key fields have a key"); + return quote! { + #ident: #parse_properties_with(properties, #key, #default).map_err(|error| { + format!("Invalid value for {}: {error}", #key) + })? + }; + } + if let Some(prefix) = &field.prefix { let value_type = field .map_value_type @@ -429,6 +471,13 @@ fn write_field(field: &PropertyField) -> TokenStream2 { }; } + if let Some(write_properties_with) = &field.write_properties_with { + let key = field.key.as_ref().expect("exact-key fields have a key"); + return quote! { + #write_properties_with(&self.#ident, properties, #key); + }; + } + if let Some(prefix) = &field.prefix { return quote! { for (suffix, value) in &self.#ident { diff --git a/crates/property-macro/tests/properties.rs b/crates/property-macro/tests/properties.rs index b360ac9890..d33fb6998f 100644 --- a/crates/property-macro/tests/properties.rs +++ b/crates/property-macro/tests/properties.rs @@ -23,6 +23,29 @@ const RETRIES: &str = "commit.retry.num-retries"; const OWNER: &str = "owner"; const FORMAT: &str = "write.format.default"; const COLUMN_FPP_PREFIX: &str = "write.parquet.bloom-filter-fpp.column."; +const WIDTH: &str = "dimensions.width"; +const HEIGHT: &str = "dimensions.height"; + +fn parse_dimensions( + properties: &HashMap, + key: &str, + default: (u64, u64), +) -> Result<(u64, u64), String> { + let parse = |property_key: &str, default| { + properties + .get(property_key) + .map(|value| value.parse::().map_err(|error| error.to_string())) + .transpose() + .map(|value| value.unwrap_or(default)) + }; + + Ok((parse(key, default.0)?, parse(HEIGHT, default.1)?)) +} + +fn write_dimensions(dimensions: &(u64, u64), properties: &mut HashMap, key: &str) { + properties.insert(key.to_string(), dimensions.0.to_string()); + properties.insert(HEIGHT.to_string(), dimensions.1.to_string()); +} #[derive(Debug, Properties)] struct TestProperties { @@ -42,6 +65,12 @@ struct TestProperties { #[prefix(COLUMN_FPP_PREFIX)] #[default(HashMap::new())] pub column_fpp: HashMap, + + #[key(WIDTH)] + #[default((640, 480))] + #[parse_properties_with(parse_dimensions)] + #[write_properties_with(write_dimensions)] + pub dimensions: (u64, u64), } #[test] @@ -53,24 +82,29 @@ fn generates_defaults_and_serde_for_public_fields() { owner: Some("iceberg".to_string()), format: "orc".to_string(), column_fpp: HashMap::from([("id".to_string(), 0.01)]), + dimensions: (1920, 1080), }; assert_eq!(properties.retries, 8); assert_eq!(properties.owner, Some("iceberg".to_string())); assert_eq!(properties.format, "orc"); assert_eq!(properties.column_fpp["id"], 0.01); + assert_eq!(properties.dimensions, (1920, 1080)); let json = serde_json::to_value(&properties).unwrap(); assert_eq!(json[RETRIES], "8"); assert_eq!(json[OWNER], "iceberg"); assert_eq!(json[FORMAT], "orc"); assert_eq!(json[format!("{COLUMN_FPP_PREFIX}id")], "0.01"); + assert_eq!(json[WIDTH], "1920"); + assert_eq!(json[HEIGHT], "1080"); let decoded: TestProperties = serde_json::from_value(json).unwrap(); assert_eq!(decoded.retries, 8); assert_eq!(decoded.owner, Some("iceberg".to_string())); assert_eq!(decoded.format, "orc"); assert_eq!(decoded.column_fpp["id"], 0.01); + assert_eq!(decoded.dimensions, (1920, 1080)); } #[derive(Clone, Debug, Properties)] From 53c4cdf4bfc7e708375818789e0502f041de8e86 Mon Sep 17 00:00:00 2001 From: Renjie Liu Date: Wed, 5 Aug 2026 15:16:52 +0800 Subject: [PATCH 09/14] Address comments --- .../iceberg/src/catalog/metadata_location.rs | 17 + crates/iceberg/src/compression.rs | 110 +++++ crates/iceberg/src/lib.rs | 2 + crates/iceberg/src/spec/parsed_table_prop.rs | 452 ++++++------------ 4 files changed, 271 insertions(+), 310 deletions(-) diff --git a/crates/iceberg/src/catalog/metadata_location.rs b/crates/iceberg/src/catalog/metadata_location.rs index d5daa02b37..cf8282d5f0 100644 --- a/crates/iceberg/src/catalog/metadata_location.rs +++ b/crates/iceberg/src/catalog/metadata_location.rs @@ -43,6 +43,23 @@ pub struct MetadataLocation { } impl MetadataLocation { + /// Parses and normalizes the `write.metadata.path` table property. + pub(crate) fn parse_write_metadata_path(value: &str) -> Result> { + if value.is_empty() { + return Err(Error::new(ErrorKind::DataInvalid, "path must not be empty")); + } + + let mut path = value; + while !path.ends_with("://") { + let Some(stripped) = path.strip_suffix('/') else { + break; + }; + path = stripped; + } + + Ok(Some(path.to_string())) + } + /// Determines the compression codec from table properties. /// Parse errors result in CompressionCodec::None. fn compression_from_properties(properties: &HashMap) -> CompressionCodec { diff --git a/crates/iceberg/src/compression.rs b/crates/iceberg/src/compression.rs index 49cd20712e..333e189427 100644 --- a/crates/iceberg/src/compression.rs +++ b/crates/iceberg/src/compression.rs @@ -17,6 +17,7 @@ //! Compression codec support for data compression and decompression. +use std::collections::HashMap; use std::fmt; use std::io::{Read, Write}; @@ -86,6 +87,115 @@ impl CompressionCodec { CompressionCodec::Zlib => "zlib", } } + + /// Parses a compression codec name used by an Iceberg table property. + pub(crate) fn parse_property(value: &str) -> Result { + serde_json::from_value(serde_json::Value::String(value.to_lowercase())).map_err(|_| { + Error::new( + ErrorKind::DataInvalid, + format!("Invalid compression codec: {value}"), + ) + }) + } + + /// Parses the metadata-file compression codec table property. + pub(crate) fn parse_metadata_property(value: &str) -> Result { + if value.is_empty() { + return Ok(Self::None); + } + + let codec = Self::parse_property(value).map_err(|_| { + Error::new( + ErrorKind::DataInvalid, + format!( + "Invalid metadata compression codec: {value}. Only '{}' and '{}' are supported.", + Self::None.name(), + Self::gzip_default().name() + ), + ) + })?; + + match codec { + Self::None | Self::Gzip(_) => Ok(codec), + _ => Err(Error::new( + ErrorKind::DataInvalid, + format!( + "Invalid metadata compression codec: {value}. Only '{}' and '{}' are supported for metadata files.", + Self::None.name(), + Self::gzip_default().name() + ), + )), + } + } + + /// Returns the codec name used by an Iceberg table property. + pub(crate) fn property_value(&self) -> String { + self.name().to_string() + } + + /// Parses a codec and its optional companion compression-level property. + pub(crate) fn parse_properties( + properties: &HashMap, + codec_key: &str, + default: Self, + ) -> Result { + let codec = properties + .get(codec_key) + .map(|value| Self::parse_property(value)) + .transpose()? + .unwrap_or(default); + let level_key = compression_level_key(codec_key); + let Some(level) = properties.get(&level_key) else { + return Ok(codec); + }; + let level = level.parse::().map_err(|error| { + Error::new( + ErrorKind::DataInvalid, + format!("Invalid compression level for {level_key}: {level}"), + ) + .with_source(error) + })?; + + match codec { + Self::Gzip(_) => Ok(Self::Gzip(level)), + Self::Zstd(_) => Ok(Self::Zstd(level)), + _ => Err(Error::new( + ErrorKind::DataInvalid, + format!( + "Compression level {level_key} is not supported for codec '{}'", + codec.name() + ), + )), + } + } + + /// Writes a codec and its optional companion compression-level property. + pub(crate) fn write_properties( + &self, + properties: &mut HashMap, + codec_key: &str, + ) { + properties.insert(codec_key.to_string(), self.property_value()); + + let level_key = compression_level_key(codec_key); + match self { + Self::Gzip(level) | Self::Zstd(level) => { + properties.insert(level_key, level.to_string()); + } + _ => { + properties.remove(&level_key); + } + } + } +} + +fn compression_level_key(codec_key: &str) -> String { + format!( + "{}level", + codec_key + .strip_suffix("codec") + .expect("compression codec property keys end with 'codec'") + ) } // Note: serialize/deserialize do not round-trip the compression level. Iceberg configuration diff --git a/crates/iceberg/src/lib.rs b/crates/iceberg/src/lib.rs index 4e346460f5..dd77beb7e9 100644 --- a/crates/iceberg/src/lib.rs +++ b/crates/iceberg/src/lib.rs @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +#![cfg_attr(test, recursion_limit = "512")] + //! Apache Iceberg Official Native Rust Implementation //! //! # Examples diff --git a/crates/iceberg/src/spec/parsed_table_prop.rs b/crates/iceberg/src/spec/parsed_table_prop.rs index 9cdb06cc79..79edfb81d2 100644 --- a/crates/iceberg/src/spec/parsed_table_prop.rs +++ b/crates/iceberg/src/spec/parsed_table_prop.rs @@ -77,8 +77,9 @@ use std::collections::HashMap; use iceberg_property_macro::Properties; use serde_with::{DeserializeFromStr, SerializeDisplay}; +use crate::catalog::MetadataLocation; use crate::compression::CompressionCodec; -use crate::error::{Error, ErrorKind, Result}; +use crate::error::Result; use crate::spec::{DataFileFormat, NameMapping}; /// Parquet data page version 1. @@ -175,133 +176,6 @@ pub enum RowLevelOperationMode { MergeOnRead, } -/// Strips trailing slashes from a location, preserving a bare URI scheme root. -fn strip_trailing_slash(path: &str) -> &str { - let mut path = path; - while !path.ends_with("://") { - let Some(stripped) = path.strip_suffix('/') else { - break; - }; - path = stripped; - } - path -} - -fn parse_metadata_location(value: &str) -> Result> { - if value.is_empty() { - return Err(Error::new(ErrorKind::DataInvalid, "path must not be empty")); - } - - Ok(Some(strip_trailing_slash(value).to_string())) -} - -fn parse_compression_codec(value: &str) -> Result { - serde_json::from_value(serde_json::Value::String(value.to_lowercase())).map_err(|_| { - Error::new( - ErrorKind::DataInvalid, - format!("Invalid compression codec: {value}"), - ) - }) -} - -fn parse_metadata_file_compression(value: &str) -> Result { - if value.is_empty() { - return Ok(CompressionCodec::None); - } - - let codec: CompressionCodec = serde_json::from_value(serde_json::Value::String( - value.to_lowercase(), - )) - .map_err(|_| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Invalid metadata compression codec: {value}. Only '{}' and '{}' are supported.", - CompressionCodec::None.name(), - CompressionCodec::gzip_default().name() - ), - ) - })?; - - match codec { - CompressionCodec::None | CompressionCodec::Gzip(_) => Ok(codec), - _ => Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Invalid metadata compression codec: {value}. Only '{}' and '{}' are supported for metadata files.", - CompressionCodec::None.name(), - CompressionCodec::gzip_default().name() - ), - )), - } -} - -fn serialize_compression_codec(codec: &CompressionCodec) -> String { - codec.name().to_string() -} - -fn compression_level_key(codec_key: &str) -> String { - format!( - "{}level", - codec_key - .strip_suffix("codec") - .expect("compression codec property keys end with 'codec'") - ) -} - -fn parse_compression_codec_properties( - properties: &HashMap, - codec_key: &str, - default: CompressionCodec, -) -> Result { - let codec = properties - .get(codec_key) - .map(|value| parse_compression_codec(value)) - .transpose()? - .unwrap_or(default); - let level_key = compression_level_key(codec_key); - let Some(level) = properties.get(&level_key) else { - return Ok(codec); - }; - let level = level.parse::().map_err(|error| { - Error::new( - ErrorKind::DataInvalid, - format!("Invalid compression level for {level_key}: {level}"), - ) - .with_source(error) - })?; - - match codec { - CompressionCodec::Gzip(_) => Ok(CompressionCodec::Gzip(level)), - CompressionCodec::Zstd(_) => Ok(CompressionCodec::Zstd(level)), - _ => Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Compression level {level_key} is not supported for codec '{}'", - codec.name() - ), - )), - } -} - -fn write_compression_codec_properties( - codec: &CompressionCodec, - properties: &mut HashMap, - codec_key: &str, -) { - properties.insert(codec_key.to_string(), serialize_compression_codec(codec)); - - let level_key = compression_level_key(codec_key); - match codec { - CompressionCodec::Gzip(level) | CompressionCodec::Zstd(level) => { - properties.insert(level_key, level.to_string()); - } - _ => { - properties.remove(&level_key); - } - } -} - fn parse_comma_separated_strings(value: &str) -> Result> { Ok(value .split(',') @@ -395,8 +269,8 @@ pub struct ParsedTableProperties { #[key = "write.manifest.compression-codec"] #[default(CompressionCodec::gzip_default())] - #[parse_properties_with(parse_compression_codec_properties)] - #[write_properties_with(write_compression_codec_properties)] + #[parse_properties_with(CompressionCodec::parse_properties)] + #[write_properties_with(CompressionCodec::write_properties)] #[doc = "Compression codec used for manifest files."] pub write_manifest_compression_codec: CompressionCodec, @@ -524,15 +398,15 @@ pub struct ParsedTableProperties { #[key = "write.parquet.compression-codec"] #[default(CompressionCodec::zstd_default())] - #[parse_properties_with(parse_compression_codec_properties)] - #[write_properties_with(write_compression_codec_properties)] + #[parse_properties_with(CompressionCodec::parse_properties)] + #[write_properties_with(CompressionCodec::write_properties)] #[doc = "Parquet compression codec used for data files."] pub write_parquet_compression_codec: CompressionCodec, #[key = "write.delete.parquet.compression-codec"] #[default(CompressionCodec::zstd_default())] - #[parse_properties_with(parse_compression_codec_properties)] - #[write_properties_with(write_compression_codec_properties)] + #[parse_properties_with(CompressionCodec::parse_properties)] + #[write_properties_with(CompressionCodec::write_properties)] #[doc = "Parquet compression codec used for delete files."] pub write_delete_parquet_compression_codec: CompressionCodec, @@ -629,15 +503,15 @@ pub struct ParsedTableProperties { // Avro properties. #[key = "write.avro.compression-codec"] #[default(CompressionCodec::gzip_default())] - #[parse_properties_with(parse_compression_codec_properties)] - #[write_properties_with(write_compression_codec_properties)] + #[parse_properties_with(CompressionCodec::parse_properties)] + #[write_properties_with(CompressionCodec::write_properties)] #[doc = "Avro compression codec used for data files."] pub write_avro_compression_codec: CompressionCodec, #[key = "write.delete.avro.compression-codec"] #[default(CompressionCodec::gzip_default())] - #[parse_properties_with(parse_compression_codec_properties)] - #[write_properties_with(write_compression_codec_properties)] + #[parse_properties_with(CompressionCodec::parse_properties)] + #[write_properties_with(CompressionCodec::write_properties)] #[doc = "Avro compression codec used for delete files."] pub write_delete_avro_compression_codec: CompressionCodec, @@ -686,15 +560,15 @@ pub struct ParsedTableProperties { #[key = "write.orc.compression-codec"] #[default(CompressionCodec::Zlib)] - #[parse_with(parse_compression_codec)] - #[serialize_with(serialize_compression_codec)] + #[parse_with(CompressionCodec::parse_property)] + #[serialize_with(CompressionCodec::property_value)] #[doc = "ORC compression codec used for data files."] pub write_orc_compression_codec: CompressionCodec, #[key = "write.delete.orc.compression-codec"] #[default(CompressionCodec::Zlib)] - #[parse_with(parse_compression_codec)] - #[serialize_with(serialize_compression_codec)] + #[parse_with(CompressionCodec::parse_property)] + #[serialize_with(CompressionCodec::property_value)] #[doc = "ORC compression codec used for delete files."] pub write_delete_orc_compression_codec: CompressionCodec, @@ -767,7 +641,7 @@ pub struct ParsedTableProperties { // Metadata properties. #[key = "write.metadata.path"] #[default(None)] - #[parse_with(parse_metadata_location)] + #[parse_with(MetadataLocation::parse_write_metadata_path)] #[doc = "Base location for metadata files written after this property is set, with trailing slashes removed."] pub write_metadata_path: Option, @@ -778,8 +652,8 @@ pub struct ParsedTableProperties { #[key = "write.metadata.compression-codec"] #[default(CompressionCodec::None)] - #[parse_with(parse_metadata_file_compression)] - #[serialize_with(serialize_compression_codec)] + #[parse_with(CompressionCodec::parse_metadata_property)] + #[serialize_with(CompressionCodec::property_value)] #[doc = "Compression codec for metadata JSON files: none or gzip."] pub write_metadata_compression_codec: CompressionCodec, @@ -941,6 +815,7 @@ pub struct ParsedTableProperties { #[cfg(test)] mod tests { use super::*; + use crate::error::{Error, ErrorKind}; use crate::spec::MappedField; fn parse(properties: HashMap) -> Result { @@ -1045,39 +920,129 @@ mod tests { "customer_id".to_string(), 0.02, )]), + write_parquet_bloom_filter_ndv_column: HashMap::from([( + "customer_id".to_string(), + 1_000_000, + )]), ..Default::default() }; let json = serde_json::to_value(&properties).unwrap(); - assert_eq!(json["commit.retry.num-retries"], "9"); - assert_eq!(json["write.format.default"], "orc"); - assert_eq!(json["write.data.path"], "s3://warehouse/table/data"); - assert_eq!(json["write.manifest.compression-codec"], "gzip"); - assert_eq!(json["write.manifest.compression-level"], "9"); - assert_eq!(json["write.parquet.compression-codec"], "zstd"); - assert_eq!(json["write.parquet.compression-level"], "5"); - assert_eq!(json["write.delete.avro.compression-codec"], "gzip"); - assert_eq!(json["write.delete.avro.compression-level"], "4"); - assert_eq!(json["write.distribution-mode"], "range"); - assert_eq!(json["write.orc.compression-codec"], "lzo"); - assert_eq!(json["write.orc.bloom.filter.columns"], "id,category"); - assert_eq!(json["write.delete.granularity"], "file"); - assert_eq!(json["write.delete.isolation-level"], "snapshot"); - assert_eq!(json["write.delete.mode"], "merge-on-read"); - assert_eq!(json["write.update.distribution-mode"], "hash"); assert_eq!( - serde_json::from_str::( - json["schema.name-mapping.default"].as_str().unwrap() - ) - .unwrap(), - serde_json::json!([{"field-id": 1, "names": ["id"]}]) + json, + serde_json::json!({ + "commit.manifest-merge.enabled": "true", + "commit.manifest.min-count-to-merge": "100", + "commit.manifest.target-size-bytes": "8388608", + "commit.retry.max-wait-ms": "60000", + "commit.retry.min-wait-ms": "100", + "commit.retry.num-retries": "9", + "commit.retry.total-timeout-ms": "1800000", + "commit.status-check.max-wait-ms": "60000", + "commit.status-check.min-wait-ms": "1000", + "commit.status-check.num-retries": "3", + "commit.status-check.total-timeout-ms": "1800000", + "compatibility.snapshot-id-inheritance.enabled": "false", + "encryption.data-key-length": "16", + "engine.hive.enabled": "false", + "engine.hive.lock-enabled": "true", + "gc.enabled": "true", + "history.expire.max-ref-age-ms": "9223372036854775807", + "history.expire.max-snapshot-age-ms": "432000000", + "history.expire.min-snapshots-to-keep": "1", + "identifier-fields.rely": "false", + "read.data-planning-mode": "auto", + "read.delete-planning-mode": "auto", + "read.orc.vectorization.batch-size": "5000", + "read.orc.vectorization.enabled": "false", + "read.parquet.vectorization.batch-size": "5000", + "read.parquet.vectorization.enabled": "true", + "read.split.adaptive-size.enabled": "true", + "read.split.metadata-target-size": "33554432", + "read.split.open-file-cost": "4194304", + "read.split.planning-lookback": "10", + "read.split.target-size": "134217728", + "schema.name-mapping.default": r#"[{"field-id":1,"names":["id"]}]"#, + "write.avro.compression-codec": "gzip", + "write.avro.compression-level": "6", + "write.data.path": "s3://warehouse/table/data", + "write.datafusion.fanout.enabled": "true", + "write.delete.avro.compression-codec": "gzip", + "write.delete.avro.compression-level": "4", + "write.delete.distribution-mode": "none", + "write.delete.format.default": "parquet", + "write.delete.granularity": "file", + "write.delete.isolation-level": "snapshot", + "write.delete.mode": "merge-on-read", + "write.delete.orc.block-size-bytes": "268435456", + "write.delete.orc.compression-codec": "zlib", + "write.delete.orc.compression-strategy": "speed", + "write.delete.orc.stripe-size-bytes": "67108864", + "write.delete.orc.vectorized.batch-size": "1024", + "write.delete.parquet.compression-codec": "zstd", + "write.delete.parquet.compression-level": "3", + "write.delete.parquet.dict-size-bytes": "2097152", + "write.delete.parquet.page-row-limit": "20000", + "write.delete.parquet.page-size-bytes": "1048576", + "write.delete.parquet.page-version": "v1", + "write.delete.parquet.row-group-check-max-record-count": "10000", + "write.delete.parquet.row-group-check-min-record-count": "100", + "write.delete.parquet.row-group-size-bytes": "134217728", + "write.delete.target-file-size-bytes": "67108864", + "write.distribution-mode": "range", + "write.format.default": "orc", + "write.manifest-lists.enabled": "true", + "write.manifest.compression-codec": "gzip", + "write.manifest.compression-level": "9", + "write.merge.distribution-mode": "none", + "write.merge.isolation-level": "serializable", + "write.merge.mode": "copy-on-write", + "write.metadata.compression-codec": "none", + "write.metadata.delete-after-commit.enabled": "false", + "write.metadata.metrics.default": "truncate(16)", + "write.metadata.metrics.max-inferred-column-defaults": "100", + "write.metadata.previous-versions-max": "100", + "write.object-storage.enabled": "false", + "write.object-storage.partitioned-paths": "true", + "write.orc.block-size-bytes": "268435456", + "write.orc.bloom.filter.columns": "id,category", + "write.orc.bloom.filter.fpp": "0.05", + "write.orc.compression-codec": "lzo", + "write.orc.compression-strategy": "speed", + "write.orc.stripe-size-bytes": "67108864", + "write.orc.vectorized.batch-size": "1024", + "write.parquet.bloom-filter-adaptive-enabled": "false", + "write.parquet.bloom-filter-fpp.column.customer_id": "0.02", + "write.parquet.bloom-filter-max-bytes": "1048576", + "write.parquet.bloom-filter-ndv.column.customer_id": "1000000", + "write.parquet.compression-codec": "zstd", + "write.parquet.compression-level": "5", + "write.parquet.content-defined-chunking.enabled": "false", + "write.parquet.content-defined-chunking.max-chunk-size": "1048576", + "write.parquet.content-defined-chunking.min-chunk-size": "262144", + "write.parquet.content-defined-chunking.norm-level": "0", + "write.parquet.dict-size-bytes": "2097152", + "write.parquet.page-row-limit": "20000", + "write.parquet.page-size-bytes": "1048576", + "write.parquet.page-version": "v1", + "write.parquet.row-group-check-max-record-count": "10000", + "write.parquet.row-group-check-min-record-count": "100", + "write.parquet.row-group-size-bytes": "134217728", + "write.parquet.row-group-size-track-uncompressed": "false", + "write.parquet.shred-variants": "false", + "write.parquet.variant-inference-buffer-size": "100", + "write.spark.accept-any-schema": "false", + "write.spark.auto-schema-evolution.enabled": "true", + "write.spark.fanout.enabled": "false", + "write.summary.partition-limit": "0", + "write.target-file-size-bytes": "536870912", + "write.update.distribution-mode": "hash", + "write.update.isolation-level": "serializable", + "write.update.mode": "copy-on-write", + "write.upsert.enabled": "false", + "write.wap.enabled": "false" + }) ); - assert_eq!( - json["write.parquet.bloom-filter-fpp.column.customer_id"], - "0.02" - ); - assert!(json.get("commit").is_none()); - assert!(json.get("write").is_none()); } #[test] @@ -1091,6 +1056,7 @@ mod tests { "write.parquet.compression-level": "5", "write.distribution-mode": "HASH", "write.orc.bloom.filter.columns": "id, category", + "write.parquet.bloom-filter-ndv.column.customer_id": "1000000", "schema.name-mapping.default": r#"[{"field-id":1,"names":["id"]}]"#, "write.delete.granularity": "FILE", "write.update.isolation-level": "snapshot", @@ -1117,6 +1083,10 @@ mod tests { "id".to_string(), "category".to_string() ]); + assert_eq!( + properties.write_parquet_bloom_filter_ndv_column, + HashMap::from([("customer_id".to_string(), 1_000_000)]) + ); assert_eq!( properties.schema_name_mapping_default, Some(NameMapping::new(vec![MappedField::new( @@ -1136,111 +1106,6 @@ mod tests { ); } - #[test] - fn every_default_round_trips() { - let defaults = ParsedTableProperties::default(); - let decoded: ParsedTableProperties = - serde_json::from_value(serde_json::to_value(&defaults).unwrap()).unwrap(); - - assert_eq!( - decoded.commit_retry_num_retries, - defaults.commit_retry_num_retries - ); - assert_eq!( - decoded.write_metadata_compression_codec, - defaults.write_metadata_compression_codec - ); - assert_eq!( - decoded.write_parquet_content_defined_chunking_max_chunk_size, - defaults.write_parquet_content_defined_chunking_max_chunk_size - ); - assert_eq!(decoded.write_merge_mode, defaults.write_merge_mode); - assert_eq!( - decoded.write_manifest_compression_codec, - defaults.write_manifest_compression_codec - ); - assert_eq!( - decoded.write_parquet_compression_codec, - defaults.write_parquet_compression_codec - ); - assert_eq!( - decoded.write_avro_compression_codec, - defaults.write_avro_compression_codec - ); - } - - #[test] - fn parses_values_across_groups() { - let properties = parse(HashMap::from([ - ("comment".to_string(), "orders table".to_string()), - ( - "commit.status-check.num-retries".to_string(), - "7".to_string(), - ), - ( - "write.delete.avro.compression-codec".to_string(), - "snappy".to_string(), - ), - ("read.split.planning-lookback".to_string(), "25".to_string()), - ( - "history.expire.min-snapshots-to-keep".to_string(), - "4".to_string(), - ), - ("write.delete.mode".to_string(), "merge-on-read".to_string()), - ("encryption.data-key-length".to_string(), "32".to_string()), - ])) - .unwrap(); - - assert_eq!(properties.comment, Some("orders table".to_string())); - assert_eq!(properties.commit_status_check_num_retries, 7); - assert_eq!( - properties.write_delete_avro_compression_codec, - CompressionCodec::Snappy - ); - assert_eq!(properties.read_split_planning_lookback, 25); - assert_eq!(properties.history_expire_min_snapshots_to_keep, 4); - assert_eq!( - properties.write_delete_mode, - RowLevelOperationMode::MergeOnRead - ); - assert_eq!(properties.encryption_data_key_length, 32); - } - - #[test] - fn metadata_values_are_normalized_and_validated() { - let properties = parse(HashMap::from([ - ( - "write.metadata.path".to_string(), - "s3://warehouse/table/metadata/".to_string(), - ), - ( - "write.metadata.compression-codec".to_string(), - "GZIP".to_string(), - ), - ])) - .unwrap(); - - assert_eq!( - properties.write_metadata_path, - Some("s3://warehouse/table/metadata".to_string()) - ); - assert_eq!( - properties.write_metadata_compression_codec, - CompressionCodec::gzip_default() - ); - - let error = parse(HashMap::from([( - "write.metadata.compression-codec".to_string(), - "zstd".to_string(), - )])) - .unwrap_err(); - assert!( - error - .message() - .contains("Invalid metadata compression codec") - ); - } - #[test] fn invalid_leaf_value_reports_its_property_key() { let error = parse(HashMap::from([( @@ -1251,37 +1116,4 @@ mod tests { assert!(error.message().contains("commit.retry.num-retries")); } - - #[test] - fn invalid_typed_value_reports_its_property_key() { - let error = parse(HashMap::from([( - "write.distribution-mode".to_string(), - "random".to_string(), - )])) - .unwrap_err(); - - assert!(error.message().contains("write.distribution-mode")); - - let error = parse(HashMap::from([( - "schema.name-mapping.default".to_string(), - "not-json".to_string(), - )])) - .unwrap_err(); - - assert!(error.message().contains("schema.name-mapping.default")); - - let error = parse(HashMap::from([ - ( - "write.parquet.compression-codec".to_string(), - "snappy".to_string(), - ), - ( - "write.parquet.compression-level".to_string(), - "7".to_string(), - ), - ])) - .unwrap_err(); - - assert!(error.message().contains("write.parquet.compression-level")); - } } From c7ab8215b4aa67d2e60bb85211117accb97953ad Mon Sep 17 00:00:00 2001 From: Renjie Liu Date: Wed, 5 Aug 2026 15:41:23 +0800 Subject: [PATCH 10/14] Address comments --- crates/iceberg/public-api.txt | 2 - .../iceberg/src/catalog/metadata_location.rs | 17 ------ crates/iceberg/src/compression.rs | 53 +++++++------------ crates/iceberg/src/lib.rs | 2 - crates/iceberg/src/spec/parsed_table_prop.rs | 44 ++++++++------- crates/property-macro/src/lib.rs | 35 ++++++++++-- crates/property-macro/tests/properties.rs | 13 +++-- 7 files changed, 86 insertions(+), 80 deletions(-) diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index e6e2134f08..4b921fcabd 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -148,7 +148,6 @@ pub iceberg::compression::CompressionCodec::Lz4 pub iceberg::compression::CompressionCodec::Lzo pub iceberg::compression::CompressionCodec::None pub iceberg::compression::CompressionCodec::Snappy -pub iceberg::compression::CompressionCodec::Uncompressed pub iceberg::compression::CompressionCodec::Zlib pub iceberg::compression::CompressionCodec::Zstd(u8) impl iceberg::compression::CompressionCodec @@ -1163,7 +1162,6 @@ pub iceberg::puffin::CompressionCodec::Lz4 pub iceberg::puffin::CompressionCodec::Lzo pub iceberg::puffin::CompressionCodec::None pub iceberg::puffin::CompressionCodec::Snappy -pub iceberg::puffin::CompressionCodec::Uncompressed pub iceberg::puffin::CompressionCodec::Zlib pub iceberg::puffin::CompressionCodec::Zstd(u8) impl iceberg::compression::CompressionCodec diff --git a/crates/iceberg/src/catalog/metadata_location.rs b/crates/iceberg/src/catalog/metadata_location.rs index cf8282d5f0..d5daa02b37 100644 --- a/crates/iceberg/src/catalog/metadata_location.rs +++ b/crates/iceberg/src/catalog/metadata_location.rs @@ -43,23 +43,6 @@ pub struct MetadataLocation { } impl MetadataLocation { - /// Parses and normalizes the `write.metadata.path` table property. - pub(crate) fn parse_write_metadata_path(value: &str) -> Result> { - if value.is_empty() { - return Err(Error::new(ErrorKind::DataInvalid, "path must not be empty")); - } - - let mut path = value; - while !path.ends_with("://") { - let Some(stripped) = path.strip_suffix('/') else { - break; - }; - path = stripped; - } - - Ok(Some(path.to_string())) - } - /// Determines the compression codec from table properties. /// Parse errors result in CompressionCodec::None. fn compression_from_properties(properties: &HashMap) -> CompressionCodec { diff --git a/crates/iceberg/src/compression.rs b/crates/iceberg/src/compression.rs index 333e189427..242a4a788d 100644 --- a/crates/iceberg/src/compression.rs +++ b/crates/iceberg/src/compression.rs @@ -41,8 +41,6 @@ pub enum CompressionCodec { #[default] /// No compression None, - /// Uncompressed file-format encoding - Uncompressed, /// Brotli compression Brotli, /// LZ4 single compression frame with content size present @@ -77,7 +75,6 @@ impl CompressionCodec { pub fn name(&self) -> &'static str { match self { CompressionCodec::None => "none", - CompressionCodec::Uncompressed => "uncompressed", CompressionCodec::Brotli => "brotli", CompressionCodec::Lz4 => "lz4", CompressionCodec::Lzo => "lzo", @@ -137,6 +134,7 @@ impl CompressionCodec { pub(crate) fn parse_properties( properties: &HashMap, codec_key: &str, + level_key: &str, default: Self, ) -> Result { let codec = properties @@ -144,8 +142,7 @@ impl CompressionCodec { .map(|value| Self::parse_property(value)) .transpose()? .unwrap_or(default); - let level_key = compression_level_key(codec_key); - let Some(level) = properties.get(&level_key) else { + let Some(level) = properties.get(level_key) else { return Ok(codec); }; let level = level.parse::().map_err(|error| { @@ -174,30 +171,21 @@ impl CompressionCodec { &self, properties: &mut HashMap, codec_key: &str, + level_key: &str, ) { properties.insert(codec_key.to_string(), self.property_value()); - let level_key = compression_level_key(codec_key); match self { Self::Gzip(level) | Self::Zstd(level) => { - properties.insert(level_key, level.to_string()); + properties.insert(level_key.to_string(), level.to_string()); } _ => { - properties.remove(&level_key); + properties.remove(level_key); } } } } -fn compression_level_key(codec_key: &str) -> String { - format!( - "{}level", - codec_key - .strip_suffix("codec") - .expect("compression codec property keys end with 'codec'") - ) -} - // Note: serialize/deserialize do not round-trip the compression level. Iceberg configuration // only the codec name (e.g. "zstd"), not the level, so deserialization always produces the // default level. A `Zstd(5)` written to metadata will be read back as `Zstd(3)`. Some @@ -212,8 +200,7 @@ impl<'de> Deserialize<'de> for CompressionCodec { fn deserialize>(deserializer: D) -> std::result::Result { let s = String::deserialize(deserializer)?; match s.to_lowercase().as_str() { - "none" => Ok(CompressionCodec::None), - "uncompressed" => Ok(CompressionCodec::Uncompressed), + "none" | "uncompressed" => Ok(CompressionCodec::None), "brotli" => Ok(CompressionCodec::Brotli), "lz4" => Ok(CompressionCodec::Lz4), "lzo" => Ok(CompressionCodec::Lzo), @@ -240,7 +227,6 @@ impl fmt::Display for CompressionCodec { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { CompressionCodec::None => write!(f, "None"), - CompressionCodec::Uncompressed => write!(f, "Uncompressed"), CompressionCodec::Brotli => write!(f, "Brotli"), CompressionCodec::Lz4 => write!(f, "Lz4"), CompressionCodec::Lzo => write!(f, "Lzo"), @@ -255,7 +241,7 @@ impl fmt::Display for CompressionCodec { impl CompressionCodec { pub(crate) fn decompress(&self, bytes: Vec) -> Result> { match self { - CompressionCodec::None | CompressionCodec::Uncompressed => Ok(bytes), + CompressionCodec::None => Ok(bytes), CompressionCodec::Lz4 => Err(Error::new( ErrorKind::FeatureUnsupported, "LZ4 decompression is not supported currently", @@ -282,7 +268,7 @@ impl CompressionCodec { pub(crate) fn compress(&self, bytes: Vec) -> Result> { match self { - CompressionCodec::None | CompressionCodec::Uncompressed => Ok(bytes), + CompressionCodec::None => Ok(bytes), CompressionCodec::Lz4 => Err(Error::new( ErrorKind::FeatureUnsupported, "LZ4 compression is not supported currently", @@ -315,10 +301,7 @@ impl CompressionCodec { } pub(crate) fn is_none(&self) -> bool { - matches!( - self, - CompressionCodec::None | CompressionCodec::Uncompressed - ) + matches!(self, CompressionCodec::None) } /// Returns the file extension suffix for this compression codec. @@ -329,7 +312,7 @@ impl CompressionCodec { /// Returns an error when the codec does not have a defined suffix. pub fn suffix(&self) -> Result<&'static str> { match self { - CompressionCodec::None | CompressionCodec::Uncompressed => Ok(""), + CompressionCodec::None => Ok(""), CompressionCodec::Gzip(_) => Ok(".gz"), codec @ (CompressionCodec::Brotli | CompressionCodec::Lz4 @@ -352,12 +335,10 @@ mod tests { async fn test_compression_codec_none() { let bytes_vec = [0_u8; 100].to_vec(); - for codec in [CompressionCodec::None, CompressionCodec::Uncompressed] { - let compressed = codec.compress(bytes_vec.clone()).unwrap(); - assert_eq!(bytes_vec, compressed); - let decompressed = codec.decompress(compressed).unwrap(); - assert_eq!(bytes_vec, decompressed); - } + let compressed = CompressionCodec::None.compress(bytes_vec.clone()).unwrap(); + assert_eq!(bytes_vec, compressed); + let decompressed = CompressionCodec::None.decompress(compressed).unwrap(); + assert_eq!(bytes_vec, decompressed); } #[tokio::test] @@ -438,7 +419,6 @@ mod tests { fn test_serde_names() { let codecs = [ ("none", CompressionCodec::None), - ("uncompressed", CompressionCodec::Uncompressed), ("brotli", CompressionCodec::Brotli), ("lz4", CompressionCodec::Lz4), ("lzo", CompressionCodec::Lzo), @@ -455,5 +435,10 @@ mod tests { codec ); } + + assert_eq!( + serde_json::from_value::(serde_json::json!("uncompressed")).unwrap(), + CompressionCodec::None + ); } } diff --git a/crates/iceberg/src/lib.rs b/crates/iceberg/src/lib.rs index dd77beb7e9..4e346460f5 100644 --- a/crates/iceberg/src/lib.rs +++ b/crates/iceberg/src/lib.rs @@ -15,8 +15,6 @@ // specific language governing permissions and limitations // under the License. -#![cfg_attr(test, recursion_limit = "512")] - //! Apache Iceberg Official Native Rust Implementation //! //! # Examples diff --git a/crates/iceberg/src/spec/parsed_table_prop.rs b/crates/iceberg/src/spec/parsed_table_prop.rs index 79edfb81d2..74d43140b1 100644 --- a/crates/iceberg/src/spec/parsed_table_prop.rs +++ b/crates/iceberg/src/spec/parsed_table_prop.rs @@ -41,16 +41,13 @@ //! ``` //! use iceberg::spec::{DataFileFormat, ParsedTableProperties}; //! -//! # fn main() -> Result<(), serde_json::Error> { //! let properties: ParsedTableProperties = serde_json::from_value(serde_json::json!({ //! "commit.retry.num-retries": "8", //! "write.format.default": "orc" -//! }))?; +//! })).unwrap(); //! //! assert_eq!(properties.commit_retry_num_retries, 8); //! assert_eq!(properties.write_format_default, DataFileFormat::Orc); -//! # Ok(()) -//! # } //! ``` //! //! # Serialize to JSON @@ -60,16 +57,13 @@ //! ``` //! use iceberg::spec::ParsedTableProperties; //! -//! # fn main() -> Result<(), serde_json::Error> { //! let mut properties = ParsedTableProperties::default(); //! properties.commit_retry_num_retries = 8; //! properties.write_data_path = Some("s3://warehouse/table/data".to_string()); //! -//! let json = serde_json::to_value(&properties)?; +//! let json = serde_json::to_value(&properties).unwrap(); //! assert_eq!(json["commit.retry.num-retries"], "8"); //! assert_eq!(json["write.data.path"], "s3://warehouse/table/data"); -//! # Ok(()) -//! # } //! ``` use std::collections::HashMap; @@ -77,7 +71,6 @@ use std::collections::HashMap; use iceberg_property_macro::Properties; use serde_with::{DeserializeFromStr, SerializeDisplay}; -use crate::catalog::MetadataLocation; use crate::compression::CompressionCodec; use crate::error::Result; use crate::spec::{DataFileFormat, NameMapping}; @@ -268,6 +261,7 @@ pub struct ParsedTableProperties { pub commit_manifest_merge_enabled: bool, #[key = "write.manifest.compression-codec"] + #[additional_key = "write.manifest.compression-level"] #[default(CompressionCodec::gzip_default())] #[parse_properties_with(CompressionCodec::parse_properties)] #[write_properties_with(CompressionCodec::write_properties)] @@ -397,6 +391,7 @@ pub struct ParsedTableProperties { pub write_delete_parquet_dict_size_bytes: usize, #[key = "write.parquet.compression-codec"] + #[additional_key = "write.parquet.compression-level"] #[default(CompressionCodec::zstd_default())] #[parse_properties_with(CompressionCodec::parse_properties)] #[write_properties_with(CompressionCodec::write_properties)] @@ -404,6 +399,7 @@ pub struct ParsedTableProperties { pub write_parquet_compression_codec: CompressionCodec, #[key = "write.delete.parquet.compression-codec"] + #[additional_key = "write.delete.parquet.compression-level"] #[default(CompressionCodec::zstd_default())] #[parse_properties_with(CompressionCodec::parse_properties)] #[write_properties_with(CompressionCodec::write_properties)] @@ -502,6 +498,7 @@ pub struct ParsedTableProperties { // Avro properties. #[key = "write.avro.compression-codec"] + #[additional_key = "write.avro.compression-level"] #[default(CompressionCodec::gzip_default())] #[parse_properties_with(CompressionCodec::parse_properties)] #[write_properties_with(CompressionCodec::write_properties)] @@ -509,6 +506,7 @@ pub struct ParsedTableProperties { pub write_avro_compression_codec: CompressionCodec, #[key = "write.delete.avro.compression-codec"] + #[additional_key = "write.delete.avro.compression-level"] #[default(CompressionCodec::gzip_default())] #[parse_properties_with(CompressionCodec::parse_properties)] #[write_properties_with(CompressionCodec::write_properties)] @@ -641,8 +639,7 @@ pub struct ParsedTableProperties { // Metadata properties. #[key = "write.metadata.path"] #[default(None)] - #[parse_with(MetadataLocation::parse_write_metadata_path)] - #[doc = "Base location for metadata files written after this property is set, with trailing slashes removed."] + #[doc = "Base location for metadata files written after this property is set."] pub write_metadata_path: Option, #[key = "write.summary.partition-limit"] @@ -928,8 +925,7 @@ mod tests { }; let json = serde_json::to_value(&properties).unwrap(); - assert_eq!( - json, + let expected_parts = [ serde_json::json!({ "commit.manifest-merge.enabled": "true", "commit.manifest.min-count-to-merge": "100", @@ -961,7 +957,9 @@ mod tests { "read.split.metadata-target-size": "33554432", "read.split.open-file-cost": "4194304", "read.split.planning-lookback": "10", - "read.split.target-size": "134217728", + "read.split.target-size": "134217728" + }), + serde_json::json!({ "schema.name-mapping.default": r#"[{"field-id":1,"names":["id"]}]"#, "write.avro.compression-codec": "gzip", "write.avro.compression-level": "6", @@ -988,7 +986,9 @@ mod tests { "write.delete.parquet.row-group-check-max-record-count": "10000", "write.delete.parquet.row-group-check-min-record-count": "100", "write.delete.parquet.row-group-size-bytes": "134217728", - "write.delete.target-file-size-bytes": "67108864", + "write.delete.target-file-size-bytes": "67108864" + }), + serde_json::json!({ "write.distribution-mode": "range", "write.format.default": "orc", "write.manifest-lists.enabled": "true", @@ -1010,7 +1010,9 @@ mod tests { "write.orc.compression-codec": "lzo", "write.orc.compression-strategy": "speed", "write.orc.stripe-size-bytes": "67108864", - "write.orc.vectorized.batch-size": "1024", + "write.orc.vectorized.batch-size": "1024" + }), + serde_json::json!({ "write.parquet.bloom-filter-adaptive-enabled": "false", "write.parquet.bloom-filter-fpp.column.customer_id": "0.02", "write.parquet.bloom-filter-max-bytes": "1048576", @@ -1041,8 +1043,14 @@ mod tests { "write.update.mode": "copy-on-write", "write.upsert.enabled": "false", "write.wap.enabled": "false" - }) - ); + }), + ]; + let mut expected = serde_json::Map::new(); + for part in expected_parts { + expected.extend(part.as_object().unwrap().clone()); + } + + assert_eq!(json, serde_json::Value::Object(expected)); } #[test] diff --git a/crates/property-macro/src/lib.rs b/crates/property-macro/src/lib.rs index 14133bd95e..b2ed947ec3 100644 --- a/crates/property-macro/src/lib.rs +++ b/crates/property-macro/src/lib.rs @@ -49,7 +49,8 @@ use syn::{ /// property map flat. `parse_with` may be used for exact-key property types that do not implement /// `FromStr` or need validation. `serialize_with` supplies their string representation in JSON. /// `parse_properties_with` and `write_properties_with` provide access to the complete property map -/// for fields represented by more than one key. +/// for fields represented by more than one key. `additional_key` declares a second key and passes +/// it to those hooks after the primary key. /// Optional fields are omitted from JSON when they are `None`. Fields need `FromStr` and `ToString` /// unless the relevant custom parsing or serialization attribute is supplied. String-literal and /// path defaults are converted into their field type with `Into`. @@ -57,6 +58,7 @@ use syn::{ Properties, attributes( key, + additional_key, prefix, nested, default, @@ -79,6 +81,7 @@ struct PropertyField { ident: Ident, ty: Type, key: Option, + additional_key: Option, prefix: Option, nested: bool, default: Option, @@ -198,6 +201,7 @@ fn parse_property_field(field: &Field) -> syn::Result { .clone() .ok_or_else(|| Error::new_spanned(field, "Properties fields must be named"))?; let key = attribute_expression_value(&field.attrs, "key")?; + let additional_key = attribute_expression_value(&field.attrs, "additional_key")?; let prefix = attribute_expression_value(&field.attrs, "prefix")?; let nested = marker_attribute(&field.attrs, "nested")?; if usize::from(key.is_some()) + usize::from(prefix.is_some()) + usize::from(nested) != 1 { @@ -231,8 +235,18 @@ fn parse_property_field(field: &Field) -> syn::Result { let serialize_with = attribute_path_value(&field.attrs, "serialize_with")?; let parse_properties_with = attribute_path_value(&field.attrs, "parse_properties_with")?; let write_properties_with = attribute_path_value(&field.attrs, "write_properties_with")?; + if additional_key.is_some() + && parse_properties_with.is_none() + && write_properties_with.is_none() + { + return Err(Error::new_spanned( + field, + "#[additional_key(...)] requires parse_properties_with or write_properties_with", + )); + } if (prefix.is_some() || nested) - && (parse_with.is_some() + && (additional_key.is_some() + || parse_with.is_some() || serialize_with.is_some() || parse_properties_with.is_some() || write_properties_with.is_some()) @@ -259,6 +273,7 @@ fn parse_property_field(field: &Field) -> syn::Result { ident, ty: field.ty.clone(), key, + additional_key, prefix, nested, default, @@ -345,8 +360,14 @@ fn parse_field(field: &PropertyField) -> TokenStream2 { if let Some(parse_properties_with) = &field.parse_properties_with { let key = field.key.as_ref().expect("exact-key fields have a key"); + let parse = match &field.additional_key { + Some(additional_key) => { + quote!(#parse_properties_with(properties, #key, #additional_key, #default)) + } + None => quote!(#parse_properties_with(properties, #key, #default)), + }; return quote! { - #ident: #parse_properties_with(properties, #key, #default).map_err(|error| { + #ident: #parse.map_err(|error| { format!("Invalid value for {}: {error}", #key) })? }; @@ -473,8 +494,14 @@ fn write_field(field: &PropertyField) -> TokenStream2 { if let Some(write_properties_with) = &field.write_properties_with { let key = field.key.as_ref().expect("exact-key fields have a key"); + let write = match &field.additional_key { + Some(additional_key) => { + quote!(#write_properties_with(&self.#ident, properties, #key, #additional_key)) + } + None => quote!(#write_properties_with(&self.#ident, properties, #key)), + }; return quote! { - #write_properties_with(&self.#ident, properties, #key); + #write; }; } diff --git a/crates/property-macro/tests/properties.rs b/crates/property-macro/tests/properties.rs index d33fb6998f..4609149657 100644 --- a/crates/property-macro/tests/properties.rs +++ b/crates/property-macro/tests/properties.rs @@ -29,6 +29,7 @@ const HEIGHT: &str = "dimensions.height"; fn parse_dimensions( properties: &HashMap, key: &str, + height_key: &str, default: (u64, u64), ) -> Result<(u64, u64), String> { let parse = |property_key: &str, default| { @@ -39,12 +40,17 @@ fn parse_dimensions( .map(|value| value.unwrap_or(default)) }; - Ok((parse(key, default.0)?, parse(HEIGHT, default.1)?)) + Ok((parse(key, default.0)?, parse(height_key, default.1)?)) } -fn write_dimensions(dimensions: &(u64, u64), properties: &mut HashMap, key: &str) { +fn write_dimensions( + dimensions: &(u64, u64), + properties: &mut HashMap, + key: &str, + height_key: &str, +) { properties.insert(key.to_string(), dimensions.0.to_string()); - properties.insert(HEIGHT.to_string(), dimensions.1.to_string()); + properties.insert(height_key.to_string(), dimensions.1.to_string()); } #[derive(Debug, Properties)] @@ -67,6 +73,7 @@ struct TestProperties { pub column_fpp: HashMap, #[key(WIDTH)] + #[additional_key(HEIGHT)] #[default((640, 480))] #[parse_properties_with(parse_dimensions)] #[write_properties_with(write_dimensions)] From 732007b8db6a44d42f91d1f301431ea791276d18 Mon Sep 17 00:00:00 2001 From: Renjie Liu Date: Wed, 5 Aug 2026 18:07:13 +0800 Subject: [PATCH 11/14] Replace --- crates/iceberg/public-api.txt | 314 +++--- .../iceberg/src/catalog/metadata_location.rs | 12 +- crates/iceberg/src/catalog/utils.rs | 2 +- crates/iceberg/src/encryption/manager.rs | 2 +- crates/iceberg/src/spec/mod.rs | 7 +- crates/iceberg/src/spec/table_metadata.rs | 73 +- .../src/spec/table_metadata_builder.rs | 31 +- crates/iceberg/src/spec/table_properties.rs | 950 ------------------ .../{parsed_table_prop.rs => table_props.rs} | 157 +-- crates/iceberg/src/table.rs | 8 +- .../src/transaction/expire_snapshots.rs | 6 +- crates/iceberg/src/transaction/mod.rs | 6 +- crates/iceberg/src/transaction/snapshot.rs | 17 +- .../src/writer/file_writer/parquet_writer.rs | 41 +- .../src/writer/file_writer/rolling_writer.rs | 2 +- .../datafusion/src/physical_plan/write.rs | 4 +- .../integrations/datafusion/src/table/mod.rs | 25 +- crates/property-macro/src/lib.rs | 38 +- crates/property-macro/tests/properties.rs | 12 +- 19 files changed, 283 insertions(+), 1424 deletions(-) delete mode 100644 crates/iceberg/src/spec/table_properties.rs rename crates/iceberg/src/spec/{parsed_table_prop.rs => table_props.rs} (84%) diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index 4b921fcabd..2a42a63904 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -2352,133 +2352,6 @@ impl serde_core::ser::Serialize for iceberg::spec::NestedField pub fn iceberg::spec::NestedField::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::NestedField pub fn iceberg::spec::NestedField::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> -pub struct iceberg::spec::ParsedTableProperties -pub iceberg::spec::ParsedTableProperties::comment: core::option::Option -pub iceberg::spec::ParsedTableProperties::commit_manifest_merge_enabled: bool -pub iceberg::spec::ParsedTableProperties::commit_manifest_min_count_to_merge: usize -pub iceberg::spec::ParsedTableProperties::commit_manifest_target_size_bytes: usize -pub iceberg::spec::ParsedTableProperties::commit_retry_max_wait_ms: u64 -pub iceberg::spec::ParsedTableProperties::commit_retry_min_wait_ms: u64 -pub iceberg::spec::ParsedTableProperties::commit_retry_num_retries: usize -pub iceberg::spec::ParsedTableProperties::commit_retry_total_timeout_ms: u64 -pub iceberg::spec::ParsedTableProperties::commit_status_check_max_wait_ms: u64 -pub iceberg::spec::ParsedTableProperties::commit_status_check_min_wait_ms: u64 -pub iceberg::spec::ParsedTableProperties::commit_status_check_num_retries: usize -pub iceberg::spec::ParsedTableProperties::commit_status_check_total_timeout_ms: u64 -pub iceberg::spec::ParsedTableProperties::compatibility_snapshot_id_inheritance_enabled: bool -pub iceberg::spec::ParsedTableProperties::encryption_data_key_length: usize -pub iceberg::spec::ParsedTableProperties::encryption_key_id: core::option::Option -pub iceberg::spec::ParsedTableProperties::engine_hive_enabled: bool -pub iceberg::spec::ParsedTableProperties::engine_hive_lock_enabled: bool -pub iceberg::spec::ParsedTableProperties::gc_enabled: bool -pub iceberg::spec::ParsedTableProperties::history_expire_max_ref_age_ms: i64 -pub iceberg::spec::ParsedTableProperties::history_expire_max_snapshot_age_ms: i64 -pub iceberg::spec::ParsedTableProperties::history_expire_min_snapshots_to_keep: usize -pub iceberg::spec::ParsedTableProperties::identifier_fields_rely: bool -pub iceberg::spec::ParsedTableProperties::read_data_planning_mode: alloc::string::String -pub iceberg::spec::ParsedTableProperties::read_delete_planning_mode: alloc::string::String -pub iceberg::spec::ParsedTableProperties::read_orc_vectorization_batch_size: usize -pub iceberg::spec::ParsedTableProperties::read_orc_vectorization_enabled: bool -pub iceberg::spec::ParsedTableProperties::read_parquet_vectorization_batch_size: usize -pub iceberg::spec::ParsedTableProperties::read_parquet_vectorization_enabled: bool -pub iceberg::spec::ParsedTableProperties::read_split_adaptive_size_enabled: bool -pub iceberg::spec::ParsedTableProperties::read_split_metadata_target_size: u64 -pub iceberg::spec::ParsedTableProperties::read_split_open_file_cost: u64 -pub iceberg::spec::ParsedTableProperties::read_split_planning_lookback: usize -pub iceberg::spec::ParsedTableProperties::read_split_target_size: u64 -pub iceberg::spec::ParsedTableProperties::schema_name_mapping_default: core::option::Option -pub iceberg::spec::ParsedTableProperties::write_avro_compression_codec: iceberg::compression::CompressionCodec -pub iceberg::spec::ParsedTableProperties::write_data_path: core::option::Option -pub iceberg::spec::ParsedTableProperties::write_datafusion_fanout_enabled: bool -pub iceberg::spec::ParsedTableProperties::write_delete_avro_compression_codec: iceberg::compression::CompressionCodec -pub iceberg::spec::ParsedTableProperties::write_delete_distribution_mode: iceberg::spec::DistributionMode -pub iceberg::spec::ParsedTableProperties::write_delete_format_default: iceberg::spec::DataFileFormat -pub iceberg::spec::ParsedTableProperties::write_delete_granularity: iceberg::spec::DeleteGranularity -pub iceberg::spec::ParsedTableProperties::write_delete_isolation_level: iceberg::spec::IsolationLevel -pub iceberg::spec::ParsedTableProperties::write_delete_mode: iceberg::spec::RowLevelOperationMode -pub iceberg::spec::ParsedTableProperties::write_delete_orc_block_size_bytes: u64 -pub iceberg::spec::ParsedTableProperties::write_delete_orc_compression_codec: iceberg::compression::CompressionCodec -pub iceberg::spec::ParsedTableProperties::write_delete_orc_compression_strategy: alloc::string::String -pub iceberg::spec::ParsedTableProperties::write_delete_orc_stripe_size_bytes: u64 -pub iceberg::spec::ParsedTableProperties::write_delete_orc_vectorized_batch_size: usize -pub iceberg::spec::ParsedTableProperties::write_delete_parquet_compression_codec: iceberg::compression::CompressionCodec -pub iceberg::spec::ParsedTableProperties::write_delete_parquet_dict_size_bytes: usize -pub iceberg::spec::ParsedTableProperties::write_delete_parquet_page_row_limit: usize -pub iceberg::spec::ParsedTableProperties::write_delete_parquet_page_size_bytes: usize -pub iceberg::spec::ParsedTableProperties::write_delete_parquet_page_version: alloc::string::String -pub iceberg::spec::ParsedTableProperties::write_delete_parquet_row_group_check_max_record_count: usize -pub iceberg::spec::ParsedTableProperties::write_delete_parquet_row_group_check_min_record_count: usize -pub iceberg::spec::ParsedTableProperties::write_delete_parquet_row_group_size_bytes: usize -pub iceberg::spec::ParsedTableProperties::write_delete_target_file_size_bytes: usize -pub iceberg::spec::ParsedTableProperties::write_distribution_mode: iceberg::spec::DistributionMode -pub iceberg::spec::ParsedTableProperties::write_folder_storage_path: core::option::Option -pub iceberg::spec::ParsedTableProperties::write_format_default: iceberg::spec::DataFileFormat -pub iceberg::spec::ParsedTableProperties::write_location_provider_impl: core::option::Option -pub iceberg::spec::ParsedTableProperties::write_manifest_compression_codec: iceberg::compression::CompressionCodec -pub iceberg::spec::ParsedTableProperties::write_manifest_lists_enabled: bool -pub iceberg::spec::ParsedTableProperties::write_merge_distribution_mode: iceberg::spec::DistributionMode -pub iceberg::spec::ParsedTableProperties::write_merge_isolation_level: iceberg::spec::IsolationLevel -pub iceberg::spec::ParsedTableProperties::write_merge_mode: iceberg::spec::RowLevelOperationMode -pub iceberg::spec::ParsedTableProperties::write_metadata_compression_codec: iceberg::compression::CompressionCodec -pub iceberg::spec::ParsedTableProperties::write_metadata_delete_after_commit_enabled: bool -pub iceberg::spec::ParsedTableProperties::write_metadata_metrics_column: std::collections::hash::map::HashMap -pub iceberg::spec::ParsedTableProperties::write_metadata_metrics_default: alloc::string::String -pub iceberg::spec::ParsedTableProperties::write_metadata_metrics_max_inferred_column_defaults: usize -pub iceberg::spec::ParsedTableProperties::write_metadata_path: core::option::Option -pub iceberg::spec::ParsedTableProperties::write_metadata_previous_versions_max: usize -pub iceberg::spec::ParsedTableProperties::write_object_storage_enabled: bool -pub iceberg::spec::ParsedTableProperties::write_object_storage_partitioned_paths: bool -pub iceberg::spec::ParsedTableProperties::write_object_storage_path: core::option::Option -pub iceberg::spec::ParsedTableProperties::write_orc_block_size_bytes: u64 -pub iceberg::spec::ParsedTableProperties::write_orc_bloom_filter_columns: alloc::vec::Vec -pub iceberg::spec::ParsedTableProperties::write_orc_bloom_filter_fpp: f64 -pub iceberg::spec::ParsedTableProperties::write_orc_compression_codec: iceberg::compression::CompressionCodec -pub iceberg::spec::ParsedTableProperties::write_orc_compression_strategy: alloc::string::String -pub iceberg::spec::ParsedTableProperties::write_orc_stripe_size_bytes: u64 -pub iceberg::spec::ParsedTableProperties::write_orc_vectorized_batch_size: usize -pub iceberg::spec::ParsedTableProperties::write_parquet_bloom_filter_adaptive_enabled: bool -pub iceberg::spec::ParsedTableProperties::write_parquet_bloom_filter_enabled_column: std::collections::hash::map::HashMap -pub iceberg::spec::ParsedTableProperties::write_parquet_bloom_filter_fpp_column: std::collections::hash::map::HashMap -pub iceberg::spec::ParsedTableProperties::write_parquet_bloom_filter_max_bytes: usize -pub iceberg::spec::ParsedTableProperties::write_parquet_bloom_filter_ndv_column: std::collections::hash::map::HashMap -pub iceberg::spec::ParsedTableProperties::write_parquet_compression_codec: iceberg::compression::CompressionCodec -pub iceberg::spec::ParsedTableProperties::write_parquet_content_defined_chunking_enabled: bool -pub iceberg::spec::ParsedTableProperties::write_parquet_content_defined_chunking_max_chunk_size: usize -pub iceberg::spec::ParsedTableProperties::write_parquet_content_defined_chunking_min_chunk_size: usize -pub iceberg::spec::ParsedTableProperties::write_parquet_content_defined_chunking_norm_level: i32 -pub iceberg::spec::ParsedTableProperties::write_parquet_dict_encoding_enabled_column: std::collections::hash::map::HashMap -pub iceberg::spec::ParsedTableProperties::write_parquet_dict_size_bytes: usize -pub iceberg::spec::ParsedTableProperties::write_parquet_page_row_limit: usize -pub iceberg::spec::ParsedTableProperties::write_parquet_page_size_bytes: usize -pub iceberg::spec::ParsedTableProperties::write_parquet_page_version: alloc::string::String -pub iceberg::spec::ParsedTableProperties::write_parquet_row_group_check_max_record_count: usize -pub iceberg::spec::ParsedTableProperties::write_parquet_row_group_check_min_record_count: usize -pub iceberg::spec::ParsedTableProperties::write_parquet_row_group_size_bytes: usize -pub iceberg::spec::ParsedTableProperties::write_parquet_row_group_size_track_uncompressed: bool -pub iceberg::spec::ParsedTableProperties::write_parquet_shred_variants: bool -pub iceberg::spec::ParsedTableProperties::write_parquet_stats_enabled_column: std::collections::hash::map::HashMap -pub iceberg::spec::ParsedTableProperties::write_parquet_variant_inference_buffer_size: usize -pub iceberg::spec::ParsedTableProperties::write_spark_accept_any_schema: bool -pub iceberg::spec::ParsedTableProperties::write_spark_advisory_partition_size_bytes: core::option::Option -pub iceberg::spec::ParsedTableProperties::write_spark_auto_schema_evolution_enabled: bool -pub iceberg::spec::ParsedTableProperties::write_spark_fanout_enabled: bool -pub iceberg::spec::ParsedTableProperties::write_summary_partition_limit: u64 -pub iceberg::spec::ParsedTableProperties::write_target_file_size_bytes: usize -pub iceberg::spec::ParsedTableProperties::write_update_distribution_mode: iceberg::spec::DistributionMode -pub iceberg::spec::ParsedTableProperties::write_update_isolation_level: iceberg::spec::IsolationLevel -pub iceberg::spec::ParsedTableProperties::write_update_mode: iceberg::spec::RowLevelOperationMode -pub iceberg::spec::ParsedTableProperties::write_upsert_enabled: bool -pub iceberg::spec::ParsedTableProperties::write_wap_enabled: bool -impl core::clone::Clone for iceberg::spec::ParsedTableProperties -pub fn iceberg::spec::ParsedTableProperties::clone(&self) -> iceberg::spec::ParsedTableProperties -impl core::default::Default for iceberg::spec::ParsedTableProperties -pub fn iceberg::spec::ParsedTableProperties::default() -> Self -impl core::fmt::Debug for iceberg::spec::ParsedTableProperties -pub fn iceberg::spec::ParsedTableProperties::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl serde_core::ser::Serialize for iceberg::spec::ParsedTableProperties -pub fn iceberg::spec::ParsedTableProperties::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> where S: serde_core::ser::Serializer -impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::ParsedTableProperties -pub fn iceberg::spec::ParsedTableProperties::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> pub struct iceberg::spec::PartitionField pub iceberg::spec::PartitionField::field_id: i32 pub iceberg::spec::PartitionField::name: alloc::string::String @@ -2887,7 +2760,6 @@ pub fn iceberg::spec::TableMetadata::metadata_location(&self) -> iceberg::Result pub fn iceberg::spec::TableMetadata::metadata_log(&self) -> &[iceberg::spec::MetadataLog] pub fn iceberg::spec::TableMetadata::next_row_id(&self) -> u64 pub fn iceberg::spec::TableMetadata::next_sequence_number(&self) -> i64 -pub fn iceberg::spec::TableMetadata::parsed_table_properties(&self) -> iceberg::Result pub fn iceberg::spec::TableMetadata::partition_spec_by_id(&self, spec_id: i32) -> core::option::Option<&iceberg::spec::PartitionSpecRef> pub fn iceberg::spec::TableMetadata::partition_specs_iter(&self) -> impl core::iter::traits::exact_size::ExactSizeIterator pub fn iceberg::spec::TableMetadata::partition_statistics_for_snapshot(&self, snapshot_id: i64) -> core::option::Option<&iceberg::spec::PartitionStatisticsFile> @@ -2971,82 +2843,135 @@ pub fn iceberg::spec::TableMetadataBuilder::clone(&self) -> iceberg::spec::Table impl core::fmt::Debug for iceberg::spec::TableMetadataBuilder pub fn iceberg::spec::TableMetadataBuilder::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result pub struct iceberg::spec::TableProperties -pub iceberg::spec::TableProperties::cdc_enabled: bool -pub iceberg::spec::TableProperties::cdc_max_chunk_size: usize -pub iceberg::spec::TableProperties::cdc_min_chunk_size: usize -pub iceberg::spec::TableProperties::cdc_norm_level: i32 -pub iceberg::spec::TableProperties::commit_max_retry_wait_ms: u64 -pub iceberg::spec::TableProperties::commit_min_retry_wait_ms: u64 -pub iceberg::spec::TableProperties::commit_num_retries: usize -pub iceberg::spec::TableProperties::commit_total_retry_timeout_ms: u64 +pub iceberg::spec::TableProperties::comment: core::option::Option +pub iceberg::spec::TableProperties::commit_manifest_merge_enabled: bool +pub iceberg::spec::TableProperties::commit_manifest_min_count_to_merge: usize +pub iceberg::spec::TableProperties::commit_manifest_target_size_bytes: usize +pub iceberg::spec::TableProperties::commit_retry_max_wait_ms: u64 +pub iceberg::spec::TableProperties::commit_retry_min_wait_ms: u64 +pub iceberg::spec::TableProperties::commit_retry_num_retries: usize +pub iceberg::spec::TableProperties::commit_retry_total_timeout_ms: u64 +pub iceberg::spec::TableProperties::commit_status_check_max_wait_ms: u64 +pub iceberg::spec::TableProperties::commit_status_check_min_wait_ms: u64 +pub iceberg::spec::TableProperties::commit_status_check_num_retries: usize +pub iceberg::spec::TableProperties::commit_status_check_total_timeout_ms: u64 +pub iceberg::spec::TableProperties::compatibility_snapshot_id_inheritance_enabled: bool pub iceberg::spec::TableProperties::encryption_data_key_length: usize pub iceberg::spec::TableProperties::encryption_key_id: core::option::Option +pub iceberg::spec::TableProperties::engine_hive_enabled: bool +pub iceberg::spec::TableProperties::engine_hive_lock_enabled: bool pub iceberg::spec::TableProperties::gc_enabled: bool -pub iceberg::spec::TableProperties::max_ref_age_ms: i64 -pub iceberg::spec::TableProperties::max_snapshot_age_ms: i64 -pub iceberg::spec::TableProperties::metadata_compression_codec: iceberg::compression::CompressionCodec -pub iceberg::spec::TableProperties::min_snapshots_to_keep: usize +pub iceberg::spec::TableProperties::history_expire_max_ref_age_ms: i64 +pub iceberg::spec::TableProperties::history_expire_max_snapshot_age_ms: i64 +pub iceberg::spec::TableProperties::history_expire_min_snapshots_to_keep: usize +pub iceberg::spec::TableProperties::identifier_fields_rely: bool +pub iceberg::spec::TableProperties::read_data_planning_mode: alloc::string::String +pub iceberg::spec::TableProperties::read_delete_planning_mode: alloc::string::String +pub iceberg::spec::TableProperties::read_orc_vectorization_batch_size: usize +pub iceberg::spec::TableProperties::read_orc_vectorization_enabled: bool +pub iceberg::spec::TableProperties::read_parquet_vectorization_batch_size: usize +pub iceberg::spec::TableProperties::read_parquet_vectorization_enabled: bool +pub iceberg::spec::TableProperties::read_split_adaptive_size_enabled: bool +pub iceberg::spec::TableProperties::read_split_metadata_target_size: u64 +pub iceberg::spec::TableProperties::read_split_open_file_cost: u64 +pub iceberg::spec::TableProperties::read_split_planning_lookback: usize +pub iceberg::spec::TableProperties::read_split_target_size: u64 +pub iceberg::spec::TableProperties::schema_name_mapping_default: core::option::Option +pub iceberg::spec::TableProperties::write_avro_compression_codec: iceberg::compression::CompressionCodec +pub iceberg::spec::TableProperties::write_data_path: core::option::Option pub iceberg::spec::TableProperties::write_datafusion_fanout_enabled: bool -pub iceberg::spec::TableProperties::write_format_default: alloc::string::String +pub iceberg::spec::TableProperties::write_delete_avro_compression_codec: iceberg::compression::CompressionCodec +pub iceberg::spec::TableProperties::write_delete_distribution_mode: iceberg::spec::DistributionMode +pub iceberg::spec::TableProperties::write_delete_format_default: iceberg::spec::DataFileFormat +pub iceberg::spec::TableProperties::write_delete_granularity: iceberg::spec::DeleteGranularity +pub iceberg::spec::TableProperties::write_delete_isolation_level: iceberg::spec::IsolationLevel +pub iceberg::spec::TableProperties::write_delete_mode: iceberg::spec::RowLevelOperationMode +pub iceberg::spec::TableProperties::write_delete_orc_block_size_bytes: u64 +pub iceberg::spec::TableProperties::write_delete_orc_compression_codec: iceberg::compression::CompressionCodec +pub iceberg::spec::TableProperties::write_delete_orc_compression_strategy: alloc::string::String +pub iceberg::spec::TableProperties::write_delete_orc_stripe_size_bytes: u64 +pub iceberg::spec::TableProperties::write_delete_orc_vectorized_batch_size: usize +pub iceberg::spec::TableProperties::write_delete_parquet_compression_codec: iceberg::compression::CompressionCodec +pub iceberg::spec::TableProperties::write_delete_parquet_dict_size_bytes: usize +pub iceberg::spec::TableProperties::write_delete_parquet_page_row_limit: usize +pub iceberg::spec::TableProperties::write_delete_parquet_page_size_bytes: usize +pub iceberg::spec::TableProperties::write_delete_parquet_page_version: alloc::string::String +pub iceberg::spec::TableProperties::write_delete_parquet_row_group_check_max_record_count: usize +pub iceberg::spec::TableProperties::write_delete_parquet_row_group_check_min_record_count: usize +pub iceberg::spec::TableProperties::write_delete_parquet_row_group_size_bytes: usize +pub iceberg::spec::TableProperties::write_delete_target_file_size_bytes: usize +pub iceberg::spec::TableProperties::write_distribution_mode: iceberg::spec::DistributionMode +pub iceberg::spec::TableProperties::write_folder_storage_path: core::option::Option +pub iceberg::spec::TableProperties::write_format_default: iceberg::spec::DataFileFormat +pub iceberg::spec::TableProperties::write_location_provider_impl: core::option::Option +pub iceberg::spec::TableProperties::write_manifest_compression_codec: iceberg::compression::CompressionCodec +pub iceberg::spec::TableProperties::write_manifest_lists_enabled: bool +pub iceberg::spec::TableProperties::write_merge_distribution_mode: iceberg::spec::DistributionMode +pub iceberg::spec::TableProperties::write_merge_isolation_level: iceberg::spec::IsolationLevel +pub iceberg::spec::TableProperties::write_merge_mode: iceberg::spec::RowLevelOperationMode +pub iceberg::spec::TableProperties::write_metadata_compression_codec: iceberg::compression::CompressionCodec +pub iceberg::spec::TableProperties::write_metadata_delete_after_commit_enabled: bool +pub iceberg::spec::TableProperties::write_metadata_metrics_column: std::collections::hash::map::HashMap +pub iceberg::spec::TableProperties::write_metadata_metrics_default: alloc::string::String +pub iceberg::spec::TableProperties::write_metadata_metrics_max_inferred_column_defaults: usize pub iceberg::spec::TableProperties::write_metadata_path: core::option::Option +pub iceberg::spec::TableProperties::write_metadata_previous_versions_max: usize +pub iceberg::spec::TableProperties::write_object_storage_enabled: bool +pub iceberg::spec::TableProperties::write_object_storage_partitioned_paths: bool +pub iceberg::spec::TableProperties::write_object_storage_path: core::option::Option +pub iceberg::spec::TableProperties::write_orc_block_size_bytes: u64 +pub iceberg::spec::TableProperties::write_orc_bloom_filter_columns: alloc::vec::Vec +pub iceberg::spec::TableProperties::write_orc_bloom_filter_fpp: f64 +pub iceberg::spec::TableProperties::write_orc_compression_codec: iceberg::compression::CompressionCodec +pub iceberg::spec::TableProperties::write_orc_compression_strategy: alloc::string::String +pub iceberg::spec::TableProperties::write_orc_stripe_size_bytes: u64 +pub iceberg::spec::TableProperties::write_orc_vectorized_batch_size: usize +pub iceberg::spec::TableProperties::write_parquet_bloom_filter_adaptive_enabled: bool +pub iceberg::spec::TableProperties::write_parquet_bloom_filter_enabled_column: std::collections::hash::map::HashMap +pub iceberg::spec::TableProperties::write_parquet_bloom_filter_fpp_column: std::collections::hash::map::HashMap +pub iceberg::spec::TableProperties::write_parquet_bloom_filter_max_bytes: usize +pub iceberg::spec::TableProperties::write_parquet_bloom_filter_ndv_column: std::collections::hash::map::HashMap +pub iceberg::spec::TableProperties::write_parquet_compression_codec: iceberg::compression::CompressionCodec +pub iceberg::spec::TableProperties::write_parquet_content_defined_chunking_enabled: bool +pub iceberg::spec::TableProperties::write_parquet_content_defined_chunking_max_chunk_size: usize +pub iceberg::spec::TableProperties::write_parquet_content_defined_chunking_min_chunk_size: usize +pub iceberg::spec::TableProperties::write_parquet_content_defined_chunking_norm_level: i32 +pub iceberg::spec::TableProperties::write_parquet_dict_encoding_enabled_column: std::collections::hash::map::HashMap +pub iceberg::spec::TableProperties::write_parquet_dict_size_bytes: usize +pub iceberg::spec::TableProperties::write_parquet_page_row_limit: usize +pub iceberg::spec::TableProperties::write_parquet_page_size_bytes: usize +pub iceberg::spec::TableProperties::write_parquet_page_version: alloc::string::String +pub iceberg::spec::TableProperties::write_parquet_row_group_check_max_record_count: usize +pub iceberg::spec::TableProperties::write_parquet_row_group_check_min_record_count: usize +pub iceberg::spec::TableProperties::write_parquet_row_group_size_bytes: usize +pub iceberg::spec::TableProperties::write_parquet_row_group_size_track_uncompressed: bool +pub iceberg::spec::TableProperties::write_parquet_shred_variants: bool +pub iceberg::spec::TableProperties::write_parquet_stats_enabled_column: std::collections::hash::map::HashMap +pub iceberg::spec::TableProperties::write_parquet_variant_inference_buffer_size: usize +pub iceberg::spec::TableProperties::write_spark_accept_any_schema: bool +pub iceberg::spec::TableProperties::write_spark_advisory_partition_size_bytes: core::option::Option +pub iceberg::spec::TableProperties::write_spark_auto_schema_evolution_enabled: bool +pub iceberg::spec::TableProperties::write_spark_fanout_enabled: bool +pub iceberg::spec::TableProperties::write_summary_partition_limit: u64 pub iceberg::spec::TableProperties::write_target_file_size_bytes: usize -impl iceberg::spec::TableProperties -pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS: &str -pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT: u64 -pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS: &str -pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT: u64 -pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_NUM_RETRIES: &str -pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT: usize -pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS: &str -pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT: u64 -pub const iceberg::spec::TableProperties::PROPERTY_CURRENT_SCHEMA: &str -pub const iceberg::spec::TableProperties::PROPERTY_CURRENT_SNAPSHOT_ID: &str -pub const iceberg::spec::TableProperties::PROPERTY_CURRENT_SNAPSHOT_SUMMARY: &str -pub const iceberg::spec::TableProperties::PROPERTY_CURRENT_SNAPSHOT_TIMESTAMP: &str -pub const iceberg::spec::TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED: &str -pub const iceberg::spec::TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT: bool -pub const iceberg::spec::TableProperties::PROPERTY_DEFAULT_FILE_FORMAT: &str -pub const iceberg::spec::TableProperties::PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT: &str -pub const iceberg::spec::TableProperties::PROPERTY_DEFAULT_PARTITION_SPEC: &str -pub const iceberg::spec::TableProperties::PROPERTY_DEFAULT_SORT_ORDER: &str -pub const iceberg::spec::TableProperties::PROPERTY_DELETE_DEFAULT_FILE_FORMAT: &str -pub const iceberg::spec::TableProperties::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH: &str -pub const iceberg::spec::TableProperties::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH_DEFAULT: usize -pub const iceberg::spec::TableProperties::PROPERTY_ENCRYPTION_KEY_ID: &str -pub const iceberg::spec::TableProperties::PROPERTY_FORMAT_VERSION: &str -pub const iceberg::spec::TableProperties::PROPERTY_GC_ENABLED: &str -pub const iceberg::spec::TableProperties::PROPERTY_GC_ENABLED_DEFAULT: bool -pub const iceberg::spec::TableProperties::PROPERTY_MAX_REF_AGE_MS: &str -pub const iceberg::spec::TableProperties::PROPERTY_MAX_REF_AGE_MS_DEFAULT: i64 -pub const iceberg::spec::TableProperties::PROPERTY_MAX_SNAPSHOT_AGE_MS: &str -pub const iceberg::spec::TableProperties::PROPERTY_MAX_SNAPSHOT_AGE_MS_DEFAULT: i64 -pub const iceberg::spec::TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC: &str -pub const iceberg::spec::TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC_DEFAULT: &str -pub const iceberg::spec::TableProperties::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX: &str -pub const iceberg::spec::TableProperties::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX_DEFAULT: usize -pub const iceberg::spec::TableProperties::PROPERTY_MIN_SNAPSHOTS_TO_KEEP: &str -pub const iceberg::spec::TableProperties::PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT: usize -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_ENABLED: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_ENABLED_DEFAULT: bool -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE_DEFAULT: usize -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT: usize -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL_DEFAULT: i32 -pub const iceberg::spec::TableProperties::PROPERTY_SNAPSHOT_COUNT: &str -pub const iceberg::spec::TableProperties::PROPERTY_UUID: &str -pub const iceberg::spec::TableProperties::PROPERTY_WRITE_METADATA_PATH: &str -pub const iceberg::spec::TableProperties::PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT: &str -pub const iceberg::spec::TableProperties::PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT_DEFAULT: u64 -pub const iceberg::spec::TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES: &str -pub const iceberg::spec::TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT: usize -pub const iceberg::spec::TableProperties::RESERVED_PROPERTIES: [&str; 9] +pub iceberg::spec::TableProperties::write_update_distribution_mode: iceberg::spec::DistributionMode +pub iceberg::spec::TableProperties::write_update_isolation_level: iceberg::spec::IsolationLevel +pub iceberg::spec::TableProperties::write_update_mode: iceberg::spec::RowLevelOperationMode +pub iceberg::spec::TableProperties::write_upsert_enabled: bool +pub iceberg::spec::TableProperties::write_wap_enabled: bool +impl core::clone::Clone for iceberg::spec::TableProperties +pub fn iceberg::spec::TableProperties::clone(&self) -> iceberg::spec::TableProperties impl core::convert::TryFrom<&std::collections::hash::map::HashMap> for iceberg::spec::TableProperties pub type iceberg::spec::TableProperties::Error = iceberg::Error -pub fn iceberg::spec::TableProperties::try_from(props: &std::collections::hash::map::HashMap) -> iceberg::Result +pub fn iceberg::spec::TableProperties::try_from(properties: &std::collections::hash::map::HashMap) -> iceberg::Result +impl core::default::Default for iceberg::spec::TableProperties +pub fn iceberg::spec::TableProperties::default() -> Self impl core::fmt::Debug for iceberg::spec::TableProperties pub fn iceberg::spec::TableProperties::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl serde_core::ser::Serialize for iceberg::spec::TableProperties +pub fn iceberg::spec::TableProperties::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> where S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::TableProperties +pub fn iceberg::spec::TableProperties::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> pub struct iceberg::spec::UnboundPartitionField pub iceberg::spec::UnboundPartitionField::field_id: core::option::Option pub iceberg::spec::UnboundPartitionField::name: alloc::string::String @@ -3510,7 +3435,6 @@ pub async fn iceberg::writer::file_writer::ParquetWriter::close(self) -> iceberg pub async fn iceberg::writer::file_writer::ParquetWriter::write(&mut self, batch: &arrow_array::record_batch::RecordBatch) -> iceberg::Result<()> pub struct iceberg::writer::file_writer::ParquetWriterBuilder impl iceberg::writer::file_writer::ParquetWriterBuilder -pub fn iceberg::writer::file_writer::ParquetWriterBuilder::from_parsed_table_properties(table_props: &iceberg::spec::ParsedTableProperties, schema: iceberg::spec::SchemaRef) -> Self pub fn iceberg::writer::file_writer::ParquetWriterBuilder::from_table_properties(table_props: &iceberg::spec::TableProperties, schema: iceberg::spec::SchemaRef) -> Self pub fn iceberg::writer::file_writer::ParquetWriterBuilder::new(props: parquet::file::properties::WriterProperties, schema: iceberg::spec::SchemaRef) -> Self pub fn iceberg::writer::file_writer::ParquetWriterBuilder::new_with_match_mode(props: parquet::file::properties::WriterProperties, schema: iceberg::spec::SchemaRef, match_mode: iceberg::arrow::FieldMatchMode) -> Self diff --git a/crates/iceberg/src/catalog/metadata_location.rs b/crates/iceberg/src/catalog/metadata_location.rs index d5daa02b37..2ab6a499e9 100644 --- a/crates/iceberg/src/catalog/metadata_location.rs +++ b/crates/iceberg/src/catalog/metadata_location.rs @@ -22,7 +22,7 @@ use std::str::FromStr; use uuid::Uuid; use crate::compression::CompressionCodec; -use crate::spec::{TableMetadata, parse_metadata_file_compression}; +use crate::spec::{TableMetadata, TableProperties}; use crate::{Error, ErrorKind, Result}; /// Default folder name for metadata files under the table location, used when the @@ -46,7 +46,9 @@ impl MetadataLocation { /// Determines the compression codec from table properties. /// Parse errors result in CompressionCodec::None. fn compression_from_properties(properties: &HashMap) -> CompressionCodec { - parse_metadata_file_compression(properties).unwrap_or(CompressionCodec::None) + TableProperties::try_from(properties) + .map(|properties| properties.write_metadata_compression_codec) + .unwrap_or(CompressionCodec::None) } /// Creates a completely new metadata location starting at version 0, deriving the @@ -158,7 +160,7 @@ mod test { use uuid::Uuid; use crate::compression::CompressionCodec; - use crate::spec::{Schema, TableMetadata, TableMetadataBuilder, TableProperties}; + use crate::spec::{Schema, TableMetadata, TableMetadataBuilder}; use crate::{MetadataLocation, TableCreation}; fn create_test_metadata(properties: HashMap) -> TableMetadata { @@ -423,7 +425,7 @@ mod test { // A configured `write.metadata.path` is honored on updates too let props = HashMap::from([( - TableProperties::PROPERTY_WRITE_METADATA_PATH.to_string(), + "write.metadata.path".to_string(), "s3://bucket/custom-meta".to_string(), )]); let with_meta_path = create_test_metadata(props); @@ -448,7 +450,7 @@ mod test { // Test a configured `write.metadata.path` is honored let props = HashMap::from([( - TableProperties::PROPERTY_WRITE_METADATA_PATH.to_string(), + "write.metadata.path".to_string(), "s3://bucket/custom-meta".to_string(), )]); let custom_meta = create_test_metadata(props); diff --git a/crates/iceberg/src/catalog/utils.rs b/crates/iceberg/src/catalog/utils.rs index f6117f71ed..853fc8d09a 100644 --- a/crates/iceberg/src/catalog/utils.rs +++ b/crates/iceberg/src/catalog/utils.rs @@ -60,7 +60,7 @@ pub async fn drop_table_data(table_info: &Table) -> Result<()> { } // Delete data files only if gc.enabled is true, to avoid corrupting shared tables - if metadata.parsed_table_properties()?.gc_enabled { + if metadata.table_properties()?.gc_enabled { delete_data_files(io, &manifests_to_delete).await?; } diff --git a/crates/iceberg/src/encryption/manager.rs b/crates/iceberg/src/encryption/manager.rs index 26e316af19..e2294c2f2c 100644 --- a/crates/iceberg/src/encryption/manager.rs +++ b/crates/iceberg/src/encryption/manager.rs @@ -118,7 +118,7 @@ impl EncryptionManager { return Ok(None); } - let table_properties = metadata.parsed_table_properties()?; + let table_properties = metadata.table_properties()?; let Some(table_key_id) = table_properties.encryption_key_id else { if kms_client.is_some() { tracing::warn!( diff --git a/crates/iceberg/src/spec/mod.rs b/crates/iceberg/src/spec/mod.rs index dc9d0f8586..e166e14bc1 100644 --- a/crates/iceberg/src/spec/mod.rs +++ b/crates/iceberg/src/spec/mod.rs @@ -22,7 +22,6 @@ mod encrypted_key; mod manifest; mod manifest_list; mod name_mapping; -mod parsed_table_prop; mod partition; mod schema; mod snapshot; @@ -31,7 +30,7 @@ mod sort; mod statistic_file; mod table_metadata; mod table_metadata_builder; -mod table_properties; +mod table_props; mod transform; mod values; mod view_metadata; @@ -43,7 +42,6 @@ pub use encrypted_key::*; pub use manifest::*; pub use manifest_list::*; pub use name_mapping::*; -pub use parsed_table_prop::*; pub use partition::*; pub use schema::*; pub use snapshot::*; @@ -52,8 +50,7 @@ pub use sort::*; pub use statistic_file::*; pub use table_metadata::*; pub(crate) use table_metadata_builder::FIRST_FIELD_ID; -pub(crate) use table_properties::parse_metadata_file_compression; -pub use table_properties::*; +pub use table_props::*; pub use transform::*; pub(crate) use values::decimal_utils; pub use values::*; diff --git a/crates/iceberg/src/spec/table_metadata.rs b/crates/iceberg/src/spec/table_metadata.rs index 5c872eeaf6..34f299a081 100644 --- a/crates/iceberg/src/spec/table_metadata.rs +++ b/crates/iceberg/src/spec/table_metadata.rs @@ -33,9 +33,9 @@ use uuid::Uuid; use super::snapshot::SnapshotReference; pub use super::table_metadata_builder::{TableMetadataBuildResult, TableMetadataBuilder}; use super::{ - DEFAULT_PARTITION_SPEC_ID, ParsedTableProperties, PartitionSpecRef, PartitionStatisticsFile, - SchemaId, SchemaRef, SnapshotRef, SnapshotRetention, SortOrder, SortOrderRef, StatisticsFile, - StructType, TableProperties, parse_metadata_file_compression, + DEFAULT_PARTITION_SPEC_ID, PartitionSpecRef, PartitionStatisticsFile, SchemaId, SchemaRef, + SnapshotRef, SnapshotRetention, SortOrder, SortOrderRef, StatisticsFile, StructType, + TableProperties, }; use crate::catalog::{METADATA_FOLDER_NAME, MetadataLocation}; use crate::compression::CompressionCodec; @@ -384,25 +384,16 @@ impl TableMetadata { /// /// Returns an error if the compression codec property has an invalid value. pub fn metadata_compression_codec(&self) -> Result { - parse_metadata_file_compression(&self.properties) + Ok(self.table_properties()?.write_metadata_compression_codec) } - /// Returns the existing typed table properties parsed from the raw property map. + /// Returns all supported table properties parsed from the raw property map. pub fn table_properties(&self) -> Result { TableProperties::try_from(&self.properties).map_err(|error| { Error::new(ErrorKind::DataInvalid, "Invalid table properties").with_source(error) }) } - /// Returns all supported table properties parsed by the property framework. - pub fn parsed_table_properties(&self) -> Result { - serde_json::to_value(&self.properties) - .and_then(serde_json::from_value) - .map_err(|error| { - Error::new(ErrorKind::DataInvalid, "Invalid table properties").with_source(error) - }) - } - /// Return location of statistics files. #[inline] pub fn statistics_iter(&self) -> impl ExactSizeIterator { @@ -507,7 +498,7 @@ impl TableMetadata { let json_data = serde_json::to_vec(self)?; // Check if compression codec from properties matches the one in metadata_location - let codec = parse_metadata_file_compression(&self.properties)?; + let codec = self.table_properties()?.write_metadata_compression_codec; if codec != metadata_location.compression_codec() { return Err(Error::new( @@ -1647,7 +1638,7 @@ mod tests { BlobMetadata, EncryptedKey, INITIAL_ROW_ID, Literal, NestedField, NullOrder, Operation, PartitionSpec, PartitionStatisticsFile, PrimitiveLiteral, PrimitiveType, Schema, Snapshot, SnapshotReference, SnapshotRetention, SortDirection, SortField, SortOrder, StatisticsFile, - Summary, TableProperties, Transform, Type, UnboundPartitionField, + Summary, Transform, Type, UnboundPartitionField, }; use crate::{ErrorKind, TableCreation}; @@ -3714,7 +3705,7 @@ mod tests { // Modify properties to enable gzip compression (using mixed case to test case-insensitive matching) let mut props = original_metadata.properties.clone(); props.insert( - TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), + "write.metadata.compression-codec".to_string(), "GziP".to_string(), ); // Use builder to create new metadata with updated properties @@ -4026,9 +4017,7 @@ mod tests { } #[test] - fn test_parsed_table_properties_with_defaults() { - use crate::spec::TableProperties; - + fn test_table_properties_with_defaults() { let schema = Schema::builder() .with_fields(vec![ NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(), @@ -4049,28 +4038,14 @@ mod tests { .unwrap() .metadata; - let props = metadata.parsed_table_properties().unwrap(); + let props = metadata.table_properties().unwrap(); - assert_eq!( - props.commit_retry_num_retries, - TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT - ); - assert_eq!( - props.write_target_file_size_bytes, - TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT - ); - - let existing_props = metadata.table_properties().unwrap(); - assert_eq!( - existing_props.commit_num_retries, - TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT - ); + assert_eq!(props.commit_retry_num_retries, 4); + assert_eq!(props.write_target_file_size_bytes, 512 * 1024 * 1024); } #[test] - fn test_parsed_table_properties_with_custom_values() { - use crate::spec::TableProperties; - + fn test_table_properties_with_custom_values() { let schema = Schema::builder() .with_fields(vec![ NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(), @@ -4079,12 +4054,9 @@ mod tests { .unwrap(); let properties = HashMap::from([ + ("commit.retry.num-retries".to_string(), "10".to_string()), ( - TableProperties::PROPERTY_COMMIT_NUM_RETRIES.to_string(), - "10".to_string(), - ), - ( - TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES.to_string(), + "write.target-file-size-bytes".to_string(), "1024".to_string(), ), ]); @@ -4102,14 +4074,14 @@ mod tests { .unwrap() .metadata; - let props = metadata.parsed_table_properties().unwrap(); + let props = metadata.table_properties().unwrap(); assert_eq!(props.commit_retry_num_retries, 10); assert_eq!(props.write_target_file_size_bytes, 1024); } #[test] - fn test_parsed_table_properties_with_invalid_value() { + fn test_table_properties_with_invalid_value() { let schema = Schema::builder() .with_fields(vec![ NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(), @@ -4135,7 +4107,7 @@ mod tests { .unwrap() .metadata; - let err = metadata.parsed_table_properties().unwrap_err(); + let err = metadata.table_properties().unwrap_err(); assert_eq!(err.kind(), ErrorKind::DataInvalid); assert!(err.message().contains("Invalid table properties")); } @@ -4347,7 +4319,7 @@ mod tests { let metadata = get_test_table_metadata("TableMetadataV2Valid.json") .into_builder(None) .set_properties(HashMap::from([( - TableProperties::PROPERTY_WRITE_METADATA_PATH.to_string(), + "write.metadata.path".to_string(), "s3://other-bucket/custom-meta".to_string(), )])) .unwrap() @@ -4361,12 +4333,11 @@ mod tests { } #[test] - fn test_metadata_location_trims_trailing_slash() { - // A configured path with a trailing slash must not yield a doubled separator + fn test_metadata_location_preserves_trailing_slash() { let metadata = get_test_table_metadata("TableMetadataV2Valid.json") .into_builder(None) .set_properties(HashMap::from([( - TableProperties::PROPERTY_WRITE_METADATA_PATH.to_string(), + "write.metadata.path".to_string(), "s3://other-bucket/custom-meta/".to_string(), )])) .unwrap() @@ -4375,7 +4346,7 @@ mod tests { .metadata; assert_eq!( metadata.metadata_location().unwrap(), - "s3://other-bucket/custom-meta" + "s3://other-bucket/custom-meta/" ); } } diff --git a/crates/iceberg/src/spec/table_metadata_builder.rs b/crates/iceberg/src/spec/table_metadata_builder.rs index 3191d6c13c..1bfb121ad1 100644 --- a/crates/iceberg/src/spec/table_metadata_builder.rs +++ b/crates/iceberg/src/spec/table_metadata_builder.rs @@ -24,7 +24,7 @@ use super::{ DEFAULT_PARTITION_SPEC_ID, DEFAULT_SCHEMA_ID, FormatVersion, MAIN_BRANCH, MetadataLog, ONE_MINUTE_MS, PartitionSpec, PartitionSpecBuilder, PartitionStatisticsFile, Schema, SchemaRef, Snapshot, SnapshotLog, SnapshotReference, SnapshotRetention, SortOrder, SortOrderRef, - StatisticsFile, StructType, TableMetadata, TableProperties, UNPARTITIONED_LAST_ASSIGNED_ID, + StatisticsFile, StructType, TableMetadata, UNPARTITIONED_LAST_ASSIGNED_ID, UnboundPartitionSpec, }; use crate::error::{Error, ErrorKind, Result}; @@ -33,6 +33,21 @@ use crate::{TableCreation, TableUpdate}; pub(crate) const FIRST_FIELD_ID: i32 = 1; +fn is_reserved_property(key: &str) -> bool { + matches!( + key, + "format-version" + | "uuid" + | "snapshot-count" + | "current-snapshot-id" + | "current-snapshot-summary" + | "current-snapshot-timestamp-ms" + | "current-schema" + | "default-partition-spec" + | "default-sort-order" + ) +} + /// Manipulating table metadata. /// /// For this builder the order of called functions matters. Functions are applied in-order. @@ -256,7 +271,7 @@ impl TableMetadataBuilder { // List of specified properties that are RESERVED and should not be persisted. let reserved_properties = properties .keys() - .filter(|key| TableProperties::RESERVED_PROPERTIES.contains(&key.as_str())) + .filter(|key| is_reserved_property(key)) .map(ToString::to_string) .collect::>(); @@ -294,7 +309,7 @@ impl TableMetadataBuilder { // disallow removal of reserved properties let reserved_properties = properties .iter() - .filter(|key| TableProperties::RESERVED_PROPERTIES.contains(&key.as_str())) + .filter(|key| is_reserved_property(key)) .map(ToString::to_string) .collect::>(); @@ -1171,9 +1186,9 @@ impl TableMetadataBuilder { let max_size = self .metadata .properties - .get(TableProperties::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX) + .get("write.metadata.previous-versions-max") .and_then(|v| v.parse::().ok()) - .unwrap_or(TableProperties::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX_DEFAULT) + .unwrap_or(100) .max(1); if self.metadata.metadata_log.len() > max_size { @@ -1469,8 +1484,8 @@ mod tests { use crate::io::FileIO; use crate::spec::{ BlobMetadata, NestedField, NullOrder, Operation, PartitionSpec, PrimitiveType, Schema, - SnapshotRetention, SortDirection, SortField, StructType, Summary, TableProperties, - Transform, Type, UnboundPartitionField, + SnapshotRetention, SortDirection, SortField, StructType, Summary, Transform, Type, + UnboundPartitionField, }; use crate::table::Table; use crate::test_utils::test_runtime; @@ -2476,7 +2491,7 @@ mod tests { let builder = builder_without_changes(FormatVersion::V2); let metadata = builder .set_properties(HashMap::from_iter(vec![( - TableProperties::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX.to_string(), + "write.metadata.previous-versions-max".to_string(), "2".to_string(), )])) .unwrap() diff --git a/crates/iceberg/src/spec/table_properties.rs b/crates/iceberg/src/spec/table_properties.rs deleted file mode 100644 index 379feee5c1..0000000000 --- a/crates/iceberg/src/spec/table_properties.rs +++ /dev/null @@ -1,950 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::collections::HashMap; -use std::fmt::Display; -use std::str::FromStr; - -use crate::compression::CompressionCodec; -use crate::error::{Error, ErrorKind, Result}; - -fn parse_property( - properties: &HashMap, - key: &str, - default: T, -) -> Result -where - ::Err: Display, -{ - properties.get(key).map_or(Ok(default), |value| { - value.parse::().map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!("Invalid value for {key}: {e}"), - ) - }) - }) -} - -/// Strips trailing slashes from a location, preserving a bare URI scheme root -fn strip_trailing_slash(path: &str) -> &str { - let mut path = path; - while !path.ends_with("://") { - let Some(stripped) = path.strip_suffix('/') else { - break; - }; - path = stripped; - } - path -} - -fn parse_location_property( - properties: &HashMap, - key: &str, -) -> Result> { - properties - .get(key) - .map(|path| { - if path.is_empty() { - return Err(Error::new( - ErrorKind::DataInvalid, - format!("Invalid value for {key}: path must not be empty"), - )); - } - - Ok(strip_trailing_slash(path).to_string()) - }) - .transpose() -} - -/// Parse compression codec for metadata files from table properties. -/// Retrieves the compression codec property, applies defaults, and parses the value. -/// Only "none" (or empty string) and "gzip" are supported for metadata compression. -/// -/// # Arguments -/// -/// * `properties` - HashMap containing table properties -/// -/// # Errors -/// -/// Returns an error if the codec is not "none", "", or "gzip" (case-insensitive). -/// Lz4 and Zstd are not supported for metadata file compression. -pub(crate) fn parse_metadata_file_compression( - properties: &HashMap, -) -> Result { - let value = properties - .get(TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC) - .map(|s| s.as_str()) - .unwrap_or(TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC_DEFAULT); - - // Handle empty string as None - if value.is_empty() { - return Ok(CompressionCodec::None); - } - - // Lowercase the value for case-insensitive parsing - let lowercase_value = value.to_lowercase(); - - // Use serde to parse the codec (which has rename_all = "lowercase") - let codec: CompressionCodec = serde_json::from_value(serde_json::Value::String( - lowercase_value, - )) - .map_err(|_| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Invalid metadata compression codec: {value}. Only '{}' and '{}' are supported.", - CompressionCodec::None.name(), - CompressionCodec::gzip_default().name() - ), - ) - })?; - - // Validate that only None and Gzip are used for metadata - match codec { - CompressionCodec::None | CompressionCodec::Gzip(_) => Ok(codec), - _ => Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Invalid metadata compression codec: {value}. Only '{}' and '{}' are supported for metadata files.", - CompressionCodec::None.name(), - CompressionCodec::gzip_default().name() - ), - )), - } -} - -/// TableProperties that contains the properties of a table. -#[derive(Debug)] -pub struct TableProperties { - /// The number of times to retry a commit. - pub commit_num_retries: usize, - /// The minimum wait time between retries. - pub commit_min_retry_wait_ms: u64, - /// The maximum wait time between retries. - pub commit_max_retry_wait_ms: u64, - /// The total timeout for commit retries. - pub commit_total_retry_timeout_ms: u64, - /// The default format for files. - pub write_format_default: String, - /// The target file size for files. - pub write_target_file_size_bytes: usize, - /// Base directory for metadata files (manifests, manifest lists), with any - /// trailing slash trimmed. `None` if `write.metadata.path` is not set. - pub write_metadata_path: Option, - /// Compression codec for metadata files (JSON) - pub metadata_compression_codec: CompressionCodec, - /// Whether to use `FanoutWriter` for partitioned tables. - pub write_datafusion_fanout_enabled: bool, - /// Whether garbage collection is enabled on drop. - /// When `false`, data files will not be deleted when a table is dropped. - pub gc_enabled: bool, - /// Default maximum age of a snapshot to keep when expiring snapshots. - pub max_snapshot_age_ms: i64, - /// Default minimum number of snapshots to keep per branch when expiring snapshots. - pub min_snapshots_to_keep: usize, - /// Default maximum age of a snapshot reference to keep when expiring snapshots. - pub max_ref_age_ms: i64, - /// Whether content-defined chunking is enabled. - /// `true` only when `write.parquet.content-defined-chunking.enabled = "true"`. - pub cdc_enabled: bool, - /// Content-defined chunking minimum chunk size in bytes. - pub cdc_min_chunk_size: usize, - /// Content-defined chunking maximum chunk size in bytes. - pub cdc_max_chunk_size: usize, - /// Content-defined chunking normalization level (gearhash bit adjustment). - pub cdc_norm_level: i32, - /// The master key id used to encrypt this table's manifest list and data - /// files. `None` if `encryption.key-id` is not set. - pub encryption_key_id: Option, - /// The encryption data encryption key length in bytes. - pub encryption_data_key_length: usize, -} - -impl TableProperties { - /// Reserved table property for table format version. - /// - /// Iceberg will default a new table's format version to the latest stable and recommended - /// version. This reserved property keyword allows users to override the Iceberg format version of - /// the table metadata. - /// - /// If this table property exists when creating a table, the table will use the specified format - /// version. If a table updates this property, it will try to upgrade to the specified format - /// version. - pub const PROPERTY_FORMAT_VERSION: &str = "format-version"; - /// Reserved table property for table UUID. - pub const PROPERTY_UUID: &str = "uuid"; - /// Reserved table property for the total number of snapshots. - pub const PROPERTY_SNAPSHOT_COUNT: &str = "snapshot-count"; - /// Reserved table property for current snapshot summary. - pub const PROPERTY_CURRENT_SNAPSHOT_SUMMARY: &str = "current-snapshot-summary"; - /// Reserved table property for current snapshot id. - pub const PROPERTY_CURRENT_SNAPSHOT_ID: &str = "current-snapshot-id"; - /// Reserved table property for current snapshot timestamp. - pub const PROPERTY_CURRENT_SNAPSHOT_TIMESTAMP: &str = "current-snapshot-timestamp-ms"; - /// Reserved table property for the JSON representation of current schema. - pub const PROPERTY_CURRENT_SCHEMA: &str = "current-schema"; - /// Reserved table property for the JSON representation of current(default) partition spec. - pub const PROPERTY_DEFAULT_PARTITION_SPEC: &str = "default-partition-spec"; - /// Reserved table property for the JSON representation of current(default) sort order. - pub const PROPERTY_DEFAULT_SORT_ORDER: &str = "default-sort-order"; - - /// Property key for max number of previous versions to keep. - pub const PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX: &str = - "write.metadata.previous-versions-max"; - /// Default value for max number of previous versions to keep. - pub const PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX_DEFAULT: usize = 100; - - /// Property key for max number of partitions to keep summary stats for. - pub const PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT: &str = "write.summary.partition-limit"; - /// Default value for the max number of partitions to keep summary stats for. - pub const PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT_DEFAULT: u64 = 0; - - /// Reserved Iceberg table properties list. - /// - /// Reserved table properties are only used to control behaviors when creating or updating a - /// table. The value of these properties are not persisted as a part of the table metadata. - pub const RESERVED_PROPERTIES: [&str; 9] = [ - Self::PROPERTY_FORMAT_VERSION, - Self::PROPERTY_UUID, - Self::PROPERTY_SNAPSHOT_COUNT, - Self::PROPERTY_CURRENT_SNAPSHOT_ID, - Self::PROPERTY_CURRENT_SNAPSHOT_SUMMARY, - Self::PROPERTY_CURRENT_SNAPSHOT_TIMESTAMP, - Self::PROPERTY_CURRENT_SCHEMA, - Self::PROPERTY_DEFAULT_PARTITION_SPEC, - Self::PROPERTY_DEFAULT_SORT_ORDER, - ]; - - /// Property key for number of commit retries. - pub const PROPERTY_COMMIT_NUM_RETRIES: &str = "commit.retry.num-retries"; - /// Default value for number of commit retries. - pub const PROPERTY_COMMIT_NUM_RETRIES_DEFAULT: usize = 4; - - /// Property key for minimum wait time (ms) between retries. - pub const PROPERTY_COMMIT_MIN_RETRY_WAIT_MS: &str = "commit.retry.min-wait-ms"; - /// Default value for minimum wait time (ms) between retries. - pub const PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT: u64 = 100; - - /// Property key for maximum wait time (ms) between retries. - pub const PROPERTY_COMMIT_MAX_RETRY_WAIT_MS: &str = "commit.retry.max-wait-ms"; - /// Default value for maximum wait time (ms) between retries. - pub const PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT: u64 = 60 * 1000; // 1 minute - - /// Property key for total maximum retry time (ms). - pub const PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS: &str = "commit.retry.total-timeout-ms"; - /// Default value for total maximum retry time (ms). - pub const PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT: u64 = 30 * 60 * 1000; // 30 minutes - - /// Default file format for data files - pub const PROPERTY_DEFAULT_FILE_FORMAT: &str = "write.format.default"; - /// Default file format for delete files - pub const PROPERTY_DELETE_DEFAULT_FILE_FORMAT: &str = "write.delete.format.default"; - /// Default value for data file format - pub const PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT: &str = "parquet"; - - /// Target file size for newly written files. - pub const PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES: &str = "write.target-file-size-bytes"; - /// Default target file size - pub const PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT: usize = 512 * 1024 * 1024; // 512 MB - - /// Base location for metadata files (manifests, manifest lists, table metadata). - /// When unset, metadata files default to the `metadata` directory under the table - /// location. - pub const PROPERTY_WRITE_METADATA_PATH: &str = "write.metadata.path"; - - /// Compression codec for metadata files (JSON) - pub const PROPERTY_METADATA_COMPRESSION_CODEC: &str = "write.metadata.compression-codec"; - /// Default metadata compression codec - uncompressed - pub const PROPERTY_METADATA_COMPRESSION_CODEC_DEFAULT: &str = "none"; - /// Whether to use `FanoutWriter` for partitioned tables (handles unsorted data). - /// If false, uses `ClusteredWriter` (requires sorted data, more memory efficient). - pub const PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED: &str = "write.datafusion.fanout.enabled"; - /// Default value for fanout writer enabled - pub const PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT: bool = true; - - /// Property key for enabling garbage collection on drop. - /// When set to `false`, data files will not be deleted when a table is dropped. - /// Defaults to `true`. - pub const PROPERTY_GC_ENABLED: &str = "gc.enabled"; - /// Default value for gc.enabled - pub const PROPERTY_GC_ENABLED_DEFAULT: bool = true; - - /// Property key for the default maximum age of a snapshot to keep when expiring snapshots. - pub const PROPERTY_MAX_SNAPSHOT_AGE_MS: &str = "history.expire.max-snapshot-age-ms"; - /// Default value for history.expire.max-snapshot-age-ms (5 days). - pub const PROPERTY_MAX_SNAPSHOT_AGE_MS_DEFAULT: i64 = 5 * 24 * 60 * 60 * 1000; - /// Property key for the default minimum number of snapshots to keep when expiring snapshots. - pub const PROPERTY_MIN_SNAPSHOTS_TO_KEEP: &str = "history.expire.min-snapshots-to-keep"; - /// Default value for history.expire.min-snapshots-to-keep. - pub const PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT: usize = 1; - /// Property key for the default maximum age of a snapshot reference to keep when expiring. - pub const PROPERTY_MAX_REF_AGE_MS: &str = "history.expire.max-ref-age-ms"; - /// Default value for history.expire.max-ref-age-ms (effectively never expire refs). - pub const PROPERTY_MAX_REF_AGE_MS_DEFAULT: i64 = i64::MAX; - - /// Enable content-defined chunking with parquet defaults (or per-property overrides). - pub const PROPERTY_PARQUET_CDC_ENABLED: &str = "write.parquet.content-defined-chunking.enabled"; - /// Default value for content-defined chunking enabled. - pub const PROPERTY_PARQUET_CDC_ENABLED_DEFAULT: bool = false; - /// Minimum chunk size in bytes for content-defined chunking. - pub const PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE: &str = - "write.parquet.content-defined-chunking.min-chunk-size"; - /// Default matches `parquet::file::properties::DEFAULT_CDC_MIN_CHUNK_SIZE`. - pub const PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT: usize = 256 * 1024; - /// Maximum chunk size in bytes for content-defined chunking. - pub const PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE: &str = - "write.parquet.content-defined-chunking.max-chunk-size"; - /// Default matches `parquet::file::properties::DEFAULT_CDC_MAX_CHUNK_SIZE`. - pub const PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE_DEFAULT: usize = 1024 * 1024; - /// Normalization level (gearhash bit adjustment) for content-defined chunking. - pub const PROPERTY_PARQUET_CDC_NORM_LEVEL: &str = - "write.parquet.content-defined-chunking.norm-level"; - /// Default matches `parquet::file::properties::DEFAULT_CDC_NORM_LEVEL`. - pub const PROPERTY_PARQUET_CDC_NORM_LEVEL_DEFAULT: i32 = 0; - - /// Property key for the master key id used to encrypt the table's manifest - /// list and data files as defined in https://iceberg.apache.org/docs/nightly/encryption/. - pub const PROPERTY_ENCRYPTION_KEY_ID: &str = "encryption.key-id"; - - /// Property key for the encryption data encryption key (DEK) length in bytes. - pub const PROPERTY_ENCRYPTION_DATA_KEY_LENGTH: &str = "encryption.data-key-length"; - /// Default value for the encryption DEK length (16 bytes = AES-128). - pub const PROPERTY_ENCRYPTION_DATA_KEY_LENGTH_DEFAULT: usize = 16; -} - -impl TryFrom<&HashMap> for TableProperties { - // parse by entry key or use default value - type Error = Error; - - fn try_from(props: &HashMap) -> Result { - Ok(TableProperties { - commit_num_retries: parse_property( - props, - TableProperties::PROPERTY_COMMIT_NUM_RETRIES, - TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT, - )?, - commit_min_retry_wait_ms: parse_property( - props, - TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS, - TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT, - )?, - commit_max_retry_wait_ms: parse_property( - props, - TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS, - TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT, - )?, - commit_total_retry_timeout_ms: parse_property( - props, - TableProperties::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS, - TableProperties::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT, - )?, - write_format_default: parse_property( - props, - TableProperties::PROPERTY_DEFAULT_FILE_FORMAT, - TableProperties::PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT.to_string(), - )?, - write_target_file_size_bytes: parse_property( - props, - TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES, - TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT, - )?, - write_metadata_path: parse_location_property( - props, - TableProperties::PROPERTY_WRITE_METADATA_PATH, - )?, - metadata_compression_codec: parse_metadata_file_compression(props)?, - write_datafusion_fanout_enabled: parse_property( - props, - TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED, - TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT, - )?, - gc_enabled: parse_property( - props, - TableProperties::PROPERTY_GC_ENABLED, - TableProperties::PROPERTY_GC_ENABLED_DEFAULT, - )?, - max_snapshot_age_ms: parse_property( - props, - TableProperties::PROPERTY_MAX_SNAPSHOT_AGE_MS, - TableProperties::PROPERTY_MAX_SNAPSHOT_AGE_MS_DEFAULT, - )?, - min_snapshots_to_keep: parse_property( - props, - TableProperties::PROPERTY_MIN_SNAPSHOTS_TO_KEEP, - TableProperties::PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT, - )?, - max_ref_age_ms: parse_property( - props, - TableProperties::PROPERTY_MAX_REF_AGE_MS, - TableProperties::PROPERTY_MAX_REF_AGE_MS_DEFAULT, - )?, - cdc_enabled: parse_property( - props, - TableProperties::PROPERTY_PARQUET_CDC_ENABLED, - TableProperties::PROPERTY_PARQUET_CDC_ENABLED_DEFAULT, - )?, - cdc_min_chunk_size: parse_property( - props, - TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE, - TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT, - )?, - cdc_max_chunk_size: parse_property( - props, - TableProperties::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE, - TableProperties::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE_DEFAULT, - )?, - cdc_norm_level: parse_property( - props, - TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL, - TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL_DEFAULT, - )?, - encryption_key_id: props - .get(TableProperties::PROPERTY_ENCRYPTION_KEY_ID) - .cloned(), - encryption_data_key_length: parse_property( - props, - TableProperties::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH, - TableProperties::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH_DEFAULT, - )?, - }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::compression::CompressionCodec; - - #[test] - fn test_table_properties_default() { - let props = HashMap::new(); - let table_properties = TableProperties::try_from(&props).unwrap(); - assert_eq!( - table_properties.commit_num_retries, - TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT - ); - assert_eq!( - table_properties.commit_min_retry_wait_ms, - TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT - ); - assert_eq!( - table_properties.commit_max_retry_wait_ms, - TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT - ); - assert_eq!( - table_properties.write_format_default, - TableProperties::PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT.to_string() - ); - assert_eq!( - table_properties.write_target_file_size_bytes, - TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT - ); - // Test compression defaults (none means CompressionCodec::None) - assert_eq!( - table_properties.metadata_compression_codec, - CompressionCodec::None - ); - assert_eq!( - table_properties.gc_enabled, - TableProperties::PROPERTY_GC_ENABLED_DEFAULT - ); - assert_eq!( - table_properties.max_snapshot_age_ms, - TableProperties::PROPERTY_MAX_SNAPSHOT_AGE_MS_DEFAULT - ); - assert_eq!( - table_properties.min_snapshots_to_keep, - TableProperties::PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT - ); - assert_eq!( - table_properties.max_ref_age_ms, - TableProperties::PROPERTY_MAX_REF_AGE_MS_DEFAULT - ); - } - - #[test] - fn test_table_properties_history_expire_overrides() { - let props = HashMap::from([ - ( - TableProperties::PROPERTY_MAX_SNAPSHOT_AGE_MS.to_string(), - "1234".to_string(), - ), - ( - TableProperties::PROPERTY_MIN_SNAPSHOTS_TO_KEEP.to_string(), - "7".to_string(), - ), - ( - TableProperties::PROPERTY_MAX_REF_AGE_MS.to_string(), - "5678".to_string(), - ), - ]); - let table_properties = TableProperties::try_from(&props).unwrap(); - assert_eq!(table_properties.max_snapshot_age_ms, 1234); - assert_eq!(table_properties.min_snapshots_to_keep, 7); - assert_eq!(table_properties.max_ref_age_ms, 5678); - } - - #[test] - fn test_table_properties_write_metadata_path() { - // Test unset - let table_properties = TableProperties::try_from(&HashMap::new()).unwrap(); - assert_eq!(table_properties.write_metadata_path, None); - - // Test empty path is invalid - let props = HashMap::from([( - TableProperties::PROPERTY_WRITE_METADATA_PATH.to_string(), - String::new(), - )]); - let error = TableProperties::try_from(&props).unwrap_err(); - assert_eq!(error.kind(), ErrorKind::DataInvalid); - assert!( - error - .message() - .contains(TableProperties::PROPERTY_WRITE_METADATA_PATH) - ); - - let props = HashMap::from([( - TableProperties::PROPERTY_WRITE_METADATA_PATH.to_string(), - "s3://other-bucket/custom-meta/".to_string(), - )]); - let table_properties = TableProperties::try_from(&props).unwrap(); - assert_eq!( - table_properties.write_metadata_path.as_deref(), - Some("s3://other-bucket/custom-meta") - ); - } - - #[test] - fn test_strip_trailing_slash() { - for (path, expected) in [ - ("s3://bucket/db/tbl", "s3://bucket/db/tbl"), - ("s3://bucket/db/tbl/", "s3://bucket/db/tbl"), - ("s3://bucket/db/tbl////", "s3://bucket/db/tbl"), - ("blobstore://", "blobstore://"), - ("blobstore:///", "blobstore://"), - ("file:///", "file://"), - ("////", ""), - ("", ""), - ] { - assert_eq!(strip_trailing_slash(path), expected); - } - } - - #[test] - fn test_table_properties_compression() { - let props = HashMap::from([( - TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), - "gzip".to_string(), - )]); - let table_properties = TableProperties::try_from(&props).unwrap(); - assert_eq!( - table_properties.metadata_compression_codec, - CompressionCodec::gzip_default() - ); - } - - #[test] - fn test_table_properties_compression_none() { - let props = HashMap::from([( - TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), - "none".to_string(), - )]); - let table_properties = TableProperties::try_from(&props).unwrap(); - assert_eq!( - table_properties.metadata_compression_codec, - CompressionCodec::None - ); - } - - #[test] - fn test_table_properties_compression_case_insensitive() { - // Test uppercase - let props_upper = HashMap::from([( - TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), - "GZIP".to_string(), - )]); - let table_properties = TableProperties::try_from(&props_upper).unwrap(); - assert_eq!( - table_properties.metadata_compression_codec, - CompressionCodec::gzip_default() - ); - - // Test mixed case - let props_mixed = HashMap::from([( - TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), - "GzIp".to_string(), - )]); - let table_properties = TableProperties::try_from(&props_mixed).unwrap(); - assert_eq!( - table_properties.metadata_compression_codec, - CompressionCodec::gzip_default() - ); - - // Test "NONE" should also be case-insensitive - let props_none_upper = HashMap::from([( - TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), - "NONE".to_string(), - )]); - let table_properties = TableProperties::try_from(&props_none_upper).unwrap(); - assert_eq!( - table_properties.metadata_compression_codec, - CompressionCodec::None - ); - } - - #[test] - fn test_table_properties_valid() { - let props = HashMap::from([ - ( - TableProperties::PROPERTY_COMMIT_NUM_RETRIES.to_string(), - "10".to_string(), - ), - ( - TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS.to_string(), - "20".to_string(), - ), - ( - TableProperties::PROPERTY_DEFAULT_FILE_FORMAT.to_string(), - "avro".to_string(), - ), - ( - TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES.to_string(), - "512".to_string(), - ), - ( - TableProperties::PROPERTY_GC_ENABLED.to_string(), - "false".to_string(), - ), - ]); - let table_properties = TableProperties::try_from(&props).unwrap(); - assert_eq!(table_properties.commit_num_retries, 10); - assert_eq!(table_properties.commit_max_retry_wait_ms, 20); - assert_eq!(table_properties.write_format_default, "avro".to_string()); - assert_eq!(table_properties.write_target_file_size_bytes, 512); - assert!(!table_properties.gc_enabled); - } - - #[test] - fn test_table_properties_invalid() { - let invalid_retries = HashMap::from([( - TableProperties::PROPERTY_COMMIT_NUM_RETRIES.to_string(), - "abc".to_string(), - )]); - - let table_properties = TableProperties::try_from(&invalid_retries).unwrap_err(); - assert!( - table_properties.to_string().contains( - "Invalid value for commit.retry.num-retries: invalid digit found in string" - ) - ); - - let invalid_min_wait = HashMap::from([( - TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS.to_string(), - "abc".to_string(), - )]); - let table_properties = TableProperties::try_from(&invalid_min_wait).unwrap_err(); - assert!( - table_properties.to_string().contains( - "Invalid value for commit.retry.min-wait-ms: invalid digit found in string" - ) - ); - - let invalid_max_wait = HashMap::from([( - TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS.to_string(), - "abc".to_string(), - )]); - let table_properties = TableProperties::try_from(&invalid_max_wait).unwrap_err(); - assert!( - table_properties.to_string().contains( - "Invalid value for commit.retry.max-wait-ms: invalid digit found in string" - ) - ); - - let invalid_target_size = HashMap::from([( - TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES.to_string(), - "abc".to_string(), - )]); - let table_properties = TableProperties::try_from(&invalid_target_size).unwrap_err(); - assert!(table_properties.to_string().contains( - "Invalid value for write.target-file-size-bytes: invalid digit found in string" - )); - - let invalid_gc_enabled = HashMap::from([( - TableProperties::PROPERTY_GC_ENABLED.to_string(), - "notabool".to_string(), - )]); - let table_properties = TableProperties::try_from(&invalid_gc_enabled).unwrap_err(); - assert!( - table_properties - .to_string() - .contains("Invalid value for gc.enabled") - ); - } - - #[test] - fn test_table_properties_compression_invalid_rejected() { - let invalid_codecs = ["lz4", "zstd", "snappy"]; - - for codec in invalid_codecs { - let props = HashMap::from([( - TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), - codec.to_string(), - )]); - let err = TableProperties::try_from(&props).unwrap_err(); - let err_msg = err.to_string(); - assert!( - err_msg.contains(&format!("Invalid metadata compression codec: {codec}")), - "Expected error message to contain codec '{codec}', got: {err_msg}" - ); - assert!( - err_msg.contains("Only 'none' and 'gzip' are supported"), - "Expected error message to contain supported codecs, got: {err_msg}" - ); - } - } - - #[test] - fn test_parse_metadata_file_compression_valid() { - // Test with "none" - let props = HashMap::from([( - TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), - "none".to_string(), - )]); - assert_eq!( - parse_metadata_file_compression(&props).unwrap(), - CompressionCodec::None - ); - - // Test with empty string - let props = HashMap::from([( - TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), - "".to_string(), - )]); - assert_eq!( - parse_metadata_file_compression(&props).unwrap(), - CompressionCodec::None - ); - - // Test with "gzip" - let props = HashMap::from([( - TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), - "gzip".to_string(), - )]); - assert_eq!( - parse_metadata_file_compression(&props).unwrap(), - CompressionCodec::gzip_default() - ); - - // Test case insensitivity - "NONE" - let props = HashMap::from([( - TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), - "NONE".to_string(), - )]); - assert_eq!( - parse_metadata_file_compression(&props).unwrap(), - CompressionCodec::None - ); - - // Test case insensitivity - "GZIP" - let props = HashMap::from([( - TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), - "GZIP".to_string(), - )]); - assert_eq!( - parse_metadata_file_compression(&props).unwrap(), - CompressionCodec::gzip_default() - ); - - // Test case insensitivity - "GzIp" - let props = HashMap::from([( - TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), - "GzIp".to_string(), - )]); - assert_eq!( - parse_metadata_file_compression(&props).unwrap(), - CompressionCodec::gzip_default() - ); - - // Test default when property is missing - let props = HashMap::new(); - assert_eq!( - parse_metadata_file_compression(&props).unwrap(), - CompressionCodec::None - ); - } - - #[test] - fn test_parse_metadata_file_compression_invalid() { - let invalid_codecs = ["lz4", "zstd", "snappy"]; - - for codec in invalid_codecs { - let props = HashMap::from([( - TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), - codec.to_string(), - )]); - let err = parse_metadata_file_compression(&props).unwrap_err(); - let err_msg = err.to_string(); - assert!( - err_msg.contains("Invalid metadata compression codec"), - "Expected error message to contain 'Invalid metadata compression codec', got: {err_msg}" - ); - assert!( - err_msg.contains("Only 'none' and 'gzip' are supported"), - "Expected error message to contain supported codecs, got: {err_msg}" - ); - } - } - - #[test] - fn test_cdc_disabled_by_default() { - let props = HashMap::new(); - let tp = TableProperties::try_from(&props).unwrap(); - assert!(!tp.cdc_enabled); - } - - #[test] - fn test_cdc_enabled_via_flag() { - let props = HashMap::from([( - TableProperties::PROPERTY_PARQUET_CDC_ENABLED.to_string(), - "true".to_string(), - )]); - let tp = TableProperties::try_from(&props).unwrap(); - assert!(tp.cdc_enabled); - assert_eq!(tp.cdc_min_chunk_size, 256 * 1024); - assert_eq!(tp.cdc_max_chunk_size, 1024 * 1024); - assert_eq!(tp.cdc_norm_level, 0); - } - - #[test] - fn test_cdc_size_props_alone_do_not_enable() { - let props = HashMap::from([( - TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE.to_string(), - "262144".to_string(), - )]); - let tp = TableProperties::try_from(&props).unwrap(); - assert!(!tp.cdc_enabled); - } - - #[test] - fn test_cdc_custom_values() { - let props = HashMap::from([ - ( - TableProperties::PROPERTY_PARQUET_CDC_ENABLED.to_string(), - "true".to_string(), - ), - ( - TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE.to_string(), - "200000".to_string(), - ), - ( - TableProperties::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE.to_string(), - "900000".to_string(), - ), - ( - TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL.to_string(), - "1".to_string(), - ), - ]); - let tp = TableProperties::try_from(&props).unwrap(); - assert!(tp.cdc_enabled); - assert_eq!(tp.cdc_min_chunk_size, 200000); - assert_eq!(tp.cdc_max_chunk_size, 900000); - assert_eq!(tp.cdc_norm_level, 1); - } - - #[test] - fn test_cdc_partial_override() { - let props = HashMap::from([ - ( - TableProperties::PROPERTY_PARQUET_CDC_ENABLED.to_string(), - "true".to_string(), - ), - ( - TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL.to_string(), - "2".to_string(), - ), - ]); - let tp = TableProperties::try_from(&props).unwrap(); - assert!(tp.cdc_enabled); - assert_eq!(tp.cdc_min_chunk_size, 256 * 1024); - assert_eq!(tp.cdc_max_chunk_size, 1024 * 1024); - assert_eq!(tp.cdc_norm_level, 2); - } - - #[test] - fn test_cdc_negative_norm_level() { - let props = HashMap::from([ - ( - TableProperties::PROPERTY_PARQUET_CDC_ENABLED.to_string(), - "true".to_string(), - ), - ( - TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL.to_string(), - "-2".to_string(), - ), - ]); - let tp = TableProperties::try_from(&props).unwrap(); - assert_eq!(tp.cdc_norm_level, -2); - } - - #[test] - fn test_cdc_invalid_min_chunk_size() { - let props = HashMap::from([ - ( - TableProperties::PROPERTY_PARQUET_CDC_ENABLED.to_string(), - "true".to_string(), - ), - ( - TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE.to_string(), - "not_a_number".to_string(), - ), - ]); - let err = TableProperties::try_from(&props).unwrap_err(); - assert!( - err.to_string().contains( - "Invalid value for write.parquet.content-defined-chunking.min-chunk-size" - ) - ); - } - - #[test] - fn test_cdc_invalid_norm_level() { - let props = HashMap::from([ - ( - TableProperties::PROPERTY_PARQUET_CDC_ENABLED.to_string(), - "true".to_string(), - ), - ( - TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL.to_string(), - "not_a_number".to_string(), - ), - ]); - let err = TableProperties::try_from(&props).unwrap_err(); - assert!( - err.to_string() - .contains("Invalid value for write.parquet.content-defined-chunking.norm-level") - ); - } - - #[test] - fn test_cdc_no_properties() { - let props = HashMap::from([("some.other.property".to_string(), "value".to_string())]); - let tp = TableProperties::try_from(&props).unwrap(); - assert!(!tp.cdc_enabled); - } -} diff --git a/crates/iceberg/src/spec/parsed_table_prop.rs b/crates/iceberg/src/spec/table_props.rs similarity index 84% rename from crates/iceberg/src/spec/parsed_table_prop.rs rename to crates/iceberg/src/spec/table_props.rs index 74d43140b1..8c60c9bd74 100644 --- a/crates/iceberg/src/spec/parsed_table_prop.rs +++ b/crates/iceberg/src/spec/table_props.rs @@ -17,7 +17,7 @@ //! Typed access to Iceberg table properties. //! -//! [`ParsedTableProperties`] exposes Iceberg's string-keyed table properties as typed public +//! [`TableProperties`] exposes Iceberg's string-keyed table properties as typed public //! fields. Its JSON representation is a flat object whose keys and values are strings. //! //! # Create from defaults @@ -25,9 +25,9 @@ //! Start with Iceberg's defaults and modify public fields directly: //! //! ``` -//! use iceberg::spec::{DataFileFormat, ParsedTableProperties}; +//! use iceberg::spec::{DataFileFormat, TableProperties}; //! -//! let mut properties = ParsedTableProperties::default(); +//! let mut properties = TableProperties::default(); //! properties.write_format_default = DataFileFormat::Orc; //! properties.write_data_path = Some("s3://warehouse/table/data".to_string()); //! @@ -39,9 +39,9 @@ //! JSON property values must be strings, matching Iceberg's table property map: //! //! ``` -//! use iceberg::spec::{DataFileFormat, ParsedTableProperties}; +//! use iceberg::spec::{DataFileFormat, TableProperties}; //! -//! let properties: ParsedTableProperties = serde_json::from_value(serde_json::json!({ +//! let properties: TableProperties = serde_json::from_value(serde_json::json!({ //! "commit.retry.num-retries": "8", //! "write.format.default": "orc" //! })).unwrap(); @@ -52,18 +52,20 @@ //! //! # Serialize to JSON //! -//! Serialization converts the typed fields back into Iceberg property keys: +//! Serialization converts non-default typed fields back into Iceberg property keys. Fields whose +//! values match their defaults are omitted: //! //! ``` -//! use iceberg::spec::ParsedTableProperties; +//! use iceberg::spec::TableProperties; //! -//! let mut properties = ParsedTableProperties::default(); +//! let mut properties = TableProperties::default(); //! properties.commit_retry_num_retries = 8; //! properties.write_data_path = Some("s3://warehouse/table/data".to_string()); //! //! let json = serde_json::to_value(&properties).unwrap(); //! assert_eq!(json["commit.retry.num-retries"], "8"); //! assert_eq!(json["write.data.path"], "s3://warehouse/table/data"); +//! assert!(json.get("write.format.default").is_none()); //! ``` use std::collections::HashMap; @@ -191,7 +193,7 @@ fn serialize_comma_separated_strings(values: &[String]) -> String { /// [Java TableProperties implementation]: https://github.com/apache/iceberg/blob/d8c10a1608170f0ba83be740d6ab0b6a3757cb3e/core/src/main/java/org/apache/iceberg/TableProperties.java /// [Apache Iceberg configuration documentation]: https://github.com/apache/iceberg/blob/d8c10a1608170f0ba83be740d6ab0b6a3757cb3e/docs/docs/configuration.md #[derive(Clone, Debug, Properties)] -pub struct ParsedTableProperties { +pub struct TableProperties { // General properties. #[key = "comment"] #[default(None)] @@ -809,20 +811,29 @@ pub struct ParsedTableProperties { pub encryption_data_key_length: usize, } +impl TryFrom<&HashMap> for TableProperties { + type Error = crate::Error; + + fn try_from(properties: &HashMap) -> Result { + Self::from_properties(properties) + .map_err(|error| crate::Error::new(crate::ErrorKind::DataInvalid, error)) + } +} + #[cfg(test)] mod tests { use super::*; use crate::error::{Error, ErrorKind}; use crate::spec::MappedField; - fn parse(properties: HashMap) -> Result { + fn parse(properties: HashMap) -> Result { serde_json::from_value(serde_json::to_value(properties).unwrap()) .map_err(|error| Error::new(ErrorKind::DataInvalid, error.to_string())) } #[test] fn creates_properties_from_defaults() { - let properties = ParsedTableProperties::default(); + let properties = TableProperties::default(); assert_eq!(properties.commit_retry_num_retries, 4); assert_eq!(properties.write_format_default, DataFileFormat::Parquet); @@ -894,7 +905,7 @@ mod tests { #[test] fn serializes_to_flat_json_object() { - let properties = ParsedTableProperties { + let properties = TableProperties { commit_retry_num_retries: 9, write_format_default: DataFileFormat::Orc, write_data_path: Some("s3://warehouse/table/data".to_string()), @@ -925,137 +936,43 @@ mod tests { }; let json = serde_json::to_value(&properties).unwrap(); - let expected_parts = [ + assert_eq!( + json, serde_json::json!({ - "commit.manifest-merge.enabled": "true", - "commit.manifest.min-count-to-merge": "100", - "commit.manifest.target-size-bytes": "8388608", - "commit.retry.max-wait-ms": "60000", - "commit.retry.min-wait-ms": "100", "commit.retry.num-retries": "9", - "commit.retry.total-timeout-ms": "1800000", - "commit.status-check.max-wait-ms": "60000", - "commit.status-check.min-wait-ms": "1000", - "commit.status-check.num-retries": "3", - "commit.status-check.total-timeout-ms": "1800000", - "compatibility.snapshot-id-inheritance.enabled": "false", - "encryption.data-key-length": "16", - "engine.hive.enabled": "false", - "engine.hive.lock-enabled": "true", - "gc.enabled": "true", - "history.expire.max-ref-age-ms": "9223372036854775807", - "history.expire.max-snapshot-age-ms": "432000000", - "history.expire.min-snapshots-to-keep": "1", - "identifier-fields.rely": "false", - "read.data-planning-mode": "auto", - "read.delete-planning-mode": "auto", - "read.orc.vectorization.batch-size": "5000", - "read.orc.vectorization.enabled": "false", - "read.parquet.vectorization.batch-size": "5000", - "read.parquet.vectorization.enabled": "true", - "read.split.adaptive-size.enabled": "true", - "read.split.metadata-target-size": "33554432", - "read.split.open-file-cost": "4194304", - "read.split.planning-lookback": "10", - "read.split.target-size": "134217728" - }), - serde_json::json!({ "schema.name-mapping.default": r#"[{"field-id":1,"names":["id"]}]"#, - "write.avro.compression-codec": "gzip", - "write.avro.compression-level": "6", "write.data.path": "s3://warehouse/table/data", - "write.datafusion.fanout.enabled": "true", "write.delete.avro.compression-codec": "gzip", "write.delete.avro.compression-level": "4", - "write.delete.distribution-mode": "none", - "write.delete.format.default": "parquet", "write.delete.granularity": "file", "write.delete.isolation-level": "snapshot", "write.delete.mode": "merge-on-read", - "write.delete.orc.block-size-bytes": "268435456", - "write.delete.orc.compression-codec": "zlib", - "write.delete.orc.compression-strategy": "speed", - "write.delete.orc.stripe-size-bytes": "67108864", - "write.delete.orc.vectorized.batch-size": "1024", - "write.delete.parquet.compression-codec": "zstd", - "write.delete.parquet.compression-level": "3", - "write.delete.parquet.dict-size-bytes": "2097152", - "write.delete.parquet.page-row-limit": "20000", - "write.delete.parquet.page-size-bytes": "1048576", - "write.delete.parquet.page-version": "v1", - "write.delete.parquet.row-group-check-max-record-count": "10000", - "write.delete.parquet.row-group-check-min-record-count": "100", - "write.delete.parquet.row-group-size-bytes": "134217728", - "write.delete.target-file-size-bytes": "67108864" - }), - serde_json::json!({ "write.distribution-mode": "range", "write.format.default": "orc", - "write.manifest-lists.enabled": "true", "write.manifest.compression-codec": "gzip", "write.manifest.compression-level": "9", - "write.merge.distribution-mode": "none", - "write.merge.isolation-level": "serializable", - "write.merge.mode": "copy-on-write", - "write.metadata.compression-codec": "none", - "write.metadata.delete-after-commit.enabled": "false", - "write.metadata.metrics.default": "truncate(16)", - "write.metadata.metrics.max-inferred-column-defaults": "100", - "write.metadata.previous-versions-max": "100", - "write.object-storage.enabled": "false", - "write.object-storage.partitioned-paths": "true", - "write.orc.block-size-bytes": "268435456", "write.orc.bloom.filter.columns": "id,category", - "write.orc.bloom.filter.fpp": "0.05", "write.orc.compression-codec": "lzo", - "write.orc.compression-strategy": "speed", - "write.orc.stripe-size-bytes": "67108864", - "write.orc.vectorized.batch-size": "1024" - }), - serde_json::json!({ - "write.parquet.bloom-filter-adaptive-enabled": "false", "write.parquet.bloom-filter-fpp.column.customer_id": "0.02", - "write.parquet.bloom-filter-max-bytes": "1048576", "write.parquet.bloom-filter-ndv.column.customer_id": "1000000", "write.parquet.compression-codec": "zstd", "write.parquet.compression-level": "5", - "write.parquet.content-defined-chunking.enabled": "false", - "write.parquet.content-defined-chunking.max-chunk-size": "1048576", - "write.parquet.content-defined-chunking.min-chunk-size": "262144", - "write.parquet.content-defined-chunking.norm-level": "0", - "write.parquet.dict-size-bytes": "2097152", - "write.parquet.page-row-limit": "20000", - "write.parquet.page-size-bytes": "1048576", - "write.parquet.page-version": "v1", - "write.parquet.row-group-check-max-record-count": "10000", - "write.parquet.row-group-check-min-record-count": "100", - "write.parquet.row-group-size-bytes": "134217728", - "write.parquet.row-group-size-track-uncompressed": "false", - "write.parquet.shred-variants": "false", - "write.parquet.variant-inference-buffer-size": "100", - "write.spark.accept-any-schema": "false", - "write.spark.auto-schema-evolution.enabled": "true", - "write.spark.fanout.enabled": "false", - "write.summary.partition-limit": "0", - "write.target-file-size-bytes": "536870912", - "write.update.distribution-mode": "hash", - "write.update.isolation-level": "serializable", - "write.update.mode": "copy-on-write", - "write.upsert.enabled": "false", - "write.wap.enabled": "false" - }), - ]; - let mut expected = serde_json::Map::new(); - for part in expected_parts { - expected.extend(part.as_object().unwrap().clone()); - } - - assert_eq!(json, serde_json::Value::Object(expected)); + "write.update.distribution-mode": "hash" + }) + ); + } + + #[test] + fn omits_default_values_when_serializing() { + assert_eq!( + serde_json::to_value(TableProperties::default()).unwrap(), + serde_json::json!({}) + ); } #[test] fn deserializes_from_flat_json_object() { - let properties: ParsedTableProperties = serde_json::from_value(serde_json::json!({ + let properties: TableProperties = serde_json::from_value(serde_json::json!({ "commit.retry.num-retries": "8", "write.format.default": "orc", "write.data.path": "s3://warehouse/table/data", diff --git a/crates/iceberg/src/table.rs b/crates/iceberg/src/table.rs index 31feade038..f8ca53bbd3 100644 --- a/crates/iceberg/src/table.rs +++ b/crates/iceberg/src/table.rs @@ -410,7 +410,6 @@ mod tests { use super::*; use crate::encryption::SensitiveBytes; use crate::encryption::kms::MemoryKeyManagementClient; - use crate::spec::TableProperties; fn load_test_metadata(filename: &str) -> TableMetadata { let path = format!( @@ -568,10 +567,9 @@ mod tests { // Encryption is a v3 spec feature; pre-v3 tables silently skip // encryption even if encryption.key-id is set. let mut metadata: TableMetadata = load_test_metadata("TableMetadataV2ValidMinimal.json"); - metadata.properties.insert( - TableProperties::PROPERTY_ENCRYPTION_KEY_ID.to_string(), - "master-1".to_string(), - ); + metadata + .properties + .insert("encryption.key-id".to_string(), "master-1".to_string()); let table = Table::builder() .file_io(FileIO::new_with_memory()) diff --git a/crates/iceberg/src/transaction/expire_snapshots.rs b/crates/iceberg/src/transaction/expire_snapshots.rs index a673b11683..533e270244 100644 --- a/crates/iceberg/src/transaction/expire_snapshots.rs +++ b/crates/iceberg/src/transaction/expire_snapshots.rs @@ -22,7 +22,7 @@ use async_trait::async_trait; use chrono::Utc; use crate::spec::{ - MAIN_BRANCH, ParsedTableProperties, SnapshotReference, SnapshotRetention, TableMetadata, + MAIN_BRANCH, SnapshotReference, SnapshotRetention, TableMetadata, TableProperties, }; use crate::table::Table; use crate::transaction::action::{ActionCommit, TransactionAction}; @@ -99,7 +99,7 @@ impl ExpireSnapshotsAction { } /// Resolves the snapshots and refs to remove, following Java `RemoveSnapshots.internalApply`. - fn plan(&self, table: &Table, properties: &ParsedTableProperties) -> Result { + fn plan(&self, table: &Table, properties: &TableProperties) -> Result { // Matches Java `RemoveSnapshots.retainLast`, which requires at least one snapshot. if self.retain_last == Some(0) { return Err(Error::new( @@ -306,7 +306,7 @@ struct ExpirePlan { impl TransactionAction for ExpireSnapshotsAction { async fn commit(self: Arc, table: &Table) -> Result { let metadata = table.metadata(); - let properties = metadata.parsed_table_properties()?; + let properties = metadata.table_properties()?; // Expiring metadata defeats a user's explicit decision to disable GC (Java refuses too). if !properties.gc_enabled { diff --git a/crates/iceberg/src/transaction/mod.rs b/crates/iceberg/src/transaction/mod.rs index 79afd34233..914a0b3503 100644 --- a/crates/iceberg/src/transaction/mod.rs +++ b/crates/iceberg/src/transaction/mod.rs @@ -70,7 +70,7 @@ use backon::{BackoffBuilder, ExponentialBackoff, ExponentialBuilder, RetryableWi pub use update_schema::AddColumn; use crate::error::Result; -use crate::spec::ParsedTableProperties; +use crate::spec::TableProperties; use crate::table::Table; use crate::transaction::action::BoxedTransactionAction; use crate::transaction::append::FastAppendAction; @@ -178,7 +178,7 @@ impl Transaction { return Ok(self.table); } - let table_props = self.table.metadata().parsed_table_properties()?; + let table_props = self.table.metadata().table_properties()?; // TODO(https://github.com/apache/iceberg-rust/issues/2034): remove once encrypted writes are supported if table_props.encryption_key_id.is_some() { @@ -203,7 +203,7 @@ impl Transaction { .1 } - fn build_backoff(props: ParsedTableProperties) -> Result { + fn build_backoff(props: TableProperties) -> Result { Ok(ExponentialBuilder::new() .with_min_delay(Duration::from_millis(props.commit_retry_min_wait_ms)) .with_max_delay(Duration::from_millis(props.commit_retry_max_wait_ms)) diff --git a/crates/iceberg/src/transaction/snapshot.rs b/crates/iceberg/src/transaction/snapshot.rs index 5685786922..6474e168e0 100644 --- a/crates/iceberg/src/transaction/snapshot.rs +++ b/crates/iceberg/src/transaction/snapshot.rs @@ -28,7 +28,7 @@ use crate::spec::{ DataFile, DataFileFormat, FormatVersion, MAIN_BRANCH, ManifestContentType, ManifestEntry, ManifestFile, ManifestListWriter, ManifestWriter, ManifestWriterBuilder, Operation, Snapshot, SnapshotReference, SnapshotRetention, SnapshotSummaryCollector, Struct, StructType, Summary, - TableProperties, update_snapshot_summaries, + update_snapshot_summaries, }; use crate::table::Table; use crate::transaction::ActionCommit; @@ -374,18 +374,9 @@ impl<'a> SnapshotProducer<'a> { let mut summary_collector = SnapshotSummaryCollector::default(); let table_metadata = self.table.metadata_ref(); - let partition_summary_limit = if let Some(limit) = table_metadata - .properties() - .get(TableProperties::PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT) - { - if let Ok(limit) = limit.parse::() { - limit - } else { - TableProperties::PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT_DEFAULT - } - } else { - TableProperties::PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT_DEFAULT - }; + let partition_summary_limit = table_metadata + .table_properties()? + .write_summary_partition_limit; summary_collector.set_partition_summary_limit(partition_summary_limit); diff --git a/crates/iceberg/src/writer/file_writer/parquet_writer.rs b/crates/iceberg/src/writer/file_writer/parquet_writer.rs index 444052bde2..bae1cfca7b 100644 --- a/crates/iceberg/src/writer/file_writer/parquet_writer.rs +++ b/crates/iceberg/src/writer/file_writer/parquet_writer.rs @@ -39,9 +39,8 @@ use crate::arrow::{ use crate::io::{FileIO, FileWrite, OutputFile}; use crate::spec::{ DataContentType, DataFileBuilder, DataFileFormat, Datum, ListType, Literal, MapType, - NestedFieldRef, ParsedTableProperties, PartitionSpec, PrimitiveType, Schema, SchemaRef, - SchemaVisitor, Struct, StructType, TableMetadata, TableProperties, Type, VariantType, - visit_schema, + NestedFieldRef, PartitionSpec, PrimitiveType, Schema, SchemaRef, SchemaVisitor, Struct, + StructType, TableMetadata, TableProperties, Type, VariantType, visit_schema, }; use crate::transform::create_transform_function; use crate::writer::{CurrentFileStatus, DataFile}; @@ -87,19 +86,6 @@ impl ParquetWriterBuilder { /// (`write.parquet.content-defined-chunking.*`); other keys fall back to /// parquet-rs defaults. pub fn from_table_properties(table_props: &TableProperties, schema: SchemaRef) -> Self { - let cdc = table_props.cdc_enabled.then_some(CdcOptions { - min_chunk_size: table_props.cdc_min_chunk_size, - max_chunk_size: table_props.cdc_max_chunk_size, - norm_level: table_props.cdc_norm_level, - }); - Self::from_cdc_options(cdc, schema) - } - - /// Build a `ParquetWriterBuilder` from properties parsed by the property framework. - pub fn from_parsed_table_properties( - table_props: &ParsedTableProperties, - schema: SchemaRef, - ) -> Self { let cdc = table_props .write_parquet_content_defined_chunking_enabled .then_some(CdcOptions { @@ -2379,20 +2365,13 @@ mod tests { ) } - fn table_props(entries: HashMap) -> ParsedTableProperties { - serde_json::from_value(serde_json::to_value(entries).unwrap()).unwrap() + fn table_props(entries: HashMap) -> TableProperties { + TableProperties::try_from(&entries).unwrap() } #[test] fn test_from_table_properties_no_cdc_by_default() { let tp = table_props(HashMap::new()); - let builder = ParquetWriterBuilder::from_parsed_table_properties(&tp, cdc_test_schema()); - assert!(builder.props.content_defined_chunking().is_none()); - } - - #[test] - fn test_existing_table_properties_constructor() { - let tp = TableProperties::try_from(&HashMap::new()).unwrap(); let builder = ParquetWriterBuilder::from_table_properties(&tp, cdc_test_schema()); assert!(builder.props.content_defined_chunking().is_none()); } @@ -2401,7 +2380,7 @@ mod tests { async fn test_from_table_properties_propagate_to_writer() { // `build()` must carry the translated `WriterProperties` through to the // `ParquetWriter` unchanged — otherwise the `write.parquet.*` settings - // derived in `from_parsed_table_properties` would never reach parquet-rs. + // derived in `from_table_properties` would never reach parquet-rs. // // Asserting on the writer's `WriterProperties` (rather than re-reading a // written file) keeps this a direct propagation check: every future @@ -2409,19 +2388,19 @@ mod tests { // `WriterProperties` getter here. let tp = table_props(HashMap::from([ ( - TableProperties::PROPERTY_PARQUET_CDC_ENABLED.to_string(), + "write.parquet.content-defined-chunking.enabled".to_string(), "true".to_string(), ), ( - TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE.to_string(), + "write.parquet.content-defined-chunking.min-chunk-size".to_string(), "4096".to_string(), ), ( - TableProperties::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE.to_string(), + "write.parquet.content-defined-chunking.max-chunk-size".to_string(), "8192".to_string(), ), ( - TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL.to_string(), + "write.parquet.content-defined-chunking.norm-level".to_string(), "2".to_string(), ), ])); @@ -2430,7 +2409,7 @@ mod tests { let output = FileIO::new_with_fs() .new_output(format!("{}/cdc.parquet", tmp.path().to_str().unwrap())) .unwrap(); - let writer = ParquetWriterBuilder::from_parsed_table_properties(&tp, cdc_test_schema()) + let writer = ParquetWriterBuilder::from_table_properties(&tp, cdc_test_schema()) .build(output) .await .unwrap(); diff --git a/crates/iceberg/src/writer/file_writer/rolling_writer.rs b/crates/iceberg/src/writer/file_writer/rolling_writer.rs index b0b2d2f191..d1e9147830 100644 --- a/crates/iceberg/src/writer/file_writer/rolling_writer.rs +++ b/crates/iceberg/src/writer/file_writer/rolling_writer.rs @@ -95,7 +95,7 @@ where ) -> Self { Self { inner_builder, - target_file_size: TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT, + target_file_size: TableProperties::default().write_target_file_size_bytes, file_io, location_generator, file_name_generator, diff --git a/crates/integrations/datafusion/src/physical_plan/write.rs b/crates/integrations/datafusion/src/physical_plan/write.rs index a3d4de3cc3..d4d80e9719 100644 --- a/crates/integrations/datafusion/src/physical_plan/write.rs +++ b/crates/integrations/datafusion/src/physical_plan/write.rs @@ -205,7 +205,7 @@ impl ExecutionPlan for IcebergWriteExec { let table_props = self .table .metadata() - .parsed_table_properties() + .table_properties() .map_err(to_datafusion_error)?; // Check data file format @@ -220,7 +220,7 @@ impl ExecutionPlan for IcebergWriteExec { // Build the writer from the already-parsed table properties so it honors // `write.parquet.*` settings (e.g. CDC). Arrow batches flowing through // DataFusion carry no field-id metadata, so match fields by name. - let parquet_file_writer_builder = ParquetWriterBuilder::from_parsed_table_properties( + let parquet_file_writer_builder = ParquetWriterBuilder::from_table_properties( &table_props, self.table.metadata().current_schema().clone(), ) diff --git a/crates/integrations/datafusion/src/table/mod.rs b/crates/integrations/datafusion/src/table/mod.rs index 2fd958dff4..3940bec72f 100644 --- a/crates/integrations/datafusion/src/table/mod.rs +++ b/crates/integrations/datafusion/src/table/mod.rs @@ -43,7 +43,6 @@ use datafusion::physical_plan::ExecutionPlan; use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; use iceberg::arrow::schema_to_arrow_schema; use iceberg::inspect::MetadataTableType; -use iceberg::spec::TableProperties; use iceberg::table::Table; use iceberg::{Catalog, Error, ErrorKind, NamespaceIdent, Result, TableIdent}; use metadata_table::IcebergMetadataTableProvider; @@ -192,25 +191,9 @@ impl TableProvider for IcebergTableProvider { // Apply sort node when it's not fanout mode let fanout_enabled = table .metadata() - .properties() - .get(TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED) - .map(|value| { - value - .parse::() - .map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Invalid value for {}, expected 'true' or 'false'", - TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED - ), - ) - .with_source(e) - }) - .map_err(to_datafusion_error) - }) - .transpose()? - .unwrap_or(TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT); + .table_properties() + .map_err(to_datafusion_error)? + .write_datafusion_fanout_enabled; let write_input = if fanout_enabled { repartitioned_plan @@ -667,7 +650,7 @@ mod tests { let mut properties = HashMap::new(); if let Some(enabled) = fanout_enabled { properties.insert( - TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED.to_string(), + "write.datafusion.fanout.enabled".to_string(), enabled.to_string(), ); } diff --git a/crates/property-macro/src/lib.rs b/crates/property-macro/src/lib.rs index b2ed947ec3..1e3b828794 100644 --- a/crates/property-macro/src/lib.rs +++ b/crates/property-macro/src/lib.rs @@ -52,7 +52,8 @@ use syn::{ /// for fields represented by more than one key. `additional_key` declares a second key and passes /// it to those hooks after the primary key. /// Optional fields are omitted from JSON when they are `None`. Fields need `FromStr` and `ToString` -/// unless the relevant custom parsing or serialization attribute is supplied. String-literal and +/// unless the relevant custom parsing or serialization attribute is supplied. Leaf fields also +/// need `PartialEq` so values equal to their defaults can be omitted from JSON. String-literal and /// path defaults are converted into their field type with `Into`. #[proc_macro_derive( Properties, @@ -353,10 +354,15 @@ fn parse_field(field: &PropertyField) -> TokenStream2 { return quote!(#ident: <#ty>::from_properties(properties)?); } + let ty = &field.ty; let default = default_value( field.default.as_ref().expect("leaf fields have defaults"), - &field.ty, + ty, ); + let default = quote!({ + let value: #ty = #default; + value + }); if let Some(parse_properties_with) = &field.parse_properties_with { let key = field.key.as_ref().expect("exact-key fields have a key"); @@ -492,6 +498,16 @@ fn write_field(field: &PropertyField) -> TokenStream2 { }; } + let ty = &field.ty; + let default = default_value( + field.default.as_ref().expect("leaf fields have defaults"), + ty, + ); + let default = quote!({ + let value: #ty = #default; + value + }); + if let Some(write_properties_with) = &field.write_properties_with { let key = field.key.as_ref().expect("exact-key fields have a key"); let write = match &field.additional_key { @@ -501,15 +517,19 @@ fn write_field(field: &PropertyField) -> TokenStream2 { None => quote!(#write_properties_with(&self.#ident, properties, #key)), }; return quote! { - #write; + if self.#ident != #default { + #write; + } }; } if let Some(prefix) = &field.prefix { return quote! { - for (suffix, value) in &self.#ident { - let key = format!("{}{}", #prefix, suffix); - properties.insert(key, ::std::string::ToString::to_string(value)); + if self.#ident != #default { + for (suffix, value) in &self.#ident { + let key = format!("{}{}", #prefix, suffix); + properties.insert(key, ::std::string::ToString::to_string(value)); + } } }; } @@ -523,7 +543,7 @@ fn write_field(field: &PropertyField) -> TokenStream2 { )), }; quote! { - if self.#ident.is_some() { + if self.#ident != #default && self.#ident.is_some() { properties.insert((#key).to_string(), #value); } } @@ -533,7 +553,9 @@ fn write_field(field: &PropertyField) -> TokenStream2 { None => quote!(::std::string::ToString::to_string(&self.#ident)), }; quote! { - properties.insert((#key).to_string(), #value); + if self.#ident != #default { + properties.insert((#key).to_string(), #value); + } } } } diff --git a/crates/property-macro/tests/properties.rs b/crates/property-macro/tests/properties.rs index 4609149657..088e191d3c 100644 --- a/crates/property-macro/tests/properties.rs +++ b/crates/property-macro/tests/properties.rs @@ -82,7 +82,12 @@ struct TestProperties { #[test] fn generates_defaults_and_serde_for_public_fields() { - assert_eq!(TestProperties::default().format, "parquet"); + let defaults = TestProperties::default(); + assert_eq!(defaults.format, "parquet"); + assert_eq!( + serde_json::to_value(&defaults).unwrap(), + serde_json::json!({}) + ); let properties = TestProperties { retries: 8, @@ -131,6 +136,11 @@ struct NestedProperties { #[test] fn nested_properties_use_a_flat_property_map() { + assert_eq!( + serde_json::to_value(NestedProperties::default()).unwrap(), + serde_json::json!({}) + ); + let mut properties = NestedProperties::default(); properties.commit.num_retries = 9; From 959093d744580f7133fe60976634463776d434ee Mon Sep 17 00:00:00 2001 From: Renjie Liu Date: Wed, 5 Aug 2026 19:34:18 +0800 Subject: [PATCH 12/14] Revert unnecessary changes --- crates/iceberg/public-api.txt | 23 +++++ .../iceberg/src/catalog/metadata_location.rs | 6 +- crates/iceberg/src/spec/table_metadata.rs | 29 +++++-- .../src/spec/table_metadata_builder.rs | 31 ++----- crates/iceberg/src/spec/table_props.rs | 83 +++++++++++++++++++ crates/iceberg/src/table.rs | 8 +- crates/iceberg/src/transaction/snapshot.rs | 17 +++- .../writer/file_writer/location_generator.rs | 18 ++-- .../src/writer/file_writer/parquet_writer.rs | 14 ++-- .../src/writer/file_writer/rolling_writer.rs | 2 +- .../integrations/datafusion/src/table/mod.rs | 25 +++++- 11 files changed, 192 insertions(+), 64 deletions(-) diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index 2a85199b9b..59ae06d763 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -2973,6 +2973,29 @@ pub iceberg::spec::TableProperties::write_update_isolation_level: iceberg::spec: pub iceberg::spec::TableProperties::write_update_mode: iceberg::spec::RowLevelOperationMode pub iceberg::spec::TableProperties::write_upsert_enabled: bool pub iceberg::spec::TableProperties::write_wap_enabled: bool +impl iceberg::spec::TableProperties +pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_NUM_RETRIES: &str +pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT: usize +pub const iceberg::spec::TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED: &str +pub const iceberg::spec::TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT: bool +pub const iceberg::spec::TableProperties::PROPERTY_ENCRYPTION_KEY_ID: &str +pub const iceberg::spec::TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC: &str +pub const iceberg::spec::TableProperties::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX: &str +pub const iceberg::spec::TableProperties::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX_DEFAULT: usize +pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_ENABLED: &str +pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE: &str +pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE: &str +pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL: &str +pub const iceberg::spec::TableProperties::PROPERTY_WRITE_DATA_LOCATION: &str +pub const iceberg::spec::TableProperties::PROPERTY_WRITE_FOLDER_STORAGE_LOCATION: &str +pub const iceberg::spec::TableProperties::PROPERTY_WRITE_METADATA_PATH: &str +pub const iceberg::spec::TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_LOCATION: &str +pub const iceberg::spec::TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS: &str +pub const iceberg::spec::TableProperties::PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT: &str +pub const iceberg::spec::TableProperties::PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT_DEFAULT: u64 +pub const iceberg::spec::TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES: &str +pub const iceberg::spec::TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT: usize +pub const iceberg::spec::TableProperties::RESERVED_PROPERTIES: [&str; 9] impl core::clone::Clone for iceberg::spec::TableProperties pub fn iceberg::spec::TableProperties::clone(&self) -> iceberg::spec::TableProperties impl core::convert::TryFrom<&std::collections::hash::map::HashMap> for iceberg::spec::TableProperties diff --git a/crates/iceberg/src/catalog/metadata_location.rs b/crates/iceberg/src/catalog/metadata_location.rs index 2ab6a499e9..5f1af2f42b 100644 --- a/crates/iceberg/src/catalog/metadata_location.rs +++ b/crates/iceberg/src/catalog/metadata_location.rs @@ -160,7 +160,7 @@ mod test { use uuid::Uuid; use crate::compression::CompressionCodec; - use crate::spec::{Schema, TableMetadata, TableMetadataBuilder}; + use crate::spec::{Schema, TableMetadata, TableMetadataBuilder, TableProperties}; use crate::{MetadataLocation, TableCreation}; fn create_test_metadata(properties: HashMap) -> TableMetadata { @@ -425,7 +425,7 @@ mod test { // A configured `write.metadata.path` is honored on updates too let props = HashMap::from([( - "write.metadata.path".to_string(), + TableProperties::PROPERTY_WRITE_METADATA_PATH.to_string(), "s3://bucket/custom-meta".to_string(), )]); let with_meta_path = create_test_metadata(props); @@ -450,7 +450,7 @@ mod test { // Test a configured `write.metadata.path` is honored let props = HashMap::from([( - "write.metadata.path".to_string(), + TableProperties::PROPERTY_WRITE_METADATA_PATH.to_string(), "s3://bucket/custom-meta".to_string(), )]); let custom_meta = create_test_metadata(props); diff --git a/crates/iceberg/src/spec/table_metadata.rs b/crates/iceberg/src/spec/table_metadata.rs index 34f299a081..3a6d94e263 100644 --- a/crates/iceberg/src/spec/table_metadata.rs +++ b/crates/iceberg/src/spec/table_metadata.rs @@ -1638,7 +1638,7 @@ mod tests { BlobMetadata, EncryptedKey, INITIAL_ROW_ID, Literal, NestedField, NullOrder, Operation, PartitionSpec, PartitionStatisticsFile, PrimitiveLiteral, PrimitiveType, Schema, Snapshot, SnapshotReference, SnapshotRetention, SortDirection, SortField, SortOrder, StatisticsFile, - Summary, Transform, Type, UnboundPartitionField, + Summary, TableProperties, Transform, Type, UnboundPartitionField, }; use crate::{ErrorKind, TableCreation}; @@ -3705,7 +3705,7 @@ mod tests { // Modify properties to enable gzip compression (using mixed case to test case-insensitive matching) let mut props = original_metadata.properties.clone(); props.insert( - "write.metadata.compression-codec".to_string(), + TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), "GziP".to_string(), ); // Use builder to create new metadata with updated properties @@ -4018,6 +4018,8 @@ mod tests { #[test] fn test_table_properties_with_defaults() { + use crate::spec::TableProperties; + let schema = Schema::builder() .with_fields(vec![ NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(), @@ -4040,12 +4042,20 @@ mod tests { let props = metadata.table_properties().unwrap(); - assert_eq!(props.commit_retry_num_retries, 4); - assert_eq!(props.write_target_file_size_bytes, 512 * 1024 * 1024); + assert_eq!( + props.commit_retry_num_retries, + TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT + ); + assert_eq!( + props.write_target_file_size_bytes, + TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT + ); } #[test] fn test_table_properties_with_custom_values() { + use crate::spec::TableProperties; + let schema = Schema::builder() .with_fields(vec![ NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(), @@ -4054,9 +4064,12 @@ mod tests { .unwrap(); let properties = HashMap::from([ - ("commit.retry.num-retries".to_string(), "10".to_string()), ( - "write.target-file-size-bytes".to_string(), + TableProperties::PROPERTY_COMMIT_NUM_RETRIES.to_string(), + "10".to_string(), + ), + ( + TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES.to_string(), "1024".to_string(), ), ]); @@ -4319,7 +4332,7 @@ mod tests { let metadata = get_test_table_metadata("TableMetadataV2Valid.json") .into_builder(None) .set_properties(HashMap::from([( - "write.metadata.path".to_string(), + TableProperties::PROPERTY_WRITE_METADATA_PATH.to_string(), "s3://other-bucket/custom-meta".to_string(), )])) .unwrap() @@ -4337,7 +4350,7 @@ mod tests { let metadata = get_test_table_metadata("TableMetadataV2Valid.json") .into_builder(None) .set_properties(HashMap::from([( - "write.metadata.path".to_string(), + TableProperties::PROPERTY_WRITE_METADATA_PATH.to_string(), "s3://other-bucket/custom-meta/".to_string(), )])) .unwrap() diff --git a/crates/iceberg/src/spec/table_metadata_builder.rs b/crates/iceberg/src/spec/table_metadata_builder.rs index 1bfb121ad1..3191d6c13c 100644 --- a/crates/iceberg/src/spec/table_metadata_builder.rs +++ b/crates/iceberg/src/spec/table_metadata_builder.rs @@ -24,7 +24,7 @@ use super::{ DEFAULT_PARTITION_SPEC_ID, DEFAULT_SCHEMA_ID, FormatVersion, MAIN_BRANCH, MetadataLog, ONE_MINUTE_MS, PartitionSpec, PartitionSpecBuilder, PartitionStatisticsFile, Schema, SchemaRef, Snapshot, SnapshotLog, SnapshotReference, SnapshotRetention, SortOrder, SortOrderRef, - StatisticsFile, StructType, TableMetadata, UNPARTITIONED_LAST_ASSIGNED_ID, + StatisticsFile, StructType, TableMetadata, TableProperties, UNPARTITIONED_LAST_ASSIGNED_ID, UnboundPartitionSpec, }; use crate::error::{Error, ErrorKind, Result}; @@ -33,21 +33,6 @@ use crate::{TableCreation, TableUpdate}; pub(crate) const FIRST_FIELD_ID: i32 = 1; -fn is_reserved_property(key: &str) -> bool { - matches!( - key, - "format-version" - | "uuid" - | "snapshot-count" - | "current-snapshot-id" - | "current-snapshot-summary" - | "current-snapshot-timestamp-ms" - | "current-schema" - | "default-partition-spec" - | "default-sort-order" - ) -} - /// Manipulating table metadata. /// /// For this builder the order of called functions matters. Functions are applied in-order. @@ -271,7 +256,7 @@ impl TableMetadataBuilder { // List of specified properties that are RESERVED and should not be persisted. let reserved_properties = properties .keys() - .filter(|key| is_reserved_property(key)) + .filter(|key| TableProperties::RESERVED_PROPERTIES.contains(&key.as_str())) .map(ToString::to_string) .collect::>(); @@ -309,7 +294,7 @@ impl TableMetadataBuilder { // disallow removal of reserved properties let reserved_properties = properties .iter() - .filter(|key| is_reserved_property(key)) + .filter(|key| TableProperties::RESERVED_PROPERTIES.contains(&key.as_str())) .map(ToString::to_string) .collect::>(); @@ -1186,9 +1171,9 @@ impl TableMetadataBuilder { let max_size = self .metadata .properties - .get("write.metadata.previous-versions-max") + .get(TableProperties::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX) .and_then(|v| v.parse::().ok()) - .unwrap_or(100) + .unwrap_or(TableProperties::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX_DEFAULT) .max(1); if self.metadata.metadata_log.len() > max_size { @@ -1484,8 +1469,8 @@ mod tests { use crate::io::FileIO; use crate::spec::{ BlobMetadata, NestedField, NullOrder, Operation, PartitionSpec, PrimitiveType, Schema, - SnapshotRetention, SortDirection, SortField, StructType, Summary, Transform, Type, - UnboundPartitionField, + SnapshotRetention, SortDirection, SortField, StructType, Summary, TableProperties, + Transform, Type, UnboundPartitionField, }; use crate::table::Table; use crate::test_utils::test_runtime; @@ -2491,7 +2476,7 @@ mod tests { let builder = builder_without_changes(FormatVersion::V2); let metadata = builder .set_properties(HashMap::from_iter(vec![( - "write.metadata.previous-versions-max".to_string(), + TableProperties::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX.to_string(), "2".to_string(), )])) .unwrap() diff --git a/crates/iceberg/src/spec/table_props.rs b/crates/iceberg/src/spec/table_props.rs index 8bcf9034ba..251b37d3ac 100644 --- a/crates/iceberg/src/spec/table_props.rs +++ b/crates/iceberg/src/spec/table_props.rs @@ -811,6 +811,89 @@ pub struct TableProperties { pub encryption_data_key_length: usize, } +impl TableProperties { + /// Property key for the number of commit retries. + pub const PROPERTY_COMMIT_NUM_RETRIES: &str = "commit.retry.num-retries"; + + /// Default number of commit retries. + pub const PROPERTY_COMMIT_NUM_RETRIES_DEFAULT: usize = 4; + + /// Property key for enabling the DataFusion fanout writer. + pub const PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED: &str = "write.datafusion.fanout.enabled"; + + /// Default value for enabling the DataFusion fanout writer. + pub const PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT: bool = true; + + /// Property key for the table encryption key identifier. + pub const PROPERTY_ENCRYPTION_KEY_ID: &str = "encryption.key-id"; + + /// Property key for the metadata compression codec. + pub const PROPERTY_METADATA_COMPRESSION_CODEC: &str = "write.metadata.compression-codec"; + + /// Property key for the maximum number of previous metadata versions to keep. + pub const PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX: &str = + "write.metadata.previous-versions-max"; + + /// Default maximum number of previous metadata versions to keep. + pub const PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX_DEFAULT: usize = 100; + + /// Property key for enabling Parquet content-defined chunking. + pub const PROPERTY_PARQUET_CDC_ENABLED: &str = "write.parquet.content-defined-chunking.enabled"; + + /// Property key for the maximum Parquet content-defined chunk size. + pub const PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE: &str = + "write.parquet.content-defined-chunking.max-chunk-size"; + + /// Property key for the minimum Parquet content-defined chunk size. + pub const PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE: &str = + "write.parquet.content-defined-chunking.min-chunk-size"; + + /// Property key for the Parquet content-defined chunking normalization level. + pub const PROPERTY_PARQUET_CDC_NORM_LEVEL: &str = + "write.parquet.content-defined-chunking.norm-level"; + + /// Property key for the base data-file location. + pub const PROPERTY_WRITE_DATA_LOCATION: &str = "write.data.path"; + + /// Property key for the deprecated folder-storage location. + pub const PROPERTY_WRITE_FOLDER_STORAGE_LOCATION: &str = "write.folder-storage.path"; + + /// Property key for the base metadata-file location. + pub const PROPERTY_WRITE_METADATA_PATH: &str = "write.metadata.path"; + + /// Property key for the deprecated object-storage location. + pub const PROPERTY_WRITE_OBJECT_STORAGE_LOCATION: &str = "write.object-storage.path"; + + /// Property key for including partition values in object-storage paths. + pub const PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS: &str = + "write.object-storage.partitioned-paths"; + + /// Property key for the snapshot-summary partition limit. + pub const PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT: &str = "write.summary.partition-limit"; + + /// Default snapshot-summary partition limit. + pub const PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT_DEFAULT: u64 = 0; + + /// Property key for the target data-file size. + pub const PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES: &str = "write.target-file-size-bytes"; + + /// Default target data-file size. + pub const PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT: usize = 512 * 1024 * 1024; + + /// Reserved table properties that must not be persisted in table metadata. + pub const RESERVED_PROPERTIES: [&str; 9] = [ + "format-version", + "uuid", + "snapshot-count", + "current-snapshot-id", + "current-snapshot-summary", + "current-snapshot-timestamp-ms", + "current-schema", + "default-partition-spec", + "default-sort-order", + ]; +} + impl TryFrom<&HashMap> for TableProperties { type Error = crate::Error; diff --git a/crates/iceberg/src/table.rs b/crates/iceberg/src/table.rs index f8ca53bbd3..31feade038 100644 --- a/crates/iceberg/src/table.rs +++ b/crates/iceberg/src/table.rs @@ -410,6 +410,7 @@ mod tests { use super::*; use crate::encryption::SensitiveBytes; use crate::encryption::kms::MemoryKeyManagementClient; + use crate::spec::TableProperties; fn load_test_metadata(filename: &str) -> TableMetadata { let path = format!( @@ -567,9 +568,10 @@ mod tests { // Encryption is a v3 spec feature; pre-v3 tables silently skip // encryption even if encryption.key-id is set. let mut metadata: TableMetadata = load_test_metadata("TableMetadataV2ValidMinimal.json"); - metadata - .properties - .insert("encryption.key-id".to_string(), "master-1".to_string()); + metadata.properties.insert( + TableProperties::PROPERTY_ENCRYPTION_KEY_ID.to_string(), + "master-1".to_string(), + ); let table = Table::builder() .file_io(FileIO::new_with_memory()) diff --git a/crates/iceberg/src/transaction/snapshot.rs b/crates/iceberg/src/transaction/snapshot.rs index 7cecb51cd3..b4f1946a7c 100644 --- a/crates/iceberg/src/transaction/snapshot.rs +++ b/crates/iceberg/src/transaction/snapshot.rs @@ -28,7 +28,7 @@ use crate::spec::{ DataFile, DataFileFormat, FormatVersion, MAIN_BRANCH, ManifestContentType, ManifestEntry, ManifestFile, ManifestListWriter, ManifestWriter, ManifestWriterBuilder, Operation, Snapshot, SnapshotReference, SnapshotRetention, SnapshotSummaryCollector, Struct, StructType, Summary, - update_snapshot_summaries, + TableProperties, update_snapshot_summaries, }; use crate::table::Table; use crate::transaction::ActionCommit; @@ -383,9 +383,18 @@ impl<'a> SnapshotProducer<'a> { let mut summary_collector = SnapshotSummaryCollector::default(); let table_metadata = self.table.metadata_ref(); - let partition_summary_limit = table_metadata - .table_properties()? - .write_summary_partition_limit; + let partition_summary_limit = if let Some(limit) = table_metadata + .properties() + .get(TableProperties::PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT) + { + if let Ok(limit) = limit.parse::() { + limit + } else { + TableProperties::PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT_DEFAULT + } + } else { + TableProperties::PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT_DEFAULT + }; summary_collector.set_partition_summary_limit(partition_summary_limit); diff --git a/crates/iceberg/src/writer/file_writer/location_generator.rs b/crates/iceberg/src/writer/file_writer/location_generator.rs index 838233e66a..e3ef61ef59 100644 --- a/crates/iceberg/src/writer/file_writer/location_generator.rs +++ b/crates/iceberg/src/writer/file_writer/location_generator.rs @@ -291,7 +291,7 @@ pub(crate) mod test { use super::LocationGenerator; use crate::spec::{ FormatVersion, Literal, NestedField, PartitionKey, PartitionSpec, PrimitiveType, Schema, - Struct, StructType, TableMetadata, Transform, Type, + Struct, StructType, TableMetadata, TableProperties, Transform, Type, }; use crate::writer::file_writer::location_generator::{ DefaultLocationGenerator, FileNameGenerator, ObjectStorageLocationGenerator, @@ -315,7 +315,7 @@ pub(crate) mod test { // test custom data location table_metadata.properties.insert( - "write.folder-storage.path".to_string(), + TableProperties::PROPERTY_WRITE_FOLDER_STORAGE_LOCATION.to_string(), "s3://data.db/table/data_1".to_string(), ); let location_generator = DefaultLocationGenerator::new(&table_metadata).unwrap(); @@ -327,7 +327,7 @@ pub(crate) mod test { ); table_metadata.properties.insert( - "write.data.path".to_string(), + TableProperties::PROPERTY_WRITE_DATA_LOCATION.to_string(), "s3://data.db/table/data_2".to_string(), ); let location_generator = DefaultLocationGenerator::new(&table_metadata).unwrap(); @@ -339,7 +339,7 @@ pub(crate) mod test { ); table_metadata.properties.insert( - "write.data.path".to_string(), + TableProperties::PROPERTY_WRITE_DATA_LOCATION.to_string(), // invalid table location "s3://data.db/data_3".to_string(), ); @@ -506,7 +506,7 @@ pub(crate) mod test { let table_metadata = table_metadata_with( "s3://data.db/table", HashMap::from([( - "write.object-storage.partitioned-paths".to_string(), + TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS.to_string(), "true".to_string(), )]), ); @@ -545,7 +545,7 @@ pub(crate) mod test { let table_metadata = table_metadata_with( "s3://data.db/table", HashMap::from([( - "write.object-storage.partitioned-paths".to_string(), + TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS.to_string(), "false".to_string(), )]), ); @@ -565,7 +565,7 @@ pub(crate) mod test { let table_metadata = table_metadata_with( "s3://data.db/table", HashMap::from([( - "write.data.path".to_string(), + TableProperties::PROPERTY_WRITE_DATA_LOCATION.to_string(), "s3://custom-bucket/objects".to_string(), )]), ); @@ -586,11 +586,11 @@ pub(crate) mod test { "s3://data.db/table", HashMap::from([ ( - "write.data.path".to_string(), + TableProperties::PROPERTY_WRITE_DATA_LOCATION.to_string(), "s3://data.db/table/data_primary".to_string(), ), ( - "write.folder-storage.path".to_string(), + TableProperties::PROPERTY_WRITE_FOLDER_STORAGE_LOCATION.to_string(), "s3://data.db/table/data_legacy".to_string(), ), ]), diff --git a/crates/iceberg/src/writer/file_writer/parquet_writer.rs b/crates/iceberg/src/writer/file_writer/parquet_writer.rs index bae1cfca7b..e302fcb51a 100644 --- a/crates/iceberg/src/writer/file_writer/parquet_writer.rs +++ b/crates/iceberg/src/writer/file_writer/parquet_writer.rs @@ -93,10 +93,6 @@ impl ParquetWriterBuilder { max_chunk_size: table_props.write_parquet_content_defined_chunking_max_chunk_size, norm_level: table_props.write_parquet_content_defined_chunking_norm_level, }); - Self::from_cdc_options(cdc, schema) - } - - fn from_cdc_options(cdc: Option, schema: SchemaRef) -> Self { // TODO: translate the remaining write.parquet.* keys (e.g. compression-codec, // row-group-size-bytes, page-size-bytes). // This constructor is intended to be the single place that maps them. @@ -2348,7 +2344,7 @@ mod tests { } // ----------------------------------------------------------------- - // ParquetWriterBuilder property constructors + // ParquetWriterBuilder::from_table_properties // ----------------------------------------------------------------- fn cdc_test_schema() -> SchemaRef { @@ -2388,19 +2384,19 @@ mod tests { // `WriterProperties` getter here. let tp = table_props(HashMap::from([ ( - "write.parquet.content-defined-chunking.enabled".to_string(), + TableProperties::PROPERTY_PARQUET_CDC_ENABLED.to_string(), "true".to_string(), ), ( - "write.parquet.content-defined-chunking.min-chunk-size".to_string(), + TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE.to_string(), "4096".to_string(), ), ( - "write.parquet.content-defined-chunking.max-chunk-size".to_string(), + TableProperties::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE.to_string(), "8192".to_string(), ), ( - "write.parquet.content-defined-chunking.norm-level".to_string(), + TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL.to_string(), "2".to_string(), ), ])); diff --git a/crates/iceberg/src/writer/file_writer/rolling_writer.rs b/crates/iceberg/src/writer/file_writer/rolling_writer.rs index d1e9147830..b0b2d2f191 100644 --- a/crates/iceberg/src/writer/file_writer/rolling_writer.rs +++ b/crates/iceberg/src/writer/file_writer/rolling_writer.rs @@ -95,7 +95,7 @@ where ) -> Self { Self { inner_builder, - target_file_size: TableProperties::default().write_target_file_size_bytes, + target_file_size: TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT, file_io, location_generator, file_name_generator, diff --git a/crates/integrations/datafusion/src/table/mod.rs b/crates/integrations/datafusion/src/table/mod.rs index 3940bec72f..2fd958dff4 100644 --- a/crates/integrations/datafusion/src/table/mod.rs +++ b/crates/integrations/datafusion/src/table/mod.rs @@ -43,6 +43,7 @@ use datafusion::physical_plan::ExecutionPlan; use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; use iceberg::arrow::schema_to_arrow_schema; use iceberg::inspect::MetadataTableType; +use iceberg::spec::TableProperties; use iceberg::table::Table; use iceberg::{Catalog, Error, ErrorKind, NamespaceIdent, Result, TableIdent}; use metadata_table::IcebergMetadataTableProvider; @@ -191,9 +192,25 @@ impl TableProvider for IcebergTableProvider { // Apply sort node when it's not fanout mode let fanout_enabled = table .metadata() - .table_properties() - .map_err(to_datafusion_error)? - .write_datafusion_fanout_enabled; + .properties() + .get(TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED) + .map(|value| { + value + .parse::() + .map_err(|e| { + Error::new( + ErrorKind::DataInvalid, + format!( + "Invalid value for {}, expected 'true' or 'false'", + TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED + ), + ) + .with_source(e) + }) + .map_err(to_datafusion_error) + }) + .transpose()? + .unwrap_or(TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT); let write_input = if fanout_enabled { repartitioned_plan @@ -650,7 +667,7 @@ mod tests { let mut properties = HashMap::new(); if let Some(enabled) = fanout_enabled { properties.insert( - "write.datafusion.fanout.enabled".to_string(), + TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED.to_string(), enabled.to_string(), ); } From a63438fc8423edfd0488acdca4882408ed51b154 Mon Sep 17 00:00:00 2001 From: Renjie Liu Date: Wed, 5 Aug 2026 20:09:36 +0800 Subject: [PATCH 13/14] Fix Cargo manifest formatting --- crates/iceberg/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/iceberg/Cargo.toml b/crates/iceberg/Cargo.toml index e95ba4ba8a..bf9c19bb63 100644 --- a/crates/iceberg/Cargo.toml +++ b/crates/iceberg/Cargo.toml @@ -59,8 +59,8 @@ flate2 = { workspace = true } fnv = { workspace = true } form_urlencoded = { workspace = true } futures = { workspace = true } -itertools = { workspace = true } iceberg-property-macro = { version = "0.10.0", path = "../property-macro" } +itertools = { workspace = true } moka = { version = "0.12.10", features = ["future"] } murmur3 = { workspace = true } once_cell = { workspace = true } From 30276656911e5eaff63fdd23ce7c1e7d98d49a15 Mon Sep 17 00:00:00 2001 From: Renjie Liu Date: Thu, 6 Aug 2026 13:16:44 +0800 Subject: [PATCH 14/14] Address comments --- crates/iceberg/public-api.txt | 355 ++++--- .../iceberg/src/catalog/metadata_location.rs | 6 +- crates/iceberg/src/catalog/utils.rs | 2 +- crates/iceberg/src/compression.rs | 39 +- crates/iceberg/src/encryption/manager.rs | 4 +- crates/iceberg/src/spec/name_mapping/mod.rs | 21 - crates/iceberg/src/spec/table_metadata.rs | 22 +- crates/iceberg/src/spec/table_props.rs | 877 ++++++++++++++---- .../src/transaction/expire_snapshots.rs | 12 +- crates/iceberg/src/transaction/mod.rs | 10 +- .../writer/file_writer/location_generator.rs | 34 +- .../src/writer/file_writer/parquet_writer.rs | 10 +- .../datafusion/src/physical_plan/write.rs | 6 +- crates/property-macro/Cargo.toml | 2 +- crates/property-macro/src/lib.rs | 131 ++- crates/property-macro/tests/properties.rs | 54 +- 16 files changed, 1185 insertions(+), 400 deletions(-) diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index 59ae06d763..7fa7ae97ae 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -2324,12 +2324,7 @@ impl core::cmp::PartialEq for iceberg::spec::NameMapping pub fn iceberg::spec::NameMapping::eq(&self, other: &iceberg::spec::NameMapping) -> bool impl core::fmt::Debug for iceberg::spec::NameMapping pub fn iceberg::spec::NameMapping::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl core::fmt::Display for iceberg::spec::NameMapping -pub fn iceberg::spec::NameMapping::fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result impl core::marker::StructuralPartialEq for iceberg::spec::NameMapping -impl core::str::traits::FromStr for iceberg::spec::NameMapping -pub type iceberg::spec::NameMapping::Err = iceberg::Error -pub fn iceberg::spec::NameMapping::from_str(value: &str) -> core::result::Result impl serde_core::ser::Serialize for iceberg::spec::NameMapping pub fn iceberg::spec::NameMapping::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::NameMapping @@ -2857,122 +2852,6 @@ pub fn iceberg::spec::TableMetadataBuilder::clone(&self) -> iceberg::spec::Table impl core::fmt::Debug for iceberg::spec::TableMetadataBuilder pub fn iceberg::spec::TableMetadataBuilder::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result pub struct iceberg::spec::TableProperties -pub iceberg::spec::TableProperties::comment: core::option::Option -pub iceberg::spec::TableProperties::commit_manifest_merge_enabled: bool -pub iceberg::spec::TableProperties::commit_manifest_min_count_to_merge: usize -pub iceberg::spec::TableProperties::commit_manifest_target_size_bytes: usize -pub iceberg::spec::TableProperties::commit_retry_max_wait_ms: u64 -pub iceberg::spec::TableProperties::commit_retry_min_wait_ms: u64 -pub iceberg::spec::TableProperties::commit_retry_num_retries: usize -pub iceberg::spec::TableProperties::commit_retry_total_timeout_ms: u64 -pub iceberg::spec::TableProperties::commit_status_check_max_wait_ms: u64 -pub iceberg::spec::TableProperties::commit_status_check_min_wait_ms: u64 -pub iceberg::spec::TableProperties::commit_status_check_num_retries: usize -pub iceberg::spec::TableProperties::commit_status_check_total_timeout_ms: u64 -pub iceberg::spec::TableProperties::compatibility_snapshot_id_inheritance_enabled: bool -pub iceberg::spec::TableProperties::encryption_data_key_length: usize -pub iceberg::spec::TableProperties::encryption_key_id: core::option::Option -pub iceberg::spec::TableProperties::engine_hive_enabled: bool -pub iceberg::spec::TableProperties::engine_hive_lock_enabled: bool -pub iceberg::spec::TableProperties::gc_enabled: bool -pub iceberg::spec::TableProperties::history_expire_max_ref_age_ms: i64 -pub iceberg::spec::TableProperties::history_expire_max_snapshot_age_ms: i64 -pub iceberg::spec::TableProperties::history_expire_min_snapshots_to_keep: usize -pub iceberg::spec::TableProperties::identifier_fields_rely: bool -pub iceberg::spec::TableProperties::read_data_planning_mode: alloc::string::String -pub iceberg::spec::TableProperties::read_delete_planning_mode: alloc::string::String -pub iceberg::spec::TableProperties::read_orc_vectorization_batch_size: usize -pub iceberg::spec::TableProperties::read_orc_vectorization_enabled: bool -pub iceberg::spec::TableProperties::read_parquet_vectorization_batch_size: usize -pub iceberg::spec::TableProperties::read_parquet_vectorization_enabled: bool -pub iceberg::spec::TableProperties::read_split_adaptive_size_enabled: bool -pub iceberg::spec::TableProperties::read_split_metadata_target_size: u64 -pub iceberg::spec::TableProperties::read_split_open_file_cost: u64 -pub iceberg::spec::TableProperties::read_split_planning_lookback: usize -pub iceberg::spec::TableProperties::read_split_target_size: u64 -pub iceberg::spec::TableProperties::schema_name_mapping_default: core::option::Option -pub iceberg::spec::TableProperties::write_avro_compression_codec: iceberg::compression::CompressionCodec -pub iceberg::spec::TableProperties::write_data_path: core::option::Option -pub iceberg::spec::TableProperties::write_datafusion_fanout_enabled: bool -pub iceberg::spec::TableProperties::write_delete_avro_compression_codec: iceberg::compression::CompressionCodec -pub iceberg::spec::TableProperties::write_delete_distribution_mode: iceberg::spec::DistributionMode -pub iceberg::spec::TableProperties::write_delete_format_default: iceberg::spec::DataFileFormat -pub iceberg::spec::TableProperties::write_delete_granularity: iceberg::spec::DeleteGranularity -pub iceberg::spec::TableProperties::write_delete_isolation_level: iceberg::spec::IsolationLevel -pub iceberg::spec::TableProperties::write_delete_mode: iceberg::spec::RowLevelOperationMode -pub iceberg::spec::TableProperties::write_delete_orc_block_size_bytes: u64 -pub iceberg::spec::TableProperties::write_delete_orc_compression_codec: iceberg::compression::CompressionCodec -pub iceberg::spec::TableProperties::write_delete_orc_compression_strategy: alloc::string::String -pub iceberg::spec::TableProperties::write_delete_orc_stripe_size_bytes: u64 -pub iceberg::spec::TableProperties::write_delete_orc_vectorized_batch_size: usize -pub iceberg::spec::TableProperties::write_delete_parquet_compression_codec: iceberg::compression::CompressionCodec -pub iceberg::spec::TableProperties::write_delete_parquet_dict_size_bytes: usize -pub iceberg::spec::TableProperties::write_delete_parquet_page_row_limit: usize -pub iceberg::spec::TableProperties::write_delete_parquet_page_size_bytes: usize -pub iceberg::spec::TableProperties::write_delete_parquet_page_version: alloc::string::String -pub iceberg::spec::TableProperties::write_delete_parquet_row_group_check_max_record_count: usize -pub iceberg::spec::TableProperties::write_delete_parquet_row_group_check_min_record_count: usize -pub iceberg::spec::TableProperties::write_delete_parquet_row_group_size_bytes: usize -pub iceberg::spec::TableProperties::write_delete_target_file_size_bytes: usize -pub iceberg::spec::TableProperties::write_distribution_mode: iceberg::spec::DistributionMode -pub iceberg::spec::TableProperties::write_folder_storage_path: core::option::Option -pub iceberg::spec::TableProperties::write_format_default: iceberg::spec::DataFileFormat -pub iceberg::spec::TableProperties::write_location_provider_impl: core::option::Option -pub iceberg::spec::TableProperties::write_manifest_compression_codec: iceberg::compression::CompressionCodec -pub iceberg::spec::TableProperties::write_manifest_lists_enabled: bool -pub iceberg::spec::TableProperties::write_merge_distribution_mode: iceberg::spec::DistributionMode -pub iceberg::spec::TableProperties::write_merge_isolation_level: iceberg::spec::IsolationLevel -pub iceberg::spec::TableProperties::write_merge_mode: iceberg::spec::RowLevelOperationMode -pub iceberg::spec::TableProperties::write_metadata_compression_codec: iceberg::compression::CompressionCodec -pub iceberg::spec::TableProperties::write_metadata_delete_after_commit_enabled: bool -pub iceberg::spec::TableProperties::write_metadata_metrics_column: std::collections::hash::map::HashMap -pub iceberg::spec::TableProperties::write_metadata_metrics_default: alloc::string::String -pub iceberg::spec::TableProperties::write_metadata_metrics_max_inferred_column_defaults: usize -pub iceberg::spec::TableProperties::write_metadata_path: core::option::Option -pub iceberg::spec::TableProperties::write_metadata_previous_versions_max: usize -pub iceberg::spec::TableProperties::write_object_storage_enabled: bool -pub iceberg::spec::TableProperties::write_object_storage_partitioned_paths: bool -pub iceberg::spec::TableProperties::write_object_storage_path: core::option::Option -pub iceberg::spec::TableProperties::write_orc_block_size_bytes: u64 -pub iceberg::spec::TableProperties::write_orc_bloom_filter_columns: alloc::vec::Vec -pub iceberg::spec::TableProperties::write_orc_bloom_filter_fpp: f64 -pub iceberg::spec::TableProperties::write_orc_compression_codec: iceberg::compression::CompressionCodec -pub iceberg::spec::TableProperties::write_orc_compression_strategy: alloc::string::String -pub iceberg::spec::TableProperties::write_orc_stripe_size_bytes: u64 -pub iceberg::spec::TableProperties::write_orc_vectorized_batch_size: usize -pub iceberg::spec::TableProperties::write_parquet_bloom_filter_adaptive_enabled: bool -pub iceberg::spec::TableProperties::write_parquet_bloom_filter_enabled_column: std::collections::hash::map::HashMap -pub iceberg::spec::TableProperties::write_parquet_bloom_filter_fpp_column: std::collections::hash::map::HashMap -pub iceberg::spec::TableProperties::write_parquet_bloom_filter_max_bytes: usize -pub iceberg::spec::TableProperties::write_parquet_bloom_filter_ndv_column: std::collections::hash::map::HashMap -pub iceberg::spec::TableProperties::write_parquet_compression_codec: iceberg::compression::CompressionCodec -pub iceberg::spec::TableProperties::write_parquet_content_defined_chunking_enabled: bool -pub iceberg::spec::TableProperties::write_parquet_content_defined_chunking_max_chunk_size: usize -pub iceberg::spec::TableProperties::write_parquet_content_defined_chunking_min_chunk_size: usize -pub iceberg::spec::TableProperties::write_parquet_content_defined_chunking_norm_level: i32 -pub iceberg::spec::TableProperties::write_parquet_dict_encoding_enabled_column: std::collections::hash::map::HashMap -pub iceberg::spec::TableProperties::write_parquet_dict_size_bytes: usize -pub iceberg::spec::TableProperties::write_parquet_page_row_limit: usize -pub iceberg::spec::TableProperties::write_parquet_page_size_bytes: usize -pub iceberg::spec::TableProperties::write_parquet_page_version: alloc::string::String -pub iceberg::spec::TableProperties::write_parquet_row_group_check_max_record_count: usize -pub iceberg::spec::TableProperties::write_parquet_row_group_check_min_record_count: usize -pub iceberg::spec::TableProperties::write_parquet_row_group_size_bytes: usize -pub iceberg::spec::TableProperties::write_parquet_row_group_size_track_uncompressed: bool -pub iceberg::spec::TableProperties::write_parquet_shred_variants: bool -pub iceberg::spec::TableProperties::write_parquet_stats_enabled_column: std::collections::hash::map::HashMap -pub iceberg::spec::TableProperties::write_parquet_variant_inference_buffer_size: usize -pub iceberg::spec::TableProperties::write_spark_accept_any_schema: bool -pub iceberg::spec::TableProperties::write_spark_advisory_partition_size_bytes: core::option::Option -pub iceberg::spec::TableProperties::write_spark_auto_schema_evolution_enabled: bool -pub iceberg::spec::TableProperties::write_spark_fanout_enabled: bool -pub iceberg::spec::TableProperties::write_summary_partition_limit: u64 -pub iceberg::spec::TableProperties::write_target_file_size_bytes: usize -pub iceberg::spec::TableProperties::write_update_distribution_mode: iceberg::spec::DistributionMode -pub iceberg::spec::TableProperties::write_update_isolation_level: iceberg::spec::IsolationLevel -pub iceberg::spec::TableProperties::write_update_mode: iceberg::spec::RowLevelOperationMode -pub iceberg::spec::TableProperties::write_upsert_enabled: bool -pub iceberg::spec::TableProperties::write_wap_enabled: bool impl iceberg::spec::TableProperties pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_NUM_RETRIES: &str pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT: usize @@ -2991,11 +2870,245 @@ pub const iceberg::spec::TableProperties::PROPERTY_WRITE_FOLDER_STORAGE_LOCATION pub const iceberg::spec::TableProperties::PROPERTY_WRITE_METADATA_PATH: &str pub const iceberg::spec::TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_LOCATION: &str pub const iceberg::spec::TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS: &str +pub const iceberg::spec::TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS_DEFAULT: bool pub const iceberg::spec::TableProperties::PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT: &str pub const iceberg::spec::TableProperties::PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT_DEFAULT: u64 pub const iceberg::spec::TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES: &str pub const iceberg::spec::TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT: usize pub const iceberg::spec::TableProperties::RESERVED_PROPERTIES: [&str; 9] +impl iceberg::spec::TableProperties +pub fn iceberg::spec::TableProperties::comment(&self) -> &core::option::Option +pub fn iceberg::spec::TableProperties::commit_manifest_merge_enabled(&self) -> &bool +pub fn iceberg::spec::TableProperties::commit_manifest_min_count_to_merge(&self) -> &usize +pub fn iceberg::spec::TableProperties::commit_manifest_target_size_bytes(&self) -> &usize +pub fn iceberg::spec::TableProperties::commit_retry_max_wait_ms(&self) -> &u64 +pub fn iceberg::spec::TableProperties::commit_retry_min_wait_ms(&self) -> &u64 +pub fn iceberg::spec::TableProperties::commit_retry_num_retries(&self) -> &usize +pub fn iceberg::spec::TableProperties::commit_retry_total_timeout_ms(&self) -> &u64 +pub fn iceberg::spec::TableProperties::commit_status_check_max_wait_ms(&self) -> &u64 +pub fn iceberg::spec::TableProperties::commit_status_check_min_wait_ms(&self) -> &u64 +pub fn iceberg::spec::TableProperties::commit_status_check_num_retries(&self) -> &usize +pub fn iceberg::spec::TableProperties::commit_status_check_total_timeout_ms(&self) -> &u64 +pub fn iceberg::spec::TableProperties::compatibility_snapshot_id_inheritance_enabled(&self) -> &bool +pub fn iceberg::spec::TableProperties::encryption_data_key_length(&self) -> &usize +pub fn iceberg::spec::TableProperties::encryption_key_id(&self) -> &core::option::Option +pub fn iceberg::spec::TableProperties::engine_hive_enabled(&self) -> &bool +pub fn iceberg::spec::TableProperties::engine_hive_lock_enabled(&self) -> &bool +pub fn iceberg::spec::TableProperties::gc_enabled(&self) -> &bool +pub fn iceberg::spec::TableProperties::history_expire_max_ref_age_ms(&self) -> &i64 +pub fn iceberg::spec::TableProperties::history_expire_max_snapshot_age_ms(&self) -> &i64 +pub fn iceberg::spec::TableProperties::history_expire_min_snapshots_to_keep(&self) -> &usize +pub fn iceberg::spec::TableProperties::identifier_fields_rely(&self) -> &bool +pub fn iceberg::spec::TableProperties::read_data_planning_mode(&self) -> &alloc::string::String +pub fn iceberg::spec::TableProperties::read_delete_planning_mode(&self) -> &alloc::string::String +pub fn iceberg::spec::TableProperties::read_orc_vectorization_batch_size(&self) -> &usize +pub fn iceberg::spec::TableProperties::read_orc_vectorization_enabled(&self) -> &bool +pub fn iceberg::spec::TableProperties::read_parquet_vectorization_batch_size(&self) -> &usize +pub fn iceberg::spec::TableProperties::read_parquet_vectorization_enabled(&self) -> &bool +pub fn iceberg::spec::TableProperties::read_split_adaptive_size_enabled(&self) -> &bool +pub fn iceberg::spec::TableProperties::read_split_metadata_target_size(&self) -> &u64 +pub fn iceberg::spec::TableProperties::read_split_open_file_cost(&self) -> &u64 +pub fn iceberg::spec::TableProperties::read_split_planning_lookback(&self) -> &usize +pub fn iceberg::spec::TableProperties::read_split_target_size(&self) -> &u64 +pub fn iceberg::spec::TableProperties::schema_name_mapping_default(&self) -> &core::option::Option +pub fn iceberg::spec::TableProperties::set_comment(&mut self, value: core::option::Option) +pub fn iceberg::spec::TableProperties::set_commit_manifest_merge_enabled(&mut self, value: bool) +pub fn iceberg::spec::TableProperties::set_commit_manifest_min_count_to_merge(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_commit_manifest_target_size_bytes(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_commit_retry_max_wait_ms(&mut self, value: u64) +pub fn iceberg::spec::TableProperties::set_commit_retry_min_wait_ms(&mut self, value: u64) +pub fn iceberg::spec::TableProperties::set_commit_retry_num_retries(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_commit_retry_total_timeout_ms(&mut self, value: u64) +pub fn iceberg::spec::TableProperties::set_commit_status_check_max_wait_ms(&mut self, value: u64) +pub fn iceberg::spec::TableProperties::set_commit_status_check_min_wait_ms(&mut self, value: u64) +pub fn iceberg::spec::TableProperties::set_commit_status_check_num_retries(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_commit_status_check_total_timeout_ms(&mut self, value: u64) +pub fn iceberg::spec::TableProperties::set_compatibility_snapshot_id_inheritance_enabled(&mut self, value: bool) +pub fn iceberg::spec::TableProperties::set_encryption_data_key_length(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_encryption_key_id(&mut self, value: core::option::Option) +pub fn iceberg::spec::TableProperties::set_engine_hive_enabled(&mut self, value: bool) +pub fn iceberg::spec::TableProperties::set_engine_hive_lock_enabled(&mut self, value: bool) +pub fn iceberg::spec::TableProperties::set_gc_enabled(&mut self, value: bool) +pub fn iceberg::spec::TableProperties::set_history_expire_max_ref_age_ms(&mut self, value: i64) +pub fn iceberg::spec::TableProperties::set_history_expire_max_snapshot_age_ms(&mut self, value: i64) +pub fn iceberg::spec::TableProperties::set_history_expire_min_snapshots_to_keep(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_identifier_fields_rely(&mut self, value: bool) +pub fn iceberg::spec::TableProperties::set_read_data_planning_mode(&mut self, value: alloc::string::String) +pub fn iceberg::spec::TableProperties::set_read_delete_planning_mode(&mut self, value: alloc::string::String) +pub fn iceberg::spec::TableProperties::set_read_orc_vectorization_batch_size(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_read_orc_vectorization_enabled(&mut self, value: bool) +pub fn iceberg::spec::TableProperties::set_read_parquet_vectorization_batch_size(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_read_parquet_vectorization_enabled(&mut self, value: bool) +pub fn iceberg::spec::TableProperties::set_read_split_adaptive_size_enabled(&mut self, value: bool) +pub fn iceberg::spec::TableProperties::set_read_split_metadata_target_size(&mut self, value: u64) +pub fn iceberg::spec::TableProperties::set_read_split_open_file_cost(&mut self, value: u64) +pub fn iceberg::spec::TableProperties::set_read_split_planning_lookback(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_read_split_target_size(&mut self, value: u64) +pub fn iceberg::spec::TableProperties::set_schema_name_mapping_default(&mut self, value: core::option::Option) +pub fn iceberg::spec::TableProperties::set_write_avro_compression_codec(&mut self, value: iceberg::compression::CompressionCodec) +pub fn iceberg::spec::TableProperties::set_write_data_path(&mut self, value: core::option::Option) +pub fn iceberg::spec::TableProperties::set_write_datafusion_fanout_enabled(&mut self, value: bool) +pub fn iceberg::spec::TableProperties::set_write_delete_avro_compression_codec(&mut self, value: iceberg::compression::CompressionCodec) +pub fn iceberg::spec::TableProperties::set_write_delete_distribution_mode(&mut self, value: core::option::Option) +pub fn iceberg::spec::TableProperties::set_write_delete_format_default(&mut self, value: iceberg::spec::DataFileFormat) +pub fn iceberg::spec::TableProperties::set_write_delete_granularity(&mut self, value: iceberg::spec::DeleteGranularity) +pub fn iceberg::spec::TableProperties::set_write_delete_isolation_level(&mut self, value: iceberg::spec::IsolationLevel) +pub fn iceberg::spec::TableProperties::set_write_delete_mode(&mut self, value: iceberg::spec::RowLevelOperationMode) +pub fn iceberg::spec::TableProperties::set_write_delete_orc_block_size_bytes(&mut self, value: u64) +pub fn iceberg::spec::TableProperties::set_write_delete_orc_compression_codec(&mut self, value: iceberg::compression::CompressionCodec) +pub fn iceberg::spec::TableProperties::set_write_delete_orc_compression_strategy(&mut self, value: alloc::string::String) +pub fn iceberg::spec::TableProperties::set_write_delete_orc_stripe_size_bytes(&mut self, value: u64) +pub fn iceberg::spec::TableProperties::set_write_delete_orc_vectorized_batch_size(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_write_delete_parquet_compression_codec(&mut self, value: iceberg::compression::CompressionCodec) +pub fn iceberg::spec::TableProperties::set_write_delete_parquet_dict_size_bytes(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_write_delete_parquet_page_row_limit(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_write_delete_parquet_page_size_bytes(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_write_delete_parquet_page_version(&mut self, value: alloc::string::String) +pub fn iceberg::spec::TableProperties::set_write_delete_parquet_row_group_check_max_record_count(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_write_delete_parquet_row_group_check_min_record_count(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_write_delete_parquet_row_group_size_bytes(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_write_delete_target_file_size_bytes(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_write_distribution_mode(&mut self, value: core::option::Option) +pub fn iceberg::spec::TableProperties::set_write_folder_storage_path(&mut self, value: core::option::Option) +pub fn iceberg::spec::TableProperties::set_write_format_default(&mut self, value: iceberg::spec::DataFileFormat) +pub fn iceberg::spec::TableProperties::set_write_location_provider_impl(&mut self, value: core::option::Option) +pub fn iceberg::spec::TableProperties::set_write_manifest_compression_codec(&mut self, value: iceberg::compression::CompressionCodec) +pub fn iceberg::spec::TableProperties::set_write_manifest_lists_enabled(&mut self, value: bool) +pub fn iceberg::spec::TableProperties::set_write_merge_distribution_mode(&mut self, value: core::option::Option) +pub fn iceberg::spec::TableProperties::set_write_merge_isolation_level(&mut self, value: iceberg::spec::IsolationLevel) +pub fn iceberg::spec::TableProperties::set_write_merge_mode(&mut self, value: iceberg::spec::RowLevelOperationMode) +pub fn iceberg::spec::TableProperties::set_write_metadata_compression_codec(&mut self, value: iceberg::compression::CompressionCodec) +pub fn iceberg::spec::TableProperties::set_write_metadata_delete_after_commit_enabled(&mut self, value: bool) +pub fn iceberg::spec::TableProperties::set_write_metadata_metrics_column(&mut self, value: std::collections::hash::map::HashMap) +pub fn iceberg::spec::TableProperties::set_write_metadata_metrics_default(&mut self, value: alloc::string::String) +pub fn iceberg::spec::TableProperties::set_write_metadata_metrics_max_inferred_column_defaults(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_write_metadata_path(&mut self, value: core::option::Option) +pub fn iceberg::spec::TableProperties::set_write_metadata_previous_versions_max(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_write_object_storage_enabled(&mut self, value: bool) +pub fn iceberg::spec::TableProperties::set_write_object_storage_partitioned_paths(&mut self, value: bool) +pub fn iceberg::spec::TableProperties::set_write_object_storage_path(&mut self, value: core::option::Option) +pub fn iceberg::spec::TableProperties::set_write_orc_block_size_bytes(&mut self, value: u64) +pub fn iceberg::spec::TableProperties::set_write_orc_bloom_filter_columns(&mut self, value: alloc::vec::Vec) +pub fn iceberg::spec::TableProperties::set_write_orc_bloom_filter_fpp(&mut self, value: f64) +pub fn iceberg::spec::TableProperties::set_write_orc_compression_codec(&mut self, value: iceberg::compression::CompressionCodec) +pub fn iceberg::spec::TableProperties::set_write_orc_compression_strategy(&mut self, value: alloc::string::String) +pub fn iceberg::spec::TableProperties::set_write_orc_stripe_size_bytes(&mut self, value: u64) +pub fn iceberg::spec::TableProperties::set_write_orc_vectorized_batch_size(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_write_parquet_bloom_filter_adaptive_enabled(&mut self, value: bool) +pub fn iceberg::spec::TableProperties::set_write_parquet_bloom_filter_enabled_column(&mut self, value: std::collections::hash::map::HashMap) +pub fn iceberg::spec::TableProperties::set_write_parquet_bloom_filter_fpp_column(&mut self, value: std::collections::hash::map::HashMap) +pub fn iceberg::spec::TableProperties::set_write_parquet_bloom_filter_max_bytes(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_write_parquet_bloom_filter_ndv_column(&mut self, value: std::collections::hash::map::HashMap) +pub fn iceberg::spec::TableProperties::set_write_parquet_compression_codec(&mut self, value: iceberg::compression::CompressionCodec) +pub fn iceberg::spec::TableProperties::set_write_parquet_content_defined_chunking_enabled(&mut self, value: bool) +pub fn iceberg::spec::TableProperties::set_write_parquet_content_defined_chunking_max_chunk_size(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_write_parquet_content_defined_chunking_min_chunk_size(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_write_parquet_content_defined_chunking_norm_level(&mut self, value: i32) +pub fn iceberg::spec::TableProperties::set_write_parquet_dict_encoding_enabled_column(&mut self, value: std::collections::hash::map::HashMap) +pub fn iceberg::spec::TableProperties::set_write_parquet_dict_size_bytes(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_write_parquet_page_row_limit(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_write_parquet_page_size_bytes(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_write_parquet_page_version(&mut self, value: alloc::string::String) +pub fn iceberg::spec::TableProperties::set_write_parquet_row_group_check_max_record_count(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_write_parquet_row_group_check_min_record_count(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_write_parquet_row_group_size_bytes(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_write_parquet_row_group_size_track_uncompressed(&mut self, value: bool) +pub fn iceberg::spec::TableProperties::set_write_parquet_shred_variants(&mut self, value: bool) +pub fn iceberg::spec::TableProperties::set_write_parquet_stats_enabled_column(&mut self, value: std::collections::hash::map::HashMap) +pub fn iceberg::spec::TableProperties::set_write_parquet_variant_inference_buffer_size(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_write_spark_accept_any_schema(&mut self, value: bool) +pub fn iceberg::spec::TableProperties::set_write_spark_advisory_partition_size_bytes(&mut self, value: core::option::Option) +pub fn iceberg::spec::TableProperties::set_write_spark_auto_schema_evolution_enabled(&mut self, value: bool) +pub fn iceberg::spec::TableProperties::set_write_spark_fanout_enabled(&mut self, value: bool) +pub fn iceberg::spec::TableProperties::set_write_summary_partition_limit(&mut self, value: u64) +pub fn iceberg::spec::TableProperties::set_write_target_file_size_bytes(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_write_update_distribution_mode(&mut self, value: core::option::Option) +pub fn iceberg::spec::TableProperties::set_write_update_isolation_level(&mut self, value: iceberg::spec::IsolationLevel) +pub fn iceberg::spec::TableProperties::set_write_update_mode(&mut self, value: iceberg::spec::RowLevelOperationMode) +pub fn iceberg::spec::TableProperties::set_write_upsert_enabled(&mut self, value: bool) +pub fn iceberg::spec::TableProperties::set_write_wap_enabled(&mut self, value: bool) +pub fn iceberg::spec::TableProperties::write_avro_compression_codec(&self) -> &iceberg::compression::CompressionCodec +pub fn iceberg::spec::TableProperties::write_data_path(&self) -> &core::option::Option +pub fn iceberg::spec::TableProperties::write_datafusion_fanout_enabled(&self) -> &bool +pub fn iceberg::spec::TableProperties::write_delete_avro_compression_codec(&self) -> &iceberg::compression::CompressionCodec +pub fn iceberg::spec::TableProperties::write_delete_distribution_mode(&self) -> &core::option::Option +pub fn iceberg::spec::TableProperties::write_delete_format_default(&self) -> &iceberg::spec::DataFileFormat +pub fn iceberg::spec::TableProperties::write_delete_granularity(&self) -> &iceberg::spec::DeleteGranularity +pub fn iceberg::spec::TableProperties::write_delete_isolation_level(&self) -> &iceberg::spec::IsolationLevel +pub fn iceberg::spec::TableProperties::write_delete_mode(&self) -> &iceberg::spec::RowLevelOperationMode +pub fn iceberg::spec::TableProperties::write_delete_orc_block_size_bytes(&self) -> &u64 +pub fn iceberg::spec::TableProperties::write_delete_orc_compression_codec(&self) -> &iceberg::compression::CompressionCodec +pub fn iceberg::spec::TableProperties::write_delete_orc_compression_strategy(&self) -> &alloc::string::String +pub fn iceberg::spec::TableProperties::write_delete_orc_stripe_size_bytes(&self) -> &u64 +pub fn iceberg::spec::TableProperties::write_delete_orc_vectorized_batch_size(&self) -> &usize +pub fn iceberg::spec::TableProperties::write_delete_parquet_compression_codec(&self) -> &iceberg::compression::CompressionCodec +pub fn iceberg::spec::TableProperties::write_delete_parquet_dict_size_bytes(&self) -> &usize +pub fn iceberg::spec::TableProperties::write_delete_parquet_page_row_limit(&self) -> &usize +pub fn iceberg::spec::TableProperties::write_delete_parquet_page_size_bytes(&self) -> &usize +pub fn iceberg::spec::TableProperties::write_delete_parquet_page_version(&self) -> &alloc::string::String +pub fn iceberg::spec::TableProperties::write_delete_parquet_row_group_check_max_record_count(&self) -> &usize +pub fn iceberg::spec::TableProperties::write_delete_parquet_row_group_check_min_record_count(&self) -> &usize +pub fn iceberg::spec::TableProperties::write_delete_parquet_row_group_size_bytes(&self) -> &usize +pub fn iceberg::spec::TableProperties::write_delete_target_file_size_bytes(&self) -> &usize +pub fn iceberg::spec::TableProperties::write_distribution_mode(&self) -> &core::option::Option +pub fn iceberg::spec::TableProperties::write_folder_storage_path(&self) -> &core::option::Option +pub fn iceberg::spec::TableProperties::write_format_default(&self) -> &iceberg::spec::DataFileFormat +pub fn iceberg::spec::TableProperties::write_location_provider_impl(&self) -> &core::option::Option +pub fn iceberg::spec::TableProperties::write_manifest_compression_codec(&self) -> &iceberg::compression::CompressionCodec +pub fn iceberg::spec::TableProperties::write_manifest_lists_enabled(&self) -> &bool +pub fn iceberg::spec::TableProperties::write_merge_distribution_mode(&self) -> &core::option::Option +pub fn iceberg::spec::TableProperties::write_merge_isolation_level(&self) -> &iceberg::spec::IsolationLevel +pub fn iceberg::spec::TableProperties::write_merge_mode(&self) -> &iceberg::spec::RowLevelOperationMode +pub fn iceberg::spec::TableProperties::write_metadata_compression_codec(&self) -> &iceberg::compression::CompressionCodec +pub fn iceberg::spec::TableProperties::write_metadata_delete_after_commit_enabled(&self) -> &bool +pub fn iceberg::spec::TableProperties::write_metadata_metrics_column(&self) -> &std::collections::hash::map::HashMap +pub fn iceberg::spec::TableProperties::write_metadata_metrics_default(&self) -> &alloc::string::String +pub fn iceberg::spec::TableProperties::write_metadata_metrics_max_inferred_column_defaults(&self) -> &usize +pub fn iceberg::spec::TableProperties::write_metadata_path(&self) -> &core::option::Option +pub fn iceberg::spec::TableProperties::write_metadata_previous_versions_max(&self) -> &usize +pub fn iceberg::spec::TableProperties::write_object_storage_enabled(&self) -> &bool +pub fn iceberg::spec::TableProperties::write_object_storage_partitioned_paths(&self) -> &bool +pub fn iceberg::spec::TableProperties::write_object_storage_path(&self) -> &core::option::Option +pub fn iceberg::spec::TableProperties::write_orc_block_size_bytes(&self) -> &u64 +pub fn iceberg::spec::TableProperties::write_orc_bloom_filter_columns(&self) -> &alloc::vec::Vec +pub fn iceberg::spec::TableProperties::write_orc_bloom_filter_fpp(&self) -> &f64 +pub fn iceberg::spec::TableProperties::write_orc_compression_codec(&self) -> &iceberg::compression::CompressionCodec +pub fn iceberg::spec::TableProperties::write_orc_compression_strategy(&self) -> &alloc::string::String +pub fn iceberg::spec::TableProperties::write_orc_stripe_size_bytes(&self) -> &u64 +pub fn iceberg::spec::TableProperties::write_orc_vectorized_batch_size(&self) -> &usize +pub fn iceberg::spec::TableProperties::write_parquet_bloom_filter_adaptive_enabled(&self) -> &bool +pub fn iceberg::spec::TableProperties::write_parquet_bloom_filter_enabled_column(&self) -> &std::collections::hash::map::HashMap +pub fn iceberg::spec::TableProperties::write_parquet_bloom_filter_fpp_column(&self) -> &std::collections::hash::map::HashMap +pub fn iceberg::spec::TableProperties::write_parquet_bloom_filter_max_bytes(&self) -> &usize +pub fn iceberg::spec::TableProperties::write_parquet_bloom_filter_ndv_column(&self) -> &std::collections::hash::map::HashMap +pub fn iceberg::spec::TableProperties::write_parquet_compression_codec(&self) -> &iceberg::compression::CompressionCodec +pub fn iceberg::spec::TableProperties::write_parquet_content_defined_chunking_enabled(&self) -> &bool +pub fn iceberg::spec::TableProperties::write_parquet_content_defined_chunking_max_chunk_size(&self) -> &usize +pub fn iceberg::spec::TableProperties::write_parquet_content_defined_chunking_min_chunk_size(&self) -> &usize +pub fn iceberg::spec::TableProperties::write_parquet_content_defined_chunking_norm_level(&self) -> &i32 +pub fn iceberg::spec::TableProperties::write_parquet_dict_encoding_enabled_column(&self) -> &std::collections::hash::map::HashMap +pub fn iceberg::spec::TableProperties::write_parquet_dict_size_bytes(&self) -> &usize +pub fn iceberg::spec::TableProperties::write_parquet_page_row_limit(&self) -> &usize +pub fn iceberg::spec::TableProperties::write_parquet_page_size_bytes(&self) -> &usize +pub fn iceberg::spec::TableProperties::write_parquet_page_version(&self) -> &alloc::string::String +pub fn iceberg::spec::TableProperties::write_parquet_row_group_check_max_record_count(&self) -> &usize +pub fn iceberg::spec::TableProperties::write_parquet_row_group_check_min_record_count(&self) -> &usize +pub fn iceberg::spec::TableProperties::write_parquet_row_group_size_bytes(&self) -> &usize +pub fn iceberg::spec::TableProperties::write_parquet_row_group_size_track_uncompressed(&self) -> &bool +pub fn iceberg::spec::TableProperties::write_parquet_shred_variants(&self) -> &bool +pub fn iceberg::spec::TableProperties::write_parquet_stats_enabled_column(&self) -> &std::collections::hash::map::HashMap +pub fn iceberg::spec::TableProperties::write_parquet_variant_inference_buffer_size(&self) -> &usize +pub fn iceberg::spec::TableProperties::write_spark_accept_any_schema(&self) -> &bool +pub fn iceberg::spec::TableProperties::write_spark_advisory_partition_size_bytes(&self) -> &core::option::Option +pub fn iceberg::spec::TableProperties::write_spark_auto_schema_evolution_enabled(&self) -> &bool +pub fn iceberg::spec::TableProperties::write_spark_fanout_enabled(&self) -> &bool +pub fn iceberg::spec::TableProperties::write_summary_partition_limit(&self) -> &u64 +pub fn iceberg::spec::TableProperties::write_target_file_size_bytes(&self) -> &usize +pub fn iceberg::spec::TableProperties::write_update_distribution_mode(&self) -> &core::option::Option +pub fn iceberg::spec::TableProperties::write_update_isolation_level(&self) -> &iceberg::spec::IsolationLevel +pub fn iceberg::spec::TableProperties::write_update_mode(&self) -> &iceberg::spec::RowLevelOperationMode +pub fn iceberg::spec::TableProperties::write_upsert_enabled(&self) -> &bool +pub fn iceberg::spec::TableProperties::write_wap_enabled(&self) -> &bool impl core::clone::Clone for iceberg::spec::TableProperties pub fn iceberg::spec::TableProperties::clone(&self) -> iceberg::spec::TableProperties impl core::convert::TryFrom<&std::collections::hash::map::HashMap> for iceberg::spec::TableProperties diff --git a/crates/iceberg/src/catalog/metadata_location.rs b/crates/iceberg/src/catalog/metadata_location.rs index 5f1af2f42b..d5daa02b37 100644 --- a/crates/iceberg/src/catalog/metadata_location.rs +++ b/crates/iceberg/src/catalog/metadata_location.rs @@ -22,7 +22,7 @@ use std::str::FromStr; use uuid::Uuid; use crate::compression::CompressionCodec; -use crate::spec::{TableMetadata, TableProperties}; +use crate::spec::{TableMetadata, parse_metadata_file_compression}; use crate::{Error, ErrorKind, Result}; /// Default folder name for metadata files under the table location, used when the @@ -46,9 +46,7 @@ impl MetadataLocation { /// Determines the compression codec from table properties. /// Parse errors result in CompressionCodec::None. fn compression_from_properties(properties: &HashMap) -> CompressionCodec { - TableProperties::try_from(properties) - .map(|properties| properties.write_metadata_compression_codec) - .unwrap_or(CompressionCodec::None) + parse_metadata_file_compression(properties).unwrap_or(CompressionCodec::None) } /// Creates a completely new metadata location starting at version 0, deriving the diff --git a/crates/iceberg/src/catalog/utils.rs b/crates/iceberg/src/catalog/utils.rs index 8e743e7d7d..065aa16e1d 100644 --- a/crates/iceberg/src/catalog/utils.rs +++ b/crates/iceberg/src/catalog/utils.rs @@ -61,7 +61,7 @@ pub async fn drop_table_data(table_info: &Table) -> Result<()> { } // Delete data files only if gc.enabled is true, to avoid corrupting shared tables - if metadata.table_properties()?.gc_enabled { + if *metadata.table_properties()?.gc_enabled() { delete_data_files(io, &manifests_to_delete).await?; } diff --git a/crates/iceberg/src/compression.rs b/crates/iceberg/src/compression.rs index 242a4a788d..3c15bdbf6f 100644 --- a/crates/iceberg/src/compression.rs +++ b/crates/iceberg/src/compression.rs @@ -97,23 +97,9 @@ impl CompressionCodec { /// Parses the metadata-file compression codec table property. pub(crate) fn parse_metadata_property(value: &str) -> Result { - if value.is_empty() { - return Ok(Self::None); - } - - let codec = Self::parse_property(value).map_err(|_| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Invalid metadata compression codec: {value}. Only '{}' and '{}' are supported.", - Self::None.name(), - Self::gzip_default().name() - ), - ) - })?; - - match codec { - Self::None | Self::Gzip(_) => Ok(codec), + match value.to_ascii_lowercase().as_str() { + "" | "none" => Ok(Self::None), + "gzip" => Ok(Self::gzip_default()), _ => Err(Error::new( ErrorKind::DataInvalid, format!( @@ -165,25 +151,6 @@ impl CompressionCodec { )), } } - - /// Writes a codec and its optional companion compression-level property. - pub(crate) fn write_properties( - &self, - properties: &mut HashMap, - codec_key: &str, - level_key: &str, - ) { - properties.insert(codec_key.to_string(), self.property_value()); - - match self { - Self::Gzip(level) | Self::Zstd(level) => { - properties.insert(level_key.to_string(), level.to_string()); - } - _ => { - properties.remove(level_key); - } - } - } } // Note: serialize/deserialize do not round-trip the compression level. Iceberg configuration diff --git a/crates/iceberg/src/encryption/manager.rs b/crates/iceberg/src/encryption/manager.rs index e2294c2f2c..a212991d64 100644 --- a/crates/iceberg/src/encryption/manager.rs +++ b/crates/iceberg/src/encryption/manager.rs @@ -119,7 +119,7 @@ impl EncryptionManager { } let table_properties = metadata.table_properties()?; - let Some(table_key_id) = table_properties.encryption_key_id else { + let Some(table_key_id) = table_properties.encryption_key_id() else { if kms_client.is_some() { tracing::warn!( "KeyManagementClient provided but table does not have encryption.key-id set" @@ -140,7 +140,7 @@ impl EncryptionManager { .table_key_id(table_key_id) .encryption_keys(metadata.encryption_keys.clone()) .key_size(AesKeySize::from_key_length( - table_properties.encryption_data_key_length, + *table_properties.encryption_data_key_length(), )?) .build(); Ok(Some(Arc::new(em))) diff --git a/crates/iceberg/src/spec/name_mapping/mod.rs b/crates/iceberg/src/spec/name_mapping/mod.rs index 3d11478e9d..db9e44c290 100644 --- a/crates/iceberg/src/spec/name_mapping/mod.rs +++ b/crates/iceberg/src/spec/name_mapping/mod.rs @@ -17,15 +17,11 @@ //! Iceberg name mapping. -use std::fmt::{Display, Formatter}; -use std::str::FromStr; use std::sync::Arc; use serde::{Deserialize, Serialize}; use serde_with::{DefaultOnNull, serde_as}; -use crate::{Error, ErrorKind}; - /// Property name for name mapping. pub const DEFAULT_SCHEMA_NAME_MAPPING: &str = "schema.name-mapping.default"; @@ -48,23 +44,6 @@ impl NameMapping { } } -impl FromStr for NameMapping { - type Err = Error; - - fn from_str(value: &str) -> Result { - serde_json::from_str(value).map_err(|error| { - Error::new(ErrorKind::DataInvalid, "Invalid name mapping").with_source(error) - }) - } -} - -impl Display for NameMapping { - fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { - let value = serde_json::to_string(self).map_err(|_| std::fmt::Error)?; - formatter.write_str(&value) - } -} - /// Maps field names to IDs. #[serde_as] #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)] diff --git a/crates/iceberg/src/spec/table_metadata.rs b/crates/iceberg/src/spec/table_metadata.rs index 3a6d94e263..0ebde6806d 100644 --- a/crates/iceberg/src/spec/table_metadata.rs +++ b/crates/iceberg/src/spec/table_metadata.rs @@ -35,7 +35,7 @@ pub use super::table_metadata_builder::{TableMetadataBuildResult, TableMetadataB use super::{ DEFAULT_PARTITION_SPEC_ID, PartitionSpecRef, PartitionStatisticsFile, SchemaId, SchemaRef, SnapshotRef, SnapshotRetention, SortOrder, SortOrderRef, StatisticsFile, StructType, - TableProperties, + TableProperties, parse_metadata_file_compression, parse_metadata_location_property, }; use crate::catalog::{METADATA_FOLDER_NAME, MetadataLocation}; use crate::compression::CompressionCodec; @@ -369,9 +369,7 @@ impl TableMetadata { /// Honors the `write.metadata.path` table property when set, otherwise defaults /// to the `metadata` subdirectory under the table location. pub fn metadata_location(&self) -> Result { - Ok(self - .table_properties()? - .write_metadata_path + Ok(parse_metadata_location_property(&self.properties)? .unwrap_or_else(|| format!("{}/{}", self.location(), METADATA_FOLDER_NAME))) } @@ -384,7 +382,7 @@ impl TableMetadata { /// /// Returns an error if the compression codec property has an invalid value. pub fn metadata_compression_codec(&self) -> Result { - Ok(self.table_properties()?.write_metadata_compression_codec) + parse_metadata_file_compression(&self.properties) } /// Returns all supported table properties parsed from the raw property map. @@ -498,7 +496,7 @@ impl TableMetadata { let json_data = serde_json::to_vec(self)?; // Check if compression codec from properties matches the one in metadata_location - let codec = self.table_properties()?.write_metadata_compression_codec; + let codec = parse_metadata_file_compression(&self.properties)?; if codec != metadata_location.compression_codec() { return Err(Error::new( @@ -4043,11 +4041,11 @@ mod tests { let props = metadata.table_properties().unwrap(); assert_eq!( - props.commit_retry_num_retries, + *props.commit_retry_num_retries(), TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT ); assert_eq!( - props.write_target_file_size_bytes, + *props.write_target_file_size_bytes(), TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT ); } @@ -4089,8 +4087,8 @@ mod tests { let props = metadata.table_properties().unwrap(); - assert_eq!(props.commit_retry_num_retries, 10); - assert_eq!(props.write_target_file_size_bytes, 1024); + assert_eq!(*props.commit_retry_num_retries(), 10); + assert_eq!(*props.write_target_file_size_bytes(), 1024); } #[test] @@ -4346,7 +4344,7 @@ mod tests { } #[test] - fn test_metadata_location_preserves_trailing_slash() { + fn test_metadata_location_trims_trailing_slash() { let metadata = get_test_table_metadata("TableMetadataV2Valid.json") .into_builder(None) .set_properties(HashMap::from([( @@ -4359,7 +4357,7 @@ mod tests { .metadata; assert_eq!( metadata.metadata_location().unwrap(), - "s3://other-bucket/custom-meta/" + "s3://other-bucket/custom-meta" ); } } diff --git a/crates/iceberg/src/spec/table_props.rs b/crates/iceberg/src/spec/table_props.rs index 251b37d3ac..21c2be9073 100644 --- a/crates/iceberg/src/spec/table_props.rs +++ b/crates/iceberg/src/spec/table_props.rs @@ -17,21 +17,21 @@ //! Typed access to Iceberg table properties. //! -//! [`TableProperties`] exposes Iceberg's string-keyed table properties as typed public -//! fields. Its JSON representation is a flat object whose keys and values are strings. +//! [`TableProperties`] exposes Iceberg's string-keyed table properties through typed getters and +//! setters. Its JSON representation is a flat object whose keys and values are strings. //! //! # Create from defaults //! -//! Start with Iceberg's defaults and modify public fields directly: +//! Start with Iceberg's defaults and modify properties through generated setters: //! //! ``` //! use iceberg::spec::{DataFileFormat, TableProperties}; //! //! let mut properties = TableProperties::default(); -//! properties.write_format_default = DataFileFormat::Orc; -//! properties.write_data_path = Some("s3://warehouse/table/data".to_string()); +//! properties.set_write_format_default(DataFileFormat::Orc); +//! properties.set_write_data_path(Some("s3://warehouse/table/data".to_string())); //! -//! assert_eq!(properties.write_format_default, DataFileFormat::Orc); +//! assert_eq!(*properties.write_format_default(), DataFileFormat::Orc); //! ``` //! //! # Deserialize from JSON @@ -46,8 +46,8 @@ //! "write.format.default": "orc" //! })).unwrap(); //! -//! assert_eq!(properties.commit_retry_num_retries, 8); -//! assert_eq!(properties.write_format_default, DataFileFormat::Orc); +//! assert_eq!(*properties.commit_retry_num_retries(), 8); +//! assert_eq!(*properties.write_format_default(), DataFileFormat::Orc); //! ``` //! //! # Serialize to JSON @@ -59,8 +59,8 @@ //! use iceberg::spec::TableProperties; //! //! let mut properties = TableProperties::default(); -//! properties.commit_retry_num_retries = 8; -//! properties.write_data_path = Some("s3://warehouse/table/data".to_string()); +//! properties.set_commit_retry_num_retries(8); +//! properties.set_write_data_path(Some("s3://warehouse/table/data".to_string())); //! //! let json = serde_json::to_value(&properties).unwrap(); //! assert_eq!(json["commit.retry.num-retries"], "8"); @@ -74,8 +74,9 @@ use iceberg_property_macro::Properties; use serde_with::{DeserializeFromStr, SerializeDisplay}; use crate::compression::CompressionCodec; -use crate::error::Result; +use crate::error::{Error, ErrorKind, Result}; use crate::spec::{DataFileFormat, NameMapping}; +use crate::util::location::strip_trailing_slash; /// Parquet data page version 1. pub const PARQUET_PAGE_VERSION_V1: &str = "v1"; @@ -184,6 +185,241 @@ fn serialize_comma_separated_strings(values: &[String]) -> String { values.join(",") } +fn parse_table_file_format(value: &str) -> Result { + let format = value.parse::()?; + match format { + DataFileFormat::Avro | DataFileFormat::Orc | DataFileFormat::Parquet => Ok(format), + DataFileFormat::Puffin => Err(Error::new( + ErrorKind::DataInvalid, + format!("Unsupported table data file format: {value}"), + )), + } +} + +fn parse_delete_file_format( + properties: &HashMap, + delete_format_key: &str, + data_format_key: &str, + default: DataFileFormat, +) -> Result { + properties + .get(delete_format_key) + .or_else(|| properties.get(data_format_key)) + .map(|value| parse_table_file_format(value)) + .transpose() + .map(|value| value.unwrap_or(default)) +} + +fn write_delete_file_format( + delete_format: &DataFileFormat, + properties: &mut HashMap, + delete_format_key: &str, + data_format_key: &str, + _default: &DataFileFormat, +) { + let data_format = properties + .get(data_format_key) + .and_then(|value| parse_table_file_format(value).ok()) + .unwrap_or(DataFileFormat::Parquet); + + if delete_format == &data_format { + properties.remove(delete_format_key); + } else { + properties.insert(delete_format_key.to_string(), delete_format.to_string()); + } +} + +fn parse_location(value: &str) -> Result { + if value.is_empty() { + return Err(Error::new(ErrorKind::DataInvalid, "path must not be empty")); + } + + Ok(strip_trailing_slash(value).to_string()) +} + +fn parse_optional_location(value: &str) -> Result> { + parse_location(value).map(Some) +} + +pub(crate) fn parse_metadata_location_property( + properties: &HashMap, +) -> Result> { + properties + .get(TableProperties::PROPERTY_WRITE_METADATA_PATH) + .map(|value| { + parse_location(value).map_err(|error| { + Error::new( + ErrorKind::DataInvalid, + format!( + "Invalid value for {}", + TableProperties::PROPERTY_WRITE_METADATA_PATH + ), + ) + .with_source(error) + }) + }) + .transpose() +} + +fn validate_compression_codec( + properties: &HashMap, + codec_key: &str, + allowed: &[&str], +) -> Result<()> { + let Some(value) = properties.get(codec_key) else { + return Ok(()); + }; + if allowed + .iter() + .any(|allowed| value.eq_ignore_ascii_case(allowed)) + { + return Ok(()); + } + + Err(Error::new( + ErrorKind::DataInvalid, + format!( + "Invalid compression codec for {codec_key}: {value}. Supported codecs: {}", + allowed.join(", ") + ), + )) +} + +fn parse_parquet_compression_properties( + properties: &HashMap, + codec_key: &str, + level_key: &str, + default: CompressionCodec, +) -> Result { + validate_compression_codec(properties, codec_key, &[ + "zstd", + "brotli", + "lz4", + "gzip", + "snappy", + "uncompressed", + ])?; + CompressionCodec::parse_properties(properties, codec_key, level_key, default) +} + +fn parse_avro_compression_properties( + properties: &HashMap, + codec_key: &str, + level_key: &str, + default: CompressionCodec, +) -> Result { + validate_compression_codec(properties, codec_key, &[ + "gzip", + "zstd", + "snappy", + "uncompressed", + ])?; + CompressionCodec::parse_properties(properties, codec_key, level_key, default) +} + +fn write_parquet_compression_properties( + codec: &CompressionCodec, + properties: &mut HashMap, + codec_key: &str, + level_key: &str, + default: &CompressionCodec, +) { + write_compression_properties( + codec, + properties, + codec_key, + level_key, + "uncompressed", + default, + ); +} + +fn write_avro_compression_properties( + codec: &CompressionCodec, + properties: &mut HashMap, + codec_key: &str, + level_key: &str, + default: &CompressionCodec, +) { + write_compression_properties( + codec, + properties, + codec_key, + level_key, + "uncompressed", + default, + ); +} + +fn write_compression_properties( + codec: &CompressionCodec, + properties: &mut HashMap, + codec_key: &str, + level_key: &str, + uncompressed_value: &str, + default: &CompressionCodec, +) { + if codec == default { + properties.remove(codec_key); + properties.remove(level_key); + return; + } + + let value = match codec { + CompressionCodec::None => uncompressed_value.to_string(), + codec => codec.property_value(), + }; + properties.insert(codec_key.to_string(), value); + + match codec { + CompressionCodec::Gzip(level) | CompressionCodec::Zstd(level) => { + properties.insert(level_key.to_string(), level.to_string()); + } + _ => { + properties.remove(level_key); + } + } +} + +fn parse_orc_compression_codec(value: &str) -> Result { + if !["zstd", "lz4", "lzo", "zlib", "snappy", "none"] + .iter() + .any(|allowed| value.eq_ignore_ascii_case(allowed)) + { + return Err(Error::new( + ErrorKind::DataInvalid, + format!("Invalid ORC compression codec: {value}"), + )); + } + + CompressionCodec::parse_property(value) +} + +fn parse_name_mapping(value: &str) -> Result> { + serde_json::from_str(value).map(Some).map_err(|error| { + Error::new(ErrorKind::DataInvalid, "Invalid name mapping").with_source(error) + }) +} + +fn serialize_name_mapping(value: &Option) -> String { + serde_json::to_string( + value + .as_ref() + .expect("checked is_some before serialization"), + ) + .expect("serializing a name mapping should not fail") +} + +pub(crate) fn parse_metadata_file_compression( + properties: &HashMap, +) -> Result { + properties + .get(TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC) + .map(|value| CompressionCodec::parse_metadata_property(value)) + .transpose() + .map(|codec| codec.unwrap_or(CompressionCodec::None)) +} + /// Typed Iceberg table properties organized into documented sections. /// /// Serde represents this struct as Iceberg's flat string-to-string property map. Property @@ -198,617 +434,740 @@ pub struct TableProperties { #[key = "comment"] #[default(None)] #[doc = "Table-level description of the table's business meaning and usage context."] - pub comment: Option, + #[property(pub(getter), pub(setter))] + comment: Option, #[key = "identifier-fields.rely"] #[default(false)] #[doc = "Whether query engines may rely on identifier fields as a primary key for optimization; this is not enforced on writes."] - pub identifier_fields_rely: bool, + #[property(pub(getter), pub(setter))] + identifier_fields_rely: bool, // Commit properties. #[key = "commit.retry.num-retries"] #[default(4)] #[doc = "Number of times to retry a commit before failing."] - pub commit_retry_num_retries: usize, + #[property(pub(getter), pub(setter))] + commit_retry_num_retries: usize, #[key = "commit.retry.min-wait-ms"] #[default(100)] #[doc = "Minimum time in milliseconds to wait before retrying a commit."] - pub commit_retry_min_wait_ms: u64, + #[property(pub(getter), pub(setter))] + commit_retry_min_wait_ms: u64, #[key = "commit.retry.max-wait-ms"] #[default(60 * 1000)] #[doc = "Maximum time in milliseconds to wait before retrying a commit."] - pub commit_retry_max_wait_ms: u64, + #[property(pub(getter), pub(setter))] + commit_retry_max_wait_ms: u64, #[key = "commit.retry.total-timeout-ms"] #[default(30 * 60 * 1000)] #[doc = "Total commit retry timeout in milliseconds."] - pub commit_retry_total_timeout_ms: u64, + #[property(pub(getter), pub(setter))] + commit_retry_total_timeout_ms: u64, #[key = "commit.status-check.num-retries"] #[default(3)] #[doc = "Number of times to check whether a commit succeeded after connectivity is lost."] - pub commit_status_check_num_retries: usize, + #[property(pub(getter), pub(setter))] + commit_status_check_num_retries: usize, #[key = "commit.status-check.min-wait-ms"] #[default(1000)] #[doc = "Minimum time in milliseconds to wait before retrying a commit status check."] - pub commit_status_check_min_wait_ms: u64, + #[property(pub(getter), pub(setter))] + commit_status_check_min_wait_ms: u64, #[key = "commit.status-check.max-wait-ms"] #[default(60 * 1000)] #[doc = "Maximum time in milliseconds to wait before retrying a commit status check."] - pub commit_status_check_max_wait_ms: u64, + #[property(pub(getter), pub(setter))] + commit_status_check_max_wait_ms: u64, #[key = "commit.status-check.total-timeout-ms"] #[default(30 * 60 * 1000)] #[doc = "Total timeout in milliseconds in which commit status checking must succeed."] - pub commit_status_check_total_timeout_ms: u64, + #[property(pub(getter), pub(setter))] + commit_status_check_total_timeout_ms: u64, // Manifest properties. #[key = "commit.manifest.target-size-bytes"] #[default(8 * 1024 * 1024)] #[doc = "Target size in bytes when merging manifest files."] - pub commit_manifest_target_size_bytes: usize, + #[property(pub(getter), pub(setter))] + commit_manifest_target_size_bytes: usize, #[key = "commit.manifest.min-count-to-merge"] #[default(100)] #[doc = "Minimum number of manifests to accumulate before merging."] - pub commit_manifest_min_count_to_merge: usize, + #[property(pub(getter), pub(setter))] + commit_manifest_min_count_to_merge: usize, #[key = "commit.manifest-merge.enabled"] #[default(true)] #[doc = "Whether manifests are automatically merged during writes."] - pub commit_manifest_merge_enabled: bool, + #[property(pub(getter), pub(setter))] + commit_manifest_merge_enabled: bool, #[key = "write.manifest.compression-codec"] #[additional_key = "write.manifest.compression-level"] #[default(CompressionCodec::gzip_default())] - #[parse_properties_with(CompressionCodec::parse_properties)] - #[write_properties_with(CompressionCodec::write_properties)] + #[parse_properties_with(parse_avro_compression_properties)] + #[write_properties_with(write_avro_compression_properties)] #[doc = "Compression codec used for manifest files."] - pub write_manifest_compression_codec: CompressionCodec, + #[property(pub(getter), pub(setter))] + write_manifest_compression_codec: CompressionCodec, #[key = "write.manifest-lists.enabled"] #[default(true)] #[doc = "Deprecated flag for writing manifest lists; manifest lists are always enabled."] - pub write_manifest_lists_enabled: bool, + #[property(pub(getter), pub(setter))] + write_manifest_lists_enabled: bool, // Write properties. #[key = "write.format.default"] #[default(DataFileFormat::Parquet)] + #[parse_with(parse_table_file_format)] #[doc = "Default data file format: Parquet, Avro, or ORC."] - pub write_format_default: DataFileFormat, + #[property(pub(getter), pub(setter))] + write_format_default: DataFileFormat, #[key = "write.delete.format.default"] + #[additional_key = "write.format.default"] #[default(DataFileFormat::Parquet)] + #[parse_properties_with(parse_delete_file_format)] + #[write_properties_with(write_delete_file_format)] #[doc = "Default delete file format: Parquet, Avro, or ORC."] - pub write_delete_format_default: DataFileFormat, + #[property(pub(getter), pub(setter))] + write_delete_format_default: DataFileFormat, #[key = "write.target-file-size-bytes"] #[default(512 * 1024 * 1024)] #[doc = "Target size in bytes for generated data files."] - pub write_target_file_size_bytes: usize, + #[property(pub(getter), pub(setter))] + write_target_file_size_bytes: usize, #[key = "write.delete.target-file-size-bytes"] #[default(64 * 1024 * 1024)] #[doc = "Target size in bytes for generated delete files."] - pub write_delete_target_file_size_bytes: usize, + #[property(pub(getter), pub(setter))] + write_delete_target_file_size_bytes: usize, #[key = "write.object-storage.enabled"] #[default(false)] #[doc = "Whether the object-storage location provider adds a hash component to file paths."] - pub write_object_storage_enabled: bool, + #[property(pub(getter), pub(setter))] + write_object_storage_enabled: bool, #[key = "write.object-storage.partitioned-paths"] #[default(true)] #[doc = "Whether object-storage file paths include partition values."] - pub write_object_storage_partitioned_paths: bool, + #[property(pub(getter), pub(setter))] + write_object_storage_partitioned_paths: bool, #[key = "write.object-storage.path"] #[default(None)] #[doc = "Deprecated base object-storage path; use write.data.path instead."] - pub write_object_storage_path: Option, + #[property(pub(getter), pub(setter))] + write_object_storage_path: Option, #[key = "write.location-provider.impl"] #[default(None)] #[doc = "Optional custom location provider implementation."] - pub write_location_provider_impl: Option, + #[property(pub(getter), pub(setter))] + write_location_provider_impl: Option, #[key = "write.folder-storage.path"] #[default(None)] #[doc = "Deprecated base folder-storage path; use write.data.path instead."] - pub write_folder_storage_path: Option, + #[property(pub(getter), pub(setter))] + write_folder_storage_path: Option, #[key = "write.data.path"] #[default(None)] #[doc = "Base location for data files written after this property is set."] - pub write_data_path: Option, + #[property(pub(getter), pub(setter))] + write_data_path: Option, #[key = "write.wap.enabled"] #[default(false)] #[doc = "Whether write-audit-publish writes are enabled."] - pub write_wap_enabled: bool, + #[property(pub(getter), pub(setter))] + write_wap_enabled: bool, #[key = "write.distribution-mode"] - #[default(DistributionMode::None)] + #[default(None)] #[doc = "Write distribution mode: none, hash, or range."] - pub write_distribution_mode: DistributionMode, + #[property(pub(getter), pub(setter))] + write_distribution_mode: Option, #[key = "write.datafusion.fanout.enabled"] #[default(true)] #[doc = "Whether DataFusion uses a fanout writer for partitioned tables."] - pub write_datafusion_fanout_enabled: bool, + #[property(pub(getter), pub(setter))] + write_datafusion_fanout_enabled: bool, // Parquet properties. #[key = "write.parquet.row-group-size-bytes"] #[default(128 * 1024 * 1024)] #[doc = "Parquet row group size in bytes for data files."] - pub write_parquet_row_group_size_bytes: usize, + #[property(pub(getter), pub(setter))] + write_parquet_row_group_size_bytes: usize, #[key = "write.delete.parquet.row-group-size-bytes"] #[default(128 * 1024 * 1024)] #[doc = "Parquet row group size in bytes for delete files."] - pub write_delete_parquet_row_group_size_bytes: usize, + #[property(pub(getter), pub(setter))] + write_delete_parquet_row_group_size_bytes: usize, #[key = "write.parquet.page-size-bytes"] #[default(1024 * 1024)] #[doc = "Parquet page size in bytes for data files."] - pub write_parquet_page_size_bytes: usize, + #[property(pub(getter), pub(setter))] + write_parquet_page_size_bytes: usize, #[key = "write.delete.parquet.page-size-bytes"] #[default(1024 * 1024)] #[doc = "Parquet page size in bytes for delete files."] - pub write_delete_parquet_page_size_bytes: usize, + #[property(pub(getter), pub(setter))] + write_delete_parquet_page_size_bytes: usize, #[key = "write.parquet.page-version"] #[default(PARQUET_PAGE_VERSION_V1)] #[doc = "Parquet data page version for data files: v1 or v2."] - pub write_parquet_page_version: String, + #[property(pub(getter), pub(setter))] + write_parquet_page_version: String, #[key = "write.delete.parquet.page-version"] #[default(PARQUET_PAGE_VERSION_V1)] #[doc = "Parquet data page version for delete files: v1 or v2."] - pub write_delete_parquet_page_version: String, + #[property(pub(getter), pub(setter))] + write_delete_parquet_page_version: String, #[key = "write.parquet.page-row-limit"] #[default(20_000)] #[doc = "Maximum number of rows per Parquet page in data files."] - pub write_parquet_page_row_limit: usize, + #[property(pub(getter), pub(setter))] + write_parquet_page_row_limit: usize, #[key = "write.delete.parquet.page-row-limit"] #[default(20_000)] #[doc = "Maximum number of rows per Parquet page in delete files."] - pub write_delete_parquet_page_row_limit: usize, + #[property(pub(getter), pub(setter))] + write_delete_parquet_page_row_limit: usize, #[key = "write.parquet.dict-size-bytes"] #[default(2 * 1024 * 1024)] #[doc = "Parquet dictionary page size in bytes for data files."] - pub write_parquet_dict_size_bytes: usize, + #[property(pub(getter), pub(setter))] + write_parquet_dict_size_bytes: usize, #[key = "write.delete.parquet.dict-size-bytes"] #[default(2 * 1024 * 1024)] #[doc = "Parquet dictionary page size in bytes for delete files."] - pub write_delete_parquet_dict_size_bytes: usize, + #[property(pub(getter), pub(setter))] + write_delete_parquet_dict_size_bytes: usize, #[key = "write.parquet.compression-codec"] #[additional_key = "write.parquet.compression-level"] #[default(CompressionCodec::zstd_default())] - #[parse_properties_with(CompressionCodec::parse_properties)] - #[write_properties_with(CompressionCodec::write_properties)] + #[parse_properties_with(parse_parquet_compression_properties)] + #[write_properties_with(write_parquet_compression_properties)] #[doc = "Parquet compression codec used for data files."] - pub write_parquet_compression_codec: CompressionCodec, + #[property(pub(getter), pub(setter))] + write_parquet_compression_codec: CompressionCodec, #[key = "write.delete.parquet.compression-codec"] #[additional_key = "write.delete.parquet.compression-level"] #[default(CompressionCodec::zstd_default())] - #[parse_properties_with(CompressionCodec::parse_properties)] - #[write_properties_with(CompressionCodec::write_properties)] + #[parse_properties_with(parse_parquet_compression_properties)] + #[write_properties_with(write_parquet_compression_properties)] #[doc = "Parquet compression codec used for delete files."] - pub write_delete_parquet_compression_codec: CompressionCodec, + #[property(pub(getter), pub(setter))] + write_delete_parquet_compression_codec: CompressionCodec, #[key = "write.parquet.shred-variants"] #[default(false)] #[doc = "Whether variant columns use shredded Parquet encoding for improved query performance."] - pub write_parquet_shred_variants: bool, + #[property(pub(getter), pub(setter))] + write_parquet_shred_variants: bool, #[key = "write.parquet.variant-inference-buffer-size"] #[default(100)] #[doc = "Number of rows buffered for schema inference when variant shredding is enabled."] - pub write_parquet_variant_inference_buffer_size: usize, + #[property(pub(getter), pub(setter))] + write_parquet_variant_inference_buffer_size: usize, #[key = "write.parquet.row-group-check-min-record-count"] #[default(100)] #[doc = "Minimum record count between Parquet data-file row group size checks."] - pub write_parquet_row_group_check_min_record_count: usize, + #[property(pub(getter), pub(setter))] + write_parquet_row_group_check_min_record_count: usize, #[key = "write.delete.parquet.row-group-check-min-record-count"] #[default(100)] #[doc = "Minimum record count between Parquet delete-file row group size checks."] - pub write_delete_parquet_row_group_check_min_record_count: usize, + #[property(pub(getter), pub(setter))] + write_delete_parquet_row_group_check_min_record_count: usize, #[key = "write.parquet.row-group-check-max-record-count"] #[default(10_000)] #[doc = "Maximum record count between Parquet data-file row group size checks."] - pub write_parquet_row_group_check_max_record_count: usize, + #[property(pub(getter), pub(setter))] + write_parquet_row_group_check_max_record_count: usize, #[key = "write.delete.parquet.row-group-check-max-record-count"] #[default(10_000)] #[doc = "Maximum record count between Parquet delete-file row group size checks."] - pub write_delete_parquet_row_group_check_max_record_count: usize, + #[property(pub(getter), pub(setter))] + write_delete_parquet_row_group_check_max_record_count: usize, #[key = "write.parquet.row-group-size-track-uncompressed"] #[default(false)] #[doc = "Whether uncompressed data size is tracked to enforce the Parquet row group target."] - pub write_parquet_row_group_size_track_uncompressed: bool, + #[property(pub(getter), pub(setter))] + write_parquet_row_group_size_track_uncompressed: bool, #[key = "write.parquet.bloom-filter-max-bytes"] #[default(1024 * 1024)] #[doc = "Maximum number of bytes for a Parquet bloom filter bitset."] - pub write_parquet_bloom_filter_max_bytes: usize, + #[property(pub(getter), pub(setter))] + write_parquet_bloom_filter_max_bytes: usize, #[key = "write.parquet.bloom-filter-adaptive-enabled"] #[default(false)] #[doc = "Whether adaptive Parquet bloom filter sizing selects the smallest suitable filter."] - pub write_parquet_bloom_filter_adaptive_enabled: bool, + #[property(pub(getter), pub(setter))] + write_parquet_bloom_filter_adaptive_enabled: bool, #[prefix = "write.parquet.bloom-filter-fpp.column."] #[default(HashMap::new())] #[doc = "Per-column Parquet bloom filter false-positive probabilities, keyed by column name."] - pub write_parquet_bloom_filter_fpp_column: HashMap, + #[property(pub(getter), pub(setter))] + write_parquet_bloom_filter_fpp_column: HashMap, #[prefix = "write.parquet.bloom-filter-ndv.column."] #[default(HashMap::new())] #[doc = "Per-column expected distinct-value counts for Parquet bloom filters."] - pub write_parquet_bloom_filter_ndv_column: HashMap, + #[property(pub(getter), pub(setter))] + write_parquet_bloom_filter_ndv_column: HashMap, #[prefix = "write.parquet.bloom-filter-enabled.column."] #[default(HashMap::new())] #[doc = "Per-column flags controlling whether Parquet bloom filters are written."] - pub write_parquet_bloom_filter_enabled_column: HashMap, + #[property(pub(getter), pub(setter))] + write_parquet_bloom_filter_enabled_column: HashMap, #[prefix = "write.parquet.stats-enabled.column."] #[default(HashMap::new())] #[doc = "Per-column flags controlling whether Parquet column statistics are collected."] - pub write_parquet_stats_enabled_column: HashMap, + #[property(pub(getter), pub(setter))] + write_parquet_stats_enabled_column: HashMap, #[prefix = "write.parquet.dict-encoding-enabled.column."] #[default(HashMap::new())] #[doc = "Per-column flags controlling whether Parquet dictionary encoding is used."] - pub write_parquet_dict_encoding_enabled_column: HashMap, + #[property(pub(getter), pub(setter))] + write_parquet_dict_encoding_enabled_column: HashMap, #[key = "write.parquet.content-defined-chunking.enabled"] #[default(false)] #[doc = "Whether Parquet content-defined chunking is enabled."] - pub write_parquet_content_defined_chunking_enabled: bool, + #[property(pub(getter), pub(setter))] + write_parquet_content_defined_chunking_enabled: bool, #[key = "write.parquet.content-defined-chunking.min-chunk-size"] #[default(256 * 1024)] #[doc = "Minimum Parquet content-defined chunk size in bytes."] - pub write_parquet_content_defined_chunking_min_chunk_size: usize, + #[property(pub(getter), pub(setter))] + write_parquet_content_defined_chunking_min_chunk_size: usize, #[key = "write.parquet.content-defined-chunking.max-chunk-size"] #[default(1024 * 1024)] #[doc = "Maximum Parquet content-defined chunk size in bytes."] - pub write_parquet_content_defined_chunking_max_chunk_size: usize, + #[property(pub(getter), pub(setter))] + write_parquet_content_defined_chunking_max_chunk_size: usize, #[key = "write.parquet.content-defined-chunking.norm-level"] #[default(0)] #[doc = "Gearhash normalization level used by Parquet content-defined chunking."] - pub write_parquet_content_defined_chunking_norm_level: i32, + #[property(pub(getter), pub(setter))] + write_parquet_content_defined_chunking_norm_level: i32, // Avro properties. #[key = "write.avro.compression-codec"] #[additional_key = "write.avro.compression-level"] #[default(CompressionCodec::gzip_default())] - #[parse_properties_with(CompressionCodec::parse_properties)] - #[write_properties_with(CompressionCodec::write_properties)] + #[parse_properties_with(parse_avro_compression_properties)] + #[write_properties_with(write_avro_compression_properties)] #[doc = "Avro compression codec used for data files."] - pub write_avro_compression_codec: CompressionCodec, + #[property(pub(getter), pub(setter))] + write_avro_compression_codec: CompressionCodec, #[key = "write.delete.avro.compression-codec"] #[additional_key = "write.delete.avro.compression-level"] #[default(CompressionCodec::gzip_default())] - #[parse_properties_with(CompressionCodec::parse_properties)] - #[write_properties_with(CompressionCodec::write_properties)] + #[parse_properties_with(parse_avro_compression_properties)] + #[write_properties_with(write_avro_compression_properties)] #[doc = "Avro compression codec used for delete files."] - pub write_delete_avro_compression_codec: CompressionCodec, + #[property(pub(getter), pub(setter))] + write_delete_avro_compression_codec: CompressionCodec, // ORC properties. #[key = "write.orc.stripe-size-bytes"] #[default(64 * 1024 * 1024)] #[doc = "Default ORC stripe size in bytes for data files."] - pub write_orc_stripe_size_bytes: u64, + #[property(pub(getter), pub(setter))] + write_orc_stripe_size_bytes: u64, #[key = "write.delete.orc.stripe-size-bytes"] #[default(64 * 1024 * 1024)] #[doc = "Default ORC stripe size in bytes for delete files."] - pub write_delete_orc_stripe_size_bytes: u64, + #[property(pub(getter), pub(setter))] + write_delete_orc_stripe_size_bytes: u64, #[key = "write.orc.bloom.filter.columns"] #[default(Vec::new())] #[parse_with(parse_comma_separated_strings)] #[serialize_with(serialize_comma_separated_strings)] #[doc = "Comma-separated column names for which ORC bloom filters are created."] - pub write_orc_bloom_filter_columns: Vec, + #[property(pub(getter), pub(setter))] + write_orc_bloom_filter_columns: Vec, #[key = "write.orc.bloom.filter.fpp"] #[default(0.05)] #[doc = "False-positive probability for ORC bloom filters."] - pub write_orc_bloom_filter_fpp: f64, + #[property(pub(getter), pub(setter))] + write_orc_bloom_filter_fpp: f64, #[key = "write.orc.block-size-bytes"] #[default(256 * 1024 * 1024)] #[doc = "Default file-system block size in bytes for ORC data files."] - pub write_orc_block_size_bytes: u64, + #[property(pub(getter), pub(setter))] + write_orc_block_size_bytes: u64, #[key = "write.delete.orc.block-size-bytes"] #[default(256 * 1024 * 1024)] #[doc = "Default file-system block size in bytes for ORC delete files."] - pub write_delete_orc_block_size_bytes: u64, + #[property(pub(getter), pub(setter))] + write_delete_orc_block_size_bytes: u64, #[key = "write.orc.vectorized.batch-size"] #[default(1024)] #[doc = "ORC vectorized write batch size for data files."] - pub write_orc_vectorized_batch_size: usize, + #[property(pub(getter), pub(setter))] + write_orc_vectorized_batch_size: usize, #[key = "write.delete.orc.vectorized.batch-size"] #[default(1024)] #[doc = "ORC vectorized write batch size for delete files."] - pub write_delete_orc_vectorized_batch_size: usize, + #[property(pub(getter), pub(setter))] + write_delete_orc_vectorized_batch_size: usize, #[key = "write.orc.compression-codec"] #[default(CompressionCodec::Zlib)] - #[parse_with(CompressionCodec::parse_property)] + #[parse_with(parse_orc_compression_codec)] #[serialize_with(CompressionCodec::property_value)] #[doc = "ORC compression codec used for data files."] - pub write_orc_compression_codec: CompressionCodec, + #[property(pub(getter), pub(setter))] + write_orc_compression_codec: CompressionCodec, #[key = "write.delete.orc.compression-codec"] #[default(CompressionCodec::Zlib)] - #[parse_with(CompressionCodec::parse_property)] + #[parse_with(parse_orc_compression_codec)] #[serialize_with(CompressionCodec::property_value)] #[doc = "ORC compression codec used for delete files."] - pub write_delete_orc_compression_codec: CompressionCodec, + #[property(pub(getter), pub(setter))] + write_delete_orc_compression_codec: CompressionCodec, #[key = "write.orc.compression-strategy"] #[default(ORC_COMPRESSION_STRATEGY_SPEED)] #[doc = "ORC compression strategy for data files: speed or compression."] - pub write_orc_compression_strategy: String, + #[property(pub(getter), pub(setter))] + write_orc_compression_strategy: String, #[key = "write.delete.orc.compression-strategy"] #[default(ORC_COMPRESSION_STRATEGY_SPEED)] #[doc = "ORC compression strategy for delete files: speed or compression."] - pub write_delete_orc_compression_strategy: String, + #[property(pub(getter), pub(setter))] + write_delete_orc_compression_strategy: String, // Read properties. #[key = "read.split.target-size"] #[default(128 * 1024 * 1024)] #[doc = "Target size in bytes when combining data input splits."] - pub read_split_target_size: u64, + #[property(pub(getter), pub(setter))] + read_split_target_size: u64, #[key = "read.split.metadata-target-size"] #[default(32 * 1024 * 1024)] #[doc = "Target size in bytes when combining metadata input splits."] - pub read_split_metadata_target_size: u64, + #[property(pub(getter), pub(setter))] + read_split_metadata_target_size: u64, #[key = "read.split.planning-lookback"] #[default(10)] #[doc = "Number of bins considered when combining input splits."] - pub read_split_planning_lookback: usize, + #[property(pub(getter), pub(setter))] + read_split_planning_lookback: usize, #[key = "read.split.open-file-cost"] #[default(4 * 1024 * 1024)] #[doc = "Estimated file-open cost used as a minimum weight when combining splits."] - pub read_split_open_file_cost: u64, + #[property(pub(getter), pub(setter))] + read_split_open_file_cost: u64, #[key = "read.split.adaptive-size.enabled"] #[default(true)] #[doc = "Whether split size is adapted to the scan size."] - pub read_split_adaptive_size_enabled: bool, + #[property(pub(getter), pub(setter))] + read_split_adaptive_size_enabled: bool, #[key = "read.parquet.vectorization.enabled"] #[default(true)] #[doc = "Whether Parquet vectorized reads are enabled."] - pub read_parquet_vectorization_enabled: bool, + #[property(pub(getter), pub(setter))] + read_parquet_vectorization_enabled: bool, #[key = "read.parquet.vectorization.batch-size"] #[default(5000)] #[doc = "Batch size for Parquet vectorized reads."] - pub read_parquet_vectorization_batch_size: usize, + #[property(pub(getter), pub(setter))] + read_parquet_vectorization_batch_size: usize, #[key = "read.orc.vectorization.enabled"] #[default(false)] #[doc = "Whether ORC vectorized reads are enabled."] - pub read_orc_vectorization_enabled: bool, + #[property(pub(getter), pub(setter))] + read_orc_vectorization_enabled: bool, #[key = "read.orc.vectorization.batch-size"] #[default(5000)] #[doc = "Batch size for ORC vectorized reads."] - pub read_orc_vectorization_batch_size: usize, + #[property(pub(getter), pub(setter))] + read_orc_vectorization_batch_size: usize, #[key = "read.data-planning-mode"] #[default("auto")] #[doc = "Planning mode used for data files."] - pub read_data_planning_mode: String, + #[property(pub(getter), pub(setter))] + read_data_planning_mode: String, #[key = "read.delete-planning-mode"] #[default("auto")] #[doc = "Planning mode used for delete files."] - pub read_delete_planning_mode: String, + #[property(pub(getter), pub(setter))] + read_delete_planning_mode: String, // Metadata properties. #[key = "write.metadata.path"] #[default(None)] + #[parse_with(parse_optional_location)] #[doc = "Base location for metadata files written after this property is set."] - pub write_metadata_path: Option, + #[property(pub(getter), pub(setter))] + write_metadata_path: Option, #[key = "write.summary.partition-limit"] #[default(0)] #[doc = "Maximum changed-partition count for including partition-level statistics in snapshot summaries."] - pub write_summary_partition_limit: u64, + #[property(pub(getter), pub(setter))] + write_summary_partition_limit: u64, #[key = "write.metadata.compression-codec"] #[default(CompressionCodec::None)] #[parse_with(CompressionCodec::parse_metadata_property)] #[serialize_with(CompressionCodec::property_value)] #[doc = "Compression codec for metadata JSON files: none or gzip."] - pub write_metadata_compression_codec: CompressionCodec, + #[property(pub(getter), pub(setter))] + write_metadata_compression_codec: CompressionCodec, #[key = "write.metadata.previous-versions-max"] #[default(100)] #[doc = "Maximum number of previous metadata file versions to track."] - pub write_metadata_previous_versions_max: usize, + #[property(pub(getter), pub(setter))] + write_metadata_previous_versions_max: usize, #[key = "write.metadata.delete-after-commit.enabled"] #[default(false)] #[doc = "Whether the oldest tracked metadata file is deleted after each commit."] - pub write_metadata_delete_after_commit_enabled: bool, + #[property(pub(getter), pub(setter))] + write_metadata_delete_after_commit_enabled: bool, #[key = "write.metadata.metrics.max-inferred-column-defaults"] #[default(100)] #[doc = "Maximum number of columns that receive inferred metrics defaults."] - pub write_metadata_metrics_max_inferred_column_defaults: usize, + #[property(pub(getter), pub(setter))] + write_metadata_metrics_max_inferred_column_defaults: usize, #[prefix = "write.metadata.metrics.column."] #[default(HashMap::new())] #[doc = "Per-column metrics modes keyed by column name."] - pub write_metadata_metrics_column: HashMap, + #[property(pub(getter), pub(setter))] + write_metadata_metrics_column: HashMap, #[key = "write.metadata.metrics.default"] #[default("truncate(16)")] #[doc = "Default metrics mode for table columns."] - pub write_metadata_metrics_default: String, + #[property(pub(getter), pub(setter))] + write_metadata_metrics_default: String, #[key = "schema.name-mapping.default"] #[default(None)] + #[parse_with(parse_name_mapping)] + #[serialize_with(serialize_name_mapping)] #[doc = "Default JSON name mapping used to resolve columns in files without field IDs."] - pub schema_name_mapping_default: Option, + #[property(pub(getter), pub(setter))] + schema_name_mapping_default: Option, // Compatibility properties. #[key = "write.spark.fanout.enabled"] #[default(false)] #[doc = "Deprecated Spark fanout-writer flag; the fanout writer accepts unclustered data but uses more memory."] - pub write_spark_fanout_enabled: bool, + #[property(pub(getter), pub(setter))] + write_spark_fanout_enabled: bool, #[key = "write.spark.accept-any-schema"] #[default(false)] #[doc = "Deprecated Spark flag allowing writes with any compatible schema."] - pub write_spark_accept_any_schema: bool, + #[property(pub(getter), pub(setter))] + write_spark_accept_any_schema: bool, #[key = "write.spark.auto-schema-evolution.enabled"] #[default(true)] #[doc = "Deprecated Spark flag enabling automatic schema evolution during writes."] - pub write_spark_auto_schema_evolution_enabled: bool, + #[property(pub(getter), pub(setter))] + write_spark_auto_schema_evolution_enabled: bool, #[key = "write.spark.advisory-partition-size-bytes"] #[default(None)] #[doc = "Deprecated Spark advisory partition size in bytes."] - pub write_spark_advisory_partition_size_bytes: Option, + #[property(pub(getter), pub(setter))] + write_spark_advisory_partition_size_bytes: Option, #[key = "compatibility.snapshot-id-inheritance.enabled"] #[default(false)] #[doc = "Whether snapshots may be committed without explicit snapshot IDs; format version 2 and later always allow this."] - pub compatibility_snapshot_id_inheritance_enabled: bool, + #[property(pub(getter), pub(setter))] + compatibility_snapshot_id_inheritance_enabled: bool, #[key = "engine.hive.enabled"] #[default(false)] #[doc = "Whether Hive engine integration behavior is enabled."] - pub engine_hive_enabled: bool, + #[property(pub(getter), pub(setter))] + engine_hive_enabled: bool, #[key = "engine.hive.lock-enabled"] #[default(true)] #[doc = "Whether Hive locking is enabled."] - pub engine_hive_lock_enabled: bool, + #[property(pub(getter), pub(setter))] + engine_hive_lock_enabled: bool, // History properties. #[key = "gc.enabled"] #[default(true)] #[doc = "Whether garbage collection operations such as snapshot expiration and orphan-file removal are allowed."] - pub gc_enabled: bool, + #[property(pub(getter), pub(setter))] + gc_enabled: bool, #[key = "history.expire.max-snapshot-age-ms"] #[default(5 * 24 * 60 * 60 * 1000)] #[doc = "Default maximum snapshot age in milliseconds while expiring snapshots."] - pub history_expire_max_snapshot_age_ms: i64, + #[property(pub(getter), pub(setter))] + history_expire_max_snapshot_age_ms: i64, #[key = "history.expire.min-snapshots-to-keep"] #[default(1)] #[doc = "Default minimum number of snapshots retained per branch while expiring snapshots."] - pub history_expire_min_snapshots_to_keep: usize, + #[property(pub(getter), pub(setter))] + history_expire_min_snapshots_to_keep: usize, #[key = "history.expire.max-ref-age-ms"] #[default(i64::MAX)] #[doc = "Default maximum age in milliseconds for snapshot references other than the main branch."] - pub history_expire_max_ref_age_ms: i64, + #[property(pub(getter), pub(setter))] + history_expire_max_ref_age_ms: i64, // Row-level operation properties. #[key = "write.delete.granularity"] #[default(DeleteGranularity::Partition)] #[doc = "Granularity of generated delete files: partition or file."] - pub write_delete_granularity: DeleteGranularity, + #[property(pub(getter), pub(setter))] + write_delete_granularity: DeleteGranularity, #[key = "write.delete.isolation-level"] #[default(IsolationLevel::Serializable)] #[doc = "Isolation level for delete commands: serializable or snapshot."] - pub write_delete_isolation_level: IsolationLevel, + #[property(pub(getter), pub(setter))] + write_delete_isolation_level: IsolationLevel, #[key = "write.delete.mode"] #[default(RowLevelOperationMode::CopyOnWrite)] #[doc = "Execution mode for delete commands: copy-on-write or merge-on-read."] - pub write_delete_mode: RowLevelOperationMode, + #[property(pub(getter), pub(setter))] + write_delete_mode: RowLevelOperationMode, #[key = "write.delete.distribution-mode"] - #[default(DistributionMode::None)] + #[default(None)] #[doc = "Distribution mode for delete command data."] - pub write_delete_distribution_mode: DistributionMode, + #[property(pub(getter), pub(setter))] + write_delete_distribution_mode: Option, #[key = "write.update.isolation-level"] #[default(IsolationLevel::Serializable)] #[doc = "Isolation level for update commands: serializable or snapshot."] - pub write_update_isolation_level: IsolationLevel, + #[property(pub(getter), pub(setter))] + write_update_isolation_level: IsolationLevel, #[key = "write.update.mode"] #[default(RowLevelOperationMode::CopyOnWrite)] #[doc = "Execution mode for update commands: copy-on-write or merge-on-read."] - pub write_update_mode: RowLevelOperationMode, + #[property(pub(getter), pub(setter))] + write_update_mode: RowLevelOperationMode, #[key = "write.update.distribution-mode"] - #[default(DistributionMode::None)] + #[default(None)] #[doc = "Distribution mode for update command data."] - pub write_update_distribution_mode: DistributionMode, + #[property(pub(getter), pub(setter))] + write_update_distribution_mode: Option, #[key = "write.merge.isolation-level"] #[default(IsolationLevel::Serializable)] #[doc = "Isolation level for merge commands: serializable or snapshot."] - pub write_merge_isolation_level: IsolationLevel, + #[property(pub(getter), pub(setter))] + write_merge_isolation_level: IsolationLevel, #[key = "write.merge.mode"] #[default(RowLevelOperationMode::CopyOnWrite)] #[doc = "Execution mode for merge commands: copy-on-write or merge-on-read."] - pub write_merge_mode: RowLevelOperationMode, + #[property(pub(getter), pub(setter))] + write_merge_mode: RowLevelOperationMode, #[key = "write.merge.distribution-mode"] - #[default(DistributionMode::None)] + #[default(None)] #[doc = "Distribution mode for merge command data."] - pub write_merge_distribution_mode: DistributionMode, + #[property(pub(getter), pub(setter))] + write_merge_distribution_mode: Option, #[key = "write.upsert.enabled"] #[default(false)] #[doc = "Whether upsert behavior is enabled."] - pub write_upsert_enabled: bool, + #[property(pub(getter), pub(setter))] + write_upsert_enabled: bool, // Encryption properties. #[key = "encryption.key-id"] #[default(None)] #[doc = "Identifier of the table's master encryption key."] - pub encryption_key_id: Option, + #[property(pub(getter), pub(setter))] + encryption_key_id: Option, #[key = "encryption.data-key-length"] #[default(16)] #[doc = "Length in bytes of data-encryption keys; valid AES lengths are 16, 24, and 32 bytes."] - pub encryption_data_key_length: usize, + #[property(pub(getter), pub(setter))] + encryption_data_key_length: usize, } impl TableProperties { @@ -868,6 +1227,9 @@ impl TableProperties { pub const PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS: &str = "write.object-storage.partitioned-paths"; + /// Default value for including partition values in object-storage paths. + pub const PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS_DEFAULT: bool = true; + /// Property key for the snapshot-summary partition limit. pub const PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT: &str = "write.summary.partition-limit"; @@ -895,11 +1257,10 @@ impl TableProperties { } impl TryFrom<&HashMap> for TableProperties { - type Error = crate::Error; + type Error = Error; fn try_from(properties: &HashMap) -> Result { - Self::from_properties(properties) - .map_err(|error| crate::Error::new(crate::ErrorKind::DataInvalid, error)) + Self::from_properties(properties).map_err(|error| Error::new(ErrorKind::DataInvalid, error)) } } @@ -936,19 +1297,10 @@ mod tests { properties.write_orc_compression_codec, CompressionCodec::Zlib ); - assert_eq!(properties.write_distribution_mode, DistributionMode::None); - assert_eq!( - properties.write_delete_distribution_mode, - DistributionMode::None - ); - assert_eq!( - properties.write_update_distribution_mode, - DistributionMode::None - ); - assert_eq!( - properties.write_merge_distribution_mode, - DistributionMode::None - ); + assert_eq!(properties.write_distribution_mode, None); + assert_eq!(properties.write_delete_distribution_mode, None); + assert_eq!(properties.write_update_distribution_mode, None); + assert_eq!(properties.write_merge_distribution_mode, None); assert_eq!( properties.write_parquet_page_version, PARQUET_PAGE_VERSION_V1 @@ -995,7 +1347,7 @@ mod tests { write_manifest_compression_codec: CompressionCodec::Gzip(9), write_parquet_compression_codec: CompressionCodec::Zstd(5), write_delete_avro_compression_codec: CompressionCodec::Gzip(4), - write_distribution_mode: DistributionMode::Range, + write_distribution_mode: Some(DistributionMode::Range), write_orc_compression_codec: CompressionCodec::Lzo, write_orc_bloom_filter_columns: vec!["id".to_string(), "category".to_string()], schema_name_mapping_default: Some(NameMapping::new(vec![MappedField::new( @@ -1006,7 +1358,7 @@ mod tests { write_delete_granularity: DeleteGranularity::File, write_delete_isolation_level: IsolationLevel::Snapshot, write_delete_mode: RowLevelOperationMode::MergeOnRead, - write_update_distribution_mode: DistributionMode::Hash, + write_update_distribution_mode: Some(DistributionMode::Hash), write_parquet_bloom_filter_fpp_column: HashMap::from([( "customer_id".to_string(), 0.02, @@ -1027,6 +1379,7 @@ mod tests { "write.data.path": "s3://warehouse/table/data", "write.delete.avro.compression-codec": "gzip", "write.delete.avro.compression-level": "4", + "write.delete.format.default": "parquet", "write.delete.granularity": "file", "write.delete.isolation-level": "snapshot", "write.delete.mode": "merge-on-read", @@ -1079,7 +1432,10 @@ mod tests { properties.write_data_path, Some("s3://warehouse/table/data".to_string()) ); - assert_eq!(properties.write_distribution_mode, DistributionMode::Hash); + assert_eq!( + properties.write_distribution_mode, + Some(DistributionMode::Hash) + ); assert!(!properties.write_object_storage_partitioned_paths); assert_eq!( properties.write_manifest_compression_codec, @@ -1126,4 +1482,201 @@ mod tests { assert!(error.message().contains("commit.retry.num-retries")); } + + #[test] + fn metadata_path_rejects_empty_values_and_trims_trailing_slashes() { + assert_eq!(TableProperties::default().write_metadata_path, None); + + let error = parse(HashMap::from([( + "write.metadata.path".to_string(), + String::new(), + )])) + .unwrap_err(); + assert!(error.message().contains("write.metadata.path")); + + let properties = parse(HashMap::from([( + "write.metadata.path".to_string(), + "s3://bucket/table/metadata/".to_string(), + )])) + .unwrap(); + assert_eq!( + properties.write_metadata_path.as_deref(), + Some("s3://bucket/table/metadata") + ); + } + + #[test] + fn metadata_compression_preserves_legacy_validation() { + for (value, expected) in [ + ("", CompressionCodec::None), + ("none", CompressionCodec::None), + ("NONE", CompressionCodec::None), + ("gzip", CompressionCodec::gzip_default()), + ("GzIp", CompressionCodec::gzip_default()), + ] { + let properties = parse(HashMap::from([( + "write.metadata.compression-codec".to_string(), + value.to_string(), + )])) + .unwrap(); + assert_eq!(properties.write_metadata_compression_codec, expected); + } + + for invalid in ["uncompressed", "lz4", "zstd", "snappy"] { + let error = parse(HashMap::from([( + "write.metadata.compression-codec".to_string(), + invalid.to_string(), + )])) + .unwrap_err(); + assert!(error.message().contains("metadata compression codec")); + } + } + + #[test] + fn rejects_invalid_numeric_and_boolean_values() { + for (key, value) in [ + ("commit.retry.min-wait-ms", "not-a-number"), + ("write.target-file-size-bytes", "not-a-number"), + ("write.datafusion.fanout.enabled", "not-a-bool"), + ] { + let error = parse(HashMap::from([(key.to_string(), value.to_string())])).unwrap_err(); + assert!(error.message().contains(key)); + } + } + + #[test] + fn parses_content_defined_chunking_defaults_overrides_and_errors() { + let defaults = TableProperties::default(); + assert!(!defaults.write_parquet_content_defined_chunking_enabled); + assert_eq!( + defaults.write_parquet_content_defined_chunking_min_chunk_size, + 256 * 1024 + ); + assert_eq!( + defaults.write_parquet_content_defined_chunking_max_chunk_size, + 1024 * 1024 + ); + assert_eq!( + defaults.write_parquet_content_defined_chunking_norm_level, + 0 + ); + + let properties = parse(HashMap::from([ + ( + "write.parquet.content-defined-chunking.enabled".to_string(), + "TRUE".to_string(), + ), + ( + "write.parquet.content-defined-chunking.norm-level".to_string(), + "-2".to_string(), + ), + ])) + .unwrap(); + assert!(properties.write_parquet_content_defined_chunking_enabled); + assert_eq!( + properties.write_parquet_content_defined_chunking_min_chunk_size, + 256 * 1024 + ); + assert_eq!( + properties.write_parquet_content_defined_chunking_max_chunk_size, + 1024 * 1024 + ); + assert_eq!( + properties.write_parquet_content_defined_chunking_norm_level, + -2 + ); + + for key in [ + "write.parquet.content-defined-chunking.min-chunk-size", + "write.parquet.content-defined-chunking.max-chunk-size", + "write.parquet.content-defined-chunking.norm-level", + ] { + let error = parse(HashMap::from([( + key.to_string(), + "not-a-number".to_string(), + )])) + .unwrap_err(); + assert!(error.message().contains(key)); + } + } + + #[test] + fn delete_format_inherits_data_format_and_round_trips() { + let properties = parse(HashMap::from([( + "write.format.default".to_string(), + "orc".to_string(), + )])) + .unwrap(); + assert_eq!(properties.write_format_default, DataFileFormat::Orc); + assert_eq!(properties.write_delete_format_default, DataFileFormat::Orc); + + let json = serde_json::to_value(&properties).unwrap(); + assert_eq!(json["write.format.default"], "orc"); + assert!(json.get("write.delete.format.default").is_none()); + + let mut properties = TableProperties::default(); + properties.set_write_format_default(DataFileFormat::Orc); + properties.set_write_delete_format_default(DataFileFormat::Parquet); + let json = serde_json::to_value(&properties).unwrap(); + assert_eq!(json["write.format.default"], "orc"); + assert_eq!(json["write.delete.format.default"], "parquet"); + } + + #[test] + fn distribution_modes_preserve_absence() { + let defaults = TableProperties::default(); + assert_eq!(defaults.write_distribution_mode, None); + assert_eq!(defaults.write_delete_distribution_mode, None); + assert_eq!(defaults.write_update_distribution_mode, None); + assert_eq!(defaults.write_merge_distribution_mode, None); + + let properties = parse(HashMap::from([( + "write.distribution-mode".to_string(), + "none".to_string(), + )])) + .unwrap(); + assert_eq!( + properties.write_distribution_mode, + Some(DistributionMode::None) + ); + assert_eq!( + serde_json::to_value(properties).unwrap()["write.distribution-mode"], + "none" + ); + } + + #[test] + fn rejects_non_table_file_formats() { + for key in ["write.format.default", "write.delete.format.default"] { + let error = + parse(HashMap::from([(key.to_string(), "puffin".to_string())])).unwrap_err(); + assert!(error.message().contains(key)); + } + } + + #[test] + fn validates_and_preserves_format_specific_compression_codecs() { + let parquet = parse(HashMap::from([( + "write.parquet.compression-codec".to_string(), + "uncompressed".to_string(), + )])) + .unwrap(); + assert_eq!( + parquet.write_parquet_compression_codec, + CompressionCodec::None + ); + assert_eq!( + serde_json::to_value(parquet).unwrap()["write.parquet.compression-codec"], + "uncompressed" + ); + + for (key, codec) in [ + ("write.parquet.compression-codec", "zlib"), + ("write.avro.compression-codec", "brotli"), + ("write.orc.compression-codec", "gzip"), + ] { + let error = parse(HashMap::from([(key.to_string(), codec.to_string())])).unwrap_err(); + assert!(error.message().contains(key)); + } + } } diff --git a/crates/iceberg/src/transaction/expire_snapshots.rs b/crates/iceberg/src/transaction/expire_snapshots.rs index 533e270244..3263c5502d 100644 --- a/crates/iceberg/src/transaction/expire_snapshots.rs +++ b/crates/iceberg/src/transaction/expire_snapshots.rs @@ -113,12 +113,12 @@ impl ExpireSnapshotsAction { // When a knob is not set explicitly, fall back to the table's `history.expire.*` properties, // matching Java `RemoveSnapshots`' constructor. With the default `max-snapshot-age-ms` (5 // days) the age path always runs, so even an explicit-id-only call applies the default cutoff. - let default_cutoff = self - .older_than_ms - .unwrap_or_else(|| now.saturating_sub(properties.history_expire_max_snapshot_age_ms)); + let default_cutoff = self.older_than_ms.unwrap_or_else(|| { + now.saturating_sub(*properties.history_expire_max_snapshot_age_ms()) + }); let default_min_to_keep = self .retain_last - .unwrap_or(properties.history_expire_min_snapshots_to_keep); + .unwrap_or(*properties.history_expire_min_snapshots_to_keep()); // Ref aging: `main` is always kept; any other ref whose head is older than its // `max_ref_age_ms` (defaulting to `history.expire.max-ref-age-ms`) is dropped, like Java's @@ -131,7 +131,7 @@ impl ExpireSnapshotsAction { metadata, snapshot_ref, now, - properties.history_expire_max_ref_age_ms, + *properties.history_expire_max_ref_age_ms(), ) { retained_refs.push(snapshot_ref); @@ -309,7 +309,7 @@ impl TransactionAction for ExpireSnapshotsAction { let properties = metadata.table_properties()?; // Expiring metadata defeats a user's explicit decision to disable GC (Java refuses too). - if !properties.gc_enabled { + if !properties.gc_enabled() { return Err(Error::new( ErrorKind::DataInvalid, "Cannot expire snapshots: gc.enabled is false", diff --git a/crates/iceberg/src/transaction/mod.rs b/crates/iceberg/src/transaction/mod.rs index 00a9e5fa8b..f3738c4ae8 100644 --- a/crates/iceberg/src/transaction/mod.rs +++ b/crates/iceberg/src/transaction/mod.rs @@ -181,7 +181,7 @@ impl Transaction { let table_props = self.table.metadata().table_properties()?; // TODO(https://github.com/apache/iceberg-rust/issues/2034): remove once encrypted writes are supported - if table_props.encryption_key_id.is_some() { + if table_props.encryption_key_id().is_some() { return Err(Error::new( ErrorKind::FeatureUnsupported, "Cannot commit to an encrypted table: encrypted writes are not yet supported", @@ -205,12 +205,12 @@ impl Transaction { fn build_backoff(props: TableProperties) -> Result { Ok(ExponentialBuilder::new() - .with_min_delay(Duration::from_millis(props.commit_retry_min_wait_ms)) - .with_max_delay(Duration::from_millis(props.commit_retry_max_wait_ms)) + .with_min_delay(Duration::from_millis(*props.commit_retry_min_wait_ms())) + .with_max_delay(Duration::from_millis(*props.commit_retry_max_wait_ms())) .with_total_delay(Some(Duration::from_millis( - props.commit_retry_total_timeout_ms, + *props.commit_retry_total_timeout_ms(), ))) - .with_max_times(props.commit_retry_num_retries) + .with_max_times(*props.commit_retry_num_retries()) .with_factor(2.0) .build()) } diff --git a/crates/iceberg/src/writer/file_writer/location_generator.rs b/crates/iceberg/src/writer/file_writer/location_generator.rs index e3ef61ef59..b419b4e9f3 100644 --- a/crates/iceberg/src/writer/file_writer/location_generator.rs +++ b/crates/iceberg/src/writer/file_writer/location_generator.rs @@ -67,10 +67,12 @@ impl DefaultLocationGenerator { /// `{table_location}/data`. pub fn new(table_metadata: &TableMetadata) -> Result { let table_location = strip_trailing_slash(table_metadata.location()); - let prop = TableProperties::try_from(table_metadata.properties())?; + let properties = table_metadata.properties(); let data_location = strip_trailing_slash( - prop.write_data_path - .or(prop.write_folder_storage_path) + properties + .get(TableProperties::PROPERTY_WRITE_DATA_LOCATION) + .or_else(|| properties.get(TableProperties::PROPERTY_WRITE_FOLDER_STORAGE_LOCATION)) + .cloned() .unwrap_or(format!("{table_location}{DEFAULT_DATA_DIR}")) .as_ref(), ) @@ -134,11 +136,13 @@ impl ObjectStorageLocationGenerator { /// `{table_location}/data`. pub fn new(table_metadata: &TableMetadata) -> Result { let table_location = strip_trailing_slash(table_metadata.location()); - let prop = TableProperties::try_from(table_metadata.properties())?; + let properties = table_metadata.properties(); let storage_location = strip_trailing_slash( - prop.write_data_path - .or(prop.write_object_storage_path) - .or(prop.write_folder_storage_path) + properties + .get(TableProperties::PROPERTY_WRITE_DATA_LOCATION) + .or_else(|| properties.get(TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_LOCATION)) + .or_else(|| properties.get(TableProperties::PROPERTY_WRITE_FOLDER_STORAGE_LOCATION)) + .cloned() .unwrap_or(format!("{table_location}{DEFAULT_DATA_DIR}")) .as_ref(), ) @@ -152,7 +156,21 @@ impl ObjectStorageLocationGenerator { Some(path_context(table_location)) }; - let include_partition_paths = prop.write_object_storage_partitioned_paths; + let include_partition_paths = properties + .get(TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS) + .map(|value| { + value.to_ascii_lowercase().parse::().map_err(|error| { + crate::Error::new( + crate::ErrorKind::DataInvalid, + format!( + "Invalid value for {}: {error}", + TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS + ), + ) + }) + }) + .transpose()? + .unwrap_or(TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS_DEFAULT); Ok(Self { storage_location, diff --git a/crates/iceberg/src/writer/file_writer/parquet_writer.rs b/crates/iceberg/src/writer/file_writer/parquet_writer.rs index e302fcb51a..c3d129c436 100644 --- a/crates/iceberg/src/writer/file_writer/parquet_writer.rs +++ b/crates/iceberg/src/writer/file_writer/parquet_writer.rs @@ -87,11 +87,13 @@ impl ParquetWriterBuilder { /// parquet-rs defaults. pub fn from_table_properties(table_props: &TableProperties, schema: SchemaRef) -> Self { let cdc = table_props - .write_parquet_content_defined_chunking_enabled + .write_parquet_content_defined_chunking_enabled() .then_some(CdcOptions { - min_chunk_size: table_props.write_parquet_content_defined_chunking_min_chunk_size, - max_chunk_size: table_props.write_parquet_content_defined_chunking_max_chunk_size, - norm_level: table_props.write_parquet_content_defined_chunking_norm_level, + min_chunk_size: *table_props + .write_parquet_content_defined_chunking_min_chunk_size(), + max_chunk_size: *table_props + .write_parquet_content_defined_chunking_max_chunk_size(), + norm_level: *table_props.write_parquet_content_defined_chunking_norm_level(), }); // TODO: translate the remaining write.parquet.* keys (e.g. compression-codec, // row-group-size-bytes, page-size-bytes). diff --git a/crates/integrations/datafusion/src/physical_plan/write.rs b/crates/integrations/datafusion/src/physical_plan/write.rs index d4d80e9719..4d8d124983 100644 --- a/crates/integrations/datafusion/src/physical_plan/write.rs +++ b/crates/integrations/datafusion/src/physical_plan/write.rs @@ -209,7 +209,7 @@ impl ExecutionPlan for IcebergWriteExec { .map_err(to_datafusion_error)?; // Check data file format - let file_format = table_props.write_format_default; + let file_format = *table_props.write_format_default(); if file_format != DataFileFormat::Parquet { return Err(to_datafusion_error(Error::new( ErrorKind::FeatureUnsupported, @@ -225,7 +225,7 @@ impl ExecutionPlan for IcebergWriteExec { self.table.metadata().current_schema().clone(), ) .with_match_mode(FieldMatchMode::Name); - let target_file_size = table_props.write_target_file_size_bytes; + let target_file_size = *table_props.write_target_file_size_bytes(); let file_io = self.table.file_io().clone(); // todo location_gen and file_name_gen should be configurable @@ -244,7 +244,7 @@ impl ExecutionPlan for IcebergWriteExec { let data_file_writer_builder = DataFileWriterBuilder::new(rolling_writer_builder); // Create TaskWriter - let fanout_enabled = table_props.write_datafusion_fanout_enabled; + let fanout_enabled = *table_props.write_datafusion_fanout_enabled(); let schema = self.table.metadata().current_schema().clone(); let partition_spec = self.table.metadata().default_partition_spec().clone(); let task_writer = TaskWriter::try_new( diff --git a/crates/property-macro/Cargo.toml b/crates/property-macro/Cargo.toml index 1fc1c73693..f67c8aabf5 100644 --- a/crates/property-macro/Cargo.toml +++ b/crates/property-macro/Cargo.toml @@ -19,7 +19,7 @@ edition = { workspace = true } homepage = { workspace = true } name = "iceberg-property-macro" -publish = false +publish = true rust-version = { workspace = true } version = { workspace = true } diff --git a/crates/property-macro/src/lib.rs b/crates/property-macro/src/lib.rs index f919c7d414..c4e5d75a7c 100644 --- a/crates/property-macro/src/lib.rs +++ b/crates/property-macro/src/lib.rs @@ -19,10 +19,12 @@ use proc_macro::TokenStream; use proc_macro2::TokenStream as TokenStream2; -use quote::quote; +use quote::{format_ident, quote}; +use syn::parse::{Parse, ParseStream}; +use syn::punctuated::Punctuated; use syn::{ Attribute, Data, DeriveInput, Error, Expr, ExprLit, ExprPath, Field, Fields, Ident, Lit, Meta, - Path, Type, parse_macro_input, + Path, Token, Type, parenthesized, parse_macro_input, }; /// Derive parsing, defaults, and JSON serialization for a typed property map. @@ -37,11 +39,14 @@ use syn::{ /// #[key = "write.format.default"] /// #[default = "parquet"] /// #[doc = "Default file format"] -/// pub write_format_default: String, +/// #[property(pub(getter), pub(setter))] +/// write_format_default: String, /// } /// -/// let properties = Properties::default(); -/// assert_eq!(properties.write_format_default, "parquet"); +/// let mut properties = Properties::default(); +/// assert_eq!(properties.write_format_default(), "parquet"); +/// properties.set_write_format_default("orc".to_string()); +/// assert_eq!(properties.write_format_default(), "orc"); /// ``` /// /// `prefix` captures a family of properties in a `HashMap`, keyed by the suffix after @@ -50,12 +55,17 @@ use syn::{ /// `FromStr` or need validation. `serialize_with` supplies their string representation in JSON. /// `parse_properties_with` and `write_properties_with` provide access to the complete property map /// for fields represented by more than one key. `additional_key` declares a second key and passes -/// it to those hooks after the primary key. +/// it to those hooks after the primary key. Write hooks are also passed the field default and are +/// responsible for omitting or removing default-valued properties. /// Optional fields are omitted from JSON when they are `None`. Fields need `FromStr` and `ToString` /// unless the relevant custom parsing or serialization attribute is supplied. Leaf fields also /// need `PartialEq` so values equal to their defaults can be omitted from JSON. String-literal and /// path defaults are converted into their field type with `Into`. Boolean property values are /// parsed case-insensitively. +/// +/// Fields remain private unless their struct declaration makes them public. The +/// `#[property(pub(getter))]` and `#[property(pub(setter))]` options generate a public getter and +/// setter respectively. Getters borrow the field, and setters are named `set_`. #[proc_macro_derive( Properties, attributes( @@ -67,7 +77,8 @@ use syn::{ parse_with, serialize_with, parse_properties_with, - write_properties_with + write_properties_with, + property ) )] pub fn derive_properties(input: TokenStream) -> TokenStream { @@ -93,6 +104,32 @@ struct PropertyField { write_properties_with: Option, option_inner_type: Option, map_value_type: Option, + public_getter: bool, + public_setter: bool, + doc_attributes: Vec, +} + +enum PublicAccessor { + Getter, + Setter, +} + +impl Parse for PublicAccessor { + fn parse(input: ParseStream<'_>) -> syn::Result { + input.parse::()?; + let content; + parenthesized!(content in input); + let accessor = content.parse::()?; + if !content.is_empty() { + return Err(content.error("expected getter or setter")); + } + + match accessor.to_string().as_str() { + "getter" => Ok(Self::Getter), + "setter" => Ok(Self::Setter), + _ => Err(Error::new_spanned(accessor, "expected getter or setter")), + } + } } fn expand_properties(input: DeriveInput) -> syn::Result { @@ -136,6 +173,8 @@ fn expand_properties(input: DeriveInput) -> syn::Result { let property_writes = fields.iter().map(write_field); + let accessors = fields.iter().map(field_accessors); + Ok(quote! { impl ::std::default::Default for #struct_name { fn default() -> Self { @@ -146,6 +185,8 @@ fn expand_properties(input: DeriveInput) -> syn::Result { } impl #struct_name { + #(#accessors)* + pub(crate) fn from_properties( properties: &::std::collections::HashMap<::std::string::String, ::std::string::String>, ) -> ::std::result::Result { @@ -270,6 +311,7 @@ fn parse_property_field(field: &Field) -> syn::Result { "fields cannot declare both serialize_with and write_properties_with", )); } + let (public_getter, public_setter) = property_accessors(&field.attrs)?; Ok(PropertyField { ident, @@ -285,9 +327,76 @@ fn parse_property_field(field: &Field) -> syn::Result { write_properties_with, option_inner_type: option_inner_type(&field.ty), map_value_type, + public_getter, + public_setter, + doc_attributes: field + .attrs + .iter() + .filter(|attribute| attribute.path().is_ident("doc")) + .cloned() + .collect(), }) } +fn property_accessors(attributes: &[Attribute]) -> syn::Result<(bool, bool)> { + let Some(attribute) = find_attribute(attributes, "property")? else { + return Ok((false, false)); + }; + + let accessors = + attribute.parse_args_with(Punctuated::::parse_terminated)?; + if accessors.is_empty() { + return Err(Error::new_spanned( + attribute, + "property must declare pub(getter), pub(setter), or both", + )); + } + + let mut public_getter = false; + let mut public_setter = false; + for accessor in accessors { + let selected = match accessor { + PublicAccessor::Getter => &mut public_getter, + PublicAccessor::Setter => &mut public_setter, + }; + if *selected { + return Err(Error::new_spanned(attribute, "duplicate property accessor")); + } + *selected = true; + } + + Ok((public_getter, public_setter)) +} + +fn field_accessors(field: &PropertyField) -> TokenStream2 { + let ident = &field.ident; + let ty = &field.ty; + let docs = &field.doc_attributes; + let getter = field.public_getter.then(|| { + quote! { + #(#docs)* + pub fn #ident(&self) -> &#ty { + &self.#ident + } + } + }); + let setter = field.public_setter.then(|| { + let setter_ident = format_ident!("set_{}", ident); + let setter_doc = format!("Sets `{ident}`."); + quote! { + #[doc = #setter_doc] + pub fn #setter_ident(&mut self, value: #ty) { + self.#ident = value; + } + } + }); + + quote! { + #getter + #setter + } +} + fn marker_attribute(attributes: &[Attribute], name: &str) -> syn::Result { let Some(attribute) = find_attribute(attributes, name)? else { return Ok(false); @@ -540,14 +649,12 @@ fn write_field(field: &PropertyField) -> TokenStream2 { let key = field.key.as_ref().expect("exact-key fields have a key"); let write = match &field.additional_key { Some(additional_key) => { - quote!(#write_properties_with(&self.#ident, properties, #key, #additional_key)) + quote!(#write_properties_with(&self.#ident, properties, #key, #additional_key, &#default)) } - None => quote!(#write_properties_with(&self.#ident, properties, #key)), + None => quote!(#write_properties_with(&self.#ident, properties, #key, &#default)), }; return quote! { - if self.#ident != #default { - #write; - } + #write; }; } diff --git a/crates/property-macro/tests/properties.rs b/crates/property-macro/tests/properties.rs index 7dc20a8092..614a47c7ee 100644 --- a/crates/property-macro/tests/properties.rs +++ b/crates/property-macro/tests/properties.rs @@ -49,9 +49,15 @@ fn write_dimensions( properties: &mut HashMap, key: &str, height_key: &str, + default: &(u64, u64), ) { - properties.insert(key.to_string(), dimensions.0.to_string()); - properties.insert(height_key.to_string(), dimensions.1.to_string()); + if dimensions == default { + properties.remove(key); + properties.remove(height_key); + } else { + properties.insert(key.to_string(), dimensions.0.to_string()); + properties.insert(height_key.to_string(), dimensions.1.to_string()); + } } #[derive(Debug, Properties)] @@ -161,3 +167,47 @@ fn nested_properties_use_a_flat_property_map() { let decoded: NestedProperties = serde_json::from_value(json).unwrap(); assert_eq!(decoded.commit.num_retries, 9); } + +mod accessor_fixture { + use iceberg_property_macro::Properties; + + #[derive(Debug, Properties)] + pub struct AccessorProperties { + #[key = "public.both"] + #[default = 1] + #[doc = "A property with public read and write access."] + #[property(pub(getter), pub(setter))] + both: u64, + + #[key = "public.getter"] + #[default = "value"] + #[doc = "A property with public read access."] + #[property(pub(getter))] + getter_only: String, + + #[key = "public.setter"] + #[default = false] + #[property(pub(setter))] + setter_only: bool, + } + + impl AccessorProperties { + pub fn setter_only_for_test(&self) -> bool { + self.setter_only + } + } +} + +#[test] +fn generates_opt_in_public_accessors_for_private_fields() { + let mut properties = accessor_fixture::AccessorProperties::default(); + + assert_eq!(*properties.both(), 1); + properties.set_both(2); + assert_eq!(*properties.both(), 2); + + assert_eq!(properties.getter_only(), "value"); + + properties.set_setter_only(true); + assert!(properties.setter_only_for_test()); +}