diff --git a/Cargo.lock b/Cargo.lock index 0f264cc1f5..df95b21bea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3787,6 +3787,7 @@ dependencies = [ "fnv", "form_urlencoded", "futures", + "iceberg-property-macro", "iceberg_test_utils", "itertools 0.13.0", "minijinja", @@ -4007,6 +4008,17 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "iceberg-property-macro" +version = "0.10.0" +dependencies = [ + "proc-macro2", + "quote", + "serde", + "serde_json", + "syn", +] + [[package]] name = "iceberg-sqllogictest" version = "0.10.0" diff --git a/Cargo.toml b/Cargo.toml index a789ef1967..bcdd080347 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ members = [ "crates/catalog/*", "crates/examples", "crates/iceberg", + "crates/property-macro", "crates/integration_tests", "crates/integrations/*", "crates/sqllogictest", diff --git a/crates/iceberg/Cargo.toml b/crates/iceberg/Cargo.toml index 66eadfb7ec..bf9c19bb63 100644 --- a/crates/iceberg/Cargo.toml +++ b/crates/iceberg/Cargo.toml @@ -59,6 +59,7 @@ flate2 = { workspace = true } fnv = { workspace = true } form_urlencoded = { workspace = true } futures = { workspace = true } +iceberg-property-macro = { version = "0.10.0", path = "../property-macro" } itertools = { workspace = true } moka = { version = "0.12.10", features = ["future"] } murmur3 = { workspace = true } diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index 530aa4cbf3..7fa7ae97ae 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -142,10 +142,13 @@ pub fn iceberg::cache::ObjectCacheProvide::manifest_list_cache(&self) -> &dyn ic pub type iceberg::cache::ObjectCacheProvider = alloc::sync::Arc pub mod iceberg::compression pub enum iceberg::compression::CompressionCodec +pub iceberg::compression::CompressionCodec::Brotli pub iceberg::compression::CompressionCodec::Gzip(u8) pub iceberg::compression::CompressionCodec::Lz4 +pub iceberg::compression::CompressionCodec::Lzo pub iceberg::compression::CompressionCodec::None pub iceberg::compression::CompressionCodec::Snappy +pub iceberg::compression::CompressionCodec::Zlib pub iceberg::compression::CompressionCodec::Zstd(u8) impl iceberg::compression::CompressionCodec pub const fn iceberg::compression::CompressionCodec::gzip_default() -> Self @@ -1162,10 +1165,13 @@ pub mod iceberg::partitioning pub fn iceberg::partitioning::compute_unified_partition_type<'a>(partition_specs: impl core::iter::traits::iterator::Iterator, schema: &iceberg::spec::Schema) -> iceberg::Result pub mod iceberg::puffin pub enum iceberg::puffin::CompressionCodec +pub iceberg::puffin::CompressionCodec::Brotli pub iceberg::puffin::CompressionCodec::Gzip(u8) pub iceberg::puffin::CompressionCodec::Lz4 +pub iceberg::puffin::CompressionCodec::Lzo pub iceberg::puffin::CompressionCodec::None pub iceberg::puffin::CompressionCodec::Snappy +pub iceberg::puffin::CompressionCodec::Zlib pub iceberg::puffin::CompressionCodec::Zstd(u8) impl iceberg::compression::CompressionCodec pub const fn iceberg::compression::CompressionCodec::gzip_default() -> Self @@ -1415,6 +1421,55 @@ impl serde_core::ser::Serialize for iceberg::spec::DataFileFormat where Self: co pub fn iceberg::spec::DataFileFormat::serialize<__S>(&self, serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::DataFileFormat where Self: core::str::traits::FromStr, ::Err: core::fmt::Display pub fn iceberg::spec::DataFileFormat::deserialize<__D>(deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub enum iceberg::spec::DeleteGranularity +pub iceberg::spec::DeleteGranularity::File +pub iceberg::spec::DeleteGranularity::Partition +impl core::clone::Clone for iceberg::spec::DeleteGranularity +pub fn iceberg::spec::DeleteGranularity::clone(&self) -> iceberg::spec::DeleteGranularity +impl core::cmp::Eq for iceberg::spec::DeleteGranularity +impl core::cmp::PartialEq for iceberg::spec::DeleteGranularity +pub fn iceberg::spec::DeleteGranularity::eq(&self, other: &iceberg::spec::DeleteGranularity) -> bool +impl core::convert::TryFrom<&str> for iceberg::spec::DeleteGranularity +pub type iceberg::spec::DeleteGranularity::Error = strum::ParseError +pub fn iceberg::spec::DeleteGranularity::try_from(s: &str) -> core::result::Result>::Error> +impl core::fmt::Debug for iceberg::spec::DeleteGranularity +pub fn iceberg::spec::DeleteGranularity::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for iceberg::spec::DeleteGranularity +pub fn iceberg::spec::DeleteGranularity::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::result::Result<(), core::fmt::Error> +impl core::marker::Copy for iceberg::spec::DeleteGranularity +impl core::marker::StructuralPartialEq for iceberg::spec::DeleteGranularity +impl core::str::traits::FromStr for iceberg::spec::DeleteGranularity +pub type iceberg::spec::DeleteGranularity::Err = strum::ParseError +pub fn iceberg::spec::DeleteGranularity::from_str(s: &str) -> core::result::Result::Err> +impl serde_core::ser::Serialize for iceberg::spec::DeleteGranularity where Self: core::fmt::Display +pub fn iceberg::spec::DeleteGranularity::serialize<__S>(&self, serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::DeleteGranularity where Self: core::str::traits::FromStr, ::Err: core::fmt::Display +pub fn iceberg::spec::DeleteGranularity::deserialize<__D>(deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub enum iceberg::spec::DistributionMode +pub iceberg::spec::DistributionMode::Hash +pub iceberg::spec::DistributionMode::None +pub iceberg::spec::DistributionMode::Range +impl core::clone::Clone for iceberg::spec::DistributionMode +pub fn iceberg::spec::DistributionMode::clone(&self) -> iceberg::spec::DistributionMode +impl core::cmp::Eq for iceberg::spec::DistributionMode +impl core::cmp::PartialEq for iceberg::spec::DistributionMode +pub fn iceberg::spec::DistributionMode::eq(&self, other: &iceberg::spec::DistributionMode) -> bool +impl core::convert::TryFrom<&str> for iceberg::spec::DistributionMode +pub type iceberg::spec::DistributionMode::Error = strum::ParseError +pub fn iceberg::spec::DistributionMode::try_from(s: &str) -> core::result::Result>::Error> +impl core::fmt::Debug for iceberg::spec::DistributionMode +pub fn iceberg::spec::DistributionMode::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for iceberg::spec::DistributionMode +pub fn iceberg::spec::DistributionMode::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::result::Result<(), core::fmt::Error> +impl core::marker::Copy for iceberg::spec::DistributionMode +impl core::marker::StructuralPartialEq for iceberg::spec::DistributionMode +impl core::str::traits::FromStr for iceberg::spec::DistributionMode +pub type iceberg::spec::DistributionMode::Err = strum::ParseError +pub fn iceberg::spec::DistributionMode::from_str(s: &str) -> core::result::Result::Err> +impl serde_core::ser::Serialize for iceberg::spec::DistributionMode where Self: core::fmt::Display +pub fn iceberg::spec::DistributionMode::serialize<__S>(&self, serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::DistributionMode where Self: core::str::traits::FromStr, ::Err: core::fmt::Display +pub fn iceberg::spec::DistributionMode::deserialize<__D>(deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> #[repr(u8)] pub enum iceberg::spec::FormatVersion pub iceberg::spec::FormatVersion::V1 = 1 pub iceberg::spec::FormatVersion::V2 = 2 @@ -1440,6 +1495,30 @@ impl serde_core::ser::Serialize for iceberg::spec::FormatVersion pub fn iceberg::spec::FormatVersion::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> where S: serde_core::ser::Serializer impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::FormatVersion pub fn iceberg::spec::FormatVersion::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> +pub enum iceberg::spec::IsolationLevel +pub iceberg::spec::IsolationLevel::Serializable +pub iceberg::spec::IsolationLevel::Snapshot +impl core::clone::Clone for iceberg::spec::IsolationLevel +pub fn iceberg::spec::IsolationLevel::clone(&self) -> iceberg::spec::IsolationLevel +impl core::cmp::Eq for iceberg::spec::IsolationLevel +impl core::cmp::PartialEq for iceberg::spec::IsolationLevel +pub fn iceberg::spec::IsolationLevel::eq(&self, other: &iceberg::spec::IsolationLevel) -> bool +impl core::convert::TryFrom<&str> for iceberg::spec::IsolationLevel +pub type iceberg::spec::IsolationLevel::Error = strum::ParseError +pub fn iceberg::spec::IsolationLevel::try_from(s: &str) -> core::result::Result>::Error> +impl core::fmt::Debug for iceberg::spec::IsolationLevel +pub fn iceberg::spec::IsolationLevel::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for iceberg::spec::IsolationLevel +pub fn iceberg::spec::IsolationLevel::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::result::Result<(), core::fmt::Error> +impl core::marker::Copy for iceberg::spec::IsolationLevel +impl core::marker::StructuralPartialEq for iceberg::spec::IsolationLevel +impl core::str::traits::FromStr for iceberg::spec::IsolationLevel +pub type iceberg::spec::IsolationLevel::Err = strum::ParseError +pub fn iceberg::spec::IsolationLevel::from_str(s: &str) -> core::result::Result::Err> +impl serde_core::ser::Serialize for iceberg::spec::IsolationLevel where Self: core::fmt::Display +pub fn iceberg::spec::IsolationLevel::serialize<__S>(&self, serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::IsolationLevel where Self: core::str::traits::FromStr, ::Err: core::fmt::Display +pub fn iceberg::spec::IsolationLevel::deserialize<__D>(deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> pub enum iceberg::spec::Literal pub iceberg::spec::Literal::List(alloc::vec::Vec>) pub iceberg::spec::Literal::Map(iceberg::spec::Map) @@ -1637,6 +1716,30 @@ impl<'de> iceberg::spec::PrimitiveType pub fn iceberg::spec::PrimitiveType::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::PrimitiveType pub fn iceberg::spec::PrimitiveType::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> +pub enum iceberg::spec::RowLevelOperationMode +pub iceberg::spec::RowLevelOperationMode::CopyOnWrite +pub iceberg::spec::RowLevelOperationMode::MergeOnRead +impl core::clone::Clone for iceberg::spec::RowLevelOperationMode +pub fn iceberg::spec::RowLevelOperationMode::clone(&self) -> iceberg::spec::RowLevelOperationMode +impl core::cmp::Eq for iceberg::spec::RowLevelOperationMode +impl core::cmp::PartialEq for iceberg::spec::RowLevelOperationMode +pub fn iceberg::spec::RowLevelOperationMode::eq(&self, other: &iceberg::spec::RowLevelOperationMode) -> bool +impl core::convert::TryFrom<&str> for iceberg::spec::RowLevelOperationMode +pub type iceberg::spec::RowLevelOperationMode::Error = strum::ParseError +pub fn iceberg::spec::RowLevelOperationMode::try_from(s: &str) -> core::result::Result>::Error> +impl core::fmt::Debug for iceberg::spec::RowLevelOperationMode +pub fn iceberg::spec::RowLevelOperationMode::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for iceberg::spec::RowLevelOperationMode +pub fn iceberg::spec::RowLevelOperationMode::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::result::Result<(), core::fmt::Error> +impl core::marker::Copy for iceberg::spec::RowLevelOperationMode +impl core::marker::StructuralPartialEq for iceberg::spec::RowLevelOperationMode +impl core::str::traits::FromStr for iceberg::spec::RowLevelOperationMode +pub type iceberg::spec::RowLevelOperationMode::Err = strum::ParseError +pub fn iceberg::spec::RowLevelOperationMode::from_str(s: &str) -> core::result::Result::Err> +impl serde_core::ser::Serialize for iceberg::spec::RowLevelOperationMode where Self: core::fmt::Display +pub fn iceberg::spec::RowLevelOperationMode::serialize<__S>(&self, serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::RowLevelOperationMode where Self: core::str::traits::FromStr, ::Err: core::fmt::Display +pub fn iceberg::spec::RowLevelOperationMode::deserialize<__D>(deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> pub enum iceberg::spec::SnapshotRetention pub iceberg::spec::SnapshotRetention::Branch pub iceberg::spec::SnapshotRetention::Branch::max_ref_age_ms: core::option::Option @@ -2749,75 +2852,19 @@ pub fn iceberg::spec::TableMetadataBuilder::clone(&self) -> iceberg::spec::Table impl core::fmt::Debug for iceberg::spec::TableMetadataBuilder pub fn iceberg::spec::TableMetadataBuilder::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result pub struct iceberg::spec::TableProperties -pub iceberg::spec::TableProperties::cdc_enabled: bool -pub iceberg::spec::TableProperties::cdc_max_chunk_size: usize -pub iceberg::spec::TableProperties::cdc_min_chunk_size: usize -pub iceberg::spec::TableProperties::cdc_norm_level: i32 -pub iceberg::spec::TableProperties::commit_max_retry_wait_ms: u64 -pub iceberg::spec::TableProperties::commit_min_retry_wait_ms: u64 -pub iceberg::spec::TableProperties::commit_num_retries: usize -pub iceberg::spec::TableProperties::commit_total_retry_timeout_ms: u64 -pub iceberg::spec::TableProperties::encryption_data_key_length: usize -pub iceberg::spec::TableProperties::encryption_key_id: core::option::Option -pub iceberg::spec::TableProperties::gc_enabled: bool -pub iceberg::spec::TableProperties::max_ref_age_ms: i64 -pub iceberg::spec::TableProperties::max_snapshot_age_ms: i64 -pub iceberg::spec::TableProperties::metadata_compression_codec: iceberg::compression::CompressionCodec -pub iceberg::spec::TableProperties::min_snapshots_to_keep: usize -pub iceberg::spec::TableProperties::write_data_location: core::option::Option -pub iceberg::spec::TableProperties::write_datafusion_fanout_enabled: bool -pub iceberg::spec::TableProperties::write_folder_storage_location: core::option::Option -pub iceberg::spec::TableProperties::write_format_default: alloc::string::String -pub iceberg::spec::TableProperties::write_metadata_path: core::option::Option -pub iceberg::spec::TableProperties::write_object_storage_location: core::option::Option -pub iceberg::spec::TableProperties::write_object_storage_partitioned_paths: bool -pub iceberg::spec::TableProperties::write_target_file_size_bytes: usize impl iceberg::spec::TableProperties -pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS: &str -pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT: u64 -pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS: &str -pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT: u64 pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_NUM_RETRIES: &str pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT: usize -pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS: &str -pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT: u64 -pub const iceberg::spec::TableProperties::PROPERTY_CURRENT_SCHEMA: &str -pub const iceberg::spec::TableProperties::PROPERTY_CURRENT_SNAPSHOT_ID: &str -pub const iceberg::spec::TableProperties::PROPERTY_CURRENT_SNAPSHOT_SUMMARY: &str -pub const iceberg::spec::TableProperties::PROPERTY_CURRENT_SNAPSHOT_TIMESTAMP: &str pub const iceberg::spec::TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED: &str pub const iceberg::spec::TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT: bool -pub const iceberg::spec::TableProperties::PROPERTY_DEFAULT_FILE_FORMAT: &str -pub const iceberg::spec::TableProperties::PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT: &str -pub const iceberg::spec::TableProperties::PROPERTY_DEFAULT_PARTITION_SPEC: &str -pub const iceberg::spec::TableProperties::PROPERTY_DEFAULT_SORT_ORDER: &str -pub const iceberg::spec::TableProperties::PROPERTY_DELETE_DEFAULT_FILE_FORMAT: &str -pub const iceberg::spec::TableProperties::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH: &str -pub const iceberg::spec::TableProperties::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH_DEFAULT: usize pub const iceberg::spec::TableProperties::PROPERTY_ENCRYPTION_KEY_ID: &str -pub const iceberg::spec::TableProperties::PROPERTY_FORMAT_VERSION: &str -pub const iceberg::spec::TableProperties::PROPERTY_GC_ENABLED: &str -pub const iceberg::spec::TableProperties::PROPERTY_GC_ENABLED_DEFAULT: bool -pub const iceberg::spec::TableProperties::PROPERTY_MAX_REF_AGE_MS: &str -pub const iceberg::spec::TableProperties::PROPERTY_MAX_REF_AGE_MS_DEFAULT: i64 -pub const iceberg::spec::TableProperties::PROPERTY_MAX_SNAPSHOT_AGE_MS: &str -pub const iceberg::spec::TableProperties::PROPERTY_MAX_SNAPSHOT_AGE_MS_DEFAULT: i64 pub const iceberg::spec::TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC: &str -pub const iceberg::spec::TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC_DEFAULT: &str pub const iceberg::spec::TableProperties::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX: &str pub const iceberg::spec::TableProperties::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX_DEFAULT: usize -pub const iceberg::spec::TableProperties::PROPERTY_MIN_SNAPSHOTS_TO_KEEP: &str -pub const iceberg::spec::TableProperties::PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT: usize pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_ENABLED: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_ENABLED_DEFAULT: bool pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE_DEFAULT: usize pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT: usize pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL_DEFAULT: i32 -pub const iceberg::spec::TableProperties::PROPERTY_SNAPSHOT_COUNT: &str -pub const iceberg::spec::TableProperties::PROPERTY_UUID: &str pub const iceberg::spec::TableProperties::PROPERTY_WRITE_DATA_LOCATION: &str pub const iceberg::spec::TableProperties::PROPERTY_WRITE_FOLDER_STORAGE_LOCATION: &str pub const iceberg::spec::TableProperties::PROPERTY_WRITE_METADATA_PATH: &str @@ -2829,11 +2876,252 @@ pub const iceberg::spec::TableProperties::PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT pub const iceberg::spec::TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES: &str pub const iceberg::spec::TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT: usize pub const iceberg::spec::TableProperties::RESERVED_PROPERTIES: [&str; 9] +impl iceberg::spec::TableProperties +pub fn iceberg::spec::TableProperties::comment(&self) -> &core::option::Option +pub fn iceberg::spec::TableProperties::commit_manifest_merge_enabled(&self) -> &bool +pub fn iceberg::spec::TableProperties::commit_manifest_min_count_to_merge(&self) -> &usize +pub fn iceberg::spec::TableProperties::commit_manifest_target_size_bytes(&self) -> &usize +pub fn iceberg::spec::TableProperties::commit_retry_max_wait_ms(&self) -> &u64 +pub fn iceberg::spec::TableProperties::commit_retry_min_wait_ms(&self) -> &u64 +pub fn iceberg::spec::TableProperties::commit_retry_num_retries(&self) -> &usize +pub fn iceberg::spec::TableProperties::commit_retry_total_timeout_ms(&self) -> &u64 +pub fn iceberg::spec::TableProperties::commit_status_check_max_wait_ms(&self) -> &u64 +pub fn iceberg::spec::TableProperties::commit_status_check_min_wait_ms(&self) -> &u64 +pub fn iceberg::spec::TableProperties::commit_status_check_num_retries(&self) -> &usize +pub fn iceberg::spec::TableProperties::commit_status_check_total_timeout_ms(&self) -> &u64 +pub fn iceberg::spec::TableProperties::compatibility_snapshot_id_inheritance_enabled(&self) -> &bool +pub fn iceberg::spec::TableProperties::encryption_data_key_length(&self) -> &usize +pub fn iceberg::spec::TableProperties::encryption_key_id(&self) -> &core::option::Option +pub fn iceberg::spec::TableProperties::engine_hive_enabled(&self) -> &bool +pub fn iceberg::spec::TableProperties::engine_hive_lock_enabled(&self) -> &bool +pub fn iceberg::spec::TableProperties::gc_enabled(&self) -> &bool +pub fn iceberg::spec::TableProperties::history_expire_max_ref_age_ms(&self) -> &i64 +pub fn iceberg::spec::TableProperties::history_expire_max_snapshot_age_ms(&self) -> &i64 +pub fn iceberg::spec::TableProperties::history_expire_min_snapshots_to_keep(&self) -> &usize +pub fn iceberg::spec::TableProperties::identifier_fields_rely(&self) -> &bool +pub fn iceberg::spec::TableProperties::read_data_planning_mode(&self) -> &alloc::string::String +pub fn iceberg::spec::TableProperties::read_delete_planning_mode(&self) -> &alloc::string::String +pub fn iceberg::spec::TableProperties::read_orc_vectorization_batch_size(&self) -> &usize +pub fn iceberg::spec::TableProperties::read_orc_vectorization_enabled(&self) -> &bool +pub fn iceberg::spec::TableProperties::read_parquet_vectorization_batch_size(&self) -> &usize +pub fn iceberg::spec::TableProperties::read_parquet_vectorization_enabled(&self) -> &bool +pub fn iceberg::spec::TableProperties::read_split_adaptive_size_enabled(&self) -> &bool +pub fn iceberg::spec::TableProperties::read_split_metadata_target_size(&self) -> &u64 +pub fn iceberg::spec::TableProperties::read_split_open_file_cost(&self) -> &u64 +pub fn iceberg::spec::TableProperties::read_split_planning_lookback(&self) -> &usize +pub fn iceberg::spec::TableProperties::read_split_target_size(&self) -> &u64 +pub fn iceberg::spec::TableProperties::schema_name_mapping_default(&self) -> &core::option::Option +pub fn iceberg::spec::TableProperties::set_comment(&mut self, value: core::option::Option) +pub fn iceberg::spec::TableProperties::set_commit_manifest_merge_enabled(&mut self, value: bool) +pub fn iceberg::spec::TableProperties::set_commit_manifest_min_count_to_merge(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_commit_manifest_target_size_bytes(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_commit_retry_max_wait_ms(&mut self, value: u64) +pub fn iceberg::spec::TableProperties::set_commit_retry_min_wait_ms(&mut self, value: u64) +pub fn iceberg::spec::TableProperties::set_commit_retry_num_retries(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_commit_retry_total_timeout_ms(&mut self, value: u64) +pub fn iceberg::spec::TableProperties::set_commit_status_check_max_wait_ms(&mut self, value: u64) +pub fn iceberg::spec::TableProperties::set_commit_status_check_min_wait_ms(&mut self, value: u64) +pub fn iceberg::spec::TableProperties::set_commit_status_check_num_retries(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_commit_status_check_total_timeout_ms(&mut self, value: u64) +pub fn iceberg::spec::TableProperties::set_compatibility_snapshot_id_inheritance_enabled(&mut self, value: bool) +pub fn iceberg::spec::TableProperties::set_encryption_data_key_length(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_encryption_key_id(&mut self, value: core::option::Option) +pub fn iceberg::spec::TableProperties::set_engine_hive_enabled(&mut self, value: bool) +pub fn iceberg::spec::TableProperties::set_engine_hive_lock_enabled(&mut self, value: bool) +pub fn iceberg::spec::TableProperties::set_gc_enabled(&mut self, value: bool) +pub fn iceberg::spec::TableProperties::set_history_expire_max_ref_age_ms(&mut self, value: i64) +pub fn iceberg::spec::TableProperties::set_history_expire_max_snapshot_age_ms(&mut self, value: i64) +pub fn iceberg::spec::TableProperties::set_history_expire_min_snapshots_to_keep(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_identifier_fields_rely(&mut self, value: bool) +pub fn iceberg::spec::TableProperties::set_read_data_planning_mode(&mut self, value: alloc::string::String) +pub fn iceberg::spec::TableProperties::set_read_delete_planning_mode(&mut self, value: alloc::string::String) +pub fn iceberg::spec::TableProperties::set_read_orc_vectorization_batch_size(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_read_orc_vectorization_enabled(&mut self, value: bool) +pub fn iceberg::spec::TableProperties::set_read_parquet_vectorization_batch_size(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_read_parquet_vectorization_enabled(&mut self, value: bool) +pub fn iceberg::spec::TableProperties::set_read_split_adaptive_size_enabled(&mut self, value: bool) +pub fn iceberg::spec::TableProperties::set_read_split_metadata_target_size(&mut self, value: u64) +pub fn iceberg::spec::TableProperties::set_read_split_open_file_cost(&mut self, value: u64) +pub fn iceberg::spec::TableProperties::set_read_split_planning_lookback(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_read_split_target_size(&mut self, value: u64) +pub fn iceberg::spec::TableProperties::set_schema_name_mapping_default(&mut self, value: core::option::Option) +pub fn iceberg::spec::TableProperties::set_write_avro_compression_codec(&mut self, value: iceberg::compression::CompressionCodec) +pub fn iceberg::spec::TableProperties::set_write_data_path(&mut self, value: core::option::Option) +pub fn iceberg::spec::TableProperties::set_write_datafusion_fanout_enabled(&mut self, value: bool) +pub fn iceberg::spec::TableProperties::set_write_delete_avro_compression_codec(&mut self, value: iceberg::compression::CompressionCodec) +pub fn iceberg::spec::TableProperties::set_write_delete_distribution_mode(&mut self, value: core::option::Option) +pub fn iceberg::spec::TableProperties::set_write_delete_format_default(&mut self, value: iceberg::spec::DataFileFormat) +pub fn iceberg::spec::TableProperties::set_write_delete_granularity(&mut self, value: iceberg::spec::DeleteGranularity) +pub fn iceberg::spec::TableProperties::set_write_delete_isolation_level(&mut self, value: iceberg::spec::IsolationLevel) +pub fn iceberg::spec::TableProperties::set_write_delete_mode(&mut self, value: iceberg::spec::RowLevelOperationMode) +pub fn iceberg::spec::TableProperties::set_write_delete_orc_block_size_bytes(&mut self, value: u64) +pub fn iceberg::spec::TableProperties::set_write_delete_orc_compression_codec(&mut self, value: iceberg::compression::CompressionCodec) +pub fn iceberg::spec::TableProperties::set_write_delete_orc_compression_strategy(&mut self, value: alloc::string::String) +pub fn iceberg::spec::TableProperties::set_write_delete_orc_stripe_size_bytes(&mut self, value: u64) +pub fn iceberg::spec::TableProperties::set_write_delete_orc_vectorized_batch_size(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_write_delete_parquet_compression_codec(&mut self, value: iceberg::compression::CompressionCodec) +pub fn iceberg::spec::TableProperties::set_write_delete_parquet_dict_size_bytes(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_write_delete_parquet_page_row_limit(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_write_delete_parquet_page_size_bytes(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_write_delete_parquet_page_version(&mut self, value: alloc::string::String) +pub fn iceberg::spec::TableProperties::set_write_delete_parquet_row_group_check_max_record_count(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_write_delete_parquet_row_group_check_min_record_count(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_write_delete_parquet_row_group_size_bytes(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_write_delete_target_file_size_bytes(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_write_distribution_mode(&mut self, value: core::option::Option) +pub fn iceberg::spec::TableProperties::set_write_folder_storage_path(&mut self, value: core::option::Option) +pub fn iceberg::spec::TableProperties::set_write_format_default(&mut self, value: iceberg::spec::DataFileFormat) +pub fn iceberg::spec::TableProperties::set_write_location_provider_impl(&mut self, value: core::option::Option) +pub fn iceberg::spec::TableProperties::set_write_manifest_compression_codec(&mut self, value: iceberg::compression::CompressionCodec) +pub fn iceberg::spec::TableProperties::set_write_manifest_lists_enabled(&mut self, value: bool) +pub fn iceberg::spec::TableProperties::set_write_merge_distribution_mode(&mut self, value: core::option::Option) +pub fn iceberg::spec::TableProperties::set_write_merge_isolation_level(&mut self, value: iceberg::spec::IsolationLevel) +pub fn iceberg::spec::TableProperties::set_write_merge_mode(&mut self, value: iceberg::spec::RowLevelOperationMode) +pub fn iceberg::spec::TableProperties::set_write_metadata_compression_codec(&mut self, value: iceberg::compression::CompressionCodec) +pub fn iceberg::spec::TableProperties::set_write_metadata_delete_after_commit_enabled(&mut self, value: bool) +pub fn iceberg::spec::TableProperties::set_write_metadata_metrics_column(&mut self, value: std::collections::hash::map::HashMap) +pub fn iceberg::spec::TableProperties::set_write_metadata_metrics_default(&mut self, value: alloc::string::String) +pub fn iceberg::spec::TableProperties::set_write_metadata_metrics_max_inferred_column_defaults(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_write_metadata_path(&mut self, value: core::option::Option) +pub fn iceberg::spec::TableProperties::set_write_metadata_previous_versions_max(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_write_object_storage_enabled(&mut self, value: bool) +pub fn iceberg::spec::TableProperties::set_write_object_storage_partitioned_paths(&mut self, value: bool) +pub fn iceberg::spec::TableProperties::set_write_object_storage_path(&mut self, value: core::option::Option) +pub fn iceberg::spec::TableProperties::set_write_orc_block_size_bytes(&mut self, value: u64) +pub fn iceberg::spec::TableProperties::set_write_orc_bloom_filter_columns(&mut self, value: alloc::vec::Vec) +pub fn iceberg::spec::TableProperties::set_write_orc_bloom_filter_fpp(&mut self, value: f64) +pub fn iceberg::spec::TableProperties::set_write_orc_compression_codec(&mut self, value: iceberg::compression::CompressionCodec) +pub fn iceberg::spec::TableProperties::set_write_orc_compression_strategy(&mut self, value: alloc::string::String) +pub fn iceberg::spec::TableProperties::set_write_orc_stripe_size_bytes(&mut self, value: u64) +pub fn iceberg::spec::TableProperties::set_write_orc_vectorized_batch_size(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_write_parquet_bloom_filter_adaptive_enabled(&mut self, value: bool) +pub fn iceberg::spec::TableProperties::set_write_parquet_bloom_filter_enabled_column(&mut self, value: std::collections::hash::map::HashMap) +pub fn iceberg::spec::TableProperties::set_write_parquet_bloom_filter_fpp_column(&mut self, value: std::collections::hash::map::HashMap) +pub fn iceberg::spec::TableProperties::set_write_parquet_bloom_filter_max_bytes(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_write_parquet_bloom_filter_ndv_column(&mut self, value: std::collections::hash::map::HashMap) +pub fn iceberg::spec::TableProperties::set_write_parquet_compression_codec(&mut self, value: iceberg::compression::CompressionCodec) +pub fn iceberg::spec::TableProperties::set_write_parquet_content_defined_chunking_enabled(&mut self, value: bool) +pub fn iceberg::spec::TableProperties::set_write_parquet_content_defined_chunking_max_chunk_size(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_write_parquet_content_defined_chunking_min_chunk_size(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_write_parquet_content_defined_chunking_norm_level(&mut self, value: i32) +pub fn iceberg::spec::TableProperties::set_write_parquet_dict_encoding_enabled_column(&mut self, value: std::collections::hash::map::HashMap) +pub fn iceberg::spec::TableProperties::set_write_parquet_dict_size_bytes(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_write_parquet_page_row_limit(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_write_parquet_page_size_bytes(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_write_parquet_page_version(&mut self, value: alloc::string::String) +pub fn iceberg::spec::TableProperties::set_write_parquet_row_group_check_max_record_count(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_write_parquet_row_group_check_min_record_count(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_write_parquet_row_group_size_bytes(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_write_parquet_row_group_size_track_uncompressed(&mut self, value: bool) +pub fn iceberg::spec::TableProperties::set_write_parquet_shred_variants(&mut self, value: bool) +pub fn iceberg::spec::TableProperties::set_write_parquet_stats_enabled_column(&mut self, value: std::collections::hash::map::HashMap) +pub fn iceberg::spec::TableProperties::set_write_parquet_variant_inference_buffer_size(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_write_spark_accept_any_schema(&mut self, value: bool) +pub fn iceberg::spec::TableProperties::set_write_spark_advisory_partition_size_bytes(&mut self, value: core::option::Option) +pub fn iceberg::spec::TableProperties::set_write_spark_auto_schema_evolution_enabled(&mut self, value: bool) +pub fn iceberg::spec::TableProperties::set_write_spark_fanout_enabled(&mut self, value: bool) +pub fn iceberg::spec::TableProperties::set_write_summary_partition_limit(&mut self, value: u64) +pub fn iceberg::spec::TableProperties::set_write_target_file_size_bytes(&mut self, value: usize) +pub fn iceberg::spec::TableProperties::set_write_update_distribution_mode(&mut self, value: core::option::Option) +pub fn iceberg::spec::TableProperties::set_write_update_isolation_level(&mut self, value: iceberg::spec::IsolationLevel) +pub fn iceberg::spec::TableProperties::set_write_update_mode(&mut self, value: iceberg::spec::RowLevelOperationMode) +pub fn iceberg::spec::TableProperties::set_write_upsert_enabled(&mut self, value: bool) +pub fn iceberg::spec::TableProperties::set_write_wap_enabled(&mut self, value: bool) +pub fn iceberg::spec::TableProperties::write_avro_compression_codec(&self) -> &iceberg::compression::CompressionCodec +pub fn iceberg::spec::TableProperties::write_data_path(&self) -> &core::option::Option +pub fn iceberg::spec::TableProperties::write_datafusion_fanout_enabled(&self) -> &bool +pub fn iceberg::spec::TableProperties::write_delete_avro_compression_codec(&self) -> &iceberg::compression::CompressionCodec +pub fn iceberg::spec::TableProperties::write_delete_distribution_mode(&self) -> &core::option::Option +pub fn iceberg::spec::TableProperties::write_delete_format_default(&self) -> &iceberg::spec::DataFileFormat +pub fn iceberg::spec::TableProperties::write_delete_granularity(&self) -> &iceberg::spec::DeleteGranularity +pub fn iceberg::spec::TableProperties::write_delete_isolation_level(&self) -> &iceberg::spec::IsolationLevel +pub fn iceberg::spec::TableProperties::write_delete_mode(&self) -> &iceberg::spec::RowLevelOperationMode +pub fn iceberg::spec::TableProperties::write_delete_orc_block_size_bytes(&self) -> &u64 +pub fn iceberg::spec::TableProperties::write_delete_orc_compression_codec(&self) -> &iceberg::compression::CompressionCodec +pub fn iceberg::spec::TableProperties::write_delete_orc_compression_strategy(&self) -> &alloc::string::String +pub fn iceberg::spec::TableProperties::write_delete_orc_stripe_size_bytes(&self) -> &u64 +pub fn iceberg::spec::TableProperties::write_delete_orc_vectorized_batch_size(&self) -> &usize +pub fn iceberg::spec::TableProperties::write_delete_parquet_compression_codec(&self) -> &iceberg::compression::CompressionCodec +pub fn iceberg::spec::TableProperties::write_delete_parquet_dict_size_bytes(&self) -> &usize +pub fn iceberg::spec::TableProperties::write_delete_parquet_page_row_limit(&self) -> &usize +pub fn iceberg::spec::TableProperties::write_delete_parquet_page_size_bytes(&self) -> &usize +pub fn iceberg::spec::TableProperties::write_delete_parquet_page_version(&self) -> &alloc::string::String +pub fn iceberg::spec::TableProperties::write_delete_parquet_row_group_check_max_record_count(&self) -> &usize +pub fn iceberg::spec::TableProperties::write_delete_parquet_row_group_check_min_record_count(&self) -> &usize +pub fn iceberg::spec::TableProperties::write_delete_parquet_row_group_size_bytes(&self) -> &usize +pub fn iceberg::spec::TableProperties::write_delete_target_file_size_bytes(&self) -> &usize +pub fn iceberg::spec::TableProperties::write_distribution_mode(&self) -> &core::option::Option +pub fn iceberg::spec::TableProperties::write_folder_storage_path(&self) -> &core::option::Option +pub fn iceberg::spec::TableProperties::write_format_default(&self) -> &iceberg::spec::DataFileFormat +pub fn iceberg::spec::TableProperties::write_location_provider_impl(&self) -> &core::option::Option +pub fn iceberg::spec::TableProperties::write_manifest_compression_codec(&self) -> &iceberg::compression::CompressionCodec +pub fn iceberg::spec::TableProperties::write_manifest_lists_enabled(&self) -> &bool +pub fn iceberg::spec::TableProperties::write_merge_distribution_mode(&self) -> &core::option::Option +pub fn iceberg::spec::TableProperties::write_merge_isolation_level(&self) -> &iceberg::spec::IsolationLevel +pub fn iceberg::spec::TableProperties::write_merge_mode(&self) -> &iceberg::spec::RowLevelOperationMode +pub fn iceberg::spec::TableProperties::write_metadata_compression_codec(&self) -> &iceberg::compression::CompressionCodec +pub fn iceberg::spec::TableProperties::write_metadata_delete_after_commit_enabled(&self) -> &bool +pub fn iceberg::spec::TableProperties::write_metadata_metrics_column(&self) -> &std::collections::hash::map::HashMap +pub fn iceberg::spec::TableProperties::write_metadata_metrics_default(&self) -> &alloc::string::String +pub fn iceberg::spec::TableProperties::write_metadata_metrics_max_inferred_column_defaults(&self) -> &usize +pub fn iceberg::spec::TableProperties::write_metadata_path(&self) -> &core::option::Option +pub fn iceberg::spec::TableProperties::write_metadata_previous_versions_max(&self) -> &usize +pub fn iceberg::spec::TableProperties::write_object_storage_enabled(&self) -> &bool +pub fn iceberg::spec::TableProperties::write_object_storage_partitioned_paths(&self) -> &bool +pub fn iceberg::spec::TableProperties::write_object_storage_path(&self) -> &core::option::Option +pub fn iceberg::spec::TableProperties::write_orc_block_size_bytes(&self) -> &u64 +pub fn iceberg::spec::TableProperties::write_orc_bloom_filter_columns(&self) -> &alloc::vec::Vec +pub fn iceberg::spec::TableProperties::write_orc_bloom_filter_fpp(&self) -> &f64 +pub fn iceberg::spec::TableProperties::write_orc_compression_codec(&self) -> &iceberg::compression::CompressionCodec +pub fn iceberg::spec::TableProperties::write_orc_compression_strategy(&self) -> &alloc::string::String +pub fn iceberg::spec::TableProperties::write_orc_stripe_size_bytes(&self) -> &u64 +pub fn iceberg::spec::TableProperties::write_orc_vectorized_batch_size(&self) -> &usize +pub fn iceberg::spec::TableProperties::write_parquet_bloom_filter_adaptive_enabled(&self) -> &bool +pub fn iceberg::spec::TableProperties::write_parquet_bloom_filter_enabled_column(&self) -> &std::collections::hash::map::HashMap +pub fn iceberg::spec::TableProperties::write_parquet_bloom_filter_fpp_column(&self) -> &std::collections::hash::map::HashMap +pub fn iceberg::spec::TableProperties::write_parquet_bloom_filter_max_bytes(&self) -> &usize +pub fn iceberg::spec::TableProperties::write_parquet_bloom_filter_ndv_column(&self) -> &std::collections::hash::map::HashMap +pub fn iceberg::spec::TableProperties::write_parquet_compression_codec(&self) -> &iceberg::compression::CompressionCodec +pub fn iceberg::spec::TableProperties::write_parquet_content_defined_chunking_enabled(&self) -> &bool +pub fn iceberg::spec::TableProperties::write_parquet_content_defined_chunking_max_chunk_size(&self) -> &usize +pub fn iceberg::spec::TableProperties::write_parquet_content_defined_chunking_min_chunk_size(&self) -> &usize +pub fn iceberg::spec::TableProperties::write_parquet_content_defined_chunking_norm_level(&self) -> &i32 +pub fn iceberg::spec::TableProperties::write_parquet_dict_encoding_enabled_column(&self) -> &std::collections::hash::map::HashMap +pub fn iceberg::spec::TableProperties::write_parquet_dict_size_bytes(&self) -> &usize +pub fn iceberg::spec::TableProperties::write_parquet_page_row_limit(&self) -> &usize +pub fn iceberg::spec::TableProperties::write_parquet_page_size_bytes(&self) -> &usize +pub fn iceberg::spec::TableProperties::write_parquet_page_version(&self) -> &alloc::string::String +pub fn iceberg::spec::TableProperties::write_parquet_row_group_check_max_record_count(&self) -> &usize +pub fn iceberg::spec::TableProperties::write_parquet_row_group_check_min_record_count(&self) -> &usize +pub fn iceberg::spec::TableProperties::write_parquet_row_group_size_bytes(&self) -> &usize +pub fn iceberg::spec::TableProperties::write_parquet_row_group_size_track_uncompressed(&self) -> &bool +pub fn iceberg::spec::TableProperties::write_parquet_shred_variants(&self) -> &bool +pub fn iceberg::spec::TableProperties::write_parquet_stats_enabled_column(&self) -> &std::collections::hash::map::HashMap +pub fn iceberg::spec::TableProperties::write_parquet_variant_inference_buffer_size(&self) -> &usize +pub fn iceberg::spec::TableProperties::write_spark_accept_any_schema(&self) -> &bool +pub fn iceberg::spec::TableProperties::write_spark_advisory_partition_size_bytes(&self) -> &core::option::Option +pub fn iceberg::spec::TableProperties::write_spark_auto_schema_evolution_enabled(&self) -> &bool +pub fn iceberg::spec::TableProperties::write_spark_fanout_enabled(&self) -> &bool +pub fn iceberg::spec::TableProperties::write_summary_partition_limit(&self) -> &u64 +pub fn iceberg::spec::TableProperties::write_target_file_size_bytes(&self) -> &usize +pub fn iceberg::spec::TableProperties::write_update_distribution_mode(&self) -> &core::option::Option +pub fn iceberg::spec::TableProperties::write_update_isolation_level(&self) -> &iceberg::spec::IsolationLevel +pub fn iceberg::spec::TableProperties::write_update_mode(&self) -> &iceberg::spec::RowLevelOperationMode +pub fn iceberg::spec::TableProperties::write_upsert_enabled(&self) -> &bool +pub fn iceberg::spec::TableProperties::write_wap_enabled(&self) -> &bool +impl core::clone::Clone for iceberg::spec::TableProperties +pub fn iceberg::spec::TableProperties::clone(&self) -> iceberg::spec::TableProperties impl core::convert::TryFrom<&std::collections::hash::map::HashMap> for iceberg::spec::TableProperties pub type iceberg::spec::TableProperties::Error = iceberg::Error -pub fn iceberg::spec::TableProperties::try_from(props: &std::collections::hash::map::HashMap) -> iceberg::Result +pub fn iceberg::spec::TableProperties::try_from(properties: &std::collections::hash::map::HashMap) -> iceberg::Result +impl core::default::Default for iceberg::spec::TableProperties +pub fn iceberg::spec::TableProperties::default() -> Self impl core::fmt::Debug for iceberg::spec::TableProperties pub fn iceberg::spec::TableProperties::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl serde_core::ser::Serialize for iceberg::spec::TableProperties +pub fn iceberg::spec::TableProperties::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> where S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::TableProperties +pub fn iceberg::spec::TableProperties::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> pub struct iceberg::spec::UnboundPartitionField pub iceberg::spec::UnboundPartitionField::field_id: core::option::Option pub iceberg::spec::UnboundPartitionField::name: alloc::string::String @@ -3029,6 +3317,10 @@ pub const iceberg::spec::MAIN_BRANCH: &str pub const iceberg::spec::MAP_KEY_FIELD_NAME: &str pub const iceberg::spec::MAP_VALUE_FIELD_NAME: &str pub const iceberg::spec::MIN_FORMAT_VERSION_ROW_LINEAGE: iceberg::spec::FormatVersion +pub const iceberg::spec::ORC_COMPRESSION_STRATEGY_COMPRESSION: &str +pub const iceberg::spec::ORC_COMPRESSION_STRATEGY_SPEED: &str +pub const iceberg::spec::PARQUET_PAGE_VERSION_V1: &str +pub const iceberg::spec::PARQUET_PAGE_VERSION_V2: &str pub const iceberg::spec::SCHEMA_NAME_DELIMITER: &str pub const iceberg::spec::UNASSIGNED_SEQUENCE_NUMBER: i64 pub const iceberg::spec::VIEW_PROPERTY_REPLACE_DROP_DIALECT_ALLOWED: &str diff --git a/crates/iceberg/src/catalog/utils.rs b/crates/iceberg/src/catalog/utils.rs index 8e743e7d7d..065aa16e1d 100644 --- a/crates/iceberg/src/catalog/utils.rs +++ b/crates/iceberg/src/catalog/utils.rs @@ -61,7 +61,7 @@ pub async fn drop_table_data(table_info: &Table) -> Result<()> { } // Delete data files only if gc.enabled is true, to avoid corrupting shared tables - if metadata.table_properties()?.gc_enabled { + if *metadata.table_properties()?.gc_enabled() { delete_data_files(io, &manifests_to_delete).await?; } diff --git a/crates/iceberg/src/compression.rs b/crates/iceberg/src/compression.rs index 929d9226e7..3c15bdbf6f 100644 --- a/crates/iceberg/src/compression.rs +++ b/crates/iceberg/src/compression.rs @@ -17,6 +17,7 @@ //! Compression codec support for data compression and decompression. +use std::collections::HashMap; use std::fmt; use std::io::{Read, Write}; @@ -40,8 +41,12 @@ pub enum CompressionCodec { #[default] /// No compression None, + /// Brotli compression + Brotli, /// LZ4 single compression frame with content size present Lz4, + /// LZO compression + Lzo, /// Zstandard single compression frame with content size present. /// Level range is 0–22, where 0 means default compression level (not no compression). /// Use [`CompressionCodec::zstd_default`] to construct with the default level. @@ -51,6 +56,8 @@ pub enum CompressionCodec { Gzip(u8), /// Snappy compression Snappy, + /// Zlib compression + Zlib, } impl CompressionCodec { @@ -68,10 +75,80 @@ impl CompressionCodec { pub fn name(&self) -> &'static str { match self { CompressionCodec::None => "none", + CompressionCodec::Brotli => "brotli", CompressionCodec::Lz4 => "lz4", + CompressionCodec::Lzo => "lzo", CompressionCodec::Zstd(_) => "zstd", CompressionCodec::Gzip(_) => "gzip", CompressionCodec::Snappy => "snappy", + CompressionCodec::Zlib => "zlib", + } + } + + /// Parses a compression codec name used by an Iceberg table property. + pub(crate) fn parse_property(value: &str) -> Result { + serde_json::from_value(serde_json::Value::String(value.to_lowercase())).map_err(|_| { + Error::new( + ErrorKind::DataInvalid, + format!("Invalid compression codec: {value}"), + ) + }) + } + + /// Parses the metadata-file compression codec table property. + pub(crate) fn parse_metadata_property(value: &str) -> Result { + match value.to_ascii_lowercase().as_str() { + "" | "none" => Ok(Self::None), + "gzip" => Ok(Self::gzip_default()), + _ => Err(Error::new( + ErrorKind::DataInvalid, + format!( + "Invalid metadata compression codec: {value}. Only '{}' and '{}' are supported for metadata files.", + Self::None.name(), + Self::gzip_default().name() + ), + )), + } + } + + /// Returns the codec name used by an Iceberg table property. + pub(crate) fn property_value(&self) -> String { + self.name().to_string() + } + + /// Parses a codec and its optional companion compression-level property. + pub(crate) fn parse_properties( + properties: &HashMap, + codec_key: &str, + level_key: &str, + default: Self, + ) -> Result { + let codec = properties + .get(codec_key) + .map(|value| Self::parse_property(value)) + .transpose()? + .unwrap_or(default); + let Some(level) = properties.get(level_key) else { + return Ok(codec); + }; + let level = level.parse::().map_err(|error| { + Error::new( + ErrorKind::DataInvalid, + format!("Invalid compression level for {level_key}: {level}"), + ) + .with_source(error) + })?; + + match codec { + Self::Gzip(_) => Ok(Self::Gzip(level)), + Self::Zstd(_) => Ok(Self::Zstd(level)), + _ => Err(Error::new( + ErrorKind::DataInvalid, + format!( + "Compression level {level_key} is not supported for codec '{}'", + codec.name() + ), + )), } } } @@ -90,13 +167,24 @@ impl<'de> Deserialize<'de> for CompressionCodec { fn deserialize>(deserializer: D) -> std::result::Result { let s = String::deserialize(deserializer)?; match s.to_lowercase().as_str() { - "none" => Ok(CompressionCodec::None), + "none" | "uncompressed" => Ok(CompressionCodec::None), + "brotli" => Ok(CompressionCodec::Brotli), "lz4" => Ok(CompressionCodec::Lz4), + "lzo" => Ok(CompressionCodec::Lzo), "zstd" => Ok(CompressionCodec::zstd_default()), "gzip" => Ok(CompressionCodec::gzip_default()), "snappy" => Ok(CompressionCodec::Snappy), + "zlib" => Ok(CompressionCodec::Zlib), other => Err(serde::de::Error::unknown_variant(other, &[ - "none", "lz4", "zstd", "gzip", "snappy", + "none", + "uncompressed", + "brotli", + "lz4", + "lzo", + "zstd", + "gzip", + "snappy", + "zlib", ])), } } @@ -106,10 +194,13 @@ impl fmt::Display for CompressionCodec { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { CompressionCodec::None => write!(f, "None"), + CompressionCodec::Brotli => write!(f, "Brotli"), CompressionCodec::Lz4 => write!(f, "Lz4"), + CompressionCodec::Lzo => write!(f, "Lzo"), CompressionCodec::Zstd(level) => write!(f, "Zstd(level={level})"), CompressionCodec::Gzip(level) => write!(f, "Gzip(level={level})"), CompressionCodec::Snappy => write!(f, "Snappy"), + CompressionCodec::Zlib => write!(f, "Zlib"), } } } @@ -122,6 +213,16 @@ impl CompressionCodec { ErrorKind::FeatureUnsupported, "LZ4 decompression is not supported currently", )), + CompressionCodec::Snappy => Err(Error::new( + ErrorKind::FeatureUnsupported, + "Snappy decompression is not supported currently", + )), + codec @ (CompressionCodec::Brotli | CompressionCodec::Lzo | CompressionCodec::Zlib) => { + Err(Error::new( + ErrorKind::FeatureUnsupported, + format!("{codec} decompression is not supported currently"), + )) + } CompressionCodec::Zstd(_) => Ok(zstd::stream::decode_all(&bytes[..])?), CompressionCodec::Gzip(_) => { let mut decoder = GzDecoder::new(&bytes[..]); @@ -129,10 +230,6 @@ impl CompressionCodec { decoder.read_to_end(&mut decompressed)?; Ok(decompressed) } - CompressionCodec::Snappy => Err(Error::new( - ErrorKind::FeatureUnsupported, - "Snappy decompression is not supported currently", - )), } } @@ -143,6 +240,16 @@ impl CompressionCodec { ErrorKind::FeatureUnsupported, "LZ4 compression is not supported currently", )), + CompressionCodec::Snappy => Err(Error::new( + ErrorKind::FeatureUnsupported, + "Snappy compression is not supported currently", + )), + codec @ (CompressionCodec::Brotli | CompressionCodec::Lzo | CompressionCodec::Zlib) => { + Err(Error::new( + ErrorKind::FeatureUnsupported, + format!("{codec} compression is not supported currently"), + )) + } CompressionCodec::Zstd(level) => { let writer = Vec::::new(); let mut encoder = zstd::stream::Encoder::new(writer, *level as i32)?; @@ -157,10 +264,6 @@ impl CompressionCodec { encoder.write_all(&bytes)?; Ok(encoder.finish()?) } - CompressionCodec::Snappy => Err(Error::new( - ErrorKind::FeatureUnsupported, - "Snappy compression is not supported currently", - )), } } @@ -173,14 +276,17 @@ impl CompressionCodec { /// /// # Errors /// - /// Returns an error for Lz4 and Zstd as they are not fully supported. + /// Returns an error when the codec does not have a defined suffix. pub fn suffix(&self) -> Result<&'static str> { match self { CompressionCodec::None => Ok(""), CompressionCodec::Gzip(_) => Ok(".gz"), - codec @ (CompressionCodec::Lz4 + codec @ (CompressionCodec::Brotli + | CompressionCodec::Lz4 + | CompressionCodec::Lzo | CompressionCodec::Zstd(_) - | CompressionCodec::Snappy) => Err(Error::new( + | CompressionCodec::Snappy + | CompressionCodec::Zlib) => Err(Error::new( ErrorKind::FeatureUnsupported, format!("suffix not defined for {codec:?}"), )), @@ -196,10 +302,9 @@ mod tests { async fn test_compression_codec_none() { let bytes_vec = [0_u8; 100].to_vec(); - let codec = CompressionCodec::None; - let compressed = codec.compress(bytes_vec.clone()).unwrap(); + let compressed = CompressionCodec::None.compress(bytes_vec.clone()).unwrap(); assert_eq!(bytes_vec, compressed); - let decompressed = codec.decompress(compressed).unwrap(); + let decompressed = CompressionCodec::None.decompress(compressed).unwrap(); assert_eq!(bytes_vec, decompressed); } @@ -225,6 +330,9 @@ mod tests { let unsupported_codecs = [ (CompressionCodec::Lz4, "LZ4"), (CompressionCodec::Snappy, "Snappy"), + (CompressionCodec::Brotli, "Brotli"), + (CompressionCodec::Lzo, "Lzo"), + (CompressionCodec::Zlib, "Zlib"), ]; let bytes_vec = [0_u8; 100].to_vec(); @@ -273,4 +381,31 @@ mod tests { assert_eq!(CompressionCodec::Gzip(9).to_string(), "Gzip(level=9)"); assert_eq!(CompressionCodec::Snappy.to_string(), "Snappy"); } + + #[test] + fn test_serde_names() { + let codecs = [ + ("none", CompressionCodec::None), + ("brotli", CompressionCodec::Brotli), + ("lz4", CompressionCodec::Lz4), + ("lzo", CompressionCodec::Lzo), + ("zstd", CompressionCodec::zstd_default()), + ("gzip", CompressionCodec::gzip_default()), + ("snappy", CompressionCodec::Snappy), + ("zlib", CompressionCodec::Zlib), + ]; + + for (name, codec) in codecs { + assert_eq!(serde_json::to_value(codec).unwrap(), name); + assert_eq!( + serde_json::from_value::(serde_json::json!(name)).unwrap(), + codec + ); + } + + assert_eq!( + serde_json::from_value::(serde_json::json!("uncompressed")).unwrap(), + CompressionCodec::None + ); + } } diff --git a/crates/iceberg/src/encryption/manager.rs b/crates/iceberg/src/encryption/manager.rs index e2294c2f2c..a212991d64 100644 --- a/crates/iceberg/src/encryption/manager.rs +++ b/crates/iceberg/src/encryption/manager.rs @@ -119,7 +119,7 @@ impl EncryptionManager { } let table_properties = metadata.table_properties()?; - let Some(table_key_id) = table_properties.encryption_key_id else { + let Some(table_key_id) = table_properties.encryption_key_id() else { if kms_client.is_some() { tracing::warn!( "KeyManagementClient provided but table does not have encryption.key-id set" @@ -140,7 +140,7 @@ impl EncryptionManager { .table_key_id(table_key_id) .encryption_keys(metadata.encryption_keys.clone()) .key_size(AesKeySize::from_key_length( - table_properties.encryption_data_key_length, + *table_properties.encryption_data_key_length(), )?) .build(); Ok(Some(Arc::new(em))) diff --git a/crates/iceberg/src/spec/mod.rs b/crates/iceberg/src/spec/mod.rs index b23ca1eda0..e166e14bc1 100644 --- a/crates/iceberg/src/spec/mod.rs +++ b/crates/iceberg/src/spec/mod.rs @@ -30,7 +30,7 @@ mod sort; mod statistic_file; mod table_metadata; mod table_metadata_builder; -mod table_properties; +mod table_props; mod transform; mod values; mod view_metadata; @@ -50,8 +50,7 @@ pub use sort::*; pub use statistic_file::*; pub use table_metadata::*; pub(crate) use table_metadata_builder::FIRST_FIELD_ID; -pub(crate) use table_properties::parse_metadata_file_compression; -pub use table_properties::*; +pub use table_props::*; pub use transform::*; pub(crate) use values::decimal_utils; pub use values::*; diff --git a/crates/iceberg/src/spec/table_metadata.rs b/crates/iceberg/src/spec/table_metadata.rs index ecc0586680..0ebde6806d 100644 --- a/crates/iceberg/src/spec/table_metadata.rs +++ b/crates/iceberg/src/spec/table_metadata.rs @@ -35,7 +35,7 @@ pub use super::table_metadata_builder::{TableMetadataBuildResult, TableMetadataB use super::{ DEFAULT_PARTITION_SPEC_ID, PartitionSpecRef, PartitionStatisticsFile, SchemaId, SchemaRef, SnapshotRef, SnapshotRetention, SortOrder, SortOrderRef, StatisticsFile, StructType, - TableProperties, parse_metadata_file_compression, + TableProperties, parse_metadata_file_compression, parse_metadata_location_property, }; use crate::catalog::{METADATA_FOLDER_NAME, MetadataLocation}; use crate::compression::CompressionCodec; @@ -369,9 +369,7 @@ impl TableMetadata { /// Honors the `write.metadata.path` table property when set, otherwise defaults /// to the `metadata` subdirectory under the table location. pub fn metadata_location(&self) -> Result { - Ok(self - .table_properties()? - .write_metadata_path + Ok(parse_metadata_location_property(&self.properties)? .unwrap_or_else(|| format!("{}/{}", self.location(), METADATA_FOLDER_NAME))) } @@ -387,10 +385,10 @@ impl TableMetadata { parse_metadata_file_compression(&self.properties) } - /// Returns typed table properties parsed from the raw properties map with defaults. + /// Returns all supported table properties parsed from the raw property map. pub fn table_properties(&self) -> Result { - TableProperties::try_from(&self.properties).map_err(|e| { - Error::new(ErrorKind::DataInvalid, "Invalid table properties").with_source(e) + TableProperties::try_from(&self.properties).map_err(|error| { + Error::new(ErrorKind::DataInvalid, "Invalid table properties").with_source(error) }) } @@ -4043,11 +4041,11 @@ mod tests { let props = metadata.table_properties().unwrap(); assert_eq!( - props.commit_num_retries, + *props.commit_retry_num_retries(), TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT ); assert_eq!( - props.write_target_file_size_bytes, + *props.write_target_file_size_bytes(), TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT ); } @@ -4089,8 +4087,8 @@ mod tests { let props = metadata.table_properties().unwrap(); - assert_eq!(props.commit_num_retries, 10); - assert_eq!(props.write_target_file_size_bytes, 1024); + assert_eq!(*props.commit_retry_num_retries(), 10); + assert_eq!(*props.write_target_file_size_bytes(), 1024); } #[test] @@ -4347,7 +4345,6 @@ mod tests { #[test] fn test_metadata_location_trims_trailing_slash() { - // A configured path with a trailing slash must not yield a doubled separator let metadata = get_test_table_metadata("TableMetadataV2Valid.json") .into_builder(None) .set_properties(HashMap::from([( diff --git a/crates/iceberg/src/spec/table_properties.rs b/crates/iceberg/src/spec/table_properties.rs deleted file mode 100644 index a35f1427a3..0000000000 --- a/crates/iceberg/src/spec/table_properties.rs +++ /dev/null @@ -1,1004 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::collections::HashMap; -use std::fmt::Display; -use std::str::FromStr; - -use crate::compression::CompressionCodec; -use crate::error::{Error, ErrorKind, Result}; -use crate::util::location::strip_trailing_slash; - -fn parse_property( - properties: &HashMap, - key: &str, - default: T, -) -> Result -where - ::Err: Display, -{ - properties.get(key).map_or(Ok(default), |value| { - value.parse::().map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!("Invalid value for {key}: {e}"), - ) - }) - }) -} - -fn parse_location_property( - properties: &HashMap, - key: &str, -) -> Result> { - properties - .get(key) - .map(|path| { - if path.is_empty() { - return Err(Error::new( - ErrorKind::DataInvalid, - format!("Invalid value for {key}: path must not be empty"), - )); - } - - Ok(strip_trailing_slash(path).to_string()) - }) - .transpose() -} - -/// Parse compression codec for metadata files from table properties. -/// Retrieves the compression codec property, applies defaults, and parses the value. -/// Only "none" (or empty string) and "gzip" are supported for metadata compression. -/// -/// # Arguments -/// -/// * `properties` - HashMap containing table properties -/// -/// # Errors -/// -/// Returns an error if the codec is not "none", "", or "gzip" (case-insensitive). -/// Lz4 and Zstd are not supported for metadata file compression. -pub(crate) fn parse_metadata_file_compression( - properties: &HashMap, -) -> Result { - let value = properties - .get(TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC) - .map(|s| s.as_str()) - .unwrap_or(TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC_DEFAULT); - - // Handle empty string as None - if value.is_empty() { - return Ok(CompressionCodec::None); - } - - // Lowercase the value for case-insensitive parsing - let lowercase_value = value.to_lowercase(); - - // Use serde to parse the codec (which has rename_all = "lowercase") - let codec: CompressionCodec = serde_json::from_value(serde_json::Value::String( - lowercase_value, - )) - .map_err(|_| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Invalid metadata compression codec: {value}. Only '{}' and '{}' are supported.", - CompressionCodec::None.name(), - CompressionCodec::gzip_default().name() - ), - ) - })?; - - // Validate that only None and Gzip are used for metadata - match codec { - CompressionCodec::None | CompressionCodec::Gzip(_) => Ok(codec), - _ => Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Invalid metadata compression codec: {value}. Only '{}' and '{}' are supported for metadata files.", - CompressionCodec::None.name(), - CompressionCodec::gzip_default().name() - ), - )), - } -} - -/// Parse boolean property case insensitively -/// Rust standard library only accepts "true" and "false", see https://doc.rust-lang.org/std/primitive.bool.html#method.from_str -/// Users might accidentally trigger fallback with valid configuration values such as "False" or "True" -fn parse_property_bool( - properties: &HashMap, - key: &str, - default: bool, -) -> Result { - properties.get(key).map_or(Ok(default), |value| { - value.to_lowercase().parse::().map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!("Invalid value for {key}: {e}"), - ) - }) - }) -} - -/// TableProperties that contains the properties of a table. -#[derive(Debug)] -pub struct TableProperties { - /// The number of times to retry a commit. - pub commit_num_retries: usize, - /// The minimum wait time between retries. - pub commit_min_retry_wait_ms: u64, - /// The maximum wait time between retries. - pub commit_max_retry_wait_ms: u64, - /// The total timeout for commit retries. - pub commit_total_retry_timeout_ms: u64, - /// The default format for files. - pub write_format_default: String, - /// The target file size for files. - pub write_target_file_size_bytes: usize, - /// Base directory for metadata files (manifests, manifest lists), with any - /// trailing slash trimmed. `None` if `write.metadata.path` is not set. - pub write_metadata_path: Option, - /// Compression codec for metadata files (JSON) - pub metadata_compression_codec: CompressionCodec, - /// Whether to use `FanoutWriter` for partitioned tables. - pub write_datafusion_fanout_enabled: bool, - /// Whether garbage collection is enabled on drop. - /// When `false`, data files will not be deleted when a table is dropped. - pub gc_enabled: bool, - /// Default maximum age of a snapshot to keep when expiring snapshots. - pub max_snapshot_age_ms: i64, - /// Default minimum number of snapshots to keep per branch when expiring snapshots. - pub min_snapshots_to_keep: usize, - /// Default maximum age of a snapshot reference to keep when expiring snapshots. - pub max_ref_age_ms: i64, - /// Whether content-defined chunking is enabled. - /// `true` only when `write.parquet.content-defined-chunking.enabled = "true"`. - pub cdc_enabled: bool, - /// Content-defined chunking minimum chunk size in bytes. - pub cdc_min_chunk_size: usize, - /// Content-defined chunking maximum chunk size in bytes. - pub cdc_max_chunk_size: usize, - /// Content-defined chunking normalization level (gearhash bit adjustment). - pub cdc_norm_level: i32, - /// The master key id used to encrypt this table's manifest list and data - /// files. `None` if `encryption.key-id` is not set. - pub encryption_key_id: Option, - /// The encryption data encryption key length in bytes. - pub encryption_data_key_length: usize, - /// Base directory for data files - pub write_data_location: Option, - /// Deprecated table property for data file write location. - /// - /// Property will be removed at a later date. - /// Superseded by [write_data_location]. - pub write_folder_storage_location: Option, - /// Deprecated table property for data file write location for object storage location generator. - /// - /// Property will be removed at a later date. - /// Superseded by [write_data_location]. - pub write_object_storage_location: Option, - /// Whether partition values are included in object storage paths. - pub write_object_storage_partitioned_paths: bool, -} - -impl TableProperties { - /// Reserved table property for table format version. - /// - /// Iceberg will default a new table's format version to the latest stable and recommended - /// version. This reserved property keyword allows users to override the Iceberg format version of - /// the table metadata. - /// - /// If this table property exists when creating a table, the table will use the specified format - /// version. If a table updates this property, it will try to upgrade to the specified format - /// version. - pub const PROPERTY_FORMAT_VERSION: &str = "format-version"; - /// Reserved table property for table UUID. - pub const PROPERTY_UUID: &str = "uuid"; - /// Reserved table property for the total number of snapshots. - pub const PROPERTY_SNAPSHOT_COUNT: &str = "snapshot-count"; - /// Reserved table property for current snapshot summary. - pub const PROPERTY_CURRENT_SNAPSHOT_SUMMARY: &str = "current-snapshot-summary"; - /// Reserved table property for current snapshot id. - pub const PROPERTY_CURRENT_SNAPSHOT_ID: &str = "current-snapshot-id"; - /// Reserved table property for current snapshot timestamp. - pub const PROPERTY_CURRENT_SNAPSHOT_TIMESTAMP: &str = "current-snapshot-timestamp-ms"; - /// Reserved table property for the JSON representation of current schema. - pub const PROPERTY_CURRENT_SCHEMA: &str = "current-schema"; - /// Reserved table property for the JSON representation of current(default) partition spec. - pub const PROPERTY_DEFAULT_PARTITION_SPEC: &str = "default-partition-spec"; - /// Reserved table property for the JSON representation of current(default) sort order. - pub const PROPERTY_DEFAULT_SORT_ORDER: &str = "default-sort-order"; - - /// Property key for max number of previous versions to keep. - pub const PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX: &str = - "write.metadata.previous-versions-max"; - /// Default value for max number of previous versions to keep. - pub const PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX_DEFAULT: usize = 100; - - /// Property key for max number of partitions to keep summary stats for. - pub const PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT: &str = "write.summary.partition-limit"; - /// Default value for the max number of partitions to keep summary stats for. - pub const PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT_DEFAULT: u64 = 0; - - /// Reserved Iceberg table properties list. - /// - /// Reserved table properties are only used to control behaviors when creating or updating a - /// table. The value of these properties are not persisted as a part of the table metadata. - pub const RESERVED_PROPERTIES: [&str; 9] = [ - Self::PROPERTY_FORMAT_VERSION, - Self::PROPERTY_UUID, - Self::PROPERTY_SNAPSHOT_COUNT, - Self::PROPERTY_CURRENT_SNAPSHOT_ID, - Self::PROPERTY_CURRENT_SNAPSHOT_SUMMARY, - Self::PROPERTY_CURRENT_SNAPSHOT_TIMESTAMP, - Self::PROPERTY_CURRENT_SCHEMA, - Self::PROPERTY_DEFAULT_PARTITION_SPEC, - Self::PROPERTY_DEFAULT_SORT_ORDER, - ]; - - /// Property key for number of commit retries. - pub const PROPERTY_COMMIT_NUM_RETRIES: &str = "commit.retry.num-retries"; - /// Default value for number of commit retries. - pub const PROPERTY_COMMIT_NUM_RETRIES_DEFAULT: usize = 4; - - /// Property key for minimum wait time (ms) between retries. - pub const PROPERTY_COMMIT_MIN_RETRY_WAIT_MS: &str = "commit.retry.min-wait-ms"; - /// Default value for minimum wait time (ms) between retries. - pub const PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT: u64 = 100; - - /// Property key for maximum wait time (ms) between retries. - pub const PROPERTY_COMMIT_MAX_RETRY_WAIT_MS: &str = "commit.retry.max-wait-ms"; - /// Default value for maximum wait time (ms) between retries. - pub const PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT: u64 = 60 * 1000; // 1 minute - - /// Property key for total maximum retry time (ms). - pub const PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS: &str = "commit.retry.total-timeout-ms"; - /// Default value for total maximum retry time (ms). - pub const PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT: u64 = 30 * 60 * 1000; // 30 minutes - - /// Default file format for data files - pub const PROPERTY_DEFAULT_FILE_FORMAT: &str = "write.format.default"; - /// Default file format for delete files - pub const PROPERTY_DELETE_DEFAULT_FILE_FORMAT: &str = "write.delete.format.default"; - /// Default value for data file format - pub const PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT: &str = "parquet"; - - /// Target file size for newly written files. - pub const PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES: &str = "write.target-file-size-bytes"; - /// Default target file size - pub const PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT: usize = 512 * 1024 * 1024; // 512 MB - - /// Base location for metadata files (manifests, manifest lists, table metadata). - /// When unset, metadata files default to the `metadata` directory under the table - /// location. - pub const PROPERTY_WRITE_METADATA_PATH: &str = "write.metadata.path"; - - /// Compression codec for metadata files (JSON) - pub const PROPERTY_METADATA_COMPRESSION_CODEC: &str = "write.metadata.compression-codec"; - /// Default metadata compression codec - uncompressed - pub const PROPERTY_METADATA_COMPRESSION_CODEC_DEFAULT: &str = "none"; - /// Whether to use `FanoutWriter` for partitioned tables (handles unsorted data). - /// If false, uses `ClusteredWriter` (requires sorted data, more memory efficient). - pub const PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED: &str = "write.datafusion.fanout.enabled"; - /// Default value for fanout writer enabled - pub const PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT: bool = true; - - /// Property key for enabling garbage collection on drop. - /// When set to `false`, data files will not be deleted when a table is dropped. - /// Defaults to `true`. - pub const PROPERTY_GC_ENABLED: &str = "gc.enabled"; - /// Default value for gc.enabled - pub const PROPERTY_GC_ENABLED_DEFAULT: bool = true; - - /// Property key for the default maximum age of a snapshot to keep when expiring snapshots. - pub const PROPERTY_MAX_SNAPSHOT_AGE_MS: &str = "history.expire.max-snapshot-age-ms"; - /// Default value for history.expire.max-snapshot-age-ms (5 days). - pub const PROPERTY_MAX_SNAPSHOT_AGE_MS_DEFAULT: i64 = 5 * 24 * 60 * 60 * 1000; - /// Property key for the default minimum number of snapshots to keep when expiring snapshots. - pub const PROPERTY_MIN_SNAPSHOTS_TO_KEEP: &str = "history.expire.min-snapshots-to-keep"; - /// Default value for history.expire.min-snapshots-to-keep. - pub const PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT: usize = 1; - /// Property key for the default maximum age of a snapshot reference to keep when expiring. - pub const PROPERTY_MAX_REF_AGE_MS: &str = "history.expire.max-ref-age-ms"; - /// Default value for history.expire.max-ref-age-ms (effectively never expire refs). - pub const PROPERTY_MAX_REF_AGE_MS_DEFAULT: i64 = i64::MAX; - - /// Enable content-defined chunking with parquet defaults (or per-property overrides). - pub const PROPERTY_PARQUET_CDC_ENABLED: &str = "write.parquet.content-defined-chunking.enabled"; - /// Default value for content-defined chunking enabled. - pub const PROPERTY_PARQUET_CDC_ENABLED_DEFAULT: bool = false; - /// Minimum chunk size in bytes for content-defined chunking. - pub const PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE: &str = - "write.parquet.content-defined-chunking.min-chunk-size"; - /// Default matches `parquet::file::properties::DEFAULT_CDC_MIN_CHUNK_SIZE`. - pub const PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT: usize = 256 * 1024; - /// Maximum chunk size in bytes for content-defined chunking. - pub const PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE: &str = - "write.parquet.content-defined-chunking.max-chunk-size"; - /// Default matches `parquet::file::properties::DEFAULT_CDC_MAX_CHUNK_SIZE`. - pub const PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE_DEFAULT: usize = 1024 * 1024; - /// Normalization level (gearhash bit adjustment) for content-defined chunking. - pub const PROPERTY_PARQUET_CDC_NORM_LEVEL: &str = - "write.parquet.content-defined-chunking.norm-level"; - /// Default matches `parquet::file::properties::DEFAULT_CDC_NORM_LEVEL`. - pub const PROPERTY_PARQUET_CDC_NORM_LEVEL_DEFAULT: i32 = 0; - - /// Property key for the master key id used to encrypt the table's manifest - /// list and data files as defined in https://iceberg.apache.org/docs/nightly/encryption/. - pub const PROPERTY_ENCRYPTION_KEY_ID: &str = "encryption.key-id"; - - /// Property key for the encryption data encryption key (DEK) length in bytes. - pub const PROPERTY_ENCRYPTION_DATA_KEY_LENGTH: &str = "encryption.data-key-length"; - /// Default value for the encryption DEK length (16 bytes = AES-128). - pub const PROPERTY_ENCRYPTION_DATA_KEY_LENGTH_DEFAULT: usize = 16; - /// Property key for the base directory for data files - pub const PROPERTY_WRITE_DATA_LOCATION: &str = "write.data.path"; - /// Property key for deprecated [write_folder_storage_location] - pub const PROPERTY_WRITE_FOLDER_STORAGE_LOCATION: &str = "write.folder-storage.path"; - /// Property key for deprecated object storage path, kept as a fallback for compatibility. - pub const PROPERTY_WRITE_OBJECT_STORAGE_LOCATION: &str = "write.object-storage.path"; - /// Property key for controlling whether partition values are included in object storage paths. - pub const PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS: &str = - "write.object-storage.partitioned-paths"; - /// Default value for [PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS] - pub const PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS_DEFAULT: bool = true; -} - -impl TryFrom<&HashMap> for TableProperties { - // parse by entry key or use default value - type Error = Error; - - fn try_from(props: &HashMap) -> Result { - Ok(TableProperties { - commit_num_retries: parse_property( - props, - TableProperties::PROPERTY_COMMIT_NUM_RETRIES, - TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT, - )?, - commit_min_retry_wait_ms: parse_property( - props, - TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS, - TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT, - )?, - commit_max_retry_wait_ms: parse_property( - props, - TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS, - TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT, - )?, - commit_total_retry_timeout_ms: parse_property( - props, - TableProperties::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS, - TableProperties::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT, - )?, - write_format_default: parse_property( - props, - TableProperties::PROPERTY_DEFAULT_FILE_FORMAT, - TableProperties::PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT.to_string(), - )?, - write_target_file_size_bytes: parse_property( - props, - TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES, - TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT, - )?, - write_metadata_path: parse_location_property( - props, - TableProperties::PROPERTY_WRITE_METADATA_PATH, - )?, - metadata_compression_codec: parse_metadata_file_compression(props)?, - write_datafusion_fanout_enabled: parse_property_bool( - props, - TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED, - TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT, - )?, - gc_enabled: parse_property_bool( - props, - TableProperties::PROPERTY_GC_ENABLED, - TableProperties::PROPERTY_GC_ENABLED_DEFAULT, - )?, - max_snapshot_age_ms: parse_property( - props, - TableProperties::PROPERTY_MAX_SNAPSHOT_AGE_MS, - TableProperties::PROPERTY_MAX_SNAPSHOT_AGE_MS_DEFAULT, - )?, - min_snapshots_to_keep: parse_property( - props, - TableProperties::PROPERTY_MIN_SNAPSHOTS_TO_KEEP, - TableProperties::PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT, - )?, - max_ref_age_ms: parse_property( - props, - TableProperties::PROPERTY_MAX_REF_AGE_MS, - TableProperties::PROPERTY_MAX_REF_AGE_MS_DEFAULT, - )?, - cdc_enabled: parse_property_bool( - props, - TableProperties::PROPERTY_PARQUET_CDC_ENABLED, - TableProperties::PROPERTY_PARQUET_CDC_ENABLED_DEFAULT, - )?, - cdc_min_chunk_size: parse_property( - props, - TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE, - TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT, - )?, - cdc_max_chunk_size: parse_property( - props, - TableProperties::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE, - TableProperties::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE_DEFAULT, - )?, - cdc_norm_level: parse_property( - props, - TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL, - TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL_DEFAULT, - )?, - encryption_key_id: props - .get(TableProperties::PROPERTY_ENCRYPTION_KEY_ID) - .cloned(), - encryption_data_key_length: parse_property( - props, - TableProperties::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH, - TableProperties::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH_DEFAULT, - )?, - write_data_location: props - .get(TableProperties::PROPERTY_WRITE_DATA_LOCATION) - .cloned(), - write_folder_storage_location: props - .get(TableProperties::PROPERTY_WRITE_FOLDER_STORAGE_LOCATION) - .cloned(), - write_object_storage_location: props - .get(TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_LOCATION) - .cloned(), - write_object_storage_partitioned_paths: parse_property_bool( - props, - TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS, - TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS_DEFAULT, - )?, - }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::compression::CompressionCodec; - - #[test] - fn test_table_properties_default() { - let props = HashMap::new(); - let table_properties = TableProperties::try_from(&props).unwrap(); - assert_eq!( - table_properties.commit_num_retries, - TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT - ); - assert_eq!( - table_properties.commit_min_retry_wait_ms, - TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT - ); - assert_eq!( - table_properties.commit_max_retry_wait_ms, - TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT - ); - assert_eq!( - table_properties.write_format_default, - TableProperties::PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT.to_string() - ); - assert_eq!( - table_properties.write_target_file_size_bytes, - TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT - ); - // Test compression defaults (none means CompressionCodec::None) - assert_eq!( - table_properties.metadata_compression_codec, - CompressionCodec::None - ); - assert_eq!( - table_properties.gc_enabled, - TableProperties::PROPERTY_GC_ENABLED_DEFAULT - ); - assert_eq!( - table_properties.max_snapshot_age_ms, - TableProperties::PROPERTY_MAX_SNAPSHOT_AGE_MS_DEFAULT - ); - assert_eq!( - table_properties.min_snapshots_to_keep, - TableProperties::PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT - ); - assert_eq!( - table_properties.max_ref_age_ms, - TableProperties::PROPERTY_MAX_REF_AGE_MS_DEFAULT - ); - } - - #[test] - fn test_table_properties_history_expire_overrides() { - let props = HashMap::from([ - ( - TableProperties::PROPERTY_MAX_SNAPSHOT_AGE_MS.to_string(), - "1234".to_string(), - ), - ( - TableProperties::PROPERTY_MIN_SNAPSHOTS_TO_KEEP.to_string(), - "7".to_string(), - ), - ( - TableProperties::PROPERTY_MAX_REF_AGE_MS.to_string(), - "5678".to_string(), - ), - ]); - let table_properties = TableProperties::try_from(&props).unwrap(); - assert_eq!(table_properties.max_snapshot_age_ms, 1234); - assert_eq!(table_properties.min_snapshots_to_keep, 7); - assert_eq!(table_properties.max_ref_age_ms, 5678); - } - - #[test] - fn test_table_properties_write_metadata_path() { - // Test unset - let table_properties = TableProperties::try_from(&HashMap::new()).unwrap(); - assert_eq!(table_properties.write_metadata_path, None); - - // Test empty path is invalid - let props = HashMap::from([( - TableProperties::PROPERTY_WRITE_METADATA_PATH.to_string(), - String::new(), - )]); - let error = TableProperties::try_from(&props).unwrap_err(); - assert_eq!(error.kind(), ErrorKind::DataInvalid); - assert!( - error - .message() - .contains(TableProperties::PROPERTY_WRITE_METADATA_PATH) - ); - - let props = HashMap::from([( - TableProperties::PROPERTY_WRITE_METADATA_PATH.to_string(), - "s3://other-bucket/custom-meta/".to_string(), - )]); - let table_properties = TableProperties::try_from(&props).unwrap(); - assert_eq!( - table_properties.write_metadata_path.as_deref(), - Some("s3://other-bucket/custom-meta") - ); - } - - #[test] - fn test_table_properties_compression() { - let props = HashMap::from([( - TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), - "gzip".to_string(), - )]); - let table_properties = TableProperties::try_from(&props).unwrap(); - assert_eq!( - table_properties.metadata_compression_codec, - CompressionCodec::gzip_default() - ); - } - - #[test] - fn test_table_properties_compression_none() { - let props = HashMap::from([( - TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), - "none".to_string(), - )]); - let table_properties = TableProperties::try_from(&props).unwrap(); - assert_eq!( - table_properties.metadata_compression_codec, - CompressionCodec::None - ); - } - - #[test] - fn test_table_properties_compression_case_insensitive() { - // Test uppercase - let props_upper = HashMap::from([( - TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), - "GZIP".to_string(), - )]); - let table_properties = TableProperties::try_from(&props_upper).unwrap(); - assert_eq!( - table_properties.metadata_compression_codec, - CompressionCodec::gzip_default() - ); - - // Test mixed case - let props_mixed = HashMap::from([( - TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), - "GzIp".to_string(), - )]); - let table_properties = TableProperties::try_from(&props_mixed).unwrap(); - assert_eq!( - table_properties.metadata_compression_codec, - CompressionCodec::gzip_default() - ); - - // Test "NONE" should also be case-insensitive - let props_none_upper = HashMap::from([( - TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), - "NONE".to_string(), - )]); - let table_properties = TableProperties::try_from(&props_none_upper).unwrap(); - assert_eq!( - table_properties.metadata_compression_codec, - CompressionCodec::None - ); - } - - #[test] - fn test_table_properties_valid() { - let props = HashMap::from([ - ( - TableProperties::PROPERTY_COMMIT_NUM_RETRIES.to_string(), - "10".to_string(), - ), - ( - TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS.to_string(), - "20".to_string(), - ), - ( - TableProperties::PROPERTY_DEFAULT_FILE_FORMAT.to_string(), - "avro".to_string(), - ), - ( - TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES.to_string(), - "512".to_string(), - ), - ( - TableProperties::PROPERTY_GC_ENABLED.to_string(), - "false".to_string(), - ), - ]); - let table_properties = TableProperties::try_from(&props).unwrap(); - assert_eq!(table_properties.commit_num_retries, 10); - assert_eq!(table_properties.commit_max_retry_wait_ms, 20); - assert_eq!(table_properties.write_format_default, "avro".to_string()); - assert_eq!(table_properties.write_target_file_size_bytes, 512); - assert!(!table_properties.gc_enabled); - } - - #[test] - fn test_table_properties_invalid() { - let invalid_retries = HashMap::from([( - TableProperties::PROPERTY_COMMIT_NUM_RETRIES.to_string(), - "abc".to_string(), - )]); - - let table_properties = TableProperties::try_from(&invalid_retries).unwrap_err(); - assert!( - table_properties.to_string().contains( - "Invalid value for commit.retry.num-retries: invalid digit found in string" - ) - ); - - let invalid_min_wait = HashMap::from([( - TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS.to_string(), - "abc".to_string(), - )]); - let table_properties = TableProperties::try_from(&invalid_min_wait).unwrap_err(); - assert!( - table_properties.to_string().contains( - "Invalid value for commit.retry.min-wait-ms: invalid digit found in string" - ) - ); - - let invalid_max_wait = HashMap::from([( - TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS.to_string(), - "abc".to_string(), - )]); - let table_properties = TableProperties::try_from(&invalid_max_wait).unwrap_err(); - assert!( - table_properties.to_string().contains( - "Invalid value for commit.retry.max-wait-ms: invalid digit found in string" - ) - ); - - let invalid_target_size = HashMap::from([( - TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES.to_string(), - "abc".to_string(), - )]); - let table_properties = TableProperties::try_from(&invalid_target_size).unwrap_err(); - assert!(table_properties.to_string().contains( - "Invalid value for write.target-file-size-bytes: invalid digit found in string" - )); - - let invalid_gc_enabled = HashMap::from([( - TableProperties::PROPERTY_GC_ENABLED.to_string(), - "notabool".to_string(), - )]); - let table_properties = TableProperties::try_from(&invalid_gc_enabled).unwrap_err(); - assert!( - table_properties - .to_string() - .contains("Invalid value for gc.enabled") - ); - } - - #[test] - fn test_table_properties_compression_invalid_rejected() { - let invalid_codecs = ["lz4", "zstd", "snappy"]; - - for codec in invalid_codecs { - let props = HashMap::from([( - TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), - codec.to_string(), - )]); - let err = TableProperties::try_from(&props).unwrap_err(); - let err_msg = err.to_string(); - assert!( - err_msg.contains(&format!("Invalid metadata compression codec: {codec}")), - "Expected error message to contain codec '{codec}', got: {err_msg}" - ); - assert!( - err_msg.contains("Only 'none' and 'gzip' are supported"), - "Expected error message to contain supported codecs, got: {err_msg}" - ); - } - } - - #[test] - fn test_parse_metadata_file_compression_valid() { - // Test with "none" - let props = HashMap::from([( - TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), - "none".to_string(), - )]); - assert_eq!( - parse_metadata_file_compression(&props).unwrap(), - CompressionCodec::None - ); - - // Test with empty string - let props = HashMap::from([( - TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), - "".to_string(), - )]); - assert_eq!( - parse_metadata_file_compression(&props).unwrap(), - CompressionCodec::None - ); - - // Test with "gzip" - let props = HashMap::from([( - TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), - "gzip".to_string(), - )]); - assert_eq!( - parse_metadata_file_compression(&props).unwrap(), - CompressionCodec::gzip_default() - ); - - // Test case insensitivity - "NONE" - let props = HashMap::from([( - TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), - "NONE".to_string(), - )]); - assert_eq!( - parse_metadata_file_compression(&props).unwrap(), - CompressionCodec::None - ); - - // Test case insensitivity - "GZIP" - let props = HashMap::from([( - TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), - "GZIP".to_string(), - )]); - assert_eq!( - parse_metadata_file_compression(&props).unwrap(), - CompressionCodec::gzip_default() - ); - - // Test case insensitivity - "GzIp" - let props = HashMap::from([( - TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), - "GzIp".to_string(), - )]); - assert_eq!( - parse_metadata_file_compression(&props).unwrap(), - CompressionCodec::gzip_default() - ); - - // Test default when property is missing - let props = HashMap::new(); - assert_eq!( - parse_metadata_file_compression(&props).unwrap(), - CompressionCodec::None - ); - } - - #[test] - fn test_parse_metadata_file_compression_invalid() { - let invalid_codecs = ["lz4", "zstd", "snappy"]; - - for codec in invalid_codecs { - let props = HashMap::from([( - TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), - codec.to_string(), - )]); - let err = parse_metadata_file_compression(&props).unwrap_err(); - let err_msg = err.to_string(); - assert!( - err_msg.contains("Invalid metadata compression codec"), - "Expected error message to contain 'Invalid metadata compression codec', got: {err_msg}" - ); - assert!( - err_msg.contains("Only 'none' and 'gzip' are supported"), - "Expected error message to contain supported codecs, got: {err_msg}" - ); - } - } - - #[test] - fn test_cdc_disabled_by_default() { - let props = HashMap::new(); - let tp = TableProperties::try_from(&props).unwrap(); - assert!(!tp.cdc_enabled); - } - - #[test] - fn test_cdc_enabled_via_flag() { - let props = HashMap::from([( - TableProperties::PROPERTY_PARQUET_CDC_ENABLED.to_string(), - "true".to_string(), - )]); - let tp = TableProperties::try_from(&props).unwrap(); - assert!(tp.cdc_enabled); - assert_eq!(tp.cdc_min_chunk_size, 256 * 1024); - assert_eq!(tp.cdc_max_chunk_size, 1024 * 1024); - assert_eq!(tp.cdc_norm_level, 0); - } - - #[test] - fn test_cdc_size_props_alone_do_not_enable() { - let props = HashMap::from([( - TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE.to_string(), - "262144".to_string(), - )]); - let tp = TableProperties::try_from(&props).unwrap(); - assert!(!tp.cdc_enabled); - } - - #[test] - fn test_cdc_custom_values() { - let props = HashMap::from([ - ( - TableProperties::PROPERTY_PARQUET_CDC_ENABLED.to_string(), - "true".to_string(), - ), - ( - TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE.to_string(), - "200000".to_string(), - ), - ( - TableProperties::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE.to_string(), - "900000".to_string(), - ), - ( - TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL.to_string(), - "1".to_string(), - ), - ]); - let tp = TableProperties::try_from(&props).unwrap(); - assert!(tp.cdc_enabled); - assert_eq!(tp.cdc_min_chunk_size, 200000); - assert_eq!(tp.cdc_max_chunk_size, 900000); - assert_eq!(tp.cdc_norm_level, 1); - } - - #[test] - fn test_cdc_partial_override() { - let props = HashMap::from([ - ( - TableProperties::PROPERTY_PARQUET_CDC_ENABLED.to_string(), - "true".to_string(), - ), - ( - TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL.to_string(), - "2".to_string(), - ), - ]); - let tp = TableProperties::try_from(&props).unwrap(); - assert!(tp.cdc_enabled); - assert_eq!(tp.cdc_min_chunk_size, 256 * 1024); - assert_eq!(tp.cdc_max_chunk_size, 1024 * 1024); - assert_eq!(tp.cdc_norm_level, 2); - } - - #[test] - fn test_cdc_negative_norm_level() { - let props = HashMap::from([ - ( - TableProperties::PROPERTY_PARQUET_CDC_ENABLED.to_string(), - "true".to_string(), - ), - ( - TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL.to_string(), - "-2".to_string(), - ), - ]); - let tp = TableProperties::try_from(&props).unwrap(); - assert_eq!(tp.cdc_norm_level, -2); - } - - #[test] - fn test_cdc_invalid_min_chunk_size() { - let props = HashMap::from([ - ( - TableProperties::PROPERTY_PARQUET_CDC_ENABLED.to_string(), - "true".to_string(), - ), - ( - TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE.to_string(), - "not_a_number".to_string(), - ), - ]); - let err = TableProperties::try_from(&props).unwrap_err(); - assert!( - err.to_string().contains( - "Invalid value for write.parquet.content-defined-chunking.min-chunk-size" - ) - ); - } - - #[test] - fn test_cdc_invalid_norm_level() { - let props = HashMap::from([ - ( - TableProperties::PROPERTY_PARQUET_CDC_ENABLED.to_string(), - "true".to_string(), - ), - ( - TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL.to_string(), - "not_a_number".to_string(), - ), - ]); - let err = TableProperties::try_from(&props).unwrap_err(); - assert!( - err.to_string() - .contains("Invalid value for write.parquet.content-defined-chunking.norm-level") - ); - } - - #[test] - fn test_cdc_no_properties() { - let props = HashMap::from([("some.other.property".to_string(), "value".to_string())]); - let tp = TableProperties::try_from(&props).unwrap(); - assert!(!tp.cdc_enabled); - } - - #[test] - fn test_parse_boolean_property_case_insensitive() { - let false_variants = ["False", "FALSE"]; - let true_variants = ["True", "TRUE"]; - - for f in false_variants { - let props = HashMap::from([( - TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS.to_string(), - f.to_string(), - )]); - let tp = TableProperties::try_from(&props).unwrap(); - assert!(!tp.write_object_storage_partitioned_paths); - } - - for t in true_variants { - let props = HashMap::from([( - TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS.to_string(), - t.to_string(), - )]); - let tp = TableProperties::try_from(&props).unwrap(); - assert!(tp.write_object_storage_partitioned_paths); - } - } -} diff --git a/crates/iceberg/src/spec/table_props.rs b/crates/iceberg/src/spec/table_props.rs new file mode 100644 index 0000000000..21c2be9073 --- /dev/null +++ b/crates/iceberg/src/spec/table_props.rs @@ -0,0 +1,1682 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Typed access to Iceberg table properties. +//! +//! [`TableProperties`] exposes Iceberg's string-keyed table properties through typed getters and +//! setters. Its JSON representation is a flat object whose keys and values are strings. +//! +//! # Create from defaults +//! +//! Start with Iceberg's defaults and modify properties through generated setters: +//! +//! ``` +//! use iceberg::spec::{DataFileFormat, TableProperties}; +//! +//! let mut properties = TableProperties::default(); +//! properties.set_write_format_default(DataFileFormat::Orc); +//! properties.set_write_data_path(Some("s3://warehouse/table/data".to_string())); +//! +//! assert_eq!(*properties.write_format_default(), DataFileFormat::Orc); +//! ``` +//! +//! # Deserialize from JSON +//! +//! JSON property values must be strings, matching Iceberg's table property map: +//! +//! ``` +//! use iceberg::spec::{DataFileFormat, TableProperties}; +//! +//! let properties: TableProperties = serde_json::from_value(serde_json::json!({ +//! "commit.retry.num-retries": "8", +//! "write.format.default": "orc" +//! })).unwrap(); +//! +//! assert_eq!(*properties.commit_retry_num_retries(), 8); +//! assert_eq!(*properties.write_format_default(), DataFileFormat::Orc); +//! ``` +//! +//! # Serialize to JSON +//! +//! Serialization converts non-default typed fields back into Iceberg property keys. Fields whose +//! values match their defaults are omitted: +//! +//! ``` +//! use iceberg::spec::TableProperties; +//! +//! let mut properties = TableProperties::default(); +//! properties.set_commit_retry_num_retries(8); +//! properties.set_write_data_path(Some("s3://warehouse/table/data".to_string())); +//! +//! let json = serde_json::to_value(&properties).unwrap(); +//! assert_eq!(json["commit.retry.num-retries"], "8"); +//! assert_eq!(json["write.data.path"], "s3://warehouse/table/data"); +//! assert!(json.get("write.format.default").is_none()); +//! ``` + +use std::collections::HashMap; + +use iceberg_property_macro::Properties; +use serde_with::{DeserializeFromStr, SerializeDisplay}; + +use crate::compression::CompressionCodec; +use crate::error::{Error, ErrorKind, Result}; +use crate::spec::{DataFileFormat, NameMapping}; +use crate::util::location::strip_trailing_slash; + +/// Parquet data page version 1. +pub const PARQUET_PAGE_VERSION_V1: &str = "v1"; + +/// Parquet data page version 2. +pub const PARQUET_PAGE_VERSION_V2: &str = "v2"; + +/// ORC compression strategy that prioritizes speed. +pub const ORC_COMPRESSION_STRATEGY_SPEED: &str = "speed"; + +/// ORC compression strategy that prioritizes compression ratio. +pub const ORC_COMPRESSION_STRATEGY_COMPRESSION: &str = "compression"; + +/// Distribution applied to rows before writing files. +#[derive( + Debug, + PartialEq, + Eq, + Clone, + Copy, + SerializeDisplay, + DeserializeFromStr, + strum::Display, + strum::EnumString, +)] +#[strum(ascii_case_insensitive, serialize_all = "kebab-case")] +pub enum DistributionMode { + /// Do not redistribute rows. + None, + /// Hash-distribute rows by partition values. + Hash, + /// Range-distribute rows by partition or sort values. + Range, +} + +/// Granularity used when creating position delete files. +#[derive( + Debug, + PartialEq, + Eq, + Clone, + Copy, + SerializeDisplay, + DeserializeFromStr, + strum::Display, + strum::EnumString, +)] +#[strum(ascii_case_insensitive, serialize_all = "kebab-case")] +pub enum DeleteGranularity { + /// Group deletes for each referenced data file separately. + File, + /// Group deletes for different data files within a partition. + Partition, +} + +/// Isolation level used by row-level operations. +#[derive( + Debug, + PartialEq, + Eq, + Clone, + Copy, + SerializeDisplay, + DeserializeFromStr, + strum::Display, + strum::EnumString, +)] +#[strum(ascii_case_insensitive, serialize_all = "kebab-case")] +pub enum IsolationLevel { + /// Fail if concurrent changes may contain rows matching the operation. + Serializable, + /// Validate only against data visible in the operation's snapshot. + Snapshot, +} + +/// Strategy used to apply row-level changes. +#[derive( + Debug, + PartialEq, + Eq, + Clone, + Copy, + SerializeDisplay, + DeserializeFromStr, + strum::Display, + strum::EnumString, +)] +#[strum(ascii_case_insensitive, serialize_all = "kebab-case")] +pub enum RowLevelOperationMode { + /// Replace affected data files immediately. + CopyOnWrite, + /// Write delete files and merge changes while reading. + MergeOnRead, +} + +fn parse_comma_separated_strings(value: &str) -> Result> { + Ok(value + .split(',') + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_string) + .collect()) +} + +fn serialize_comma_separated_strings(values: &[String]) -> String { + values.join(",") +} + +fn parse_table_file_format(value: &str) -> Result { + let format = value.parse::()?; + match format { + DataFileFormat::Avro | DataFileFormat::Orc | DataFileFormat::Parquet => Ok(format), + DataFileFormat::Puffin => Err(Error::new( + ErrorKind::DataInvalid, + format!("Unsupported table data file format: {value}"), + )), + } +} + +fn parse_delete_file_format( + properties: &HashMap, + delete_format_key: &str, + data_format_key: &str, + default: DataFileFormat, +) -> Result { + properties + .get(delete_format_key) + .or_else(|| properties.get(data_format_key)) + .map(|value| parse_table_file_format(value)) + .transpose() + .map(|value| value.unwrap_or(default)) +} + +fn write_delete_file_format( + delete_format: &DataFileFormat, + properties: &mut HashMap, + delete_format_key: &str, + data_format_key: &str, + _default: &DataFileFormat, +) { + let data_format = properties + .get(data_format_key) + .and_then(|value| parse_table_file_format(value).ok()) + .unwrap_or(DataFileFormat::Parquet); + + if delete_format == &data_format { + properties.remove(delete_format_key); + } else { + properties.insert(delete_format_key.to_string(), delete_format.to_string()); + } +} + +fn parse_location(value: &str) -> Result { + if value.is_empty() { + return Err(Error::new(ErrorKind::DataInvalid, "path must not be empty")); + } + + Ok(strip_trailing_slash(value).to_string()) +} + +fn parse_optional_location(value: &str) -> Result> { + parse_location(value).map(Some) +} + +pub(crate) fn parse_metadata_location_property( + properties: &HashMap, +) -> Result> { + properties + .get(TableProperties::PROPERTY_WRITE_METADATA_PATH) + .map(|value| { + parse_location(value).map_err(|error| { + Error::new( + ErrorKind::DataInvalid, + format!( + "Invalid value for {}", + TableProperties::PROPERTY_WRITE_METADATA_PATH + ), + ) + .with_source(error) + }) + }) + .transpose() +} + +fn validate_compression_codec( + properties: &HashMap, + codec_key: &str, + allowed: &[&str], +) -> Result<()> { + let Some(value) = properties.get(codec_key) else { + return Ok(()); + }; + if allowed + .iter() + .any(|allowed| value.eq_ignore_ascii_case(allowed)) + { + return Ok(()); + } + + Err(Error::new( + ErrorKind::DataInvalid, + format!( + "Invalid compression codec for {codec_key}: {value}. Supported codecs: {}", + allowed.join(", ") + ), + )) +} + +fn parse_parquet_compression_properties( + properties: &HashMap, + codec_key: &str, + level_key: &str, + default: CompressionCodec, +) -> Result { + validate_compression_codec(properties, codec_key, &[ + "zstd", + "brotli", + "lz4", + "gzip", + "snappy", + "uncompressed", + ])?; + CompressionCodec::parse_properties(properties, codec_key, level_key, default) +} + +fn parse_avro_compression_properties( + properties: &HashMap, + codec_key: &str, + level_key: &str, + default: CompressionCodec, +) -> Result { + validate_compression_codec(properties, codec_key, &[ + "gzip", + "zstd", + "snappy", + "uncompressed", + ])?; + CompressionCodec::parse_properties(properties, codec_key, level_key, default) +} + +fn write_parquet_compression_properties( + codec: &CompressionCodec, + properties: &mut HashMap, + codec_key: &str, + level_key: &str, + default: &CompressionCodec, +) { + write_compression_properties( + codec, + properties, + codec_key, + level_key, + "uncompressed", + default, + ); +} + +fn write_avro_compression_properties( + codec: &CompressionCodec, + properties: &mut HashMap, + codec_key: &str, + level_key: &str, + default: &CompressionCodec, +) { + write_compression_properties( + codec, + properties, + codec_key, + level_key, + "uncompressed", + default, + ); +} + +fn write_compression_properties( + codec: &CompressionCodec, + properties: &mut HashMap, + codec_key: &str, + level_key: &str, + uncompressed_value: &str, + default: &CompressionCodec, +) { + if codec == default { + properties.remove(codec_key); + properties.remove(level_key); + return; + } + + let value = match codec { + CompressionCodec::None => uncompressed_value.to_string(), + codec => codec.property_value(), + }; + properties.insert(codec_key.to_string(), value); + + match codec { + CompressionCodec::Gzip(level) | CompressionCodec::Zstd(level) => { + properties.insert(level_key.to_string(), level.to_string()); + } + _ => { + properties.remove(level_key); + } + } +} + +fn parse_orc_compression_codec(value: &str) -> Result { + if !["zstd", "lz4", "lzo", "zlib", "snappy", "none"] + .iter() + .any(|allowed| value.eq_ignore_ascii_case(allowed)) + { + return Err(Error::new( + ErrorKind::DataInvalid, + format!("Invalid ORC compression codec: {value}"), + )); + } + + CompressionCodec::parse_property(value) +} + +fn parse_name_mapping(value: &str) -> Result> { + serde_json::from_str(value).map(Some).map_err(|error| { + Error::new(ErrorKind::DataInvalid, "Invalid name mapping").with_source(error) + }) +} + +fn serialize_name_mapping(value: &Option) -> String { + serde_json::to_string( + value + .as_ref() + .expect("checked is_some before serialization"), + ) + .expect("serializing a name mapping should not fail") +} + +pub(crate) fn parse_metadata_file_compression( + properties: &HashMap, +) -> Result { + properties + .get(TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC) + .map(|value| CompressionCodec::parse_metadata_property(value)) + .transpose() + .map(|codec| codec.unwrap_or(CompressionCodec::None)) +} + +/// Typed Iceberg table properties organized into documented sections. +/// +/// Serde represents this struct as Iceberg's flat string-to-string property map. Property +/// definitions and descriptions are based on the pinned [Java TableProperties implementation] +/// and [Apache Iceberg configuration documentation]. +/// +/// [Java TableProperties implementation]: https://github.com/apache/iceberg/blob/d8c10a1608170f0ba83be740d6ab0b6a3757cb3e/core/src/main/java/org/apache/iceberg/TableProperties.java +/// [Apache Iceberg configuration documentation]: https://github.com/apache/iceberg/blob/d8c10a1608170f0ba83be740d6ab0b6a3757cb3e/docs/docs/configuration.md +#[derive(Clone, Debug, Properties)] +pub struct TableProperties { + // General properties. + #[key = "comment"] + #[default(None)] + #[doc = "Table-level description of the table's business meaning and usage context."] + #[property(pub(getter), pub(setter))] + comment: Option, + + #[key = "identifier-fields.rely"] + #[default(false)] + #[doc = "Whether query engines may rely on identifier fields as a primary key for optimization; this is not enforced on writes."] + #[property(pub(getter), pub(setter))] + identifier_fields_rely: bool, + + // Commit properties. + #[key = "commit.retry.num-retries"] + #[default(4)] + #[doc = "Number of times to retry a commit before failing."] + #[property(pub(getter), pub(setter))] + commit_retry_num_retries: usize, + + #[key = "commit.retry.min-wait-ms"] + #[default(100)] + #[doc = "Minimum time in milliseconds to wait before retrying a commit."] + #[property(pub(getter), pub(setter))] + commit_retry_min_wait_ms: u64, + + #[key = "commit.retry.max-wait-ms"] + #[default(60 * 1000)] + #[doc = "Maximum time in milliseconds to wait before retrying a commit."] + #[property(pub(getter), pub(setter))] + commit_retry_max_wait_ms: u64, + + #[key = "commit.retry.total-timeout-ms"] + #[default(30 * 60 * 1000)] + #[doc = "Total commit retry timeout in milliseconds."] + #[property(pub(getter), pub(setter))] + commit_retry_total_timeout_ms: u64, + + #[key = "commit.status-check.num-retries"] + #[default(3)] + #[doc = "Number of times to check whether a commit succeeded after connectivity is lost."] + #[property(pub(getter), pub(setter))] + commit_status_check_num_retries: usize, + + #[key = "commit.status-check.min-wait-ms"] + #[default(1000)] + #[doc = "Minimum time in milliseconds to wait before retrying a commit status check."] + #[property(pub(getter), pub(setter))] + commit_status_check_min_wait_ms: u64, + + #[key = "commit.status-check.max-wait-ms"] + #[default(60 * 1000)] + #[doc = "Maximum time in milliseconds to wait before retrying a commit status check."] + #[property(pub(getter), pub(setter))] + commit_status_check_max_wait_ms: u64, + + #[key = "commit.status-check.total-timeout-ms"] + #[default(30 * 60 * 1000)] + #[doc = "Total timeout in milliseconds in which commit status checking must succeed."] + #[property(pub(getter), pub(setter))] + commit_status_check_total_timeout_ms: u64, + + // Manifest properties. + #[key = "commit.manifest.target-size-bytes"] + #[default(8 * 1024 * 1024)] + #[doc = "Target size in bytes when merging manifest files."] + #[property(pub(getter), pub(setter))] + commit_manifest_target_size_bytes: usize, + + #[key = "commit.manifest.min-count-to-merge"] + #[default(100)] + #[doc = "Minimum number of manifests to accumulate before merging."] + #[property(pub(getter), pub(setter))] + commit_manifest_min_count_to_merge: usize, + + #[key = "commit.manifest-merge.enabled"] + #[default(true)] + #[doc = "Whether manifests are automatically merged during writes."] + #[property(pub(getter), pub(setter))] + commit_manifest_merge_enabled: bool, + + #[key = "write.manifest.compression-codec"] + #[additional_key = "write.manifest.compression-level"] + #[default(CompressionCodec::gzip_default())] + #[parse_properties_with(parse_avro_compression_properties)] + #[write_properties_with(write_avro_compression_properties)] + #[doc = "Compression codec used for manifest files."] + #[property(pub(getter), pub(setter))] + write_manifest_compression_codec: CompressionCodec, + + #[key = "write.manifest-lists.enabled"] + #[default(true)] + #[doc = "Deprecated flag for writing manifest lists; manifest lists are always enabled."] + #[property(pub(getter), pub(setter))] + write_manifest_lists_enabled: bool, + + // Write properties. + #[key = "write.format.default"] + #[default(DataFileFormat::Parquet)] + #[parse_with(parse_table_file_format)] + #[doc = "Default data file format: Parquet, Avro, or ORC."] + #[property(pub(getter), pub(setter))] + write_format_default: DataFileFormat, + + #[key = "write.delete.format.default"] + #[additional_key = "write.format.default"] + #[default(DataFileFormat::Parquet)] + #[parse_properties_with(parse_delete_file_format)] + #[write_properties_with(write_delete_file_format)] + #[doc = "Default delete file format: Parquet, Avro, or ORC."] + #[property(pub(getter), pub(setter))] + write_delete_format_default: DataFileFormat, + + #[key = "write.target-file-size-bytes"] + #[default(512 * 1024 * 1024)] + #[doc = "Target size in bytes for generated data files."] + #[property(pub(getter), pub(setter))] + write_target_file_size_bytes: usize, + + #[key = "write.delete.target-file-size-bytes"] + #[default(64 * 1024 * 1024)] + #[doc = "Target size in bytes for generated delete files."] + #[property(pub(getter), pub(setter))] + write_delete_target_file_size_bytes: usize, + + #[key = "write.object-storage.enabled"] + #[default(false)] + #[doc = "Whether the object-storage location provider adds a hash component to file paths."] + #[property(pub(getter), pub(setter))] + write_object_storage_enabled: bool, + + #[key = "write.object-storage.partitioned-paths"] + #[default(true)] + #[doc = "Whether object-storage file paths include partition values."] + #[property(pub(getter), pub(setter))] + write_object_storage_partitioned_paths: bool, + + #[key = "write.object-storage.path"] + #[default(None)] + #[doc = "Deprecated base object-storage path; use write.data.path instead."] + #[property(pub(getter), pub(setter))] + write_object_storage_path: Option, + + #[key = "write.location-provider.impl"] + #[default(None)] + #[doc = "Optional custom location provider implementation."] + #[property(pub(getter), pub(setter))] + write_location_provider_impl: Option, + + #[key = "write.folder-storage.path"] + #[default(None)] + #[doc = "Deprecated base folder-storage path; use write.data.path instead."] + #[property(pub(getter), pub(setter))] + write_folder_storage_path: Option, + + #[key = "write.data.path"] + #[default(None)] + #[doc = "Base location for data files written after this property is set."] + #[property(pub(getter), pub(setter))] + write_data_path: Option, + + #[key = "write.wap.enabled"] + #[default(false)] + #[doc = "Whether write-audit-publish writes are enabled."] + #[property(pub(getter), pub(setter))] + write_wap_enabled: bool, + + #[key = "write.distribution-mode"] + #[default(None)] + #[doc = "Write distribution mode: none, hash, or range."] + #[property(pub(getter), pub(setter))] + write_distribution_mode: Option, + + #[key = "write.datafusion.fanout.enabled"] + #[default(true)] + #[doc = "Whether DataFusion uses a fanout writer for partitioned tables."] + #[property(pub(getter), pub(setter))] + write_datafusion_fanout_enabled: bool, + + // Parquet properties. + #[key = "write.parquet.row-group-size-bytes"] + #[default(128 * 1024 * 1024)] + #[doc = "Parquet row group size in bytes for data files."] + #[property(pub(getter), pub(setter))] + write_parquet_row_group_size_bytes: usize, + + #[key = "write.delete.parquet.row-group-size-bytes"] + #[default(128 * 1024 * 1024)] + #[doc = "Parquet row group size in bytes for delete files."] + #[property(pub(getter), pub(setter))] + write_delete_parquet_row_group_size_bytes: usize, + + #[key = "write.parquet.page-size-bytes"] + #[default(1024 * 1024)] + #[doc = "Parquet page size in bytes for data files."] + #[property(pub(getter), pub(setter))] + write_parquet_page_size_bytes: usize, + + #[key = "write.delete.parquet.page-size-bytes"] + #[default(1024 * 1024)] + #[doc = "Parquet page size in bytes for delete files."] + #[property(pub(getter), pub(setter))] + write_delete_parquet_page_size_bytes: usize, + + #[key = "write.parquet.page-version"] + #[default(PARQUET_PAGE_VERSION_V1)] + #[doc = "Parquet data page version for data files: v1 or v2."] + #[property(pub(getter), pub(setter))] + write_parquet_page_version: String, + + #[key = "write.delete.parquet.page-version"] + #[default(PARQUET_PAGE_VERSION_V1)] + #[doc = "Parquet data page version for delete files: v1 or v2."] + #[property(pub(getter), pub(setter))] + write_delete_parquet_page_version: String, + + #[key = "write.parquet.page-row-limit"] + #[default(20_000)] + #[doc = "Maximum number of rows per Parquet page in data files."] + #[property(pub(getter), pub(setter))] + write_parquet_page_row_limit: usize, + + #[key = "write.delete.parquet.page-row-limit"] + #[default(20_000)] + #[doc = "Maximum number of rows per Parquet page in delete files."] + #[property(pub(getter), pub(setter))] + write_delete_parquet_page_row_limit: usize, + + #[key = "write.parquet.dict-size-bytes"] + #[default(2 * 1024 * 1024)] + #[doc = "Parquet dictionary page size in bytes for data files."] + #[property(pub(getter), pub(setter))] + write_parquet_dict_size_bytes: usize, + + #[key = "write.delete.parquet.dict-size-bytes"] + #[default(2 * 1024 * 1024)] + #[doc = "Parquet dictionary page size in bytes for delete files."] + #[property(pub(getter), pub(setter))] + write_delete_parquet_dict_size_bytes: usize, + + #[key = "write.parquet.compression-codec"] + #[additional_key = "write.parquet.compression-level"] + #[default(CompressionCodec::zstd_default())] + #[parse_properties_with(parse_parquet_compression_properties)] + #[write_properties_with(write_parquet_compression_properties)] + #[doc = "Parquet compression codec used for data files."] + #[property(pub(getter), pub(setter))] + write_parquet_compression_codec: CompressionCodec, + + #[key = "write.delete.parquet.compression-codec"] + #[additional_key = "write.delete.parquet.compression-level"] + #[default(CompressionCodec::zstd_default())] + #[parse_properties_with(parse_parquet_compression_properties)] + #[write_properties_with(write_parquet_compression_properties)] + #[doc = "Parquet compression codec used for delete files."] + #[property(pub(getter), pub(setter))] + write_delete_parquet_compression_codec: CompressionCodec, + + #[key = "write.parquet.shred-variants"] + #[default(false)] + #[doc = "Whether variant columns use shredded Parquet encoding for improved query performance."] + #[property(pub(getter), pub(setter))] + write_parquet_shred_variants: bool, + + #[key = "write.parquet.variant-inference-buffer-size"] + #[default(100)] + #[doc = "Number of rows buffered for schema inference when variant shredding is enabled."] + #[property(pub(getter), pub(setter))] + write_parquet_variant_inference_buffer_size: usize, + + #[key = "write.parquet.row-group-check-min-record-count"] + #[default(100)] + #[doc = "Minimum record count between Parquet data-file row group size checks."] + #[property(pub(getter), pub(setter))] + write_parquet_row_group_check_min_record_count: usize, + + #[key = "write.delete.parquet.row-group-check-min-record-count"] + #[default(100)] + #[doc = "Minimum record count between Parquet delete-file row group size checks."] + #[property(pub(getter), pub(setter))] + write_delete_parquet_row_group_check_min_record_count: usize, + + #[key = "write.parquet.row-group-check-max-record-count"] + #[default(10_000)] + #[doc = "Maximum record count between Parquet data-file row group size checks."] + #[property(pub(getter), pub(setter))] + write_parquet_row_group_check_max_record_count: usize, + + #[key = "write.delete.parquet.row-group-check-max-record-count"] + #[default(10_000)] + #[doc = "Maximum record count between Parquet delete-file row group size checks."] + #[property(pub(getter), pub(setter))] + write_delete_parquet_row_group_check_max_record_count: usize, + + #[key = "write.parquet.row-group-size-track-uncompressed"] + #[default(false)] + #[doc = "Whether uncompressed data size is tracked to enforce the Parquet row group target."] + #[property(pub(getter), pub(setter))] + write_parquet_row_group_size_track_uncompressed: bool, + + #[key = "write.parquet.bloom-filter-max-bytes"] + #[default(1024 * 1024)] + #[doc = "Maximum number of bytes for a Parquet bloom filter bitset."] + #[property(pub(getter), pub(setter))] + write_parquet_bloom_filter_max_bytes: usize, + + #[key = "write.parquet.bloom-filter-adaptive-enabled"] + #[default(false)] + #[doc = "Whether adaptive Parquet bloom filter sizing selects the smallest suitable filter."] + #[property(pub(getter), pub(setter))] + write_parquet_bloom_filter_adaptive_enabled: bool, + + #[prefix = "write.parquet.bloom-filter-fpp.column."] + #[default(HashMap::new())] + #[doc = "Per-column Parquet bloom filter false-positive probabilities, keyed by column name."] + #[property(pub(getter), pub(setter))] + write_parquet_bloom_filter_fpp_column: HashMap, + + #[prefix = "write.parquet.bloom-filter-ndv.column."] + #[default(HashMap::new())] + #[doc = "Per-column expected distinct-value counts for Parquet bloom filters."] + #[property(pub(getter), pub(setter))] + write_parquet_bloom_filter_ndv_column: HashMap, + + #[prefix = "write.parquet.bloom-filter-enabled.column."] + #[default(HashMap::new())] + #[doc = "Per-column flags controlling whether Parquet bloom filters are written."] + #[property(pub(getter), pub(setter))] + write_parquet_bloom_filter_enabled_column: HashMap, + + #[prefix = "write.parquet.stats-enabled.column."] + #[default(HashMap::new())] + #[doc = "Per-column flags controlling whether Parquet column statistics are collected."] + #[property(pub(getter), pub(setter))] + write_parquet_stats_enabled_column: HashMap, + + #[prefix = "write.parquet.dict-encoding-enabled.column."] + #[default(HashMap::new())] + #[doc = "Per-column flags controlling whether Parquet dictionary encoding is used."] + #[property(pub(getter), pub(setter))] + write_parquet_dict_encoding_enabled_column: HashMap, + + #[key = "write.parquet.content-defined-chunking.enabled"] + #[default(false)] + #[doc = "Whether Parquet content-defined chunking is enabled."] + #[property(pub(getter), pub(setter))] + write_parquet_content_defined_chunking_enabled: bool, + + #[key = "write.parquet.content-defined-chunking.min-chunk-size"] + #[default(256 * 1024)] + #[doc = "Minimum Parquet content-defined chunk size in bytes."] + #[property(pub(getter), pub(setter))] + write_parquet_content_defined_chunking_min_chunk_size: usize, + + #[key = "write.parquet.content-defined-chunking.max-chunk-size"] + #[default(1024 * 1024)] + #[doc = "Maximum Parquet content-defined chunk size in bytes."] + #[property(pub(getter), pub(setter))] + write_parquet_content_defined_chunking_max_chunk_size: usize, + + #[key = "write.parquet.content-defined-chunking.norm-level"] + #[default(0)] + #[doc = "Gearhash normalization level used by Parquet content-defined chunking."] + #[property(pub(getter), pub(setter))] + write_parquet_content_defined_chunking_norm_level: i32, + + // Avro properties. + #[key = "write.avro.compression-codec"] + #[additional_key = "write.avro.compression-level"] + #[default(CompressionCodec::gzip_default())] + #[parse_properties_with(parse_avro_compression_properties)] + #[write_properties_with(write_avro_compression_properties)] + #[doc = "Avro compression codec used for data files."] + #[property(pub(getter), pub(setter))] + write_avro_compression_codec: CompressionCodec, + + #[key = "write.delete.avro.compression-codec"] + #[additional_key = "write.delete.avro.compression-level"] + #[default(CompressionCodec::gzip_default())] + #[parse_properties_with(parse_avro_compression_properties)] + #[write_properties_with(write_avro_compression_properties)] + #[doc = "Avro compression codec used for delete files."] + #[property(pub(getter), pub(setter))] + write_delete_avro_compression_codec: CompressionCodec, + + // ORC properties. + #[key = "write.orc.stripe-size-bytes"] + #[default(64 * 1024 * 1024)] + #[doc = "Default ORC stripe size in bytes for data files."] + #[property(pub(getter), pub(setter))] + write_orc_stripe_size_bytes: u64, + + #[key = "write.delete.orc.stripe-size-bytes"] + #[default(64 * 1024 * 1024)] + #[doc = "Default ORC stripe size in bytes for delete files."] + #[property(pub(getter), pub(setter))] + write_delete_orc_stripe_size_bytes: u64, + + #[key = "write.orc.bloom.filter.columns"] + #[default(Vec::new())] + #[parse_with(parse_comma_separated_strings)] + #[serialize_with(serialize_comma_separated_strings)] + #[doc = "Comma-separated column names for which ORC bloom filters are created."] + #[property(pub(getter), pub(setter))] + write_orc_bloom_filter_columns: Vec, + + #[key = "write.orc.bloom.filter.fpp"] + #[default(0.05)] + #[doc = "False-positive probability for ORC bloom filters."] + #[property(pub(getter), pub(setter))] + write_orc_bloom_filter_fpp: f64, + + #[key = "write.orc.block-size-bytes"] + #[default(256 * 1024 * 1024)] + #[doc = "Default file-system block size in bytes for ORC data files."] + #[property(pub(getter), pub(setter))] + write_orc_block_size_bytes: u64, + + #[key = "write.delete.orc.block-size-bytes"] + #[default(256 * 1024 * 1024)] + #[doc = "Default file-system block size in bytes for ORC delete files."] + #[property(pub(getter), pub(setter))] + write_delete_orc_block_size_bytes: u64, + + #[key = "write.orc.vectorized.batch-size"] + #[default(1024)] + #[doc = "ORC vectorized write batch size for data files."] + #[property(pub(getter), pub(setter))] + write_orc_vectorized_batch_size: usize, + + #[key = "write.delete.orc.vectorized.batch-size"] + #[default(1024)] + #[doc = "ORC vectorized write batch size for delete files."] + #[property(pub(getter), pub(setter))] + write_delete_orc_vectorized_batch_size: usize, + + #[key = "write.orc.compression-codec"] + #[default(CompressionCodec::Zlib)] + #[parse_with(parse_orc_compression_codec)] + #[serialize_with(CompressionCodec::property_value)] + #[doc = "ORC compression codec used for data files."] + #[property(pub(getter), pub(setter))] + write_orc_compression_codec: CompressionCodec, + + #[key = "write.delete.orc.compression-codec"] + #[default(CompressionCodec::Zlib)] + #[parse_with(parse_orc_compression_codec)] + #[serialize_with(CompressionCodec::property_value)] + #[doc = "ORC compression codec used for delete files."] + #[property(pub(getter), pub(setter))] + write_delete_orc_compression_codec: CompressionCodec, + + #[key = "write.orc.compression-strategy"] + #[default(ORC_COMPRESSION_STRATEGY_SPEED)] + #[doc = "ORC compression strategy for data files: speed or compression."] + #[property(pub(getter), pub(setter))] + write_orc_compression_strategy: String, + + #[key = "write.delete.orc.compression-strategy"] + #[default(ORC_COMPRESSION_STRATEGY_SPEED)] + #[doc = "ORC compression strategy for delete files: speed or compression."] + #[property(pub(getter), pub(setter))] + write_delete_orc_compression_strategy: String, + + // Read properties. + #[key = "read.split.target-size"] + #[default(128 * 1024 * 1024)] + #[doc = "Target size in bytes when combining data input splits."] + #[property(pub(getter), pub(setter))] + read_split_target_size: u64, + + #[key = "read.split.metadata-target-size"] + #[default(32 * 1024 * 1024)] + #[doc = "Target size in bytes when combining metadata input splits."] + #[property(pub(getter), pub(setter))] + read_split_metadata_target_size: u64, + + #[key = "read.split.planning-lookback"] + #[default(10)] + #[doc = "Number of bins considered when combining input splits."] + #[property(pub(getter), pub(setter))] + read_split_planning_lookback: usize, + + #[key = "read.split.open-file-cost"] + #[default(4 * 1024 * 1024)] + #[doc = "Estimated file-open cost used as a minimum weight when combining splits."] + #[property(pub(getter), pub(setter))] + read_split_open_file_cost: u64, + + #[key = "read.split.adaptive-size.enabled"] + #[default(true)] + #[doc = "Whether split size is adapted to the scan size."] + #[property(pub(getter), pub(setter))] + read_split_adaptive_size_enabled: bool, + + #[key = "read.parquet.vectorization.enabled"] + #[default(true)] + #[doc = "Whether Parquet vectorized reads are enabled."] + #[property(pub(getter), pub(setter))] + read_parquet_vectorization_enabled: bool, + + #[key = "read.parquet.vectorization.batch-size"] + #[default(5000)] + #[doc = "Batch size for Parquet vectorized reads."] + #[property(pub(getter), pub(setter))] + read_parquet_vectorization_batch_size: usize, + + #[key = "read.orc.vectorization.enabled"] + #[default(false)] + #[doc = "Whether ORC vectorized reads are enabled."] + #[property(pub(getter), pub(setter))] + read_orc_vectorization_enabled: bool, + + #[key = "read.orc.vectorization.batch-size"] + #[default(5000)] + #[doc = "Batch size for ORC vectorized reads."] + #[property(pub(getter), pub(setter))] + read_orc_vectorization_batch_size: usize, + + #[key = "read.data-planning-mode"] + #[default("auto")] + #[doc = "Planning mode used for data files."] + #[property(pub(getter), pub(setter))] + read_data_planning_mode: String, + + #[key = "read.delete-planning-mode"] + #[default("auto")] + #[doc = "Planning mode used for delete files."] + #[property(pub(getter), pub(setter))] + read_delete_planning_mode: String, + + // Metadata properties. + #[key = "write.metadata.path"] + #[default(None)] + #[parse_with(parse_optional_location)] + #[doc = "Base location for metadata files written after this property is set."] + #[property(pub(getter), pub(setter))] + write_metadata_path: Option, + + #[key = "write.summary.partition-limit"] + #[default(0)] + #[doc = "Maximum changed-partition count for including partition-level statistics in snapshot summaries."] + #[property(pub(getter), pub(setter))] + write_summary_partition_limit: u64, + + #[key = "write.metadata.compression-codec"] + #[default(CompressionCodec::None)] + #[parse_with(CompressionCodec::parse_metadata_property)] + #[serialize_with(CompressionCodec::property_value)] + #[doc = "Compression codec for metadata JSON files: none or gzip."] + #[property(pub(getter), pub(setter))] + write_metadata_compression_codec: CompressionCodec, + + #[key = "write.metadata.previous-versions-max"] + #[default(100)] + #[doc = "Maximum number of previous metadata file versions to track."] + #[property(pub(getter), pub(setter))] + write_metadata_previous_versions_max: usize, + + #[key = "write.metadata.delete-after-commit.enabled"] + #[default(false)] + #[doc = "Whether the oldest tracked metadata file is deleted after each commit."] + #[property(pub(getter), pub(setter))] + write_metadata_delete_after_commit_enabled: bool, + + #[key = "write.metadata.metrics.max-inferred-column-defaults"] + #[default(100)] + #[doc = "Maximum number of columns that receive inferred metrics defaults."] + #[property(pub(getter), pub(setter))] + write_metadata_metrics_max_inferred_column_defaults: usize, + + #[prefix = "write.metadata.metrics.column."] + #[default(HashMap::new())] + #[doc = "Per-column metrics modes keyed by column name."] + #[property(pub(getter), pub(setter))] + write_metadata_metrics_column: HashMap, + + #[key = "write.metadata.metrics.default"] + #[default("truncate(16)")] + #[doc = "Default metrics mode for table columns."] + #[property(pub(getter), pub(setter))] + write_metadata_metrics_default: String, + + #[key = "schema.name-mapping.default"] + #[default(None)] + #[parse_with(parse_name_mapping)] + #[serialize_with(serialize_name_mapping)] + #[doc = "Default JSON name mapping used to resolve columns in files without field IDs."] + #[property(pub(getter), pub(setter))] + schema_name_mapping_default: Option, + + // Compatibility properties. + #[key = "write.spark.fanout.enabled"] + #[default(false)] + #[doc = "Deprecated Spark fanout-writer flag; the fanout writer accepts unclustered data but uses more memory."] + #[property(pub(getter), pub(setter))] + write_spark_fanout_enabled: bool, + + #[key = "write.spark.accept-any-schema"] + #[default(false)] + #[doc = "Deprecated Spark flag allowing writes with any compatible schema."] + #[property(pub(getter), pub(setter))] + write_spark_accept_any_schema: bool, + + #[key = "write.spark.auto-schema-evolution.enabled"] + #[default(true)] + #[doc = "Deprecated Spark flag enabling automatic schema evolution during writes."] + #[property(pub(getter), pub(setter))] + write_spark_auto_schema_evolution_enabled: bool, + + #[key = "write.spark.advisory-partition-size-bytes"] + #[default(None)] + #[doc = "Deprecated Spark advisory partition size in bytes."] + #[property(pub(getter), pub(setter))] + write_spark_advisory_partition_size_bytes: Option, + + #[key = "compatibility.snapshot-id-inheritance.enabled"] + #[default(false)] + #[doc = "Whether snapshots may be committed without explicit snapshot IDs; format version 2 and later always allow this."] + #[property(pub(getter), pub(setter))] + compatibility_snapshot_id_inheritance_enabled: bool, + + #[key = "engine.hive.enabled"] + #[default(false)] + #[doc = "Whether Hive engine integration behavior is enabled."] + #[property(pub(getter), pub(setter))] + engine_hive_enabled: bool, + + #[key = "engine.hive.lock-enabled"] + #[default(true)] + #[doc = "Whether Hive locking is enabled."] + #[property(pub(getter), pub(setter))] + engine_hive_lock_enabled: bool, + + // History properties. + #[key = "gc.enabled"] + #[default(true)] + #[doc = "Whether garbage collection operations such as snapshot expiration and orphan-file removal are allowed."] + #[property(pub(getter), pub(setter))] + gc_enabled: bool, + + #[key = "history.expire.max-snapshot-age-ms"] + #[default(5 * 24 * 60 * 60 * 1000)] + #[doc = "Default maximum snapshot age in milliseconds while expiring snapshots."] + #[property(pub(getter), pub(setter))] + history_expire_max_snapshot_age_ms: i64, + + #[key = "history.expire.min-snapshots-to-keep"] + #[default(1)] + #[doc = "Default minimum number of snapshots retained per branch while expiring snapshots."] + #[property(pub(getter), pub(setter))] + history_expire_min_snapshots_to_keep: usize, + + #[key = "history.expire.max-ref-age-ms"] + #[default(i64::MAX)] + #[doc = "Default maximum age in milliseconds for snapshot references other than the main branch."] + #[property(pub(getter), pub(setter))] + history_expire_max_ref_age_ms: i64, + + // Row-level operation properties. + #[key = "write.delete.granularity"] + #[default(DeleteGranularity::Partition)] + #[doc = "Granularity of generated delete files: partition or file."] + #[property(pub(getter), pub(setter))] + write_delete_granularity: DeleteGranularity, + + #[key = "write.delete.isolation-level"] + #[default(IsolationLevel::Serializable)] + #[doc = "Isolation level for delete commands: serializable or snapshot."] + #[property(pub(getter), pub(setter))] + write_delete_isolation_level: IsolationLevel, + + #[key = "write.delete.mode"] + #[default(RowLevelOperationMode::CopyOnWrite)] + #[doc = "Execution mode for delete commands: copy-on-write or merge-on-read."] + #[property(pub(getter), pub(setter))] + write_delete_mode: RowLevelOperationMode, + + #[key = "write.delete.distribution-mode"] + #[default(None)] + #[doc = "Distribution mode for delete command data."] + #[property(pub(getter), pub(setter))] + write_delete_distribution_mode: Option, + + #[key = "write.update.isolation-level"] + #[default(IsolationLevel::Serializable)] + #[doc = "Isolation level for update commands: serializable or snapshot."] + #[property(pub(getter), pub(setter))] + write_update_isolation_level: IsolationLevel, + + #[key = "write.update.mode"] + #[default(RowLevelOperationMode::CopyOnWrite)] + #[doc = "Execution mode for update commands: copy-on-write or merge-on-read."] + #[property(pub(getter), pub(setter))] + write_update_mode: RowLevelOperationMode, + + #[key = "write.update.distribution-mode"] + #[default(None)] + #[doc = "Distribution mode for update command data."] + #[property(pub(getter), pub(setter))] + write_update_distribution_mode: Option, + + #[key = "write.merge.isolation-level"] + #[default(IsolationLevel::Serializable)] + #[doc = "Isolation level for merge commands: serializable or snapshot."] + #[property(pub(getter), pub(setter))] + write_merge_isolation_level: IsolationLevel, + + #[key = "write.merge.mode"] + #[default(RowLevelOperationMode::CopyOnWrite)] + #[doc = "Execution mode for merge commands: copy-on-write or merge-on-read."] + #[property(pub(getter), pub(setter))] + write_merge_mode: RowLevelOperationMode, + + #[key = "write.merge.distribution-mode"] + #[default(None)] + #[doc = "Distribution mode for merge command data."] + #[property(pub(getter), pub(setter))] + write_merge_distribution_mode: Option, + + #[key = "write.upsert.enabled"] + #[default(false)] + #[doc = "Whether upsert behavior is enabled."] + #[property(pub(getter), pub(setter))] + write_upsert_enabled: bool, + + // Encryption properties. + #[key = "encryption.key-id"] + #[default(None)] + #[doc = "Identifier of the table's master encryption key."] + #[property(pub(getter), pub(setter))] + encryption_key_id: Option, + + #[key = "encryption.data-key-length"] + #[default(16)] + #[doc = "Length in bytes of data-encryption keys; valid AES lengths are 16, 24, and 32 bytes."] + #[property(pub(getter), pub(setter))] + encryption_data_key_length: usize, +} + +impl TableProperties { + /// Property key for the number of commit retries. + pub const PROPERTY_COMMIT_NUM_RETRIES: &str = "commit.retry.num-retries"; + + /// Default number of commit retries. + pub const PROPERTY_COMMIT_NUM_RETRIES_DEFAULT: usize = 4; + + /// Property key for enabling the DataFusion fanout writer. + pub const PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED: &str = "write.datafusion.fanout.enabled"; + + /// Default value for enabling the DataFusion fanout writer. + pub const PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT: bool = true; + + /// Property key for the table encryption key identifier. + pub const PROPERTY_ENCRYPTION_KEY_ID: &str = "encryption.key-id"; + + /// Property key for the metadata compression codec. + pub const PROPERTY_METADATA_COMPRESSION_CODEC: &str = "write.metadata.compression-codec"; + + /// Property key for the maximum number of previous metadata versions to keep. + pub const PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX: &str = + "write.metadata.previous-versions-max"; + + /// Default maximum number of previous metadata versions to keep. + pub const PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX_DEFAULT: usize = 100; + + /// Property key for enabling Parquet content-defined chunking. + pub const PROPERTY_PARQUET_CDC_ENABLED: &str = "write.parquet.content-defined-chunking.enabled"; + + /// Property key for the maximum Parquet content-defined chunk size. + pub const PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE: &str = + "write.parquet.content-defined-chunking.max-chunk-size"; + + /// Property key for the minimum Parquet content-defined chunk size. + pub const PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE: &str = + "write.parquet.content-defined-chunking.min-chunk-size"; + + /// Property key for the Parquet content-defined chunking normalization level. + pub const PROPERTY_PARQUET_CDC_NORM_LEVEL: &str = + "write.parquet.content-defined-chunking.norm-level"; + + /// Property key for the base data-file location. + pub const PROPERTY_WRITE_DATA_LOCATION: &str = "write.data.path"; + + /// Property key for the deprecated folder-storage location. + pub const PROPERTY_WRITE_FOLDER_STORAGE_LOCATION: &str = "write.folder-storage.path"; + + /// Property key for the base metadata-file location. + pub const PROPERTY_WRITE_METADATA_PATH: &str = "write.metadata.path"; + + /// Property key for the deprecated object-storage location. + pub const PROPERTY_WRITE_OBJECT_STORAGE_LOCATION: &str = "write.object-storage.path"; + + /// Property key for including partition values in object-storage paths. + pub const PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS: &str = + "write.object-storage.partitioned-paths"; + + /// Default value for including partition values in object-storage paths. + pub const PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS_DEFAULT: bool = true; + + /// Property key for the snapshot-summary partition limit. + pub const PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT: &str = "write.summary.partition-limit"; + + /// Default snapshot-summary partition limit. + pub const PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT_DEFAULT: u64 = 0; + + /// Property key for the target data-file size. + pub const PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES: &str = "write.target-file-size-bytes"; + + /// Default target data-file size. + pub const PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT: usize = 512 * 1024 * 1024; + + /// Reserved table properties that must not be persisted in table metadata. + pub const RESERVED_PROPERTIES: [&str; 9] = [ + "format-version", + "uuid", + "snapshot-count", + "current-snapshot-id", + "current-snapshot-summary", + "current-snapshot-timestamp-ms", + "current-schema", + "default-partition-spec", + "default-sort-order", + ]; +} + +impl TryFrom<&HashMap> for TableProperties { + type Error = Error; + + fn try_from(properties: &HashMap) -> Result { + Self::from_properties(properties).map_err(|error| Error::new(ErrorKind::DataInvalid, error)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::error::{Error, ErrorKind}; + use crate::spec::MappedField; + + fn parse(properties: HashMap) -> Result { + serde_json::from_value(serde_json::to_value(properties).unwrap()) + .map_err(|error| Error::new(ErrorKind::DataInvalid, error.to_string())) + } + + #[test] + fn creates_properties_from_defaults() { + let properties = TableProperties::default(); + + assert_eq!(properties.commit_retry_num_retries, 4); + assert_eq!(properties.write_format_default, DataFileFormat::Parquet); + assert_eq!( + properties.write_manifest_compression_codec, + CompressionCodec::gzip_default() + ); + assert_eq!( + properties.write_parquet_compression_codec, + CompressionCodec::zstd_default() + ); + assert_eq!( + properties.write_avro_compression_codec, + CompressionCodec::gzip_default() + ); + assert_eq!( + properties.write_orc_compression_codec, + CompressionCodec::Zlib + ); + assert_eq!(properties.write_distribution_mode, None); + assert_eq!(properties.write_delete_distribution_mode, None); + assert_eq!(properties.write_update_distribution_mode, None); + assert_eq!(properties.write_merge_distribution_mode, None); + assert_eq!( + properties.write_parquet_page_version, + PARQUET_PAGE_VERSION_V1 + ); + assert_eq!( + properties.write_delete_parquet_page_version, + PARQUET_PAGE_VERSION_V1 + ); + assert_eq!(PARQUET_PAGE_VERSION_V2, "v2"); + assert_eq!( + properties.write_orc_compression_strategy, + ORC_COMPRESSION_STRATEGY_SPEED + ); + assert_eq!(ORC_COMPRESSION_STRATEGY_COMPRESSION, "compression"); + assert!(properties.write_orc_bloom_filter_columns.is_empty()); + assert_eq!(properties.schema_name_mapping_default, None); + assert_eq!( + properties.write_delete_granularity, + DeleteGranularity::Partition + ); + assert_eq!( + properties.write_delete_isolation_level, + IsolationLevel::Serializable + ); + assert_eq!( + properties.write_delete_mode, + RowLevelOperationMode::CopyOnWrite + ); + assert_eq!( + properties.write_parquet_row_group_size_bytes, + 128 * 1024 * 1024 + ); + assert_eq!(properties.read_split_target_size, 128 * 1024 * 1024); + assert!(properties.gc_enabled); + assert_eq!(properties.encryption_data_key_length, 16); + } + + #[test] + fn serializes_to_flat_json_object() { + let properties = TableProperties { + commit_retry_num_retries: 9, + write_format_default: DataFileFormat::Orc, + write_data_path: Some("s3://warehouse/table/data".to_string()), + write_manifest_compression_codec: CompressionCodec::Gzip(9), + write_parquet_compression_codec: CompressionCodec::Zstd(5), + write_delete_avro_compression_codec: CompressionCodec::Gzip(4), + write_distribution_mode: Some(DistributionMode::Range), + write_orc_compression_codec: CompressionCodec::Lzo, + write_orc_bloom_filter_columns: vec!["id".to_string(), "category".to_string()], + schema_name_mapping_default: Some(NameMapping::new(vec![MappedField::new( + Some(1), + vec!["id".to_string()], + vec![], + )])), + write_delete_granularity: DeleteGranularity::File, + write_delete_isolation_level: IsolationLevel::Snapshot, + write_delete_mode: RowLevelOperationMode::MergeOnRead, + write_update_distribution_mode: Some(DistributionMode::Hash), + write_parquet_bloom_filter_fpp_column: HashMap::from([( + "customer_id".to_string(), + 0.02, + )]), + write_parquet_bloom_filter_ndv_column: HashMap::from([( + "customer_id".to_string(), + 1_000_000, + )]), + ..Default::default() + }; + + let json = serde_json::to_value(&properties).unwrap(); + assert_eq!( + json, + serde_json::json!({ + "commit.retry.num-retries": "9", + "schema.name-mapping.default": r#"[{"field-id":1,"names":["id"]}]"#, + "write.data.path": "s3://warehouse/table/data", + "write.delete.avro.compression-codec": "gzip", + "write.delete.avro.compression-level": "4", + "write.delete.format.default": "parquet", + "write.delete.granularity": "file", + "write.delete.isolation-level": "snapshot", + "write.delete.mode": "merge-on-read", + "write.distribution-mode": "range", + "write.format.default": "orc", + "write.manifest.compression-codec": "gzip", + "write.manifest.compression-level": "9", + "write.orc.bloom.filter.columns": "id,category", + "write.orc.compression-codec": "lzo", + "write.parquet.bloom-filter-fpp.column.customer_id": "0.02", + "write.parquet.bloom-filter-ndv.column.customer_id": "1000000", + "write.parquet.compression-codec": "zstd", + "write.parquet.compression-level": "5", + "write.update.distribution-mode": "hash" + }) + ); + } + + #[test] + fn omits_default_values_when_serializing() { + assert_eq!( + serde_json::to_value(TableProperties::default()).unwrap(), + serde_json::json!({}) + ); + } + + #[test] + fn deserializes_from_flat_json_object() { + let properties: TableProperties = serde_json::from_value(serde_json::json!({ + "commit.retry.num-retries": "8", + "write.format.default": "orc", + "write.data.path": "s3://warehouse/table/data", + "write.manifest.compression-codec": "gzip", + "write.manifest.compression-level": "8", + "write.parquet.compression-level": "5", + "write.distribution-mode": "HASH", + "write.object-storage.partitioned-paths": "FALSE", + "write.orc.bloom.filter.columns": "id, category", + "write.parquet.bloom-filter-ndv.column.customer_id": "1000000", + "schema.name-mapping.default": r#"[{"field-id":1,"names":["id"]}]"#, + "write.delete.granularity": "FILE", + "write.update.isolation-level": "snapshot", + "write.merge.mode": "merge-on-read" + })) + .unwrap(); + + assert_eq!(properties.commit_retry_num_retries, 8); + assert_eq!(properties.write_format_default, DataFileFormat::Orc); + assert_eq!( + properties.write_data_path, + Some("s3://warehouse/table/data".to_string()) + ); + assert_eq!( + properties.write_distribution_mode, + Some(DistributionMode::Hash) + ); + assert!(!properties.write_object_storage_partitioned_paths); + assert_eq!( + properties.write_manifest_compression_codec, + CompressionCodec::Gzip(8) + ); + assert_eq!( + properties.write_parquet_compression_codec, + CompressionCodec::Zstd(5) + ); + assert_eq!(properties.write_orc_bloom_filter_columns, vec![ + "id".to_string(), + "category".to_string() + ]); + assert_eq!( + properties.write_parquet_bloom_filter_ndv_column, + HashMap::from([("customer_id".to_string(), 1_000_000)]) + ); + assert_eq!( + properties.schema_name_mapping_default, + Some(NameMapping::new(vec![MappedField::new( + Some(1), + vec!["id".to_string()], + vec![], + )])) + ); + assert_eq!(properties.write_delete_granularity, DeleteGranularity::File); + assert_eq!( + properties.write_update_isolation_level, + IsolationLevel::Snapshot + ); + assert_eq!( + properties.write_merge_mode, + RowLevelOperationMode::MergeOnRead + ); + } + + #[test] + fn invalid_leaf_value_reports_its_property_key() { + let error = parse(HashMap::from([( + "commit.retry.num-retries".to_string(), + "not-a-number".to_string(), + )])) + .unwrap_err(); + + assert!(error.message().contains("commit.retry.num-retries")); + } + + #[test] + fn metadata_path_rejects_empty_values_and_trims_trailing_slashes() { + assert_eq!(TableProperties::default().write_metadata_path, None); + + let error = parse(HashMap::from([( + "write.metadata.path".to_string(), + String::new(), + )])) + .unwrap_err(); + assert!(error.message().contains("write.metadata.path")); + + let properties = parse(HashMap::from([( + "write.metadata.path".to_string(), + "s3://bucket/table/metadata/".to_string(), + )])) + .unwrap(); + assert_eq!( + properties.write_metadata_path.as_deref(), + Some("s3://bucket/table/metadata") + ); + } + + #[test] + fn metadata_compression_preserves_legacy_validation() { + for (value, expected) in [ + ("", CompressionCodec::None), + ("none", CompressionCodec::None), + ("NONE", CompressionCodec::None), + ("gzip", CompressionCodec::gzip_default()), + ("GzIp", CompressionCodec::gzip_default()), + ] { + let properties = parse(HashMap::from([( + "write.metadata.compression-codec".to_string(), + value.to_string(), + )])) + .unwrap(); + assert_eq!(properties.write_metadata_compression_codec, expected); + } + + for invalid in ["uncompressed", "lz4", "zstd", "snappy"] { + let error = parse(HashMap::from([( + "write.metadata.compression-codec".to_string(), + invalid.to_string(), + )])) + .unwrap_err(); + assert!(error.message().contains("metadata compression codec")); + } + } + + #[test] + fn rejects_invalid_numeric_and_boolean_values() { + for (key, value) in [ + ("commit.retry.min-wait-ms", "not-a-number"), + ("write.target-file-size-bytes", "not-a-number"), + ("write.datafusion.fanout.enabled", "not-a-bool"), + ] { + let error = parse(HashMap::from([(key.to_string(), value.to_string())])).unwrap_err(); + assert!(error.message().contains(key)); + } + } + + #[test] + fn parses_content_defined_chunking_defaults_overrides_and_errors() { + let defaults = TableProperties::default(); + assert!(!defaults.write_parquet_content_defined_chunking_enabled); + assert_eq!( + defaults.write_parquet_content_defined_chunking_min_chunk_size, + 256 * 1024 + ); + assert_eq!( + defaults.write_parquet_content_defined_chunking_max_chunk_size, + 1024 * 1024 + ); + assert_eq!( + defaults.write_parquet_content_defined_chunking_norm_level, + 0 + ); + + let properties = parse(HashMap::from([ + ( + "write.parquet.content-defined-chunking.enabled".to_string(), + "TRUE".to_string(), + ), + ( + "write.parquet.content-defined-chunking.norm-level".to_string(), + "-2".to_string(), + ), + ])) + .unwrap(); + assert!(properties.write_parquet_content_defined_chunking_enabled); + assert_eq!( + properties.write_parquet_content_defined_chunking_min_chunk_size, + 256 * 1024 + ); + assert_eq!( + properties.write_parquet_content_defined_chunking_max_chunk_size, + 1024 * 1024 + ); + assert_eq!( + properties.write_parquet_content_defined_chunking_norm_level, + -2 + ); + + for key in [ + "write.parquet.content-defined-chunking.min-chunk-size", + "write.parquet.content-defined-chunking.max-chunk-size", + "write.parquet.content-defined-chunking.norm-level", + ] { + let error = parse(HashMap::from([( + key.to_string(), + "not-a-number".to_string(), + )])) + .unwrap_err(); + assert!(error.message().contains(key)); + } + } + + #[test] + fn delete_format_inherits_data_format_and_round_trips() { + let properties = parse(HashMap::from([( + "write.format.default".to_string(), + "orc".to_string(), + )])) + .unwrap(); + assert_eq!(properties.write_format_default, DataFileFormat::Orc); + assert_eq!(properties.write_delete_format_default, DataFileFormat::Orc); + + let json = serde_json::to_value(&properties).unwrap(); + assert_eq!(json["write.format.default"], "orc"); + assert!(json.get("write.delete.format.default").is_none()); + + let mut properties = TableProperties::default(); + properties.set_write_format_default(DataFileFormat::Orc); + properties.set_write_delete_format_default(DataFileFormat::Parquet); + let json = serde_json::to_value(&properties).unwrap(); + assert_eq!(json["write.format.default"], "orc"); + assert_eq!(json["write.delete.format.default"], "parquet"); + } + + #[test] + fn distribution_modes_preserve_absence() { + let defaults = TableProperties::default(); + assert_eq!(defaults.write_distribution_mode, None); + assert_eq!(defaults.write_delete_distribution_mode, None); + assert_eq!(defaults.write_update_distribution_mode, None); + assert_eq!(defaults.write_merge_distribution_mode, None); + + let properties = parse(HashMap::from([( + "write.distribution-mode".to_string(), + "none".to_string(), + )])) + .unwrap(); + assert_eq!( + properties.write_distribution_mode, + Some(DistributionMode::None) + ); + assert_eq!( + serde_json::to_value(properties).unwrap()["write.distribution-mode"], + "none" + ); + } + + #[test] + fn rejects_non_table_file_formats() { + for key in ["write.format.default", "write.delete.format.default"] { + let error = + parse(HashMap::from([(key.to_string(), "puffin".to_string())])).unwrap_err(); + assert!(error.message().contains(key)); + } + } + + #[test] + fn validates_and_preserves_format_specific_compression_codecs() { + let parquet = parse(HashMap::from([( + "write.parquet.compression-codec".to_string(), + "uncompressed".to_string(), + )])) + .unwrap(); + assert_eq!( + parquet.write_parquet_compression_codec, + CompressionCodec::None + ); + assert_eq!( + serde_json::to_value(parquet).unwrap()["write.parquet.compression-codec"], + "uncompressed" + ); + + for (key, codec) in [ + ("write.parquet.compression-codec", "zlib"), + ("write.avro.compression-codec", "brotli"), + ("write.orc.compression-codec", "gzip"), + ] { + let error = parse(HashMap::from([(key.to_string(), codec.to_string())])).unwrap_err(); + assert!(error.message().contains(key)); + } + } +} diff --git a/crates/iceberg/src/transaction/expire_snapshots.rs b/crates/iceberg/src/transaction/expire_snapshots.rs index b2420a1dff..3263c5502d 100644 --- a/crates/iceberg/src/transaction/expire_snapshots.rs +++ b/crates/iceberg/src/transaction/expire_snapshots.rs @@ -113,10 +113,12 @@ impl ExpireSnapshotsAction { // When a knob is not set explicitly, fall back to the table's `history.expire.*` properties, // matching Java `RemoveSnapshots`' constructor. With the default `max-snapshot-age-ms` (5 // days) the age path always runs, so even an explicit-id-only call applies the default cutoff. - let default_cutoff = self - .older_than_ms - .unwrap_or_else(|| now.saturating_sub(properties.max_snapshot_age_ms)); - let default_min_to_keep = self.retain_last.unwrap_or(properties.min_snapshots_to_keep); + let default_cutoff = self.older_than_ms.unwrap_or_else(|| { + now.saturating_sub(*properties.history_expire_max_snapshot_age_ms()) + }); + let default_min_to_keep = self + .retain_last + .unwrap_or(*properties.history_expire_min_snapshots_to_keep()); // Ref aging: `main` is always kept; any other ref whose head is older than its // `max_ref_age_ms` (defaulting to `history.expire.max-ref-age-ms`) is dropped, like Java's @@ -125,7 +127,12 @@ impl ExpireSnapshotsAction { let mut retained_refs: Vec<&SnapshotReference> = vec![]; for (ref_name, snapshot_ref) in &metadata.refs { if ref_name == MAIN_BRANCH - || !Self::ref_aged_out(metadata, snapshot_ref, now, properties.max_ref_age_ms) + || !Self::ref_aged_out( + metadata, + snapshot_ref, + now, + *properties.history_expire_max_ref_age_ms(), + ) { retained_refs.push(snapshot_ref); } else { @@ -302,7 +309,7 @@ impl TransactionAction for ExpireSnapshotsAction { let properties = metadata.table_properties()?; // Expiring metadata defeats a user's explicit decision to disable GC (Java refuses too). - if !properties.gc_enabled { + if !properties.gc_enabled() { return Err(Error::new( ErrorKind::DataInvalid, "Cannot expire snapshots: gc.enabled is false", diff --git a/crates/iceberg/src/transaction/mod.rs b/crates/iceberg/src/transaction/mod.rs index 3e0a4e9391..f3738c4ae8 100644 --- a/crates/iceberg/src/transaction/mod.rs +++ b/crates/iceberg/src/transaction/mod.rs @@ -181,7 +181,7 @@ impl Transaction { let table_props = self.table.metadata().table_properties()?; // TODO(https://github.com/apache/iceberg-rust/issues/2034): remove once encrypted writes are supported - if table_props.encryption_key_id.is_some() { + if table_props.encryption_key_id().is_some() { return Err(Error::new( ErrorKind::FeatureUnsupported, "Cannot commit to an encrypted table: encrypted writes are not yet supported", @@ -205,12 +205,12 @@ impl Transaction { fn build_backoff(props: TableProperties) -> Result { Ok(ExponentialBuilder::new() - .with_min_delay(Duration::from_millis(props.commit_min_retry_wait_ms)) - .with_max_delay(Duration::from_millis(props.commit_max_retry_wait_ms)) + .with_min_delay(Duration::from_millis(*props.commit_retry_min_wait_ms())) + .with_max_delay(Duration::from_millis(*props.commit_retry_max_wait_ms())) .with_total_delay(Some(Duration::from_millis( - props.commit_total_retry_timeout_ms, + *props.commit_retry_total_timeout_ms(), ))) - .with_max_times(props.commit_num_retries) + .with_max_times(*props.commit_retry_num_retries()) .with_factor(2.0) .build()) } diff --git a/crates/iceberg/src/writer/file_writer/location_generator.rs b/crates/iceberg/src/writer/file_writer/location_generator.rs index caaeb7b683..b419b4e9f3 100644 --- a/crates/iceberg/src/writer/file_writer/location_generator.rs +++ b/crates/iceberg/src/writer/file_writer/location_generator.rs @@ -67,10 +67,12 @@ impl DefaultLocationGenerator { /// `{table_location}/data`. pub fn new(table_metadata: &TableMetadata) -> Result { let table_location = strip_trailing_slash(table_metadata.location()); - let prop = TableProperties::try_from(table_metadata.properties())?; + let properties = table_metadata.properties(); let data_location = strip_trailing_slash( - prop.write_data_location - .or(prop.write_folder_storage_location) + properties + .get(TableProperties::PROPERTY_WRITE_DATA_LOCATION) + .or_else(|| properties.get(TableProperties::PROPERTY_WRITE_FOLDER_STORAGE_LOCATION)) + .cloned() .unwrap_or(format!("{table_location}{DEFAULT_DATA_DIR}")) .as_ref(), ) @@ -134,11 +136,13 @@ impl ObjectStorageLocationGenerator { /// `{table_location}/data`. pub fn new(table_metadata: &TableMetadata) -> Result { let table_location = strip_trailing_slash(table_metadata.location()); - let prop = TableProperties::try_from(table_metadata.properties())?; + let properties = table_metadata.properties(); let storage_location = strip_trailing_slash( - prop.write_data_location - .or(prop.write_object_storage_location) - .or(prop.write_folder_storage_location) + properties + .get(TableProperties::PROPERTY_WRITE_DATA_LOCATION) + .or_else(|| properties.get(TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_LOCATION)) + .or_else(|| properties.get(TableProperties::PROPERTY_WRITE_FOLDER_STORAGE_LOCATION)) + .cloned() .unwrap_or(format!("{table_location}{DEFAULT_DATA_DIR}")) .as_ref(), ) @@ -152,7 +156,21 @@ impl ObjectStorageLocationGenerator { Some(path_context(table_location)) }; - let include_partition_paths = prop.write_object_storage_partitioned_paths; + let include_partition_paths = properties + .get(TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS) + .map(|value| { + value.to_ascii_lowercase().parse::().map_err(|error| { + crate::Error::new( + crate::ErrorKind::DataInvalid, + format!( + "Invalid value for {}: {error}", + TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS + ), + ) + }) + }) + .transpose()? + .unwrap_or(TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS_DEFAULT); Ok(Self { storage_location, diff --git a/crates/iceberg/src/writer/file_writer/parquet_writer.rs b/crates/iceberg/src/writer/file_writer/parquet_writer.rs index db9f170938..c3d129c436 100644 --- a/crates/iceberg/src/writer/file_writer/parquet_writer.rs +++ b/crates/iceberg/src/writer/file_writer/parquet_writer.rs @@ -86,11 +86,15 @@ impl ParquetWriterBuilder { /// (`write.parquet.content-defined-chunking.*`); other keys fall back to /// parquet-rs defaults. pub fn from_table_properties(table_props: &TableProperties, schema: SchemaRef) -> Self { - let cdc = table_props.cdc_enabled.then_some(CdcOptions { - min_chunk_size: table_props.cdc_min_chunk_size, - max_chunk_size: table_props.cdc_max_chunk_size, - norm_level: table_props.cdc_norm_level, - }); + let cdc = table_props + .write_parquet_content_defined_chunking_enabled() + .then_some(CdcOptions { + min_chunk_size: *table_props + .write_parquet_content_defined_chunking_min_chunk_size(), + max_chunk_size: *table_props + .write_parquet_content_defined_chunking_max_chunk_size(), + norm_level: *table_props.write_parquet_content_defined_chunking_norm_level(), + }); // TODO: translate the remaining write.parquet.* keys (e.g. compression-codec, // row-group-size-bytes, page-size-bytes). // This constructor is intended to be the single place that maps them. diff --git a/crates/integrations/datafusion/src/physical_plan/write.rs b/crates/integrations/datafusion/src/physical_plan/write.rs index a7d771ec1b..4d8d124983 100644 --- a/crates/integrations/datafusion/src/physical_plan/write.rs +++ b/crates/integrations/datafusion/src/physical_plan/write.rs @@ -16,7 +16,6 @@ // under the License. use std::fmt::{Debug, Formatter}; -use std::str::FromStr; use std::sync::Arc; use datafusion::arrow::array::{ArrayRef, RecordBatch, StringArray}; @@ -210,8 +209,7 @@ impl ExecutionPlan for IcebergWriteExec { .map_err(to_datafusion_error)?; // Check data file format - let file_format = DataFileFormat::from_str(&table_props.write_format_default) - .map_err(to_datafusion_error)?; + let file_format = *table_props.write_format_default(); if file_format != DataFileFormat::Parquet { return Err(to_datafusion_error(Error::new( ErrorKind::FeatureUnsupported, @@ -227,7 +225,7 @@ impl ExecutionPlan for IcebergWriteExec { self.table.metadata().current_schema().clone(), ) .with_match_mode(FieldMatchMode::Name); - let target_file_size = table_props.write_target_file_size_bytes; + let target_file_size = *table_props.write_target_file_size_bytes(); let file_io = self.table.file_io().clone(); // todo location_gen and file_name_gen should be configurable @@ -246,7 +244,7 @@ impl ExecutionPlan for IcebergWriteExec { let data_file_writer_builder = DataFileWriterBuilder::new(rolling_writer_builder); // Create TaskWriter - let fanout_enabled = table_props.write_datafusion_fanout_enabled; + let fanout_enabled = *table_props.write_datafusion_fanout_enabled(); let schema = self.table.metadata().current_schema().clone(); let partition_spec = self.table.metadata().default_partition_spec().clone(); let task_writer = TaskWriter::try_new( diff --git a/crates/property-macro/Cargo.toml b/crates/property-macro/Cargo.toml new file mode 100644 index 0000000000..f67c8aabf5 --- /dev/null +++ b/crates/property-macro/Cargo.toml @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +edition = { workspace = true } +homepage = { workspace = true } +name = "iceberg-property-macro" +publish = true +rust-version = { workspace = true } +version = { workspace = true } + +license = { workspace = true } +repository = { workspace = true } + +categories = ["database"] +description = "Derive macros for Apache Iceberg Rust" +keywords = ["iceberg"] + +[lib] +proc-macro = true + +[dependencies] +proc-macro2 = "1" +quote = "1" +syn = { version = "2", features = ["full"] } + +[dev-dependencies] +serde = { workspace = true } +serde_json = { workspace = true } + +[lints] +workspace = true diff --git a/crates/property-macro/src/lib.rs b/crates/property-macro/src/lib.rs new file mode 100644 index 0000000000..c4e5d75a7c --- /dev/null +++ b/crates/property-macro/src/lib.rs @@ -0,0 +1,696 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Derives for Iceberg's string-keyed property maps. + +use proc_macro::TokenStream; +use proc_macro2::TokenStream as TokenStream2; +use quote::{format_ident, quote}; +use syn::parse::{Parse, ParseStream}; +use syn::punctuated::Punctuated; +use syn::{ + Attribute, Data, DeriveInput, Error, Expr, ExprLit, ExprPath, Field, Fields, Ident, Lit, Meta, + Path, Token, Type, parenthesized, parse_macro_input, +}; + +/// Derive parsing, defaults, and JSON serialization for a typed property map. +/// +/// Leaf fields must declare the table-property key and its default: +/// +/// ``` +/// use iceberg_property_macro::Properties; +/// +/// #[derive(Properties)] +/// struct Properties { +/// #[key = "write.format.default"] +/// #[default = "parquet"] +/// #[doc = "Default file format"] +/// #[property(pub(getter), pub(setter))] +/// write_format_default: String, +/// } +/// +/// let mut properties = Properties::default(); +/// assert_eq!(properties.write_format_default(), "parquet"); +/// properties.set_write_format_default("orc".to_string()); +/// assert_eq!(properties.write_format_default(), "orc"); +/// ``` +/// +/// `prefix` captures a family of properties in a `HashMap`, keyed by the suffix after +/// the declared prefix. `nested` embeds another `Properties` struct while keeping its serialized +/// property map flat. `parse_with` may be used for exact-key property types that do not implement +/// `FromStr` or need validation. `serialize_with` supplies their string representation in JSON. +/// `parse_properties_with` and `write_properties_with` provide access to the complete property map +/// for fields represented by more than one key. `additional_key` declares a second key and passes +/// it to those hooks after the primary key. Write hooks are also passed the field default and are +/// responsible for omitting or removing default-valued properties. +/// Optional fields are omitted from JSON when they are `None`. Fields need `FromStr` and `ToString` +/// unless the relevant custom parsing or serialization attribute is supplied. Leaf fields also +/// need `PartialEq` so values equal to their defaults can be omitted from JSON. String-literal and +/// path defaults are converted into their field type with `Into`. Boolean property values are +/// parsed case-insensitively. +/// +/// Fields remain private unless their struct declaration makes them public. The +/// `#[property(pub(getter))]` and `#[property(pub(setter))]` options generate a public getter and +/// setter respectively. Getters borrow the field, and setters are named `set_`. +#[proc_macro_derive( + Properties, + attributes( + key, + additional_key, + prefix, + nested, + default, + parse_with, + serialize_with, + parse_properties_with, + write_properties_with, + property + ) +)] +pub fn derive_properties(input: TokenStream) -> TokenStream { + let input = parse_macro_input!(input as DeriveInput); + + match expand_properties(input) { + Ok(tokens) => tokens.into(), + Err(error) => error.into_compile_error().into(), + } +} + +struct PropertyField { + ident: Ident, + ty: Type, + key: Option, + additional_key: Option, + prefix: Option, + nested: bool, + default: Option, + parse_with: Option, + serialize_with: Option, + parse_properties_with: Option, + write_properties_with: Option, + option_inner_type: Option, + map_value_type: Option, + public_getter: bool, + public_setter: bool, + doc_attributes: Vec, +} + +enum PublicAccessor { + Getter, + Setter, +} + +impl Parse for PublicAccessor { + fn parse(input: ParseStream<'_>) -> syn::Result { + input.parse::()?; + let content; + parenthesized!(content in input); + let accessor = content.parse::()?; + if !content.is_empty() { + return Err(content.error("expected getter or setter")); + } + + match accessor.to_string().as_str() { + "getter" => Ok(Self::Getter), + "setter" => Ok(Self::Setter), + _ => Err(Error::new_spanned(accessor, "expected getter or setter")), + } + } +} + +fn expand_properties(input: DeriveInput) -> syn::Result { + let struct_name = input.ident; + let fields = match input.data { + Data::Struct(data) => match data.fields { + Fields::Named(fields) => fields.named, + _ => { + return Err(Error::new_spanned( + struct_name, + "Properties can only be derived for structs with named fields", + )); + } + }, + _ => { + return Err(Error::new_spanned( + struct_name, + "Properties can only be derived for structs", + )); + } + }; + + let fields = fields + .iter() + .map(parse_property_field) + .collect::>>()?; + + let defaults = fields.iter().map(|field| { + let ident = &field.ident; + if field.nested { + quote!(#ident: ::std::default::Default::default()) + } else { + let default = field.default.as_ref().expect("leaf fields have defaults"); + let ty = &field.ty; + let default = default_value(default, ty); + quote!(#ident: #default) + } + }); + + let parses = fields.iter().map(parse_field); + + let property_writes = fields.iter().map(write_field); + + let accessors = fields.iter().map(field_accessors); + + Ok(quote! { + impl ::std::default::Default for #struct_name { + fn default() -> Self { + Self { + #(#defaults,)* + } + } + } + + impl #struct_name { + #(#accessors)* + + pub(crate) fn from_properties( + properties: &::std::collections::HashMap<::std::string::String, ::std::string::String>, + ) -> ::std::result::Result { + Ok(Self { + #(#parses,)* + }) + } + + pub(crate) fn write_properties( + &self, + properties: &mut ::std::collections::HashMap< + ::std::string::String, + ::std::string::String, + >, + ) { + #(#property_writes)* + } + + fn to_properties( + &self, + ) -> ::std::collections::HashMap< + ::std::string::String, + ::std::string::String, + > { + let mut properties = ::std::collections::HashMap::new(); + self.write_properties(&mut properties); + properties + } + } + + impl ::serde::Serialize for #struct_name { + fn serialize(&self, serializer: S) -> ::std::result::Result + where + S: ::serde::Serializer, + { + ::serde::Serialize::serialize(&self.to_properties(), serializer) + } + } + + impl<'de> ::serde::Deserialize<'de> for #struct_name { + fn deserialize(deserializer: D) -> ::std::result::Result + where + D: ::serde::Deserializer<'de>, + { + let properties = <::std::collections::HashMap<::std::string::String, ::std::string::String> as ::serde::Deserialize>::deserialize(deserializer)?; + Self::from_properties(&properties).map_err(::serde::de::Error::custom) + } + } + }) +} + +fn parse_property_field(field: &Field) -> syn::Result { + let ident = field + .ident + .clone() + .ok_or_else(|| Error::new_spanned(field, "Properties fields must be named"))?; + let key = attribute_expression_value(&field.attrs, "key")?; + let additional_key = attribute_expression_value(&field.attrs, "additional_key")?; + let prefix = attribute_expression_value(&field.attrs, "prefix")?; + let nested = marker_attribute(&field.attrs, "nested")?; + if usize::from(key.is_some()) + usize::from(prefix.is_some()) + usize::from(nested) != 1 { + return Err(Error::new_spanned( + field, + "Properties fields must declare exactly one of #[key(...)], #[prefix(...)], or #[nested]", + )); + } + let default = attribute_expression_value(&field.attrs, "default")?; + if nested && default.is_some() { + return Err(Error::new_spanned( + field, + "#[nested] fields use the nested type's Default implementation and cannot declare #[default(...)]", + )); + } + if !nested && default.is_none() { + return Err(Error::new_spanned( + field, + "Properties leaf fields must declare #[default(...)]", + )); + } + + let map_value_type = map_value_type(&field.ty); + if prefix.is_some() && map_value_type.is_none() { + return Err(Error::new_spanned( + &field.ty, + "#[prefix(...)] fields must have type HashMap", + )); + } + let parse_with = attribute_path_value(&field.attrs, "parse_with")?; + let serialize_with = attribute_path_value(&field.attrs, "serialize_with")?; + let parse_properties_with = attribute_path_value(&field.attrs, "parse_properties_with")?; + let write_properties_with = attribute_path_value(&field.attrs, "write_properties_with")?; + if additional_key.is_some() + && parse_properties_with.is_none() + && write_properties_with.is_none() + { + return Err(Error::new_spanned( + field, + "#[additional_key(...)] requires parse_properties_with or write_properties_with", + )); + } + if (prefix.is_some() || nested) + && (additional_key.is_some() + || parse_with.is_some() + || serialize_with.is_some() + || parse_properties_with.is_some() + || write_properties_with.is_some()) + { + return Err(Error::new_spanned( + field, + "#[prefix(...)] and #[nested] fields do not support custom parse or write functions", + )); + } + if parse_with.is_some() && parse_properties_with.is_some() { + return Err(Error::new_spanned( + field, + "fields cannot declare both parse_with and parse_properties_with", + )); + } + if serialize_with.is_some() && write_properties_with.is_some() { + return Err(Error::new_spanned( + field, + "fields cannot declare both serialize_with and write_properties_with", + )); + } + let (public_getter, public_setter) = property_accessors(&field.attrs)?; + + Ok(PropertyField { + ident, + ty: field.ty.clone(), + key, + additional_key, + prefix, + nested, + default, + parse_with, + serialize_with, + parse_properties_with, + write_properties_with, + option_inner_type: option_inner_type(&field.ty), + map_value_type, + public_getter, + public_setter, + doc_attributes: field + .attrs + .iter() + .filter(|attribute| attribute.path().is_ident("doc")) + .cloned() + .collect(), + }) +} + +fn property_accessors(attributes: &[Attribute]) -> syn::Result<(bool, bool)> { + let Some(attribute) = find_attribute(attributes, "property")? else { + return Ok((false, false)); + }; + + let accessors = + attribute.parse_args_with(Punctuated::::parse_terminated)?; + if accessors.is_empty() { + return Err(Error::new_spanned( + attribute, + "property must declare pub(getter), pub(setter), or both", + )); + } + + let mut public_getter = false; + let mut public_setter = false; + for accessor in accessors { + let selected = match accessor { + PublicAccessor::Getter => &mut public_getter, + PublicAccessor::Setter => &mut public_setter, + }; + if *selected { + return Err(Error::new_spanned(attribute, "duplicate property accessor")); + } + *selected = true; + } + + Ok((public_getter, public_setter)) +} + +fn field_accessors(field: &PropertyField) -> TokenStream2 { + let ident = &field.ident; + let ty = &field.ty; + let docs = &field.doc_attributes; + let getter = field.public_getter.then(|| { + quote! { + #(#docs)* + pub fn #ident(&self) -> &#ty { + &self.#ident + } + } + }); + let setter = field.public_setter.then(|| { + let setter_ident = format_ident!("set_{}", ident); + let setter_doc = format!("Sets `{ident}`."); + quote! { + #[doc = #setter_doc] + pub fn #setter_ident(&mut self, value: #ty) { + self.#ident = value; + } + } + }); + + quote! { + #getter + #setter + } +} + +fn marker_attribute(attributes: &[Attribute], name: &str) -> syn::Result { + let Some(attribute) = find_attribute(attributes, name)? else { + return Ok(false); + }; + + match &attribute.meta { + Meta::Path(_) => Ok(true), + _ => Err(Error::new_spanned( + attribute, + format!("{name} must use the form #[{name}]"), + )), + } +} + +fn attribute_expression_value(attributes: &[Attribute], name: &str) -> syn::Result> { + let Some(attribute) = find_attribute(attributes, name)? else { + return Ok(None); + }; + + match &attribute.meta { + Meta::NameValue(name_value) => Ok(Some(name_value.value.clone())), + Meta::List(_) => attribute.parse_args::().map(Some), + _ => Err(Error::new_spanned( + attribute, + format!("{name} must use the form #[{name}(...)]"), + )), + } +} + +fn attribute_path_value(attributes: &[Attribute], name: &str) -> syn::Result> { + let Some(expression) = attribute_expression_value(attributes, name)? else { + return Ok(None); + }; + + match expression { + Expr::Path(ExprPath { path, .. }) => Ok(Some(path)), + _ => Err(Error::new_spanned( + expression, + format!("{name} must be a path"), + )), + } +} + +fn find_attribute<'a>( + attributes: &'a [Attribute], + name: &str, +) -> syn::Result> { + let mut matching = attributes + .iter() + .filter(|attribute| attribute.path().is_ident(name)); + let first = matching.next(); + if let Some(duplicate) = matching.next() { + return Err(Error::new_spanned( + duplicate, + format!("duplicate #[{name}] attribute"), + )); + } + Ok(first) +} + +fn parse_field(field: &PropertyField) -> TokenStream2 { + let ident = &field.ident; + if field.nested { + let ty = &field.ty; + return quote!(#ident: <#ty>::from_properties(properties)?); + } + + let ty = &field.ty; + let default = default_value( + field.default.as_ref().expect("leaf fields have defaults"), + ty, + ); + let default = quote!({ + let value: #ty = #default; + value + }); + + if let Some(parse_properties_with) = &field.parse_properties_with { + let key = field.key.as_ref().expect("exact-key fields have a key"); + let parse = match &field.additional_key { + Some(additional_key) => { + quote!(#parse_properties_with(properties, #key, #additional_key, #default)) + } + None => quote!(#parse_properties_with(properties, #key, #default)), + }; + return quote! { + #ident: #parse.map_err(|error| { + format!("Invalid value for {}: {error}", #key) + })? + }; + } + + if let Some(prefix) = &field.prefix { + let value_type = field + .map_value_type + .as_ref() + .expect("prefix fields are validated as maps"); + let parse = if is_bool(value_type) { + quote!(value.to_ascii_lowercase().parse::<#value_type>()) + } else { + quote!(value.parse::<#value_type>()) + }; + return quote! { + #ident: { + let parsed = properties + .iter() + .filter_map(|(key, value)| { + key.strip_prefix(#prefix).map(|suffix| { + #parse + .map(|parsed| (suffix.to_string(), parsed)) + .map_err(|error| format!("Invalid value for {key}: {error}")) + }) + }) + .collect::<::std::result::Result< + ::std::collections::HashMap<_, _>, + ::std::string::String, + >>()?; + if parsed.is_empty() { + #default + } else { + parsed + } + } + }; + } + + let ty = &field.ty; + let key = field.key.as_ref().expect("exact-key fields have a key"); + let parse = match (&field.parse_with, &field.option_inner_type) { + (Some(parse_with), _) => quote! { + #parse_with(value).map_err(|error| { + format!("Invalid value for {}: {error}", #key) + })? + }, + (None, Some(inner_type)) if is_bool(inner_type) => quote! { + Some(value.to_ascii_lowercase().parse::<#inner_type>().map_err(|error| { + format!("Invalid value for {}: {error}", #key) + })?) + }, + (None, Some(inner_type)) => quote! { + Some(value.parse::<#inner_type>().map_err(|error| { + format!("Invalid value for {}: {error}", #key) + })?) + }, + (None, None) if is_bool(ty) => quote! { + value.to_ascii_lowercase().parse::<#ty>().map_err(|error| { + format!("Invalid value for {}: {error}", #key) + })? + }, + (None, None) => quote! { + value.parse::<#ty>().map_err(|error| { + format!("Invalid value for {}: {error}", #key) + })? + }, + }; + + quote! { + #ident: match properties.get(#key) { + Some(value) => #parse, + None => #default, + } + } +} + +fn default_value(default: &Expr, ty: &Type) -> TokenStream2 { + if matches!( + default, + Expr::Lit(ExprLit { + lit: Lit::Str(_), + .. + }) | Expr::Path(_) + ) { + quote!(::std::convert::Into::<#ty>::into(#default)) + } else { + quote!(#default) + } +} + +fn option_inner_type(ty: &Type) -> Option { + let Type::Path(type_path) = ty else { + return None; + }; + + let segment = type_path.path.segments.last()?; + if segment.ident != "Option" { + return None; + } + + let syn::PathArguments::AngleBracketed(arguments) = &segment.arguments else { + return None; + }; + let Some(syn::GenericArgument::Type(inner_type)) = arguments.args.first() else { + return None; + }; + + Some(inner_type.clone()) +} + +fn map_value_type(ty: &Type) -> Option { + let Type::Path(type_path) = ty else { + return None; + }; + + let segment = type_path.path.segments.last()?; + if segment.ident != "HashMap" { + return None; + } + + let syn::PathArguments::AngleBracketed(arguments) = &segment.arguments else { + return None; + }; + let Some(syn::GenericArgument::Type(value_type)) = arguments.args.iter().nth(1) else { + return None; + }; + + Some(value_type.clone()) +} + +fn is_bool(ty: &Type) -> bool { + let Type::Path(type_path) = ty else { + return false; + }; + + type_path + .path + .segments + .last() + .is_some_and(|segment| segment.ident == "bool") +} + +fn write_field(field: &PropertyField) -> TokenStream2 { + let ident = &field.ident; + if field.nested { + return quote! { + self.#ident.write_properties(properties); + }; + } + + let ty = &field.ty; + let default = default_value( + field.default.as_ref().expect("leaf fields have defaults"), + ty, + ); + let default = quote!({ + let value: #ty = #default; + value + }); + + if let Some(write_properties_with) = &field.write_properties_with { + let key = field.key.as_ref().expect("exact-key fields have a key"); + let write = match &field.additional_key { + Some(additional_key) => { + quote!(#write_properties_with(&self.#ident, properties, #key, #additional_key, &#default)) + } + None => quote!(#write_properties_with(&self.#ident, properties, #key, &#default)), + }; + return quote! { + #write; + }; + } + + if let Some(prefix) = &field.prefix { + return quote! { + if self.#ident != #default { + for (suffix, value) in &self.#ident { + let key = format!("{}{}", #prefix, suffix); + properties.insert(key, ::std::string::ToString::to_string(value)); + } + } + }; + } + + let key = field.key.as_ref().expect("exact-key fields have a key"); + if field.option_inner_type.is_some() { + let value = match &field.serialize_with { + Some(serialize_with) => quote!(#serialize_with(&self.#ident)), + None => quote!(::std::string::ToString::to_string( + self.#ident.as_ref().expect("checked is_some above") + )), + }; + quote! { + if self.#ident != #default && self.#ident.is_some() { + properties.insert((#key).to_string(), #value); + } + } + } else { + let value = match &field.serialize_with { + Some(serialize_with) => quote!(#serialize_with(&self.#ident)), + None => quote!(::std::string::ToString::to_string(&self.#ident)), + }; + quote! { + if self.#ident != #default { + properties.insert((#key).to_string(), #value); + } + } + } +} diff --git a/crates/property-macro/tests/properties.rs b/crates/property-macro/tests/properties.rs new file mode 100644 index 0000000000..614a47c7ee --- /dev/null +++ b/crates/property-macro/tests/properties.rs @@ -0,0 +1,213 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::HashMap; + +use iceberg_property_macro::Properties; + +const RETRIES: &str = "commit.retry.num-retries"; +const OWNER: &str = "owner"; +const FORMAT: &str = "write.format.default"; +const FANOUT_ENABLED: &str = "write.fanout.enabled"; +const COLUMN_FPP_PREFIX: &str = "write.parquet.bloom-filter-fpp.column."; +const WIDTH: &str = "dimensions.width"; +const HEIGHT: &str = "dimensions.height"; + +fn parse_dimensions( + properties: &HashMap, + key: &str, + height_key: &str, + default: (u64, u64), +) -> Result<(u64, u64), String> { + let parse = |property_key: &str, default| { + properties + .get(property_key) + .map(|value| value.parse::().map_err(|error| error.to_string())) + .transpose() + .map(|value| value.unwrap_or(default)) + }; + + Ok((parse(key, default.0)?, parse(height_key, default.1)?)) +} + +fn write_dimensions( + dimensions: &(u64, u64), + properties: &mut HashMap, + key: &str, + height_key: &str, + default: &(u64, u64), +) { + if dimensions == default { + properties.remove(key); + properties.remove(height_key); + } else { + properties.insert(key.to_string(), dimensions.0.to_string()); + properties.insert(height_key.to_string(), dimensions.1.to_string()); + } +} + +#[derive(Debug, Properties)] +struct TestProperties { + #[key(RETRIES)] + #[default(4)] + #[doc = "Number of retries."] + pub retries: u64, + + #[key(OWNER)] + #[default(None)] + pub owner: Option, + + #[key(FORMAT)] + #[default("parquet")] + pub format: String, + + #[key(FANOUT_ENABLED)] + #[default(true)] + pub fanout_enabled: bool, + + #[prefix(COLUMN_FPP_PREFIX)] + #[default(HashMap::new())] + pub column_fpp: HashMap, + + #[key(WIDTH)] + #[additional_key(HEIGHT)] + #[default((640, 480))] + #[parse_properties_with(parse_dimensions)] + #[write_properties_with(write_dimensions)] + pub dimensions: (u64, u64), +} + +#[test] +fn generates_defaults_and_serde_for_public_fields() { + let defaults = TestProperties::default(); + assert_eq!(defaults.format, "parquet"); + assert_eq!( + serde_json::to_value(&defaults).unwrap(), + serde_json::json!({}) + ); + + let properties = TestProperties { + retries: 8, + owner: Some("iceberg".to_string()), + format: "orc".to_string(), + fanout_enabled: false, + column_fpp: HashMap::from([("id".to_string(), 0.01)]), + dimensions: (1920, 1080), + }; + + assert_eq!(properties.retries, 8); + assert_eq!(properties.owner, Some("iceberg".to_string())); + assert_eq!(properties.format, "orc"); + assert!(!properties.fanout_enabled); + assert_eq!(properties.column_fpp["id"], 0.01); + assert_eq!(properties.dimensions, (1920, 1080)); + + let json = serde_json::to_value(&properties).unwrap(); + assert_eq!(json[RETRIES], "8"); + assert_eq!(json[OWNER], "iceberg"); + assert_eq!(json[FORMAT], "orc"); + assert_eq!(json[FANOUT_ENABLED], "false"); + assert_eq!(json[format!("{COLUMN_FPP_PREFIX}id")], "0.01"); + assert_eq!(json[WIDTH], "1920"); + assert_eq!(json[HEIGHT], "1080"); + + let mut json = json; + json[FANOUT_ENABLED] = "FALSE".into(); + let decoded: TestProperties = serde_json::from_value(json).unwrap(); + assert_eq!(decoded.retries, 8); + assert_eq!(decoded.owner, Some("iceberg".to_string())); + assert_eq!(decoded.format, "orc"); + assert!(!decoded.fanout_enabled); + assert_eq!(decoded.column_fpp["id"], 0.01); + assert_eq!(decoded.dimensions, (1920, 1080)); +} + +#[derive(Clone, Debug, Properties)] +struct CommitProperties { + #[key = "commit.retry.num-retries"] + #[default = 4] + #[doc = "Number of times to retry a commit before failing."] + pub num_retries: u64, +} + +#[derive(Debug, Properties)] +struct NestedProperties { + #[nested] + #[doc = "Commit behavior properties."] + pub commit: CommitProperties, +} + +#[test] +fn nested_properties_use_a_flat_property_map() { + assert_eq!( + serde_json::to_value(NestedProperties::default()).unwrap(), + serde_json::json!({}) + ); + + let mut properties = NestedProperties::default(); + properties.commit.num_retries = 9; + + let json = serde_json::to_value(&properties).unwrap(); + assert_eq!(json["commit.retry.num-retries"], "9"); + + let decoded: NestedProperties = serde_json::from_value(json).unwrap(); + assert_eq!(decoded.commit.num_retries, 9); +} + +mod accessor_fixture { + use iceberg_property_macro::Properties; + + #[derive(Debug, Properties)] + pub struct AccessorProperties { + #[key = "public.both"] + #[default = 1] + #[doc = "A property with public read and write access."] + #[property(pub(getter), pub(setter))] + both: u64, + + #[key = "public.getter"] + #[default = "value"] + #[doc = "A property with public read access."] + #[property(pub(getter))] + getter_only: String, + + #[key = "public.setter"] + #[default = false] + #[property(pub(setter))] + setter_only: bool, + } + + impl AccessorProperties { + pub fn setter_only_for_test(&self) -> bool { + self.setter_only + } + } +} + +#[test] +fn generates_opt_in_public_accessors_for_private_fields() { + let mut properties = accessor_fixture::AccessorProperties::default(); + + assert_eq!(*properties.both(), 1); + properties.set_both(2); + assert_eq!(*properties.both(), 2); + + assert_eq!(properties.getter_only(), "value"); + + properties.set_setter_only(true); + assert!(properties.setter_only_for_test()); +}