From ffd695f9cdb7e699a62bcb4a5356ed63f3543ceb Mon Sep 17 00:00:00 2001 From: Ray Liu <257669749+blackmwk@users.noreply.github.com> Date: Wed, 12 Aug 2026 10:52:17 +0800 Subject: [PATCH 1/3] feat: Derive TableProperties parsing Port the existing TableProperties keys, defaults, and custom parsers to the Properties derive macro while preserving the TryFrom API and public fields for compatibility. Add generated getters and update the public API snapshot.\n\nCloses #2968. --- Cargo.lock | 1 + Cargo.toml | 1 + crates/iceberg/Cargo.toml | 1 + crates/iceberg/public-api.txt | 32 +- crates/iceberg/src/spec/table_properties.rs | 421 +++++++++----------- crates/property-macro/src/properties.rs | 1 + 6 files changed, 232 insertions(+), 225 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f1d00378f7..d6b897a8cf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3788,6 +3788,7 @@ dependencies = [ "fnv", "form_urlencoded", "futures", + "iceberg-property-macro", "iceberg_test_utils", "itertools 0.13.0", "minijinja", diff --git a/Cargo.toml b/Cargo.toml index aec84da8eb..32667b20c7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -108,6 +108,7 @@ iceberg-catalog-rest = { version = "0.10.0", path = "./crates/catalog/rest" } iceberg-catalog-s3tables = { version = "0.10.0", path = "./crates/catalog/s3tables" } iceberg-catalog-sql = { version = "0.10.0", path = "./crates/catalog/sql" } iceberg-datafusion = { version = "0.10.0", path = "./crates/integrations/datafusion" } +iceberg-property-macro = { version = "0.10.0", path = "./crates/property-macro" } iceberg-storage-opendal = { version = "0.10.0", path = "./crates/storage/opendal" } indicatif = "0.18" itertools = "0.13" diff --git a/crates/iceberg/Cargo.toml b/crates/iceberg/Cargo.toml index 9219c728a5..56e11b1804 100644 --- a/crates/iceberg/Cargo.toml +++ b/crates/iceberg/Cargo.toml @@ -60,6 +60,7 @@ flate2 = { workspace = true } fnv = { workspace = true } form_urlencoded = { workspace = true } futures = { workspace = true } +iceberg-property-macro = { workspace = true } itertools = { workspace = true } moka = { version = "0.12.10", features = ["future"] } murmur3 = { workspace = true } diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index 9ef584678e..7deec16a9d 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -2888,10 +2888,40 @@ pub const iceberg::spec::TableProperties::PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT pub const iceberg::spec::TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES: &str pub const iceberg::spec::TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT: usize pub const iceberg::spec::TableProperties::RESERVED_PROPERTIES: [&str; 9] +impl iceberg::spec::TableProperties +pub fn iceberg::spec::TableProperties::cdc_enabled(&self) -> bool +pub fn iceberg::spec::TableProperties::cdc_max_chunk_size(&self) -> usize +pub fn iceberg::spec::TableProperties::cdc_min_chunk_size(&self) -> usize +pub fn iceberg::spec::TableProperties::cdc_norm_level(&self) -> i32 +pub fn iceberg::spec::TableProperties::commit_max_retry_wait_ms(&self) -> u64 +pub fn iceberg::spec::TableProperties::commit_min_retry_wait_ms(&self) -> u64 +pub fn iceberg::spec::TableProperties::commit_num_retries(&self) -> usize +pub fn iceberg::spec::TableProperties::commit_total_retry_timeout_ms(&self) -> u64 pub fn iceberg::spec::TableProperties::data_encryption_key_size(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties::encryption_data_key_length(&self) -> usize +pub fn iceberg::spec::TableProperties::encryption_key_id(&self) -> &core::option::Option +pub fn iceberg::spec::TableProperties::from_properties(properties: &std::collections::hash::map::HashMap) -> iceberg::Result +pub fn iceberg::spec::TableProperties::gc_enabled(&self) -> bool +pub fn iceberg::spec::TableProperties::max_ref_age_ms(&self) -> i64 +pub fn iceberg::spec::TableProperties::max_snapshot_age_ms(&self) -> i64 +pub fn iceberg::spec::TableProperties::metadata_compression_codec(&self) -> &iceberg::compression::CompressionCodec +pub fn iceberg::spec::TableProperties::min_snapshots_to_keep(&self) -> usize +pub fn iceberg::spec::TableProperties::parquet_compression_codec(&self) -> &iceberg::compression::CompressionCodec +pub fn iceberg::spec::TableProperties::parquet_dict_size_bytes(&self) -> usize +pub fn iceberg::spec::TableProperties::parquet_page_row_limit(&self) -> usize +pub fn iceberg::spec::TableProperties::parquet_page_size_bytes(&self) -> usize +pub fn iceberg::spec::TableProperties::parquet_row_group_size_bytes(&self) -> usize +pub fn iceberg::spec::TableProperties::write_data_location(&self) -> &core::option::Option +pub fn iceberg::spec::TableProperties::write_datafusion_fanout_enabled(&self) -> bool +pub fn iceberg::spec::TableProperties::write_folder_storage_location(&self) -> &core::option::Option +pub fn iceberg::spec::TableProperties::write_format_default(&self) -> &alloc::string::String +pub fn iceberg::spec::TableProperties::write_metadata_path(&self) -> &core::option::Option +pub fn iceberg::spec::TableProperties::write_object_storage_location(&self) -> &core::option::Option +pub fn iceberg::spec::TableProperties::write_object_storage_partitioned_paths(&self) -> bool +pub fn iceberg::spec::TableProperties::write_target_file_size_bytes(&self) -> usize impl core::convert::TryFrom<&std::collections::hash::map::HashMap> for iceberg::spec::TableProperties pub type iceberg::spec::TableProperties::Error = iceberg::Error -pub fn iceberg::spec::TableProperties::try_from(props: &std::collections::hash::map::HashMap) -> iceberg::Result +pub fn iceberg::spec::TableProperties::try_from(properties: &std::collections::hash::map::HashMap) -> iceberg::Result impl core::fmt::Debug for iceberg::spec::TableProperties pub fn iceberg::spec::TableProperties::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result pub struct iceberg::spec::UnboundPartitionField diff --git a/crates/iceberg/src/spec/table_properties.rs b/crates/iceberg/src/spec/table_properties.rs index 06dbe366cc..a5a51e28cc 100644 --- a/crates/iceberg/src/spec/table_properties.rs +++ b/crates/iceberg/src/spec/table_properties.rs @@ -16,71 +16,26 @@ // under the License. use std::collections::HashMap; -use std::fmt::Display; -use std::str::FromStr; + +use iceberg_property_macro::Properties; use crate::compression::CompressionCodec; use crate::encryption::AesKeySize; use crate::error::{Error, ErrorKind, Result}; use crate::util::location::strip_trailing_slash; -fn parse_property( - properties: &HashMap, - key: &str, - default: T, -) -> Result -where - ::Err: Display, -{ - properties.get(key).map_or(Ok(default), |value| { - value.parse::().map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!("Invalid value for {key}: {e}"), - ) - }) - }) -} - -/// Parse an optional property, returning `None` when the key is absent and an -/// error when the value is present but fails to parse. -fn parse_optional_property( - properties: &HashMap, - key: &str, -) -> Result> -where - ::Err: Display, -{ - properties - .get(key) - .map(|value| { - value.parse::().map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!("Invalid value for {key}: {e}"), - ) - }) - }) - .transpose() -} - -fn parse_location_property( - properties: &HashMap, - key: &str, -) -> Result> { - properties - .get(key) - .map(|path| { - if path.is_empty() { - return Err(Error::new( - ErrorKind::DataInvalid, - format!("Invalid value for {key}: path must not be empty"), - )); - } +fn parse_location_property(path: &str) -> Result { + if path.is_empty() { + return Err(Error::new( + ErrorKind::DataInvalid, + format!( + "Invalid value for {}: path must not be empty", + TableProperties::PROPERTY_WRITE_METADATA_PATH + ), + )); + } - Ok(strip_trailing_slash(path).to_string()) - }) - .transpose() + Ok(strip_trailing_slash(path).to_string()) } /// Parse compression codec for metadata files from table properties. @@ -103,6 +58,10 @@ pub(crate) fn parse_metadata_file_compression( .map(|s| s.as_str()) .unwrap_or(TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC_DEFAULT); + parse_metadata_compression(value) +} + +fn parse_metadata_compression(value: &str) -> Result { // Handle empty string as None if value.is_empty() { return Ok(CompressionCodec::None); @@ -143,27 +102,40 @@ pub(crate) fn parse_metadata_file_compression( /// Parse the Parquet data-file compression codec (`write.parquet.compression-codec`) /// and fold in the compression level (`write.parquet.compression-level`) for the /// codecs that accept one (`zstd`, `gzip`, `brotli`). -fn parse_parquet_compression(properties: &HashMap) -> Result { - let value = properties - .get(TableProperties::PROPERTY_PARQUET_COMPRESSION_CODEC) - .map(|s| s.as_str()) - .unwrap_or(TableProperties::PROPERTY_PARQUET_COMPRESSION_CODEC_DEFAULT); - - let codec: CompressionCodec = - serde_json::from_value(serde_json::Value::String(value.to_lowercase())).map_err(|_| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Invalid Parquet compression codec: {value}. Supported codecs: \ - uncompressed, snappy, gzip, lzo, brotli, lz4, lz4_raw, zstd" - ), - ) - })?; +fn parse_parquet_compression( + properties: &HashMap, + codec_key: &str, + additional_keys: &[&str], + default: CompressionCodec, +) -> Result { + let level_key = additional_keys[0]; + let codec = properties + .get(codec_key) + .map(|value| { + serde_json::from_value(serde_json::Value::String(value.to_lowercase())).map_err(|_| { + Error::new( + ErrorKind::DataInvalid, + format!( + "Invalid Parquet compression codec: {value}. Supported codecs: \ + uncompressed, snappy, gzip, lzo, brotli, lz4, lz4_raw, zstd" + ), + ) + }) + }) + .transpose()? + .unwrap_or(default); - let level: Option = parse_optional_property( - properties, - TableProperties::PROPERTY_PARQUET_COMPRESSION_LEVEL, - )?; + let level = properties + .get(level_key) + .map(|value| { + value.parse::().map_err(|error| { + Error::new( + ErrorKind::DataInvalid, + format!("Invalid value for {level_key}: {error}"), + ) + }) + }) + .transpose()?; Ok(match (codec, level) { (CompressionCodec::Zstd(_), Some(level)) => CompressionCodec::Zstd(level), @@ -173,94 +145,220 @@ fn parse_parquet_compression(properties: &HashMap) -> Result, - key: &str, - default: bool, -) -> Result { - properties.get(key).map_or(Ok(default), |value| { - value.to_lowercase().parse::().map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!("Invalid value for {key}: {e}"), - ) - }) - }) -} - /// TableProperties that contains the properties of a table. -#[derive(Debug)] +#[derive(Debug, Properties)] pub struct TableProperties { /// The number of times to retry a commit. + #[property( + key = Self::PROPERTY_COMMIT_NUM_RETRIES, + default = Self::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT, + getter + )] pub commit_num_retries: usize, /// The minimum wait time between retries. + #[property( + key = Self::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS, + default = Self::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT, + getter + )] pub commit_min_retry_wait_ms: u64, /// The maximum wait time between retries. + #[property( + key = Self::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS, + default = Self::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT, + getter + )] pub commit_max_retry_wait_ms: u64, /// The total timeout for commit retries. + #[property( + key = Self::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS, + default = Self::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT, + getter + )] pub commit_total_retry_timeout_ms: u64, /// The default format for files. + #[property( + key = Self::PROPERTY_DEFAULT_FILE_FORMAT, + default = Self::PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT, + getter + )] pub write_format_default: String, /// The target file size for files. + #[property( + key = Self::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES, + default = Self::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT, + getter + )] pub write_target_file_size_bytes: usize, /// Base directory for metadata files (manifests, manifest lists), with any /// trailing slash trimmed. `None` if `write.metadata.path` is not set. + #[property( + key = Self::PROPERTY_WRITE_METADATA_PATH, + default = None, + parse_with = parse_location_property, + getter + )] pub write_metadata_path: Option, /// Compression codec for metadata files (JSON) + #[property( + key = Self::PROPERTY_METADATA_COMPRESSION_CODEC, + default = CompressionCodec::None, + parse_with = parse_metadata_compression, + getter + )] pub metadata_compression_codec: CompressionCodec, /// Whether to use `FanoutWriter` for partitioned tables. + #[property( + key = Self::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED, + default = Self::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT, + getter + )] pub write_datafusion_fanout_enabled: bool, /// Whether garbage collection is enabled on drop. /// When `false`, data files will not be deleted when a table is dropped. + #[property( + key = Self::PROPERTY_GC_ENABLED, + default = Self::PROPERTY_GC_ENABLED_DEFAULT, + getter + )] pub gc_enabled: bool, /// Default maximum age of a snapshot to keep when expiring snapshots. + #[property( + key = Self::PROPERTY_MAX_SNAPSHOT_AGE_MS, + default = Self::PROPERTY_MAX_SNAPSHOT_AGE_MS_DEFAULT, + getter + )] pub max_snapshot_age_ms: i64, /// Default minimum number of snapshots to keep per branch when expiring snapshots. + #[property( + key = Self::PROPERTY_MIN_SNAPSHOTS_TO_KEEP, + default = Self::PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT, + getter + )] pub min_snapshots_to_keep: usize, /// Default maximum age of a snapshot reference to keep when expiring snapshots. + #[property( + key = Self::PROPERTY_MAX_REF_AGE_MS, + default = Self::PROPERTY_MAX_REF_AGE_MS_DEFAULT, + getter + )] pub max_ref_age_ms: i64, /// Whether content-defined chunking is enabled. /// `true` only when `write.parquet.content-defined-chunking.enabled = "true"`. + #[property( + key = Self::PROPERTY_PARQUET_CDC_ENABLED, + default = Self::PROPERTY_PARQUET_CDC_ENABLED_DEFAULT, + getter + )] pub cdc_enabled: bool, /// Content-defined chunking minimum chunk size in bytes. + #[property( + key = Self::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE, + default = Self::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT, + getter + )] pub cdc_min_chunk_size: usize, /// Content-defined chunking maximum chunk size in bytes. + #[property( + key = Self::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE, + default = Self::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE_DEFAULT, + getter + )] pub cdc_max_chunk_size: usize, /// Content-defined chunking normalization level (gearhash bit adjustment). + #[property( + key = Self::PROPERTY_PARQUET_CDC_NORM_LEVEL, + default = Self::PROPERTY_PARQUET_CDC_NORM_LEVEL_DEFAULT, + getter + )] pub cdc_norm_level: i32, /// Parquet compression codec for data files, with the resolved compression /// level folded in (from `write.parquet.compression-level`, or the codec's /// default when unset). + #[property( + key = Self::PROPERTY_PARQUET_COMPRESSION_CODEC, + additional_keys = [Self::PROPERTY_PARQUET_COMPRESSION_LEVEL], + default = CompressionCodec::zstd_default(), + parse_properties_with = parse_parquet_compression, + getter + )] pub parquet_compression_codec: CompressionCodec, /// Approximate maximum Parquet row group size in bytes. + #[property( + key = Self::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES, + default = Self::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT, + getter + )] pub parquet_row_group_size_bytes: usize, /// Approximate maximum Parquet data page size in bytes. + #[property( + key = Self::PROPERTY_PARQUET_PAGE_SIZE_BYTES, + default = Self::PROPERTY_PARQUET_PAGE_SIZE_BYTES_DEFAULT, + getter + )] pub parquet_page_size_bytes: usize, /// Maximum number of rows per Parquet data page. + #[property( + key = Self::PROPERTY_PARQUET_PAGE_ROW_LIMIT, + default = Self::PROPERTY_PARQUET_PAGE_ROW_LIMIT_DEFAULT, + getter + )] pub parquet_page_row_limit: usize, /// Approximate maximum Parquet dictionary page size in bytes. + #[property( + key = Self::PROPERTY_PARQUET_DICT_SIZE_BYTES, + default = Self::PROPERTY_PARQUET_DICT_SIZE_BYTES_DEFAULT, + getter + )] pub parquet_dict_size_bytes: usize, /// The master key id used to encrypt this table's manifest list and data /// files. `None` if `encryption.key-id` is not set. + #[property( + key = Self::PROPERTY_ENCRYPTION_KEY_ID, + default = None, + getter + )] pub encryption_key_id: Option, /// The encryption data encryption key length in bytes. + #[property( + key = Self::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH, + default = Self::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH_DEFAULT, + getter + )] pub encryption_data_key_length: usize, /// Base directory for data files + #[property( + key = Self::PROPERTY_WRITE_DATA_LOCATION, + default = None, + getter + )] pub write_data_location: Option, /// Deprecated table property for data file write location. /// /// Property will be removed at a later date. /// Superseded by [write_data_location]. + #[property( + key = Self::PROPERTY_WRITE_FOLDER_STORAGE_LOCATION, + default = None, + getter + )] pub write_folder_storage_location: Option, /// Deprecated table property for data file write location for object storage location generator. /// /// Property will be removed at a later date. /// Superseded by [write_data_location]. + #[property( + key = Self::PROPERTY_WRITE_OBJECT_STORAGE_LOCATION, + default = None, + getter + )] pub write_object_storage_location: Option, /// Whether partition values are included in object storage paths. + #[property( + key = Self::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS, + default = Self::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS_DEFAULT, + getter + )] pub write_object_storage_partitioned_paths: bool, } @@ -465,135 +563,10 @@ impl TableProperties { } impl TryFrom<&HashMap> for TableProperties { - // parse by entry key or use default value type Error = Error; - fn try_from(props: &HashMap) -> Result { - Ok(TableProperties { - commit_num_retries: parse_property( - props, - TableProperties::PROPERTY_COMMIT_NUM_RETRIES, - TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT, - )?, - commit_min_retry_wait_ms: parse_property( - props, - TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS, - TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT, - )?, - commit_max_retry_wait_ms: parse_property( - props, - TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS, - TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT, - )?, - commit_total_retry_timeout_ms: parse_property( - props, - TableProperties::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS, - TableProperties::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT, - )?, - write_format_default: parse_property( - props, - TableProperties::PROPERTY_DEFAULT_FILE_FORMAT, - TableProperties::PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT.to_string(), - )?, - write_target_file_size_bytes: parse_property( - props, - TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES, - TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT, - )?, - write_metadata_path: parse_location_property( - props, - TableProperties::PROPERTY_WRITE_METADATA_PATH, - )?, - metadata_compression_codec: parse_metadata_file_compression(props)?, - write_datafusion_fanout_enabled: parse_property_bool( - props, - TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED, - TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT, - )?, - gc_enabled: parse_property_bool( - props, - TableProperties::PROPERTY_GC_ENABLED, - TableProperties::PROPERTY_GC_ENABLED_DEFAULT, - )?, - max_snapshot_age_ms: parse_property( - props, - TableProperties::PROPERTY_MAX_SNAPSHOT_AGE_MS, - TableProperties::PROPERTY_MAX_SNAPSHOT_AGE_MS_DEFAULT, - )?, - min_snapshots_to_keep: parse_property( - props, - TableProperties::PROPERTY_MIN_SNAPSHOTS_TO_KEEP, - TableProperties::PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT, - )?, - max_ref_age_ms: parse_property( - props, - TableProperties::PROPERTY_MAX_REF_AGE_MS, - TableProperties::PROPERTY_MAX_REF_AGE_MS_DEFAULT, - )?, - cdc_enabled: parse_property_bool( - props, - TableProperties::PROPERTY_PARQUET_CDC_ENABLED, - TableProperties::PROPERTY_PARQUET_CDC_ENABLED_DEFAULT, - )?, - cdc_min_chunk_size: parse_property( - props, - TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE, - TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT, - )?, - cdc_max_chunk_size: parse_property( - props, - TableProperties::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE, - TableProperties::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE_DEFAULT, - )?, - cdc_norm_level: parse_property( - props, - TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL, - TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL_DEFAULT, - )?, - parquet_compression_codec: parse_parquet_compression(props)?, - parquet_row_group_size_bytes: parse_property( - props, - TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES, - TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT, - )?, - parquet_page_size_bytes: parse_property( - props, - TableProperties::PROPERTY_PARQUET_PAGE_SIZE_BYTES, - TableProperties::PROPERTY_PARQUET_PAGE_SIZE_BYTES_DEFAULT, - )?, - parquet_page_row_limit: parse_property( - props, - TableProperties::PROPERTY_PARQUET_PAGE_ROW_LIMIT, - TableProperties::PROPERTY_PARQUET_PAGE_ROW_LIMIT_DEFAULT, - )?, - parquet_dict_size_bytes: parse_property( - props, - TableProperties::PROPERTY_PARQUET_DICT_SIZE_BYTES, - TableProperties::PROPERTY_PARQUET_DICT_SIZE_BYTES_DEFAULT, - )?, - encryption_key_id: props - .get(TableProperties::PROPERTY_ENCRYPTION_KEY_ID) - .cloned(), - encryption_data_key_length: parse_property( - props, - TableProperties::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH, - TableProperties::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH_DEFAULT, - )?, - write_data_location: props - .get(TableProperties::PROPERTY_WRITE_DATA_LOCATION) - .cloned(), - write_folder_storage_location: props - .get(TableProperties::PROPERTY_WRITE_FOLDER_STORAGE_LOCATION) - .cloned(), - write_object_storage_location: props - .get(TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_LOCATION) - .cloned(), - write_object_storage_partitioned_paths: parse_property_bool( - props, - TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS, - TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS_DEFAULT, - )?, - }) + fn try_from(properties: &HashMap) -> Result { + Self::from_properties(properties) } } diff --git a/crates/property-macro/src/properties.rs b/crates/property-macro/src/properties.rs index 03820311fb..b4fcf2d63e 100644 --- a/crates/property-macro/src/properties.rs +++ b/crates/property-macro/src/properties.rs @@ -128,6 +128,7 @@ pub(crate) fn expand_properties(input: DeriveInput) -> syn::Result impl #impl_generics #struct_name #type_generics #where_clause { #(#accessors)* + /// Parses this typed property set from a flat string-to-string map. pub fn from_properties( properties: &::std::collections::HashMap< ::std::string::String, From f12768683f2ef0487f87356c5bb71a4654679872 Mon Sep 17 00:00:00 2001 From: Ray Liu <257669749+blackmwk@users.noreply.github.com> Date: Wed, 19 Aug 2026 17:36:30 +0800 Subject: [PATCH 2/3] fix: Address TableProperties review feedback --- crates/iceberg/public-api.txt | 2 +- crates/iceberg/src/spec/table_properties.rs | 357 ++++++++++++-------- crates/property-macro/Cargo.toml | 2 +- crates/property-macro/public-api.txt | 2 + 4 files changed, 219 insertions(+), 144 deletions(-) create mode 100644 crates/property-macro/public-api.txt diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index 7deec16a9d..77b736b976 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -2888,6 +2888,7 @@ pub const iceberg::spec::TableProperties::PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT pub const iceberg::spec::TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES: &str pub const iceberg::spec::TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT: usize pub const iceberg::spec::TableProperties::RESERVED_PROPERTIES: [&str; 9] +pub fn iceberg::spec::TableProperties::data_encryption_key_size(&self) -> iceberg::Result impl iceberg::spec::TableProperties pub fn iceberg::spec::TableProperties::cdc_enabled(&self) -> bool pub fn iceberg::spec::TableProperties::cdc_max_chunk_size(&self) -> usize @@ -2897,7 +2898,6 @@ pub fn iceberg::spec::TableProperties::commit_max_retry_wait_ms(&self) -> u64 pub fn iceberg::spec::TableProperties::commit_min_retry_wait_ms(&self) -> u64 pub fn iceberg::spec::TableProperties::commit_num_retries(&self) -> usize pub fn iceberg::spec::TableProperties::commit_total_retry_timeout_ms(&self) -> u64 -pub fn iceberg::spec::TableProperties::data_encryption_key_size(&self) -> iceberg::Result pub fn iceberg::spec::TableProperties::encryption_data_key_length(&self) -> usize pub fn iceberg::spec::TableProperties::encryption_key_id(&self) -> &core::option::Option pub fn iceberg::spec::TableProperties::from_properties(properties: &std::collections::hash::map::HashMap) -> iceberg::Result diff --git a/crates/iceberg/src/spec/table_properties.rs b/crates/iceberg/src/spec/table_properties.rs index a5a51e28cc..6c92152a31 100644 --- a/crates/iceberg/src/spec/table_properties.rs +++ b/crates/iceberg/src/spec/table_properties.rs @@ -24,15 +24,62 @@ use crate::encryption::AesKeySize; use crate::error::{Error, ErrorKind, Result}; use crate::util::location::strip_trailing_slash; +const COMMIT_NUM_RETRIES: &str = "commit.retry.num-retries"; +const COMMIT_NUM_RETRIES_DEFAULT: usize = 4; +const COMMIT_MIN_RETRY_WAIT_MS: &str = "commit.retry.min-wait-ms"; +const COMMIT_MIN_RETRY_WAIT_MS_DEFAULT: u64 = 100; +const COMMIT_MAX_RETRY_WAIT_MS: &str = "commit.retry.max-wait-ms"; +const COMMIT_MAX_RETRY_WAIT_MS_DEFAULT: u64 = 60 * 1000; +const COMMIT_TOTAL_RETRY_TIME_MS: &str = "commit.retry.total-timeout-ms"; +const COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT: u64 = 30 * 60 * 1000; +const DEFAULT_FILE_FORMAT: &str = "write.format.default"; +const DEFAULT_FILE_FORMAT_DEFAULT: &str = "parquet"; +const WRITE_TARGET_FILE_SIZE_BYTES: &str = "write.target-file-size-bytes"; +const WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT: usize = 512 * 1024 * 1024; +const WRITE_METADATA_PATH: &str = "write.metadata.path"; +const METADATA_COMPRESSION_CODEC: &str = "write.metadata.compression-codec"; +const METADATA_COMPRESSION_CODEC_DEFAULT: &str = "none"; +const DATAFUSION_WRITE_FANOUT_ENABLED: &str = "write.datafusion.fanout.enabled"; +const DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT: bool = true; +const GC_ENABLED: &str = "gc.enabled"; +const GC_ENABLED_DEFAULT: bool = true; +const MAX_SNAPSHOT_AGE_MS: &str = "history.expire.max-snapshot-age-ms"; +const MAX_SNAPSHOT_AGE_MS_DEFAULT: i64 = 5 * 24 * 60 * 60 * 1000; +const MIN_SNAPSHOTS_TO_KEEP: &str = "history.expire.min-snapshots-to-keep"; +const MIN_SNAPSHOTS_TO_KEEP_DEFAULT: usize = 1; +const MAX_REF_AGE_MS: &str = "history.expire.max-ref-age-ms"; +const MAX_REF_AGE_MS_DEFAULT: i64 = i64::MAX; +const PARQUET_CDC_ENABLED: &str = "write.parquet.content-defined-chunking.enabled"; +const PARQUET_CDC_ENABLED_DEFAULT: bool = false; +const PARQUET_CDC_MIN_CHUNK_SIZE: &str = "write.parquet.content-defined-chunking.min-chunk-size"; +const PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT: usize = 256 * 1024; +const PARQUET_CDC_MAX_CHUNK_SIZE: &str = "write.parquet.content-defined-chunking.max-chunk-size"; +const PARQUET_CDC_MAX_CHUNK_SIZE_DEFAULT: usize = 1024 * 1024; +const PARQUET_CDC_NORM_LEVEL: &str = "write.parquet.content-defined-chunking.norm-level"; +const PARQUET_CDC_NORM_LEVEL_DEFAULT: i32 = 0; +const PARQUET_COMPRESSION_CODEC: &str = "write.parquet.compression-codec"; +const PARQUET_COMPRESSION_CODEC_DEFAULT: &str = "zstd"; +const PARQUET_COMPRESSION_LEVEL: &str = "write.parquet.compression-level"; +const PARQUET_ROW_GROUP_SIZE_BYTES: &str = "write.parquet.row-group-size-bytes"; +const PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT: usize = 128 * 1024 * 1024; +const PARQUET_PAGE_SIZE_BYTES: &str = "write.parquet.page-size-bytes"; +const PARQUET_PAGE_SIZE_BYTES_DEFAULT: usize = 1024 * 1024; +const PARQUET_PAGE_ROW_LIMIT: &str = "write.parquet.page-row-limit"; +const PARQUET_PAGE_ROW_LIMIT_DEFAULT: usize = 20000; +const PARQUET_DICT_SIZE_BYTES: &str = "write.parquet.dict-size-bytes"; +const PARQUET_DICT_SIZE_BYTES_DEFAULT: usize = 2 * 1024 * 1024; +const ENCRYPTION_KEY_ID: &str = "encryption.key-id"; +const ENCRYPTION_DATA_KEY_LENGTH: &str = "encryption.data-key-length"; +const ENCRYPTION_DATA_KEY_LENGTH_DEFAULT: usize = 16; +const WRITE_DATA_LOCATION: &str = "write.data.path"; +const WRITE_FOLDER_STORAGE_LOCATION: &str = "write.folder-storage.path"; +const WRITE_OBJECT_STORAGE_LOCATION: &str = "write.object-storage.path"; +const WRITE_OBJECT_STORAGE_PARTITIONED_PATHS: &str = "write.object-storage.partitioned-paths"; +const WRITE_OBJECT_STORAGE_PARTITIONED_PATHS_DEFAULT: bool = true; + fn parse_location_property(path: &str) -> Result { if path.is_empty() { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Invalid value for {}: path must not be empty", - TableProperties::PROPERTY_WRITE_METADATA_PATH - ), - )); + return Err(Error::new(ErrorKind::DataInvalid, "path must not be empty")); } Ok(strip_trailing_slash(path).to_string()) @@ -54,9 +101,9 @@ pub(crate) fn parse_metadata_file_compression( properties: &HashMap, ) -> Result { let value = properties - .get(TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC) + .get(METADATA_COMPRESSION_CODEC) .map(|s| s.as_str()) - .unwrap_or(TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC_DEFAULT); + .unwrap_or(METADATA_COMPRESSION_CODEC_DEFAULT); parse_metadata_compression(value) } @@ -150,50 +197,50 @@ fn parse_parquet_compression( pub struct TableProperties { /// The number of times to retry a commit. #[property( - key = Self::PROPERTY_COMMIT_NUM_RETRIES, - default = Self::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT, + key = COMMIT_NUM_RETRIES, + default = COMMIT_NUM_RETRIES_DEFAULT, getter )] pub commit_num_retries: usize, /// The minimum wait time between retries. #[property( - key = Self::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS, - default = Self::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT, + key = COMMIT_MIN_RETRY_WAIT_MS, + default = COMMIT_MIN_RETRY_WAIT_MS_DEFAULT, getter )] pub commit_min_retry_wait_ms: u64, /// The maximum wait time between retries. #[property( - key = Self::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS, - default = Self::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT, + key = COMMIT_MAX_RETRY_WAIT_MS, + default = COMMIT_MAX_RETRY_WAIT_MS_DEFAULT, getter )] pub commit_max_retry_wait_ms: u64, /// The total timeout for commit retries. #[property( - key = Self::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS, - default = Self::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT, + key = COMMIT_TOTAL_RETRY_TIME_MS, + default = COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT, getter )] pub commit_total_retry_timeout_ms: u64, /// The default format for files. #[property( - key = Self::PROPERTY_DEFAULT_FILE_FORMAT, - default = Self::PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT, + key = DEFAULT_FILE_FORMAT, + default = DEFAULT_FILE_FORMAT_DEFAULT, getter )] pub write_format_default: String, /// The target file size for files. #[property( - key = Self::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES, - default = Self::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT, + key = WRITE_TARGET_FILE_SIZE_BYTES, + default = WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT, getter )] pub write_target_file_size_bytes: usize, /// Base directory for metadata files (manifests, manifest lists), with any /// trailing slash trimmed. `None` if `write.metadata.path` is not set. #[property( - key = Self::PROPERTY_WRITE_METADATA_PATH, + key = WRITE_METADATA_PATH, default = None, parse_with = parse_location_property, getter @@ -201,7 +248,7 @@ pub struct TableProperties { pub write_metadata_path: Option, /// Compression codec for metadata files (JSON) #[property( - key = Self::PROPERTY_METADATA_COMPRESSION_CODEC, + key = METADATA_COMPRESSION_CODEC, default = CompressionCodec::None, parse_with = parse_metadata_compression, getter @@ -209,66 +256,66 @@ pub struct TableProperties { pub metadata_compression_codec: CompressionCodec, /// Whether to use `FanoutWriter` for partitioned tables. #[property( - key = Self::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED, - default = Self::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT, + key = DATAFUSION_WRITE_FANOUT_ENABLED, + default = DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT, getter )] pub write_datafusion_fanout_enabled: bool, /// Whether garbage collection is enabled on drop. /// When `false`, data files will not be deleted when a table is dropped. #[property( - key = Self::PROPERTY_GC_ENABLED, - default = Self::PROPERTY_GC_ENABLED_DEFAULT, + key = GC_ENABLED, + default = GC_ENABLED_DEFAULT, getter )] pub gc_enabled: bool, /// Default maximum age of a snapshot to keep when expiring snapshots. #[property( - key = Self::PROPERTY_MAX_SNAPSHOT_AGE_MS, - default = Self::PROPERTY_MAX_SNAPSHOT_AGE_MS_DEFAULT, + key = MAX_SNAPSHOT_AGE_MS, + default = MAX_SNAPSHOT_AGE_MS_DEFAULT, getter )] pub max_snapshot_age_ms: i64, /// Default minimum number of snapshots to keep per branch when expiring snapshots. #[property( - key = Self::PROPERTY_MIN_SNAPSHOTS_TO_KEEP, - default = Self::PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT, + key = MIN_SNAPSHOTS_TO_KEEP, + default = MIN_SNAPSHOTS_TO_KEEP_DEFAULT, getter )] pub min_snapshots_to_keep: usize, /// Default maximum age of a snapshot reference to keep when expiring snapshots. #[property( - key = Self::PROPERTY_MAX_REF_AGE_MS, - default = Self::PROPERTY_MAX_REF_AGE_MS_DEFAULT, + key = MAX_REF_AGE_MS, + default = MAX_REF_AGE_MS_DEFAULT, getter )] pub max_ref_age_ms: i64, /// Whether content-defined chunking is enabled. /// `true` only when `write.parquet.content-defined-chunking.enabled = "true"`. #[property( - key = Self::PROPERTY_PARQUET_CDC_ENABLED, - default = Self::PROPERTY_PARQUET_CDC_ENABLED_DEFAULT, + key = PARQUET_CDC_ENABLED, + default = PARQUET_CDC_ENABLED_DEFAULT, getter )] pub cdc_enabled: bool, /// Content-defined chunking minimum chunk size in bytes. #[property( - key = Self::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE, - default = Self::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT, + key = PARQUET_CDC_MIN_CHUNK_SIZE, + default = PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT, getter )] pub cdc_min_chunk_size: usize, /// Content-defined chunking maximum chunk size in bytes. #[property( - key = Self::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE, - default = Self::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE_DEFAULT, + key = PARQUET_CDC_MAX_CHUNK_SIZE, + default = PARQUET_CDC_MAX_CHUNK_SIZE_DEFAULT, getter )] pub cdc_max_chunk_size: usize, /// Content-defined chunking normalization level (gearhash bit adjustment). #[property( - key = Self::PROPERTY_PARQUET_CDC_NORM_LEVEL, - default = Self::PROPERTY_PARQUET_CDC_NORM_LEVEL_DEFAULT, + key = PARQUET_CDC_NORM_LEVEL, + default = PARQUET_CDC_NORM_LEVEL_DEFAULT, getter )] pub cdc_norm_level: i32, @@ -276,8 +323,8 @@ pub struct TableProperties { /// level folded in (from `write.parquet.compression-level`, or the codec's /// default when unset). #[property( - key = Self::PROPERTY_PARQUET_COMPRESSION_CODEC, - additional_keys = [Self::PROPERTY_PARQUET_COMPRESSION_LEVEL], + key = PARQUET_COMPRESSION_CODEC, + additional_keys = [PARQUET_COMPRESSION_LEVEL], default = CompressionCodec::zstd_default(), parse_properties_with = parse_parquet_compression, getter @@ -285,78 +332,82 @@ pub struct TableProperties { pub parquet_compression_codec: CompressionCodec, /// Approximate maximum Parquet row group size in bytes. #[property( - key = Self::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES, - default = Self::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT, + key = PARQUET_ROW_GROUP_SIZE_BYTES, + default = PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT, getter )] pub parquet_row_group_size_bytes: usize, /// Approximate maximum Parquet data page size in bytes. #[property( - key = Self::PROPERTY_PARQUET_PAGE_SIZE_BYTES, - default = Self::PROPERTY_PARQUET_PAGE_SIZE_BYTES_DEFAULT, + key = PARQUET_PAGE_SIZE_BYTES, + default = PARQUET_PAGE_SIZE_BYTES_DEFAULT, getter )] pub parquet_page_size_bytes: usize, /// Maximum number of rows per Parquet data page. #[property( - key = Self::PROPERTY_PARQUET_PAGE_ROW_LIMIT, - default = Self::PROPERTY_PARQUET_PAGE_ROW_LIMIT_DEFAULT, + key = PARQUET_PAGE_ROW_LIMIT, + default = PARQUET_PAGE_ROW_LIMIT_DEFAULT, getter )] pub parquet_page_row_limit: usize, /// Approximate maximum Parquet dictionary page size in bytes. #[property( - key = Self::PROPERTY_PARQUET_DICT_SIZE_BYTES, - default = Self::PROPERTY_PARQUET_DICT_SIZE_BYTES_DEFAULT, + key = PARQUET_DICT_SIZE_BYTES, + default = PARQUET_DICT_SIZE_BYTES_DEFAULT, getter )] pub parquet_dict_size_bytes: usize, /// The master key id used to encrypt this table's manifest list and data /// files. `None` if `encryption.key-id` is not set. #[property( - key = Self::PROPERTY_ENCRYPTION_KEY_ID, + key = ENCRYPTION_KEY_ID, default = None, getter )] pub encryption_key_id: Option, /// The encryption data encryption key length in bytes. #[property( - key = Self::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH, - default = Self::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH_DEFAULT, + key = ENCRYPTION_DATA_KEY_LENGTH, + default = ENCRYPTION_DATA_KEY_LENGTH_DEFAULT, getter )] pub encryption_data_key_length: usize, - /// Base directory for data files + /// Base directory for data files, with any trailing slash trimmed. #[property( - key = Self::PROPERTY_WRITE_DATA_LOCATION, + key = WRITE_DATA_LOCATION, default = None, + parse_with = parse_location_property, getter )] pub write_data_location: Option, - /// Deprecated table property for data file write location. + /// Deprecated table property for data file write location, with any trailing slash trimmed. /// /// Property will be removed at a later date. /// Superseded by [write_data_location]. #[property( - key = Self::PROPERTY_WRITE_FOLDER_STORAGE_LOCATION, + key = WRITE_FOLDER_STORAGE_LOCATION, default = None, + parse_with = parse_location_property, getter )] pub write_folder_storage_location: Option, - /// Deprecated table property for data file write location for object storage location generator. + /// Deprecated table property for data file write location for object storage location generator, + /// with any trailing slash trimmed. /// /// Property will be removed at a later date. /// Superseded by [write_data_location]. #[property( - key = Self::PROPERTY_WRITE_OBJECT_STORAGE_LOCATION, + key = WRITE_OBJECT_STORAGE_LOCATION, default = None, + parse_with = parse_location_property, getter )] pub write_object_storage_location: Option, /// Whether partition values are included in object storage paths. #[property( - key = Self::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS, - default = Self::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS_DEFAULT, + key = WRITE_OBJECT_STORAGE_PARTITIONED_PATHS, + default = WRITE_OBJECT_STORAGE_PARTITIONED_PATHS_DEFAULT, getter )] pub write_object_storage_partitioned_paths: bool, @@ -418,142 +469,147 @@ impl TableProperties { ]; /// Property key for number of commit retries. - pub const PROPERTY_COMMIT_NUM_RETRIES: &str = "commit.retry.num-retries"; + pub const PROPERTY_COMMIT_NUM_RETRIES: &str = COMMIT_NUM_RETRIES; /// Default value for number of commit retries. - pub const PROPERTY_COMMIT_NUM_RETRIES_DEFAULT: usize = 4; + pub const PROPERTY_COMMIT_NUM_RETRIES_DEFAULT: usize = COMMIT_NUM_RETRIES_DEFAULT; /// Property key for minimum wait time (ms) between retries. - pub const PROPERTY_COMMIT_MIN_RETRY_WAIT_MS: &str = "commit.retry.min-wait-ms"; + pub const PROPERTY_COMMIT_MIN_RETRY_WAIT_MS: &str = COMMIT_MIN_RETRY_WAIT_MS; /// Default value for minimum wait time (ms) between retries. - pub const PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT: u64 = 100; + pub const PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT: u64 = COMMIT_MIN_RETRY_WAIT_MS_DEFAULT; /// Property key for maximum wait time (ms) between retries. - pub const PROPERTY_COMMIT_MAX_RETRY_WAIT_MS: &str = "commit.retry.max-wait-ms"; + pub const PROPERTY_COMMIT_MAX_RETRY_WAIT_MS: &str = COMMIT_MAX_RETRY_WAIT_MS; /// Default value for maximum wait time (ms) between retries. - pub const PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT: u64 = 60 * 1000; // 1 minute + pub const PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT: u64 = COMMIT_MAX_RETRY_WAIT_MS_DEFAULT; /// Property key for total maximum retry time (ms). - pub const PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS: &str = "commit.retry.total-timeout-ms"; + pub const PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS: &str = COMMIT_TOTAL_RETRY_TIME_MS; /// Default value for total maximum retry time (ms). - pub const PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT: u64 = 30 * 60 * 1000; // 30 minutes + pub const PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT: u64 = COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT; /// Default file format for data files - pub const PROPERTY_DEFAULT_FILE_FORMAT: &str = "write.format.default"; + pub const PROPERTY_DEFAULT_FILE_FORMAT: &str = DEFAULT_FILE_FORMAT; /// Default file format for delete files pub const PROPERTY_DELETE_DEFAULT_FILE_FORMAT: &str = "write.delete.format.default"; /// Default value for data file format - pub const PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT: &str = "parquet"; + pub const PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT: &str = DEFAULT_FILE_FORMAT_DEFAULT; /// Target file size for newly written files. - pub const PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES: &str = "write.target-file-size-bytes"; + pub const PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES: &str = WRITE_TARGET_FILE_SIZE_BYTES; /// Default target file size - pub const PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT: usize = 512 * 1024 * 1024; // 512 MB + pub const PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT: usize = + WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT; /// Base location for metadata files (manifests, manifest lists, table metadata). /// When unset, metadata files default to the `metadata` directory under the table /// location. - pub const PROPERTY_WRITE_METADATA_PATH: &str = "write.metadata.path"; + pub const PROPERTY_WRITE_METADATA_PATH: &str = WRITE_METADATA_PATH; /// Compression codec for metadata files (JSON) - pub const PROPERTY_METADATA_COMPRESSION_CODEC: &str = "write.metadata.compression-codec"; + pub const PROPERTY_METADATA_COMPRESSION_CODEC: &str = METADATA_COMPRESSION_CODEC; /// Default metadata compression codec - uncompressed - pub const PROPERTY_METADATA_COMPRESSION_CODEC_DEFAULT: &str = "none"; + pub const PROPERTY_METADATA_COMPRESSION_CODEC_DEFAULT: &str = + METADATA_COMPRESSION_CODEC_DEFAULT; /// Whether to use `FanoutWriter` for partitioned tables (handles unsorted data). /// If false, uses `ClusteredWriter` (requires sorted data, more memory efficient). - pub const PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED: &str = "write.datafusion.fanout.enabled"; + pub const PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED: &str = DATAFUSION_WRITE_FANOUT_ENABLED; /// Default value for fanout writer enabled - pub const PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT: bool = true; + pub const PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT: bool = + DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT; /// Property key for enabling garbage collection on drop. /// When set to `false`, data files will not be deleted when a table is dropped. /// Defaults to `true`. - pub const PROPERTY_GC_ENABLED: &str = "gc.enabled"; + pub const PROPERTY_GC_ENABLED: &str = GC_ENABLED; /// Default value for gc.enabled - pub const PROPERTY_GC_ENABLED_DEFAULT: bool = true; + pub const PROPERTY_GC_ENABLED_DEFAULT: bool = GC_ENABLED_DEFAULT; /// Property key for the default maximum age of a snapshot to keep when expiring snapshots. - pub const PROPERTY_MAX_SNAPSHOT_AGE_MS: &str = "history.expire.max-snapshot-age-ms"; + pub const PROPERTY_MAX_SNAPSHOT_AGE_MS: &str = MAX_SNAPSHOT_AGE_MS; /// Default value for history.expire.max-snapshot-age-ms (5 days). - pub const PROPERTY_MAX_SNAPSHOT_AGE_MS_DEFAULT: i64 = 5 * 24 * 60 * 60 * 1000; + pub const PROPERTY_MAX_SNAPSHOT_AGE_MS_DEFAULT: i64 = MAX_SNAPSHOT_AGE_MS_DEFAULT; /// Property key for the default minimum number of snapshots to keep when expiring snapshots. - pub const PROPERTY_MIN_SNAPSHOTS_TO_KEEP: &str = "history.expire.min-snapshots-to-keep"; + pub const PROPERTY_MIN_SNAPSHOTS_TO_KEEP: &str = MIN_SNAPSHOTS_TO_KEEP; /// Default value for history.expire.min-snapshots-to-keep. - pub const PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT: usize = 1; + pub const PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT: usize = MIN_SNAPSHOTS_TO_KEEP_DEFAULT; /// Property key for the default maximum age of a snapshot reference to keep when expiring. - pub const PROPERTY_MAX_REF_AGE_MS: &str = "history.expire.max-ref-age-ms"; + pub const PROPERTY_MAX_REF_AGE_MS: &str = MAX_REF_AGE_MS; /// Default value for history.expire.max-ref-age-ms (effectively never expire refs). - pub const PROPERTY_MAX_REF_AGE_MS_DEFAULT: i64 = i64::MAX; + pub const PROPERTY_MAX_REF_AGE_MS_DEFAULT: i64 = MAX_REF_AGE_MS_DEFAULT; /// Enable content-defined chunking with parquet defaults (or per-property overrides). - pub const PROPERTY_PARQUET_CDC_ENABLED: &str = "write.parquet.content-defined-chunking.enabled"; + pub const PROPERTY_PARQUET_CDC_ENABLED: &str = PARQUET_CDC_ENABLED; /// Default value for content-defined chunking enabled. - pub const PROPERTY_PARQUET_CDC_ENABLED_DEFAULT: bool = false; + pub const PROPERTY_PARQUET_CDC_ENABLED_DEFAULT: bool = PARQUET_CDC_ENABLED_DEFAULT; /// Minimum chunk size in bytes for content-defined chunking. - pub const PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE: &str = - "write.parquet.content-defined-chunking.min-chunk-size"; + pub const PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE: &str = PARQUET_CDC_MIN_CHUNK_SIZE; /// Default matches `parquet::file::properties::DEFAULT_CDC_MIN_CHUNK_SIZE`. - pub const PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT: usize = 256 * 1024; + pub const PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT: usize = + PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT; /// Maximum chunk size in bytes for content-defined chunking. - pub const PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE: &str = - "write.parquet.content-defined-chunking.max-chunk-size"; + pub const PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE: &str = PARQUET_CDC_MAX_CHUNK_SIZE; /// Default matches `parquet::file::properties::DEFAULT_CDC_MAX_CHUNK_SIZE`. - pub const PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE_DEFAULT: usize = 1024 * 1024; + pub const PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE_DEFAULT: usize = + PARQUET_CDC_MAX_CHUNK_SIZE_DEFAULT; /// Normalization level (gearhash bit adjustment) for content-defined chunking. - pub const PROPERTY_PARQUET_CDC_NORM_LEVEL: &str = - "write.parquet.content-defined-chunking.norm-level"; + pub const PROPERTY_PARQUET_CDC_NORM_LEVEL: &str = PARQUET_CDC_NORM_LEVEL; /// Default matches `parquet::file::properties::DEFAULT_CDC_NORM_LEVEL`. - pub const PROPERTY_PARQUET_CDC_NORM_LEVEL_DEFAULT: i32 = 0; + pub const PROPERTY_PARQUET_CDC_NORM_LEVEL_DEFAULT: i32 = PARQUET_CDC_NORM_LEVEL_DEFAULT; /// Compression codec for Parquet data files (e.g. `zstd`, `gzip`, `snappy`, /// `lz4`, `lz4_raw`, `brotli`, `lzo`, `uncompressed`). The codec name is /// parsed into a [`CompressionCodec`] when properties are parsed; the level's /// range is validated when the writer is built. - pub const PROPERTY_PARQUET_COMPRESSION_CODEC: &str = "write.parquet.compression-codec"; + pub const PROPERTY_PARQUET_COMPRESSION_CODEC: &str = PARQUET_COMPRESSION_CODEC; /// Default Parquet compression codec. - pub const PROPERTY_PARQUET_COMPRESSION_CODEC_DEFAULT: &str = "zstd"; + pub const PROPERTY_PARQUET_COMPRESSION_CODEC_DEFAULT: &str = PARQUET_COMPRESSION_CODEC_DEFAULT; /// Compression level for Parquet data files, for codecs that take one /// (`gzip`, `zstd`, `brotli`). When unset, the codec's default level is used. - pub const PROPERTY_PARQUET_COMPRESSION_LEVEL: &str = "write.parquet.compression-level"; + pub const PROPERTY_PARQUET_COMPRESSION_LEVEL: &str = PARQUET_COMPRESSION_LEVEL; /// Approximate maximum size of a Parquet row group in bytes. - pub const PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES: &str = "write.parquet.row-group-size-bytes"; + pub const PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES: &str = PARQUET_ROW_GROUP_SIZE_BYTES; /// Default Parquet row group size in bytes. - pub const PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT: usize = 128 * 1024 * 1024; + pub const PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT: usize = + PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT; /// Approximate maximum size of a Parquet data page in bytes. - pub const PROPERTY_PARQUET_PAGE_SIZE_BYTES: &str = "write.parquet.page-size-bytes"; + pub const PROPERTY_PARQUET_PAGE_SIZE_BYTES: &str = PARQUET_PAGE_SIZE_BYTES; /// Default Parquet page size in bytes. - pub const PROPERTY_PARQUET_PAGE_SIZE_BYTES_DEFAULT: usize = 1024 * 1024; + pub const PROPERTY_PARQUET_PAGE_SIZE_BYTES_DEFAULT: usize = PARQUET_PAGE_SIZE_BYTES_DEFAULT; /// Maximum number of rows per Parquet data page. - pub const PROPERTY_PARQUET_PAGE_ROW_LIMIT: &str = "write.parquet.page-row-limit"; + pub const PROPERTY_PARQUET_PAGE_ROW_LIMIT: &str = PARQUET_PAGE_ROW_LIMIT; /// Default Parquet page row limit. - pub const PROPERTY_PARQUET_PAGE_ROW_LIMIT_DEFAULT: usize = 20000; + pub const PROPERTY_PARQUET_PAGE_ROW_LIMIT_DEFAULT: usize = PARQUET_PAGE_ROW_LIMIT_DEFAULT; /// Approximate maximum size of the Parquet dictionary page in bytes. - pub const PROPERTY_PARQUET_DICT_SIZE_BYTES: &str = "write.parquet.dict-size-bytes"; + pub const PROPERTY_PARQUET_DICT_SIZE_BYTES: &str = PARQUET_DICT_SIZE_BYTES; /// Default Parquet dictionary page size in bytes. - pub const PROPERTY_PARQUET_DICT_SIZE_BYTES_DEFAULT: usize = 2 * 1024 * 1024; + pub const PROPERTY_PARQUET_DICT_SIZE_BYTES_DEFAULT: usize = PARQUET_DICT_SIZE_BYTES_DEFAULT; /// Property key for the master key id used to encrypt the table's manifest /// list and data files as defined in https://iceberg.apache.org/docs/nightly/encryption/. - pub const PROPERTY_ENCRYPTION_KEY_ID: &str = "encryption.key-id"; + pub const PROPERTY_ENCRYPTION_KEY_ID: &str = ENCRYPTION_KEY_ID; /// Property key for the encryption data encryption key (DEK) length in bytes. - pub const PROPERTY_ENCRYPTION_DATA_KEY_LENGTH: &str = "encryption.data-key-length"; + pub const PROPERTY_ENCRYPTION_DATA_KEY_LENGTH: &str = ENCRYPTION_DATA_KEY_LENGTH; /// Default value for the encryption DEK length (16 bytes = AES-128). - pub const PROPERTY_ENCRYPTION_DATA_KEY_LENGTH_DEFAULT: usize = 16; + pub const PROPERTY_ENCRYPTION_DATA_KEY_LENGTH_DEFAULT: usize = + ENCRYPTION_DATA_KEY_LENGTH_DEFAULT; /// Property key for the base directory for data files - pub const PROPERTY_WRITE_DATA_LOCATION: &str = "write.data.path"; + pub const PROPERTY_WRITE_DATA_LOCATION: &str = WRITE_DATA_LOCATION; /// Property key for deprecated [write_folder_storage_location] - pub const PROPERTY_WRITE_FOLDER_STORAGE_LOCATION: &str = "write.folder-storage.path"; + pub const PROPERTY_WRITE_FOLDER_STORAGE_LOCATION: &str = WRITE_FOLDER_STORAGE_LOCATION; /// Property key for deprecated object storage path, kept as a fallback for compatibility. - pub const PROPERTY_WRITE_OBJECT_STORAGE_LOCATION: &str = "write.object-storage.path"; + pub const PROPERTY_WRITE_OBJECT_STORAGE_LOCATION: &str = WRITE_OBJECT_STORAGE_LOCATION; /// Property key for controlling whether partition values are included in object storage paths. pub const PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS: &str = - "write.object-storage.partitioned-paths"; + WRITE_OBJECT_STORAGE_PARTITIONED_PATHS; /// Default value for [PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS] - pub const PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS_DEFAULT: bool = true; + pub const PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS_DEFAULT: bool = + WRITE_OBJECT_STORAGE_PARTITIONED_PATHS_DEFAULT; /// The AES key size to use when generating data encryption keys, derived /// from `encryption.data-key-length`. @@ -645,33 +701,50 @@ mod tests { } #[test] - fn test_table_properties_write_metadata_path() { - // Test unset + fn test_table_properties_location_paths() { + // Test unset. let table_properties = TableProperties::try_from(&HashMap::new()).unwrap(); assert_eq!(table_properties.write_metadata_path, None); - - // Test empty path is invalid - let props = HashMap::from([( - TableProperties::PROPERTY_WRITE_METADATA_PATH.to_string(), - String::new(), - )]); - let error = TableProperties::try_from(&props).unwrap_err(); - assert_eq!(error.kind(), ErrorKind::DataInvalid); - assert!( - error - .message() - .contains(TableProperties::PROPERTY_WRITE_METADATA_PATH) - ); - - let props = HashMap::from([( - TableProperties::PROPERTY_WRITE_METADATA_PATH.to_string(), - "s3://other-bucket/custom-meta/".to_string(), - )]); - let table_properties = TableProperties::try_from(&props).unwrap(); - assert_eq!( - table_properties.write_metadata_path.as_deref(), - Some("s3://other-bucket/custom-meta") - ); + assert_eq!(table_properties.write_data_location, None); + assert_eq!(table_properties.write_folder_storage_location, None); + assert_eq!(table_properties.write_object_storage_location, None); + + for key in [ + TableProperties::PROPERTY_WRITE_METADATA_PATH, + TableProperties::PROPERTY_WRITE_DATA_LOCATION, + TableProperties::PROPERTY_WRITE_FOLDER_STORAGE_LOCATION, + TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_LOCATION, + ] { + // Test empty paths are invalid and retain the property key as error context. + let error = + TableProperties::try_from(&HashMap::from([(key.to_string(), String::new())])) + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::DataInvalid); + assert!(format!("{error}").contains(key)); + + // Test all supported location properties share trailing-slash normalization. + let table_properties = TableProperties::try_from(&HashMap::from([( + key.to_string(), + "s3://other-bucket/custom-path/".to_string(), + )])) + .unwrap(); + let parsed = match key { + TableProperties::PROPERTY_WRITE_METADATA_PATH => { + table_properties.write_metadata_path.as_deref() + } + TableProperties::PROPERTY_WRITE_DATA_LOCATION => { + table_properties.write_data_location.as_deref() + } + TableProperties::PROPERTY_WRITE_FOLDER_STORAGE_LOCATION => { + table_properties.write_folder_storage_location.as_deref() + } + TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_LOCATION => { + table_properties.write_object_storage_location.as_deref() + } + _ => unreachable!(), + }; + assert_eq!(parsed, Some("s3://other-bucket/custom-path")); + } } #[test] diff --git a/crates/property-macro/Cargo.toml b/crates/property-macro/Cargo.toml index 700b2ecf44..cc2e4edd6b 100644 --- a/crates/property-macro/Cargo.toml +++ b/crates/property-macro/Cargo.toml @@ -19,7 +19,7 @@ edition = { workspace = true } homepage = { workspace = true } name = "iceberg-property-macro" -publish = false +publish = true readme = "README.md" rust-version = { workspace = true } version = { workspace = true } diff --git a/crates/property-macro/public-api.txt b/crates/property-macro/public-api.txt new file mode 100644 index 0000000000..8edb5d0952 --- /dev/null +++ b/crates/property-macro/public-api.txt @@ -0,0 +1,2 @@ +pub mod iceberg_property_macro +pub proc macro iceberg_property_macro::#[derive(Properties)] From 80ad493df51f9e2dc2c336d25111789fa6d28a32 Mon Sep 17 00:00:00 2001 From: Ray Liu <257669749+blackmwk@users.noreply.github.com> Date: Wed, 19 Aug 2026 18:00:22 +0800 Subject: [PATCH 3/3] fix: Keep TableProperties constants associated --- crates/iceberg/src/spec/table_properties.rs | 272 ++++++++------------ 1 file changed, 107 insertions(+), 165 deletions(-) diff --git a/crates/iceberg/src/spec/table_properties.rs b/crates/iceberg/src/spec/table_properties.rs index 6c92152a31..0a9393f12d 100644 --- a/crates/iceberg/src/spec/table_properties.rs +++ b/crates/iceberg/src/spec/table_properties.rs @@ -24,59 +24,6 @@ use crate::encryption::AesKeySize; use crate::error::{Error, ErrorKind, Result}; use crate::util::location::strip_trailing_slash; -const COMMIT_NUM_RETRIES: &str = "commit.retry.num-retries"; -const COMMIT_NUM_RETRIES_DEFAULT: usize = 4; -const COMMIT_MIN_RETRY_WAIT_MS: &str = "commit.retry.min-wait-ms"; -const COMMIT_MIN_RETRY_WAIT_MS_DEFAULT: u64 = 100; -const COMMIT_MAX_RETRY_WAIT_MS: &str = "commit.retry.max-wait-ms"; -const COMMIT_MAX_RETRY_WAIT_MS_DEFAULT: u64 = 60 * 1000; -const COMMIT_TOTAL_RETRY_TIME_MS: &str = "commit.retry.total-timeout-ms"; -const COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT: u64 = 30 * 60 * 1000; -const DEFAULT_FILE_FORMAT: &str = "write.format.default"; -const DEFAULT_FILE_FORMAT_DEFAULT: &str = "parquet"; -const WRITE_TARGET_FILE_SIZE_BYTES: &str = "write.target-file-size-bytes"; -const WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT: usize = 512 * 1024 * 1024; -const WRITE_METADATA_PATH: &str = "write.metadata.path"; -const METADATA_COMPRESSION_CODEC: &str = "write.metadata.compression-codec"; -const METADATA_COMPRESSION_CODEC_DEFAULT: &str = "none"; -const DATAFUSION_WRITE_FANOUT_ENABLED: &str = "write.datafusion.fanout.enabled"; -const DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT: bool = true; -const GC_ENABLED: &str = "gc.enabled"; -const GC_ENABLED_DEFAULT: bool = true; -const MAX_SNAPSHOT_AGE_MS: &str = "history.expire.max-snapshot-age-ms"; -const MAX_SNAPSHOT_AGE_MS_DEFAULT: i64 = 5 * 24 * 60 * 60 * 1000; -const MIN_SNAPSHOTS_TO_KEEP: &str = "history.expire.min-snapshots-to-keep"; -const MIN_SNAPSHOTS_TO_KEEP_DEFAULT: usize = 1; -const MAX_REF_AGE_MS: &str = "history.expire.max-ref-age-ms"; -const MAX_REF_AGE_MS_DEFAULT: i64 = i64::MAX; -const PARQUET_CDC_ENABLED: &str = "write.parquet.content-defined-chunking.enabled"; -const PARQUET_CDC_ENABLED_DEFAULT: bool = false; -const PARQUET_CDC_MIN_CHUNK_SIZE: &str = "write.parquet.content-defined-chunking.min-chunk-size"; -const PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT: usize = 256 * 1024; -const PARQUET_CDC_MAX_CHUNK_SIZE: &str = "write.parquet.content-defined-chunking.max-chunk-size"; -const PARQUET_CDC_MAX_CHUNK_SIZE_DEFAULT: usize = 1024 * 1024; -const PARQUET_CDC_NORM_LEVEL: &str = "write.parquet.content-defined-chunking.norm-level"; -const PARQUET_CDC_NORM_LEVEL_DEFAULT: i32 = 0; -const PARQUET_COMPRESSION_CODEC: &str = "write.parquet.compression-codec"; -const PARQUET_COMPRESSION_CODEC_DEFAULT: &str = "zstd"; -const PARQUET_COMPRESSION_LEVEL: &str = "write.parquet.compression-level"; -const PARQUET_ROW_GROUP_SIZE_BYTES: &str = "write.parquet.row-group-size-bytes"; -const PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT: usize = 128 * 1024 * 1024; -const PARQUET_PAGE_SIZE_BYTES: &str = "write.parquet.page-size-bytes"; -const PARQUET_PAGE_SIZE_BYTES_DEFAULT: usize = 1024 * 1024; -const PARQUET_PAGE_ROW_LIMIT: &str = "write.parquet.page-row-limit"; -const PARQUET_PAGE_ROW_LIMIT_DEFAULT: usize = 20000; -const PARQUET_DICT_SIZE_BYTES: &str = "write.parquet.dict-size-bytes"; -const PARQUET_DICT_SIZE_BYTES_DEFAULT: usize = 2 * 1024 * 1024; -const ENCRYPTION_KEY_ID: &str = "encryption.key-id"; -const ENCRYPTION_DATA_KEY_LENGTH: &str = "encryption.data-key-length"; -const ENCRYPTION_DATA_KEY_LENGTH_DEFAULT: usize = 16; -const WRITE_DATA_LOCATION: &str = "write.data.path"; -const WRITE_FOLDER_STORAGE_LOCATION: &str = "write.folder-storage.path"; -const WRITE_OBJECT_STORAGE_LOCATION: &str = "write.object-storage.path"; -const WRITE_OBJECT_STORAGE_PARTITIONED_PATHS: &str = "write.object-storage.partitioned-paths"; -const WRITE_OBJECT_STORAGE_PARTITIONED_PATHS_DEFAULT: bool = true; - fn parse_location_property(path: &str) -> Result { if path.is_empty() { return Err(Error::new(ErrorKind::DataInvalid, "path must not be empty")); @@ -101,9 +48,9 @@ pub(crate) fn parse_metadata_file_compression( properties: &HashMap, ) -> Result { let value = properties - .get(METADATA_COMPRESSION_CODEC) + .get(TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC) .map(|s| s.as_str()) - .unwrap_or(METADATA_COMPRESSION_CODEC_DEFAULT); + .unwrap_or(TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC_DEFAULT); parse_metadata_compression(value) } @@ -197,50 +144,50 @@ fn parse_parquet_compression( pub struct TableProperties { /// The number of times to retry a commit. #[property( - key = COMMIT_NUM_RETRIES, - default = COMMIT_NUM_RETRIES_DEFAULT, + key = Self::PROPERTY_COMMIT_NUM_RETRIES, + default = Self::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT, getter )] pub commit_num_retries: usize, /// The minimum wait time between retries. #[property( - key = COMMIT_MIN_RETRY_WAIT_MS, - default = COMMIT_MIN_RETRY_WAIT_MS_DEFAULT, + key = Self::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS, + default = Self::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT, getter )] pub commit_min_retry_wait_ms: u64, /// The maximum wait time between retries. #[property( - key = COMMIT_MAX_RETRY_WAIT_MS, - default = COMMIT_MAX_RETRY_WAIT_MS_DEFAULT, + key = Self::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS, + default = Self::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT, getter )] pub commit_max_retry_wait_ms: u64, /// The total timeout for commit retries. #[property( - key = COMMIT_TOTAL_RETRY_TIME_MS, - default = COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT, + key = Self::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS, + default = Self::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT, getter )] pub commit_total_retry_timeout_ms: u64, /// The default format for files. #[property( - key = DEFAULT_FILE_FORMAT, - default = DEFAULT_FILE_FORMAT_DEFAULT, + key = Self::PROPERTY_DEFAULT_FILE_FORMAT, + default = Self::PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT, getter )] pub write_format_default: String, /// The target file size for files. #[property( - key = WRITE_TARGET_FILE_SIZE_BYTES, - default = WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT, + key = Self::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES, + default = Self::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT, getter )] pub write_target_file_size_bytes: usize, /// Base directory for metadata files (manifests, manifest lists), with any /// trailing slash trimmed. `None` if `write.metadata.path` is not set. #[property( - key = WRITE_METADATA_PATH, + key = Self::PROPERTY_WRITE_METADATA_PATH, default = None, parse_with = parse_location_property, getter @@ -248,7 +195,7 @@ pub struct TableProperties { pub write_metadata_path: Option, /// Compression codec for metadata files (JSON) #[property( - key = METADATA_COMPRESSION_CODEC, + key = Self::PROPERTY_METADATA_COMPRESSION_CODEC, default = CompressionCodec::None, parse_with = parse_metadata_compression, getter @@ -256,66 +203,66 @@ pub struct TableProperties { pub metadata_compression_codec: CompressionCodec, /// Whether to use `FanoutWriter` for partitioned tables. #[property( - key = DATAFUSION_WRITE_FANOUT_ENABLED, - default = DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT, + key = Self::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED, + default = Self::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT, getter )] pub write_datafusion_fanout_enabled: bool, /// Whether garbage collection is enabled on drop. /// When `false`, data files will not be deleted when a table is dropped. #[property( - key = GC_ENABLED, - default = GC_ENABLED_DEFAULT, + key = Self::PROPERTY_GC_ENABLED, + default = Self::PROPERTY_GC_ENABLED_DEFAULT, getter )] pub gc_enabled: bool, /// Default maximum age of a snapshot to keep when expiring snapshots. #[property( - key = MAX_SNAPSHOT_AGE_MS, - default = MAX_SNAPSHOT_AGE_MS_DEFAULT, + key = Self::PROPERTY_MAX_SNAPSHOT_AGE_MS, + default = Self::PROPERTY_MAX_SNAPSHOT_AGE_MS_DEFAULT, getter )] pub max_snapshot_age_ms: i64, /// Default minimum number of snapshots to keep per branch when expiring snapshots. #[property( - key = MIN_SNAPSHOTS_TO_KEEP, - default = MIN_SNAPSHOTS_TO_KEEP_DEFAULT, + key = Self::PROPERTY_MIN_SNAPSHOTS_TO_KEEP, + default = Self::PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT, getter )] pub min_snapshots_to_keep: usize, /// Default maximum age of a snapshot reference to keep when expiring snapshots. #[property( - key = MAX_REF_AGE_MS, - default = MAX_REF_AGE_MS_DEFAULT, + key = Self::PROPERTY_MAX_REF_AGE_MS, + default = Self::PROPERTY_MAX_REF_AGE_MS_DEFAULT, getter )] pub max_ref_age_ms: i64, /// Whether content-defined chunking is enabled. /// `true` only when `write.parquet.content-defined-chunking.enabled = "true"`. #[property( - key = PARQUET_CDC_ENABLED, - default = PARQUET_CDC_ENABLED_DEFAULT, + key = Self::PROPERTY_PARQUET_CDC_ENABLED, + default = Self::PROPERTY_PARQUET_CDC_ENABLED_DEFAULT, getter )] pub cdc_enabled: bool, /// Content-defined chunking minimum chunk size in bytes. #[property( - key = PARQUET_CDC_MIN_CHUNK_SIZE, - default = PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT, + key = Self::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE, + default = Self::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT, getter )] pub cdc_min_chunk_size: usize, /// Content-defined chunking maximum chunk size in bytes. #[property( - key = PARQUET_CDC_MAX_CHUNK_SIZE, - default = PARQUET_CDC_MAX_CHUNK_SIZE_DEFAULT, + key = Self::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE, + default = Self::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE_DEFAULT, getter )] pub cdc_max_chunk_size: usize, /// Content-defined chunking normalization level (gearhash bit adjustment). #[property( - key = PARQUET_CDC_NORM_LEVEL, - default = PARQUET_CDC_NORM_LEVEL_DEFAULT, + key = Self::PROPERTY_PARQUET_CDC_NORM_LEVEL, + default = Self::PROPERTY_PARQUET_CDC_NORM_LEVEL_DEFAULT, getter )] pub cdc_norm_level: i32, @@ -323,8 +270,8 @@ pub struct TableProperties { /// level folded in (from `write.parquet.compression-level`, or the codec's /// default when unset). #[property( - key = PARQUET_COMPRESSION_CODEC, - additional_keys = [PARQUET_COMPRESSION_LEVEL], + key = Self::PROPERTY_PARQUET_COMPRESSION_CODEC, + additional_keys = [Self::PROPERTY_PARQUET_COMPRESSION_LEVEL], default = CompressionCodec::zstd_default(), parse_properties_with = parse_parquet_compression, getter @@ -332,50 +279,50 @@ pub struct TableProperties { pub parquet_compression_codec: CompressionCodec, /// Approximate maximum Parquet row group size in bytes. #[property( - key = PARQUET_ROW_GROUP_SIZE_BYTES, - default = PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT, + key = Self::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES, + default = Self::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT, getter )] pub parquet_row_group_size_bytes: usize, /// Approximate maximum Parquet data page size in bytes. #[property( - key = PARQUET_PAGE_SIZE_BYTES, - default = PARQUET_PAGE_SIZE_BYTES_DEFAULT, + key = Self::PROPERTY_PARQUET_PAGE_SIZE_BYTES, + default = Self::PROPERTY_PARQUET_PAGE_SIZE_BYTES_DEFAULT, getter )] pub parquet_page_size_bytes: usize, /// Maximum number of rows per Parquet data page. #[property( - key = PARQUET_PAGE_ROW_LIMIT, - default = PARQUET_PAGE_ROW_LIMIT_DEFAULT, + key = Self::PROPERTY_PARQUET_PAGE_ROW_LIMIT, + default = Self::PROPERTY_PARQUET_PAGE_ROW_LIMIT_DEFAULT, getter )] pub parquet_page_row_limit: usize, /// Approximate maximum Parquet dictionary page size in bytes. #[property( - key = PARQUET_DICT_SIZE_BYTES, - default = PARQUET_DICT_SIZE_BYTES_DEFAULT, + key = Self::PROPERTY_PARQUET_DICT_SIZE_BYTES, + default = Self::PROPERTY_PARQUET_DICT_SIZE_BYTES_DEFAULT, getter )] pub parquet_dict_size_bytes: usize, /// The master key id used to encrypt this table's manifest list and data /// files. `None` if `encryption.key-id` is not set. #[property( - key = ENCRYPTION_KEY_ID, + key = Self::PROPERTY_ENCRYPTION_KEY_ID, default = None, getter )] pub encryption_key_id: Option, /// The encryption data encryption key length in bytes. #[property( - key = ENCRYPTION_DATA_KEY_LENGTH, - default = ENCRYPTION_DATA_KEY_LENGTH_DEFAULT, + key = Self::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH, + default = Self::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH_DEFAULT, getter )] pub encryption_data_key_length: usize, /// Base directory for data files, with any trailing slash trimmed. #[property( - key = WRITE_DATA_LOCATION, + key = Self::PROPERTY_WRITE_DATA_LOCATION, default = None, parse_with = parse_location_property, getter @@ -386,7 +333,7 @@ pub struct TableProperties { /// Property will be removed at a later date. /// Superseded by [write_data_location]. #[property( - key = WRITE_FOLDER_STORAGE_LOCATION, + key = Self::PROPERTY_WRITE_FOLDER_STORAGE_LOCATION, default = None, parse_with = parse_location_property, getter @@ -398,7 +345,7 @@ pub struct TableProperties { /// Property will be removed at a later date. /// Superseded by [write_data_location]. #[property( - key = WRITE_OBJECT_STORAGE_LOCATION, + key = Self::PROPERTY_WRITE_OBJECT_STORAGE_LOCATION, default = None, parse_with = parse_location_property, getter @@ -406,8 +353,8 @@ pub struct TableProperties { pub write_object_storage_location: Option, /// Whether partition values are included in object storage paths. #[property( - key = WRITE_OBJECT_STORAGE_PARTITIONED_PATHS, - default = WRITE_OBJECT_STORAGE_PARTITIONED_PATHS_DEFAULT, + key = Self::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS, + default = Self::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS_DEFAULT, getter )] pub write_object_storage_partitioned_paths: bool, @@ -469,147 +416,142 @@ impl TableProperties { ]; /// Property key for number of commit retries. - pub const PROPERTY_COMMIT_NUM_RETRIES: &str = COMMIT_NUM_RETRIES; + pub const PROPERTY_COMMIT_NUM_RETRIES: &str = "commit.retry.num-retries"; /// Default value for number of commit retries. - pub const PROPERTY_COMMIT_NUM_RETRIES_DEFAULT: usize = COMMIT_NUM_RETRIES_DEFAULT; + pub const PROPERTY_COMMIT_NUM_RETRIES_DEFAULT: usize = 4; /// Property key for minimum wait time (ms) between retries. - pub const PROPERTY_COMMIT_MIN_RETRY_WAIT_MS: &str = COMMIT_MIN_RETRY_WAIT_MS; + pub const PROPERTY_COMMIT_MIN_RETRY_WAIT_MS: &str = "commit.retry.min-wait-ms"; /// Default value for minimum wait time (ms) between retries. - pub const PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT: u64 = COMMIT_MIN_RETRY_WAIT_MS_DEFAULT; + pub const PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT: u64 = 100; /// Property key for maximum wait time (ms) between retries. - pub const PROPERTY_COMMIT_MAX_RETRY_WAIT_MS: &str = COMMIT_MAX_RETRY_WAIT_MS; + pub const PROPERTY_COMMIT_MAX_RETRY_WAIT_MS: &str = "commit.retry.max-wait-ms"; /// Default value for maximum wait time (ms) between retries. - pub const PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT: u64 = COMMIT_MAX_RETRY_WAIT_MS_DEFAULT; + pub const PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT: u64 = 60 * 1000; // 1 minute /// Property key for total maximum retry time (ms). - pub const PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS: &str = COMMIT_TOTAL_RETRY_TIME_MS; + pub const PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS: &str = "commit.retry.total-timeout-ms"; /// Default value for total maximum retry time (ms). - pub const PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT: u64 = COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT; + pub const PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT: u64 = 30 * 60 * 1000; // 30 minutes /// Default file format for data files - pub const PROPERTY_DEFAULT_FILE_FORMAT: &str = DEFAULT_FILE_FORMAT; + pub const PROPERTY_DEFAULT_FILE_FORMAT: &str = "write.format.default"; /// Default file format for delete files pub const PROPERTY_DELETE_DEFAULT_FILE_FORMAT: &str = "write.delete.format.default"; /// Default value for data file format - pub const PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT: &str = DEFAULT_FILE_FORMAT_DEFAULT; + pub const PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT: &str = "parquet"; /// Target file size for newly written files. - pub const PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES: &str = WRITE_TARGET_FILE_SIZE_BYTES; + pub const PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES: &str = "write.target-file-size-bytes"; /// Default target file size - pub const PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT: usize = - WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT; + pub const PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT: usize = 512 * 1024 * 1024; // 512 MB /// Base location for metadata files (manifests, manifest lists, table metadata). /// When unset, metadata files default to the `metadata` directory under the table /// location. - pub const PROPERTY_WRITE_METADATA_PATH: &str = WRITE_METADATA_PATH; + pub const PROPERTY_WRITE_METADATA_PATH: &str = "write.metadata.path"; /// Compression codec for metadata files (JSON) - pub const PROPERTY_METADATA_COMPRESSION_CODEC: &str = METADATA_COMPRESSION_CODEC; + pub const PROPERTY_METADATA_COMPRESSION_CODEC: &str = "write.metadata.compression-codec"; /// Default metadata compression codec - uncompressed - pub const PROPERTY_METADATA_COMPRESSION_CODEC_DEFAULT: &str = - METADATA_COMPRESSION_CODEC_DEFAULT; + pub const PROPERTY_METADATA_COMPRESSION_CODEC_DEFAULT: &str = "none"; /// Whether to use `FanoutWriter` for partitioned tables (handles unsorted data). /// If false, uses `ClusteredWriter` (requires sorted data, more memory efficient). - pub const PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED: &str = DATAFUSION_WRITE_FANOUT_ENABLED; + pub const PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED: &str = "write.datafusion.fanout.enabled"; /// Default value for fanout writer enabled - pub const PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT: bool = - DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT; + pub const PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT: bool = true; /// Property key for enabling garbage collection on drop. /// When set to `false`, data files will not be deleted when a table is dropped. /// Defaults to `true`. - pub const PROPERTY_GC_ENABLED: &str = GC_ENABLED; + pub const PROPERTY_GC_ENABLED: &str = "gc.enabled"; /// Default value for gc.enabled - pub const PROPERTY_GC_ENABLED_DEFAULT: bool = GC_ENABLED_DEFAULT; + pub const PROPERTY_GC_ENABLED_DEFAULT: bool = true; /// Property key for the default maximum age of a snapshot to keep when expiring snapshots. - pub const PROPERTY_MAX_SNAPSHOT_AGE_MS: &str = MAX_SNAPSHOT_AGE_MS; + pub const PROPERTY_MAX_SNAPSHOT_AGE_MS: &str = "history.expire.max-snapshot-age-ms"; /// Default value for history.expire.max-snapshot-age-ms (5 days). - pub const PROPERTY_MAX_SNAPSHOT_AGE_MS_DEFAULT: i64 = MAX_SNAPSHOT_AGE_MS_DEFAULT; + pub const PROPERTY_MAX_SNAPSHOT_AGE_MS_DEFAULT: i64 = 5 * 24 * 60 * 60 * 1000; /// Property key for the default minimum number of snapshots to keep when expiring snapshots. - pub const PROPERTY_MIN_SNAPSHOTS_TO_KEEP: &str = MIN_SNAPSHOTS_TO_KEEP; + pub const PROPERTY_MIN_SNAPSHOTS_TO_KEEP: &str = "history.expire.min-snapshots-to-keep"; /// Default value for history.expire.min-snapshots-to-keep. - pub const PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT: usize = MIN_SNAPSHOTS_TO_KEEP_DEFAULT; + pub const PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT: usize = 1; /// Property key for the default maximum age of a snapshot reference to keep when expiring. - pub const PROPERTY_MAX_REF_AGE_MS: &str = MAX_REF_AGE_MS; + pub const PROPERTY_MAX_REF_AGE_MS: &str = "history.expire.max-ref-age-ms"; /// Default value for history.expire.max-ref-age-ms (effectively never expire refs). - pub const PROPERTY_MAX_REF_AGE_MS_DEFAULT: i64 = MAX_REF_AGE_MS_DEFAULT; + pub const PROPERTY_MAX_REF_AGE_MS_DEFAULT: i64 = i64::MAX; /// Enable content-defined chunking with parquet defaults (or per-property overrides). - pub const PROPERTY_PARQUET_CDC_ENABLED: &str = PARQUET_CDC_ENABLED; + pub const PROPERTY_PARQUET_CDC_ENABLED: &str = "write.parquet.content-defined-chunking.enabled"; /// Default value for content-defined chunking enabled. - pub const PROPERTY_PARQUET_CDC_ENABLED_DEFAULT: bool = PARQUET_CDC_ENABLED_DEFAULT; + pub const PROPERTY_PARQUET_CDC_ENABLED_DEFAULT: bool = false; /// Minimum chunk size in bytes for content-defined chunking. - pub const PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE: &str = PARQUET_CDC_MIN_CHUNK_SIZE; + pub const PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE: &str = + "write.parquet.content-defined-chunking.min-chunk-size"; /// Default matches `parquet::file::properties::DEFAULT_CDC_MIN_CHUNK_SIZE`. - pub const PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT: usize = - PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT; + pub const PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT: usize = 256 * 1024; /// Maximum chunk size in bytes for content-defined chunking. - pub const PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE: &str = PARQUET_CDC_MAX_CHUNK_SIZE; + pub const PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE: &str = + "write.parquet.content-defined-chunking.max-chunk-size"; /// Default matches `parquet::file::properties::DEFAULT_CDC_MAX_CHUNK_SIZE`. - pub const PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE_DEFAULT: usize = - PARQUET_CDC_MAX_CHUNK_SIZE_DEFAULT; + pub const PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE_DEFAULT: usize = 1024 * 1024; /// Normalization level (gearhash bit adjustment) for content-defined chunking. - pub const PROPERTY_PARQUET_CDC_NORM_LEVEL: &str = PARQUET_CDC_NORM_LEVEL; + pub const PROPERTY_PARQUET_CDC_NORM_LEVEL: &str = + "write.parquet.content-defined-chunking.norm-level"; /// Default matches `parquet::file::properties::DEFAULT_CDC_NORM_LEVEL`. - pub const PROPERTY_PARQUET_CDC_NORM_LEVEL_DEFAULT: i32 = PARQUET_CDC_NORM_LEVEL_DEFAULT; + pub const PROPERTY_PARQUET_CDC_NORM_LEVEL_DEFAULT: i32 = 0; /// Compression codec for Parquet data files (e.g. `zstd`, `gzip`, `snappy`, /// `lz4`, `lz4_raw`, `brotli`, `lzo`, `uncompressed`). The codec name is /// parsed into a [`CompressionCodec`] when properties are parsed; the level's /// range is validated when the writer is built. - pub const PROPERTY_PARQUET_COMPRESSION_CODEC: &str = PARQUET_COMPRESSION_CODEC; + pub const PROPERTY_PARQUET_COMPRESSION_CODEC: &str = "write.parquet.compression-codec"; /// Default Parquet compression codec. - pub const PROPERTY_PARQUET_COMPRESSION_CODEC_DEFAULT: &str = PARQUET_COMPRESSION_CODEC_DEFAULT; + pub const PROPERTY_PARQUET_COMPRESSION_CODEC_DEFAULT: &str = "zstd"; /// Compression level for Parquet data files, for codecs that take one /// (`gzip`, `zstd`, `brotli`). When unset, the codec's default level is used. - pub const PROPERTY_PARQUET_COMPRESSION_LEVEL: &str = PARQUET_COMPRESSION_LEVEL; + pub const PROPERTY_PARQUET_COMPRESSION_LEVEL: &str = "write.parquet.compression-level"; /// Approximate maximum size of a Parquet row group in bytes. - pub const PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES: &str = PARQUET_ROW_GROUP_SIZE_BYTES; + pub const PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES: &str = "write.parquet.row-group-size-bytes"; /// Default Parquet row group size in bytes. - pub const PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT: usize = - PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT; + pub const PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT: usize = 128 * 1024 * 1024; /// Approximate maximum size of a Parquet data page in bytes. - pub const PROPERTY_PARQUET_PAGE_SIZE_BYTES: &str = PARQUET_PAGE_SIZE_BYTES; + pub const PROPERTY_PARQUET_PAGE_SIZE_BYTES: &str = "write.parquet.page-size-bytes"; /// Default Parquet page size in bytes. - pub const PROPERTY_PARQUET_PAGE_SIZE_BYTES_DEFAULT: usize = PARQUET_PAGE_SIZE_BYTES_DEFAULT; + pub const PROPERTY_PARQUET_PAGE_SIZE_BYTES_DEFAULT: usize = 1024 * 1024; /// Maximum number of rows per Parquet data page. - pub const PROPERTY_PARQUET_PAGE_ROW_LIMIT: &str = PARQUET_PAGE_ROW_LIMIT; + pub const PROPERTY_PARQUET_PAGE_ROW_LIMIT: &str = "write.parquet.page-row-limit"; /// Default Parquet page row limit. - pub const PROPERTY_PARQUET_PAGE_ROW_LIMIT_DEFAULT: usize = PARQUET_PAGE_ROW_LIMIT_DEFAULT; + pub const PROPERTY_PARQUET_PAGE_ROW_LIMIT_DEFAULT: usize = 20000; /// Approximate maximum size of the Parquet dictionary page in bytes. - pub const PROPERTY_PARQUET_DICT_SIZE_BYTES: &str = PARQUET_DICT_SIZE_BYTES; + pub const PROPERTY_PARQUET_DICT_SIZE_BYTES: &str = "write.parquet.dict-size-bytes"; /// Default Parquet dictionary page size in bytes. - pub const PROPERTY_PARQUET_DICT_SIZE_BYTES_DEFAULT: usize = PARQUET_DICT_SIZE_BYTES_DEFAULT; + pub const PROPERTY_PARQUET_DICT_SIZE_BYTES_DEFAULT: usize = 2 * 1024 * 1024; /// Property key for the master key id used to encrypt the table's manifest /// list and data files as defined in https://iceberg.apache.org/docs/nightly/encryption/. - pub const PROPERTY_ENCRYPTION_KEY_ID: &str = ENCRYPTION_KEY_ID; + pub const PROPERTY_ENCRYPTION_KEY_ID: &str = "encryption.key-id"; /// Property key for the encryption data encryption key (DEK) length in bytes. - pub const PROPERTY_ENCRYPTION_DATA_KEY_LENGTH: &str = ENCRYPTION_DATA_KEY_LENGTH; + pub const PROPERTY_ENCRYPTION_DATA_KEY_LENGTH: &str = "encryption.data-key-length"; /// Default value for the encryption DEK length (16 bytes = AES-128). - pub const PROPERTY_ENCRYPTION_DATA_KEY_LENGTH_DEFAULT: usize = - ENCRYPTION_DATA_KEY_LENGTH_DEFAULT; + pub const PROPERTY_ENCRYPTION_DATA_KEY_LENGTH_DEFAULT: usize = 16; /// Property key for the base directory for data files - pub const PROPERTY_WRITE_DATA_LOCATION: &str = WRITE_DATA_LOCATION; + pub const PROPERTY_WRITE_DATA_LOCATION: &str = "write.data.path"; /// Property key for deprecated [write_folder_storage_location] - pub const PROPERTY_WRITE_FOLDER_STORAGE_LOCATION: &str = WRITE_FOLDER_STORAGE_LOCATION; + pub const PROPERTY_WRITE_FOLDER_STORAGE_LOCATION: &str = "write.folder-storage.path"; /// Property key for deprecated object storage path, kept as a fallback for compatibility. - pub const PROPERTY_WRITE_OBJECT_STORAGE_LOCATION: &str = WRITE_OBJECT_STORAGE_LOCATION; + pub const PROPERTY_WRITE_OBJECT_STORAGE_LOCATION: &str = "write.object-storage.path"; /// Property key for controlling whether partition values are included in object storage paths. pub const PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS: &str = - WRITE_OBJECT_STORAGE_PARTITIONED_PATHS; + "write.object-storage.partitioned-paths"; /// Default value for [PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS] - pub const PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS_DEFAULT: bool = - WRITE_OBJECT_STORAGE_PARTITIONED_PATHS_DEFAULT; + pub const PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS_DEFAULT: bool = true; /// The AES key size to use when generating data encryption keys, derived /// from `encryption.data-key-length`.