From 84b1d4576399a3353c556ed5f98b585385c7d534 Mon Sep 17 00:00:00 2001 From: Tianhao Wang Date: Tue, 14 Jul 2026 08:54:35 +0300 Subject: [PATCH] require public numeric bounds Require pipeline numerical attributes to provide public NumericalAttribute bounds before DP quantile derivation instead of deriving exact min/max from sensitive data. --- .../dataset_encoding.py | 11 ++++ .../numerical_values_derivation.py | 52 ++++++------------- .../dataset_encoding_test.py | 33 +++++++++++- .../numerical_values_derivation_test.py | 17 ++++-- 4 files changed, 71 insertions(+), 42 deletions(-) diff --git a/dpsynth/pipeline_transformations/dataset_encoding.py b/dpsynth/pipeline_transformations/dataset_encoding.py index 75ace8d..97aacfb 100644 --- a/dpsynth/pipeline_transformations/dataset_encoding.py +++ b/dpsynth/pipeline_transformations/dataset_encoding.py @@ -59,6 +59,16 @@ def encode_dataset( numerical_attr_indices_to_derive, ) = get_indices_to_discretisize(descriptor) + public_numerical_attributes = {} + for index in numerical_attr_indices_to_derive: + numerical_attribute = descriptor.attributes[index].numerical_attribute + if numerical_attribute is None: + raise ValueError( + 'Public numerical bounds must be provided for every numerical' + f' attribute. Missing bounds for: {descriptor.attributes[index].name}.' + ) + public_numerical_attributes[index] = numerical_attribute + # Derive categorical values. derived_categorical_values = ( categorical_values_derivation.derive_categorical_values( @@ -73,6 +83,7 @@ def encode_dataset( dp_engine, numerical_attr_indices_to_derive, num_quantiles, + public_numerical_attributes, ) ) # (Collection[key, attribute, quantiles]) i.e. # (Collection[str | int, domain.NumericalAttribute, tuple[float,..]]) diff --git a/dpsynth/pipeline_transformations/numerical_values_derivation.py b/dpsynth/pipeline_transformations/numerical_values_derivation.py index a2d9877..bfecf67 100644 --- a/dpsynth/pipeline_transformations/numerical_values_derivation.py +++ b/dpsynth/pipeline_transformations/numerical_values_derivation.py @@ -27,7 +27,6 @@ from dpsynth.pipeline_transformations import types import numpy as np import pipeline_dp -from pipeline_dp import pipeline_functions Key: TypeAlias = TypeVar('Key', bound=str | int) Record: TypeAlias = tuple[Any, ...] | dict[Key, Any] @@ -46,6 +45,7 @@ def derive_numerical_attributes( dp_engine: pipeline_dp.DPEngine, attribute_keys_to_derive: list[Key], num_quantile_buckets: int, + public_bounds: dict[Key, domain.NumericalAttribute] | None = None, ) -> types.Collection[NumericalAttributeOutput] | None: """Derives new NumericalAttribute objects by computing DP quantiles. @@ -61,10 +61,12 @@ def derive_numerical_attributes( num_quantile_buckets: The number of quantile buckets to use for discretization. This means `num_quantile_buckets - 1` boundaries will be computed. + public_bounds: Public numerical bounds for each attribute. These bounds + must be provided by the caller instead of inferred from sensitive data. Returns: A collection of `NumericalAttributeOutput` objects, where - each object contains the field key, the min/max-based + each object contains the field key, the public-bounds-based `domain.NumericalAttribute`, and the DP-computed quantiles. Returns None if there are no attributes to derive values for. """ @@ -72,40 +74,18 @@ def derive_numerical_attributes( # No attributes to derive values for. return None - def extract_field_values(row): - for key in attribute_keys_to_derive: - yield key, row[key] - - key_val_pairs = backend.flat_map( - input_data, - extract_field_values, - 'Extract key-value pairs for all keys in attribute_keys_to_derive.', - ) # (key, value) - - min_max = pipeline_functions.min_max_per_key( - backend, key_val_pairs, 'Compute min and max per key.' - ) # (key, (min, max)) - - def _make_numerical_attr(min_max): - lo, hi = min_max - if lo >= hi: - # Constant column: pad range so the attribute is valid. - hi = lo + 1.0 - return domain.NumericalAttribute(min_value=lo, max_value=hi) - - key_to_attr = backend.map_values( - min_max, - _make_numerical_attr, - 'Create numerical attribute from min/max', - ) # (key, domain.NumericalAttribute) - - # Conversion of key_to_attr to a singleton collection - key_to_attr = backend.to_list( - key_to_attr, 'key_to_attr to list' - ) # singleton [(key, domain.NumericalAttribute)] - key_to_attr = backend.map( - key_to_attr, dict, 'key_to_attr list to dict' - ) # singleton {key: domain.NumericalAttribute} + if public_bounds is None: + public_bounds = {} + for key in attribute_keys_to_derive: + if key not in public_bounds: + raise ValueError( + 'Public numerical bounds must be provided for every numerical' + f' attribute. Missing bounds for: {key}.' + ) + + key_to_attr = backend.to_collection( + [public_bounds], input_data, 'Create public numerical attributes' + ) quantiles = _compute_dp_quantiles( input_data, diff --git a/tests/pipeline_transformations/dataset_encoding_test.py b/tests/pipeline_transformations/dataset_encoding_test.py index 8b6d2c9..e090a39 100644 --- a/tests/pipeline_transformations/dataset_encoding_test.py +++ b/tests/pipeline_transformations/dataset_encoding_test.py @@ -60,7 +60,11 @@ def test_encode_decode_mixed(self, mock_derive_numerical): ), # numerical attribute dataset_descriptor.AttributeDescriptor( - name="col3", data_type=dataset_descriptor.DataType.FLOAT + name="col3", + data_type=dataset_descriptor.DataType.FLOAT, + numerical_attribute=domain.NumericalAttribute( + min_value=0.0, max_value=12.0, clip_to_range=False + ), ), ], data_record_converter=FakeDataRecordConverter(), @@ -137,8 +141,33 @@ def test_encode_decode_mixed(self, mock_derive_numerical): self.assertEqual(decoded_data, expected_decoded_data) mock_derive_numerical.assert_called_once_with( - data, backend, dp_engine, [2], num_quantiles + data, + backend, + dp_engine, + [2], + num_quantiles, + {2: mock_numerical_attribute}, + ) + + def test_encode_dataset_raises_for_numerical_attribute_missing_bounds(self): + backend = pipeline_dp.LocalBackend() + accountant = pipeline_dp.NaiveBudgetAccountant( + total_epsilon=5.0, total_delta=1e-10 ) + dp_engine = pipeline_dp.DPEngine(accountant, backend) + descriptors = dataset_descriptor.DatasetDescriptor( + attributes=[ + dataset_descriptor.AttributeDescriptor( + name="col1", data_type=dataset_descriptor.DataType.FLOAT + ), + ], + data_record_converter=FakeDataRecordConverter(), + ) + + with self.assertRaisesRegex(ValueError, "Missing bounds for"): + dataset_encoding.encode_dataset( + [(1.0,), (2.0,)], backend, dp_engine, descriptors + ) def test_get_indices_to_discretisize(self): descriptors = dataset_descriptor.DatasetDescriptor( diff --git a/tests/pipeline_transformations/numerical_values_derivation_test.py b/tests/pipeline_transformations/numerical_values_derivation_test.py index b7554d4..f9a42af 100644 --- a/tests/pipeline_transformations/numerical_values_derivation_test.py +++ b/tests/pipeline_transformations/numerical_values_derivation_test.py @@ -182,6 +182,10 @@ def test_derive_numerical_attributes(self): dp_engine=dp_engine, attribute_keys_to_derive=attribute_keys, num_quantile_buckets=num_buckets, + public_bounds={ + "feat1": domain.NumericalAttribute(min_value=0, max_value=9), + "feat2": domain.NumericalAttribute(min_value=10, max_value=100), + }, ) accountant.compute_budgets() @@ -191,7 +195,7 @@ def test_derive_numerical_attributes(self): self.assertIn("feat1", derived_attrs_dict) self.assertIn("feat2", derived_attrs_dict) - # Assert feat1 (Range [0, 9]) + # Assert feat1 uses the provided public range. feat1_output = derived_attrs_dict["feat1"] self.assertEqual(feat1_output.attribute.min_value, 0) self.assertEqual(feat1_output.attribute.max_value, 9) @@ -201,7 +205,7 @@ def test_derive_numerical_attributes(self): feat1_output.quantiles, expected_quantiles_feat1, delta=10 ) - # Assert F2 (Range [10, 100]) + # Assert F2 uses the provided public range. f2_output = derived_attrs_dict["feat2"] self.assertEqual(f2_output.attribute.min_value, 10) self.assertEqual(f2_output.attribute.max_value, 100) @@ -226,6 +230,9 @@ def test_derive_numerical_attributes_empty_input(self): dp_engine=dp_engine, attribute_keys_to_derive=["field"], num_quantile_buckets=3, + public_bounds={ + "field": domain.NumericalAttribute(min_value=0, max_value=1) + }, ) accountant.compute_budgets() derived_attrs_list = list(derived_attrs) @@ -245,6 +252,9 @@ def test_derive_numerical_attributes_constant_val(self): dp_engine=dp_engine, attribute_keys_to_derive=["field"], num_quantile_buckets=4, + public_bounds={ + "field": domain.NumericalAttribute(min_value=10, max_value=11) + }, ) accountant.compute_budgets() @@ -254,8 +264,7 @@ def test_derive_numerical_attributes_constant_val(self): output = derived_attrs_list[0] self.assertEqual(output.key, "field") self.assertEqual(output.attribute.min_value, 10) - # Constant column: derivation pads max_value to min_value + 1. - self.assertEqual(output.attribute.max_value, 11.0) + self.assertEqual(output.attribute.max_value, 11) self.assertLen(output.quantiles, 3) self.assertSequenceAlmostEqual( output.quantiles, [10.0, 10.0, 10.0], delta=1.0