From e840cdeaa68695964386961d51f41e57784da0f7 Mon Sep 17 00:00:00 2001 From: R-Palazzo Date: Thu, 20 Aug 2026 09:45:58 +0100 Subject: [PATCH 01/13] remove 'transformer' param from 'ht._remove_column_in_multi_column_fields' --- rdt/hyper_transformer.py | 96 ++++++-------------- tests/integration/test__deprecations.py | 26 ++++++ tests/integration/test_hyper_transformer.py | 6 +- tests/unit/test_hyper_transformer.py | 97 ++------------------- 4 files changed, 64 insertions(+), 161 deletions(-) create mode 100644 tests/integration/test__deprecations.py diff --git a/rdt/hyper_transformer.py b/rdt/hyper_transformer.py index 754c0a576..043e1f59c 100644 --- a/rdt/hyper_transformer.py +++ b/rdt/hyper_transformer.py @@ -288,7 +288,7 @@ def set_config(self, config): warnings.warn(self._REFIT_MESSAGE) def _validate_update_transformers_by_sdtype( - self, sdtype, transformer, transformer_name, transformer_parameters + self, sdtype, transformer_name, transformer_parameters ): if not self.field_sdtypes: raise ConfigNotSetError( @@ -296,60 +296,25 @@ def _validate_update_transformers_by_sdtype( 'pre-populate all the sdtypes and transformers from your dataset.' ) - if transformer_name is None: - if transformer is None: - raise InvalidConfigError("Missing required parameter 'transformer_name'.") - - if not isinstance(transformer, BaseTransformer): - raise InvalidConfigError( - 'Invalid transformer. Please input an rdt transformer object.' - ) - - if sdtype not in transformer.get_supported_sdtypes(): - raise InvalidConfigError( - "The transformer you've assigned is incompatible with the sdtype." - ) - - else: - if ( - transformer_name not in get_class_by_transformer_name() - or sdtype - not in get_class_by_transformer_name()[transformer_name].get_supported_sdtypes() - ): - raise InvalidConfigError( - f"Invalid transformer name '{transformer_name}' for the '{sdtype}' sdtype." - ) - - if transformer_parameters is not None: - transformer = get_class_by_transformer_name()[transformer_name] - valid = inspect.signature(transformer).parameters - invalid_parameters = {arg for arg in transformer_parameters if arg not in valid} - if invalid_parameters: - raise TransformerInputError( - f'Invalid parameters {tuple(sorted(invalid_parameters))} ' - f"for the '{transformer_name}'." - ) - - def _warn_update_transformers_by_sdtype(self, transformer, transformer_name): - if self._fitted: - warnings.warn(self._REFIT_MESSAGE) + if ( + transformer_name not in get_class_by_transformer_name() + or sdtype + not in get_class_by_transformer_name()[transformer_name].get_supported_sdtypes() + ): + raise InvalidConfigError( + f"Invalid transformer name '{transformer_name}' for the '{sdtype}' sdtype." + ) - if transformer_name is not None: - if transformer is not None: - warnings.warn( - "The 'transformer' parameter will no longer be supported in future versions " - "of the RDT. Using the 'transformer_name' parameter instead.", - FutureWarning, + if transformer_parameters is not None: + transformer = get_class_by_transformer_name()[transformer_name] + valid = inspect.signature(transformer).parameters + invalid_parameters = {arg for arg in transformer_parameters if arg not in valid} + if invalid_parameters: + raise TransformerInputError( + f'Invalid parameters {tuple(sorted(invalid_parameters))} ' + f"for the '{transformer_name}'." ) - else: - warnings.warn( - "The 'transformer' parameter will no longer be supported in future versions " - "of the RDT. Please use the 'transformer_name' and 'transformer_parameters' " - 'parameters instead.', - FutureWarning, - ) - def _remove_column_in_multi_column_fields(self, column): """Remove a column that is part of a multi-column field. @@ -401,42 +366,35 @@ def _update_multi_column_transformer(self): def update_transformers_by_sdtype( self, sdtype, - transformer=None, transformer_name=None, transformer_parameters=None, ): """Update the transformers for the specified ``sdtype``. - Given an ``sdtype`` and a ``transformer``, change all the fields of the ``sdtype`` + Given an ``sdtype`` and a ``transformer_name``, change all the fields of the ``sdtype`` to use the given transformer. Args: sdtype (str): Semantic data type for the transformer. - transformer (rdt.transformers.BaseTransformer): - Transformer class or instance to be used for the given ``sdtype``. - Note: this parameter is deprecated, use ``transformer_name`` and - ``transformer_parameters`` instead. transformer_name (str): A string with the class name of the transformer. transformer_parameters (dict): A dict of the kwargs of the transformer. """ self._validate_update_transformers_by_sdtype( - sdtype, transformer, transformer_name, transformer_parameters + sdtype, transformer_name, transformer_parameters ) - self._warn_update_transformers_by_sdtype(transformer, transformer_name) - - transformer_instance = transformer + if self._fitted: + warnings.warn(self._REFIT_MESSAGE) - if transformer_name is not None: - if transformer_parameters is not None: - transformer_instance = get_class_by_transformer_name()[transformer_name]( - **transformer_parameters - ) + if transformer_parameters is not None: + transformer_instance = get_class_by_transformer_name()[transformer_name]( + **transformer_parameters + ) - else: - transformer_instance = get_class_by_transformer_name()[transformer_name]() + else: + transformer_instance = get_class_by_transformer_name()[transformer_name]() for field, field_sdtype in self.field_sdtypes.items(): if field_sdtype == sdtype: diff --git a/tests/integration/test__deprecations.py b/tests/integration/test__deprecations.py new file mode 100644 index 000000000..725f905cb --- /dev/null +++ b/tests/integration/test__deprecations.py @@ -0,0 +1,26 @@ +import re + +import pytest + +from rdt.hyper_transformer import HyperTransformer + + +@pytest.mark.parametrize( + 'method, parameter, expected_message', + [ + ( + 'update_transformers_by_sdtype', + 'transformer', + 'HyperTransformer.update_transformers_by_sdtype() got an unexpected keyword' + " argument 'transformer'. Did you mean 'transformer_name'?", + ), + ], +) +def test_deprecated_parameters(method, parameter, expected_message): + """Test that deprecated parameters raise an error.""" + # Setup + ht = HyperTransformer() + + # Run and Assert + with pytest.raises(TypeError, match=re.escape(expected_message)): + getattr(ht, method)(**{parameter: 'value'}) diff --git a/tests/integration/test_hyper_transformer.py b/tests/integration/test_hyper_transformer.py index ff6b82a02..4603ec4f9 100644 --- a/tests/integration/test_hyper_transformer.py +++ b/tests/integration/test_hyper_transformer.py @@ -884,7 +884,9 @@ def test_with_multiple_supported_sdtypes(self): ht = HyperTransformer() ht.detect_initial_config(data) ht.update_transformers_by_sdtype( - sdtype='boolean', transformer=FrequencyEncoder(add_noise=True) + sdtype='boolean', + transformer_name='FrequencyEncoder', + transformer_parameters={'add_noise': True}, ) # Run @@ -2077,7 +2079,7 @@ def test_with_tuple_returned_by_faker(self): ), ( 'update_transformers_by_sdtype', - {'sdtype': 'boolean', 'transformer': UniformEncoder()}, + {'sdtype': 'boolean', 'transformer_name': 'UniformEncoder'}, expected_update, ), ('remove_transformers', {'column_names': 'C'}, expected_remove), diff --git a/tests/unit/test_hyper_transformer.py b/tests/unit/test_hyper_transformer.py index 782bceaab..3e132e2cf 100644 --- a/tests/unit/test_hyper_transformer.py +++ b/tests/unit/test_hyper_transformer.py @@ -2055,7 +2055,7 @@ def test_update_transformers_by_sdtype_no_config(self): 'pre-populate all the sdtypes and transformers from your dataset.' ) with pytest.raises(ConfigNotSetError, match=expected_msg): - ht.update_transformers_by_sdtype('categorical', object()) + ht.update_transformers_by_sdtype('categorical', 'LabelEncoder') # Assert assert ht.field_transformers == {} @@ -2082,10 +2082,9 @@ def test_update_transformers_by_sdtype_field_sdtypes_not_fitted(self): 'categorical_column': 'categorical', 'numerical_column': 'numerical', } - transformer = LabelEncoder() # Run - ht.update_transformers_by_sdtype('categorical', transformer) + ht.update_transformers_by_sdtype('categorical', 'LabelEncoder') # Assert assert isinstance(ht.field_transformers['categorical_column'], LabelEncoder) @@ -2113,23 +2112,16 @@ def test_update_transformers_by_sdtype_field_sdtypes_fitted(self, mock_warnings) ht._fitted = True ht.field_transformers = {'categorical_column': FrequencyEncoder()} ht.field_sdtypes = {'categorical_column': 'categorical'} - transformer = LabelEncoder() # Run - ht.update_transformers_by_sdtype('categorical', transformer) + ht.update_transformers_by_sdtype('categorical', 'LabelEncoder') # Assert expected_warnings_msgs = [ call( 'For this change to take effect, please refit your data using ' "'fit' or 'fit_transform'." - ), - call( - "The 'transformer' parameter will no longer be supported in future " - "versions of the RDT. Please use the 'transformer_name' and " - "'transformer_parameters' parameters instead.", - FutureWarning, - ), + ) ] mock_warnings.warn.assert_has_calls(expected_warnings_msgs) @@ -2155,31 +2147,6 @@ def test_update_transformers_by_sdtype_unsupported_sdtype_raises_error( with pytest.raises(InvalidConfigError, match=expected_msg): ht.update_transformers_by_sdtype('fake_type', transformer_name='LabelEncoder') - def test_update_transformers_by_sdtype_bad_transformer_raises_error(self): - """Test ``update_transformers_by_sdtype`` with an object that isn't a transformer instance. - - Setup: - - HyperTransformer instance with ``field_transformers`` and ``field-data_types``. - - Side Effects: - - Error is raised with a message about using a transformer instance. - """ - # Setup - ht = HyperTransformer() - ht.field_transformers = { - 'categorical_column': Mock(), - 'numerical_column': Mock(), - } - ht.field_sdtypes = { - 'categorical_column': 'categorical', - 'numerical_column': 'numerical', - } - - # Run / Assert - expected_msg = 'Invalid transformer. Please input an rdt transformer object.' - with pytest.raises(InvalidConfigError, match=expected_msg): - ht.update_transformers_by_sdtype('categorical', Mock()) - def test_update_transformers_by_sdtype_mismatched_sdtype_raises_error( self, ): @@ -2202,23 +2169,10 @@ def test_update_transformers_by_sdtype_mismatched_sdtype_raises_error( 'numerical_column': 'numerical', } - # Run / Assert - expected_msg = "The transformer you've assigned is incompatible with the sdtype." - with pytest.raises(InvalidConfigError, match=expected_msg): - ht.update_transformers_by_sdtype('categorical', FloatFormatter()) - - def test_update_transformers_by_sdtype_with_transformer_none_transformer_name_none( - self, - ): - """When ``transformer_name`` and ``transformer`` are both ``None``, it should crash.""" - # Setup - ht = HyperTransformer() - ht.field_sdtypes = {'doest matter'} - # Run and Assert - err_msg = "Missing required parameter 'transformer_name'." - with pytest.raises(InvalidConfigError, match=err_msg): - ht.update_transformers_by_sdtype('categorical', None, None, None) + expected_msg = "Invalid transformer name 'FloatFormatter' for the 'categorical' sdtype." + with pytest.raises(InvalidConfigError, match=expected_msg): + ht.update_transformers_by_sdtype('categorical', transformer_name='FloatFormatter') def test_update_transformers_by_sdtype_incorrect_transformer_name(self): """When ``transformer_name`` is not a valid transformer, it should crash.""" @@ -2302,43 +2256,6 @@ def test_update_transformers_by_sdtype_transformer_name(self): assert ht.field_transformers['numerical_column'] == ff assert isinstance(ht.field_transformers['categorical_column'], LabelEncoder) - @patch('rdt.hyper_transformer.warnings') - def test_update_transformers_by_sdtype_transformer_name_and_transformer(self, mock_warning): - """Test setting ``transformer_name`` ignores ``transformer`` parameter. - - Expect the ``transformer`` parameter to be ignored, a warning to be raised, - and the ``field_transformers`` to be updated with an instance of the passed - ``transformer_name`` like normal. - """ - # Setup - ht = HyperTransformer() - ff = FloatFormatter() - ht.field_transformers = { - 'categorical_column': FrequencyEncoder(), - 'numerical_column': ff, - } - ht.field_sdtypes = { - 'categorical_column': 'categorical', - 'numerical_column': 'numerical', - } - - # Run - ht.update_transformers_by_sdtype( - 'categorical', - transformer='doesnt matter', - transformer_name='LabelEncoder', - ) - - # Assert - expected_msg = ( - "The 'transformer' parameter will no longer be supported in future versions " - "of the RDT. Using the 'transformer_name' parameter instead." - ) - mock_warning.warn.assert_called_once_with(expected_msg, FutureWarning) - assert len(ht.field_transformers) == 2 - assert ht.field_transformers['numerical_column'] == ff - assert isinstance(ht.field_transformers['categorical_column'], LabelEncoder) - def test_update_transformers_by_sdtype_with_transformer_name_transformer_parameters( self, ): From 83f6fb15606c90d00cac078ba41dff6482e5a062 Mon Sep 17 00:00:00 2001 From: R-Palazzo Date: Thu, 20 Aug 2026 10:40:05 +0100 Subject: [PATCH 02/13] remove 'model_missing_values' --- rdt/transformers/base.py | 28 +-------- rdt/transformers/boolean.py | 8 --- rdt/transformers/datetime.py | 13 ---- rdt/transformers/numerical.py | 23 ------- tests/integration/test__deprecations.py | 26 -------- .../integration/transformers/test_datetime.py | 25 +------- .../transformers/test_numerical.py | 22 +------ tests/unit/test__deprecations.py | 45 ++++++++++++++ tests/unit/test_hyper_transformer.py | 2 +- tests/unit/transformers/test_base.py | 62 ------------------- tests/unit/transformers/test_boolean.py | 8 --- tests/unit/transformers/test_datetime.py | 20 +----- 12 files changed, 54 insertions(+), 228 deletions(-) delete mode 100644 tests/integration/test__deprecations.py create mode 100644 tests/unit/test__deprecations.py diff --git a/rdt/transformers/base.py b/rdt/transformers/base.py index 3922fb065..518e21005 100644 --- a/rdt/transformers/base.py +++ b/rdt/transformers/base.py @@ -111,16 +111,6 @@ def reset_randomization(self): 'reverse_transform': np.random.RandomState(self.random_seed + 1), } - @property - def model_missing_values(self): - """Whether or not a new column is being used to model missing values.""" - warnings.warn( - "Future versions of RDT will not support the 'model_missing_values' parameter. " - "Please switch to using the 'missing_value_generation' parameter instead.", - FutureWarning, - ) - return self.missing_value_generation == 'from_column' - def _set_missing_value_generation(self, missing_value_generation): if missing_value_generation not in (None, 'from_column', 'random'): raise TransformerInputError( @@ -130,18 +120,6 @@ def _set_missing_value_generation(self, missing_value_generation): self.missing_value_generation = missing_value_generation - def _set_model_missing_values(self, model_missing_values): - warnings.warn( - "Future versions of RDT will not support the 'model_missing_values' parameter. " - "Please switch to using the 'missing_value_generation' parameter to select your " - 'strategy.', - FutureWarning, - ) - if model_missing_values is True: - self._set_missing_value_generation('from_column') - elif model_missing_values is False: - self._set_missing_value_generation('random') - @classmethod def get_name(cls): """Return transformer name. @@ -327,11 +305,7 @@ def __repr__(self): custom_args = [] args = inspect.getfullargspec(self.__init__) keys = args.args[1:] - instanced = { - key: getattr(self, key) - for key in keys - if key != 'model_missing_values' and hasattr(self, key) # Remove after deprecation - } + instanced = {key: getattr(self, key) for key in keys if hasattr(self, key)} default_values_list = args.defaults or [] default_arg_to_default_value = {} diff --git a/rdt/transformers/boolean.py b/rdt/transformers/boolean.py index a1eb0e940..61f575524 100644 --- a/rdt/transformers/boolean.py +++ b/rdt/transformers/boolean.py @@ -20,11 +20,6 @@ class BinaryEncoder(BaseTransformer): Indicate what to replace the null values with. If the string ``'mode'`` is given, replace them with the most common value. Defaults to ``mode``. - model_missing_values (bool): - **DEPRECATED** Whether to create a new column to indicate which values were null or - not. The column will be created only if there are null values. If ``True``, create - the new column if there are null values. If ``False``, do not create the new column - even if there are null values. Defaults to ``False``. missing_value_generation (str or None): The way missing values are being handled. There are three strategies: @@ -42,14 +37,11 @@ class BinaryEncoder(BaseTransformer): def __init__( self, missing_value_replacement='mode', - model_missing_values=None, missing_value_generation='random', ): super().__init__() self.missing_value_replacement = missing_value_replacement self._set_missing_value_generation(missing_value_generation) - if model_missing_values is not None: - self._set_model_missing_values(model_missing_values) def _fit(self, data): """Fit the transformer to the data. diff --git a/rdt/transformers/datetime.py b/rdt/transformers/datetime.py index 097ded338..05ca62f2c 100644 --- a/rdt/transformers/datetime.py +++ b/rdt/transformers/datetime.py @@ -27,11 +27,6 @@ class UnixTimestampEncoder(BaseTransformer): are given, replace them with the corresponding aggregation, if ``'random'``, use random values from the dataset to fill the nan values. Defaults to ``mean``. - model_missing_values (bool): - **DEPRECATED** Whether to create a new column to indicate which values were null or - not. The column will be created only if there are null values. If ``True``, create - the new column if there are null values. If ``False``, do not create the new column - even if there are null values. Defaults to ``False``. datetime_format (str): The strftime to use for parsing time. For more information, see https://docs.python.org/3/library/datetime.html#strftime-and-strptime-behavior. @@ -57,7 +52,6 @@ class UnixTimestampEncoder(BaseTransformer): def __init__( self, missing_value_replacement='mean', - model_missing_values=None, datetime_format=None, missing_value_generation='random', enforce_min_max_values=False, @@ -66,8 +60,6 @@ def __init__( self.missing_value_replacement = missing_value_replacement self._set_missing_value_generation(missing_value_generation) self.enforce_min_max_values = enforce_min_max_values - if model_missing_values is not None: - self._set_model_missing_values(model_missing_values) self.datetime_format = datetime_format self._dtype = None @@ -310,11 +302,6 @@ class OptimizedTimestampEncoder(UnixTimestampEncoder): are given, replace them with the corresponding aggregation, if ``'random'``, use random values from the dataset to fill the nan values. Defaults to ``mean``. - model_missing_values (bool): - **DEPRECATED** Whether to create a new column to indicate which values were null or - not. The column will be created only if there are null values. If ``True``, create - the new column if there are null values. If ``False``, do not create the new column - even if there are null values. Defaults to ``False``. datetime_format (str): The strftime to use for parsing time. For more information, see https://docs.python.org/3/library/datetime.html#strftime-and-strptime-behavior. diff --git a/rdt/transformers/numerical.py b/rdt/transformers/numerical.py index 477f09cf7..d5067eb79 100644 --- a/rdt/transformers/numerical.py +++ b/rdt/transformers/numerical.py @@ -43,11 +43,6 @@ class FloatFormatter(BaseTransformer): replace them with the given value. If the strings ``'mean'`` or ``'mode'`` are given, replace them with the corresponding aggregation and if ``'random'`` replace each null value with a random value in the data range. Defaults to ``mean``. - model_missing_values (bool): - **DEPRECATED** Whether to create a new column to indicate which values were null or - not. The column will be created only if there are null values. If ``True``, create - the new column if there are null values. If ``False``, do not create the new column - even if there are null values. Defaults to ``False``. learn_rounding_scheme (bool): Whether or not to learn what place to round to based on the data seen during ``fit``. If ``True``, the data returned by ``reverse_transform`` will be rounded to that place. @@ -81,7 +76,6 @@ class FloatFormatter(BaseTransformer): def __init__( self, missing_value_replacement='mean', - model_missing_values=None, learn_rounding_scheme=False, enforce_min_max_values=False, computer_representation='Float', @@ -90,9 +84,6 @@ def __init__( super().__init__() self.missing_value_replacement = missing_value_replacement self._set_missing_value_generation(missing_value_generation) - if model_missing_values is not None: - self._set_model_missing_values(model_missing_values) - self.learn_rounding_scheme = learn_rounding_scheme self.enforce_min_max_values = enforce_min_max_values self.computer_representation = computer_representation @@ -266,11 +257,6 @@ class GaussianNormalizer(FloatFormatter): to :math:`u` and then to :math:`x`. Args: - model_missing_values (bool): - **DEPRECATED** Whether to create a new column to indicate which values were null or - not. The column will be created only if there are null values. If ``True``, create - the new column if there are null values. If ``False``, do not create the new column - even if there are null values. Defaults to ``False``. learn_rounding_scheme (bool): Whether or not to learn what place to round to based on the data seen during ``fit``. If ``True``, the data returned by ``reverse_transform`` will be rounded to that place. @@ -336,7 +322,6 @@ def _get_distributions(): def __init__( self, - model_missing_values=None, learn_rounding_scheme=False, enforce_min_max_values=False, distribution='truncnorm', @@ -346,7 +331,6 @@ def __init__( # as this may lead to different outcomes in certain synthesizers # affecting the synthesizers directly and this is out of scope for now. super().__init__( - model_missing_values=model_missing_values, missing_value_generation=missing_value_generation, missing_value_replacement='mean', learn_rounding_scheme=learn_rounding_scheme, @@ -483,11 +467,6 @@ class ClusterBasedNormalizer(FloatFormatter): based on the mean and std of the selected component. Args: - model_missing_values (bool): - **DEPRECATED** Whether to create a new column to indicate which values were null or - not. The column will be created only if there are null values. If ``True``, create - the new column if there are null values. If ``False``, do not create the new column - even if there are null values. Defaults to ``False``. learn_rounding_scheme (bool): Whether or not to learn what place to round to based on the data seen during ``fit``. If ``True``, the data returned by ``reverse_transform`` will be rounded to that place. @@ -527,7 +506,6 @@ class ClusterBasedNormalizer(FloatFormatter): def __init__( self, - model_missing_values=None, learn_rounding_scheme=False, enforce_min_max_values=False, max_clusters=10, @@ -538,7 +516,6 @@ def __init__( # as this may lead to different outcomes in certain synthesizers # affecting the synthesizers directly and this is out of scope for now. super().__init__( - model_missing_values=model_missing_values, missing_value_generation=missing_value_generation, missing_value_replacement='mean', learn_rounding_scheme=learn_rounding_scheme, diff --git a/tests/integration/test__deprecations.py b/tests/integration/test__deprecations.py deleted file mode 100644 index 725f905cb..000000000 --- a/tests/integration/test__deprecations.py +++ /dev/null @@ -1,26 +0,0 @@ -import re - -import pytest - -from rdt.hyper_transformer import HyperTransformer - - -@pytest.mark.parametrize( - 'method, parameter, expected_message', - [ - ( - 'update_transformers_by_sdtype', - 'transformer', - 'HyperTransformer.update_transformers_by_sdtype() got an unexpected keyword' - " argument 'transformer'. Did you mean 'transformer_name'?", - ), - ], -) -def test_deprecated_parameters(method, parameter, expected_message): - """Test that deprecated parameters raise an error.""" - # Setup - ht = HyperTransformer() - - # Run and Assert - with pytest.raises(TypeError, match=re.escape(expected_message)): - getattr(ht, method)(**{parameter: 'value'}) diff --git a/tests/integration/transformers/test_datetime.py b/tests/integration/transformers/test_datetime.py index e8575546a..8756db25f 100644 --- a/tests/integration/transformers/test_datetime.py +++ b/tests/integration/transformers/test_datetime.py @@ -94,31 +94,10 @@ def test_unixtimestampencoder_with_missing_value_replacement_random(self): pd.testing.assert_frame_equal(transformed, expect_transformed) pd.testing.assert_frame_equal(reverted, expected_reversed) - def test_unixtimestampencoder_with_model_missing_values(self): - """Test that `model_missing_values` is accepted by the transformer.""" - # Setup - ute = UnixTimestampEncoder('mean', True) - data = pd.DataFrame({'column': pd.to_datetime([None, '1996-10-17', '1965-05-23'])}) - - # Run - ute.fit(data, column='column') - ute.set_random_state(np.random.RandomState(7), 'reverse_transform') - transformed = ute.transform(data) - reverted = ute.reverse_transform(transformed) - - # Asserts - expected_transformed = pd.DataFrame({ - 'column': [3.500064e17, 845510400000000000, -145497600000000000], - 'column.is_null': [1.0, 0.0, 0.0], - }) - - pd.testing.assert_frame_equal(expected_transformed, transformed) - pd.testing.assert_frame_equal(reverted, data) - def test_unixtimestampencoder_with_integer_datetimes(self): """Test that the transformer properly handles integer columns.""" # Setup - ute = UnixTimestampEncoder('mean', True, datetime_format='%m%d%Y') + ute = UnixTimestampEncoder('mean', datetime_format='%m%d%Y') data = pd.DataFrame({'column': [1201992, 11022028, 10011990]}) # Run @@ -138,7 +117,7 @@ def test_unixtimestampencoder_with_integer_datetimes(self): def test_unixtimestampencoder_with_nans(self): """Test that the transformer properly handles null columns.""" # Setup - ute = UnixTimestampEncoder('mean', True) + ute = UnixTimestampEncoder('mean', missing_value_generation='from_column') data = pd.DataFrame({'column': [np.nan, np.nan, np.nan]}) # Run diff --git a/tests/integration/transformers/test_numerical.py b/tests/integration/transformers/test_numerical.py index 932cbb1bd..cd2574834 100644 --- a/tests/integration/transformers/test_numerical.py +++ b/tests/integration/transformers/test_numerical.py @@ -132,25 +132,7 @@ def test_missing_value_generation_random_and_nans(self): # Assert pd.testing.assert_frame_equal(result[['col1', 'col2']], data) - def test_model_missing_value(self): - """Test that we are still able to use ``model_missing_value``.""" - # Setup - data = pd.DataFrame([1, 2, 1, 2, np.nan, 1], columns=['a']) - column = 'a' - - # Run - nt = FloatFormatter('mean', True) - nt.fit(data, column) - transformed = nt.transform(data) - reverse = nt.reverse_transform(transformed) - - # Assert - assert isinstance(transformed, pd.DataFrame) - assert transformed.shape == (6, 2) - assert list(transformed.iloc[:, 1]) == [0, 0, 0, 0, 1, 0] - np.testing.assert_array_almost_equal(reverse, data, decimal=2) - - def test_missing_value_replacement_set_to_random_and_model_missing_values( + def test_missing_value_replacement_set_to_random( self, ): """Test that we are still able to use ``missing_value_replacement`` when is ``random``.""" @@ -158,7 +140,7 @@ def test_missing_value_replacement_set_to_random_and_model_missing_values( data = pd.DataFrame({'a': [1, 2, 3, np.nan, np.nan, 4]}) # Run - ft = FloatFormatter('random', True) + ft = FloatFormatter('random', missing_value_generation='from_column') ft.fit(data, 'a') transformed = ft.transform(data) reverse = ft.reverse_transform(transformed) diff --git a/tests/unit/test__deprecations.py b/tests/unit/test__deprecations.py new file mode 100644 index 000000000..f96007911 --- /dev/null +++ b/tests/unit/test__deprecations.py @@ -0,0 +1,45 @@ +import re + +import pytest + +from rdt.hyper_transformer import HyperTransformer +from rdt.transformers import ( + BinaryEncoder, + ClusterBasedNormalizer, + FloatFormatter, + GaussianNormalizer, + UnixTimestampEncoder, +) + + +@pytest.mark.parametrize( + 'class_, method, parameter', + [ + ( + HyperTransformer, + 'update_transformers_by_sdtype', + 'transformer', + ), + *[ + (class_, '__init__', 'model_missing_values') + for class_ in ( + FloatFormatter, + BinaryEncoder, + UnixTimestampEncoder, + GaussianNormalizer, + ClusterBasedNormalizer, + ) + ], + ], +) +def test_deprecated_parameters(class_, method, parameter): + """Test that deprecated parameters raise an error.""" + # Setup + instance = class_() + expected_message = ( + f"{class_.__name__}.{method}() got an unexpected keyword argument '{parameter}'" + ) + + # Run and Assert + with pytest.raises(TypeError, match=re.escape(expected_message)): + getattr(instance, method)(**{parameter: 'value'}) diff --git a/tests/unit/test_hyper_transformer.py b/tests/unit/test_hyper_transformer.py index 3e132e2cf..2294ab8a3 100644 --- a/tests/unit/test_hyper_transformer.py +++ b/tests/unit/test_hyper_transformer.py @@ -1134,7 +1134,7 @@ def test_fit_warns_columns_in_data(self): ht = HyperTransformer() data = pd.DataFrame({'col': [1, np.nan, 3], 'col.is_null': [1, 2, 3]}) ht.detect_initial_config(data) - ht.field_transformers['col'] = FloatFormatter(model_missing_values=True) + ht.field_transformers['col'] = FloatFormatter(missing_value_generation='from_column') # Run and Assert warn_msg = re.escape( diff --git a/tests/unit/transformers/test_base.py b/tests/unit/transformers/test_base.py index c1a773465..3445f7e12 100644 --- a/tests/unit/transformers/test_base.py +++ b/tests/unit/transformers/test_base.py @@ -274,68 +274,6 @@ def test__set_missing_value_generation_invalid(self): with pytest.raises(TransformerInputError, match=error_msg): BaseTransformer._set_missing_value_generation(instance, 'None') - @patch('rdt.transformers.base.warnings') - def test_model_missing_values(self, mock_warnings): - """Test ``model_missing_values`` property. - - Test that when ``instance.model_missing_values`` is being called a ``boolean`` value - is returned whether ``missing_value_generation`` is ``from_column`` or not. - """ - # Setup - instance = BaseTransformer() - instance.missing_value_generation = 'from_column' - - # Run - result = instance.model_missing_values - - # Assert - assert result is True - mock_warnings.warn.assert_called_once_with( - ( - "Future versions of RDT will not support the 'model_missing_values' parameter. " - "Please switch to using the 'missing_value_generation' parameter instead." - ), - FutureWarning, - ) - - @patch('rdt.transformers.base.warnings') - def test__set_model_missing_values_true(self, mock_warnings): - """Test that a ``FutureWarning`` is being raised.""" - # Setup - instance = Mock() - # Run - BaseTransformer._set_model_missing_values(instance, True) - - # Assert - mock_warnings.warn.assert_called_once_with( - ( - "Future versions of RDT will not support the 'model_missing_values' parameter. " - "Please switch to using the 'missing_value_generation' parameter to select your " - 'strategy.' - ), - FutureWarning, - ) - instance._set_missing_value_generation.assert_called_once_with('from_column') - - @patch('rdt.transformers.base.warnings') - def test__set_model_missing_values_false(self, mock_warnings): - """Test that a ``FutureWarning`` is being raised.""" - # Setup - instance = Mock() - # Run - BaseTransformer._set_model_missing_values(instance, False) - - # Assert - mock_warnings.warn.assert_called_once_with( - ( - "Future versions of RDT will not support the 'model_missing_values' parameter. " - "Please switch to using the 'missing_value_generation' parameter to select your " - 'strategy.' - ), - FutureWarning, - ) - instance._set_missing_value_generation.assert_called_once_with('random') - def test___repr___no_parameters(self): """Test that the ``__str__`` method returns the class name. diff --git a/tests/unit/transformers/test_boolean.py b/tests/unit/transformers/test_boolean.py index 22a0c54a4..e429fcf94 100644 --- a/tests/unit/transformers/test_boolean.py +++ b/tests/unit/transformers/test_boolean.py @@ -20,14 +20,6 @@ def test___init__(self): assert transformer.missing_value_replacement == 'mode', error_message assert transformer.missing_value_generation == 'random', error_generation - def test___init___model_missing_value_passed(self): - """Test when model missing value is passed to the init.""" - # Run - transformer = BinaryEncoder(model_missing_values=True) - - # Assert - transformer.missing_value_generation == 'from_column' - def test__fit_missing_value_replacement_not_ignore(self): """Test _fit missing_value_replacement not equal to ignore""" # Setup diff --git a/tests/unit/transformers/test_datetime.py b/tests/unit/transformers/test_datetime.py index 8e2c8c9e7..f04188572 100644 --- a/tests/unit/transformers/test_datetime.py +++ b/tests/unit/transformers/test_datetime.py @@ -32,20 +32,6 @@ def test___init__(self): assert transformer.datetime_format == '%M-%d-%Y' assert transformer.enforce_min_max_values is True - def test___init__with_model_missing_values(self): - """Test the ``__init__`` method and the passed arguments are stored as attributes.""" - # Run - transformer = UnixTimestampEncoder( - missing_value_replacement='mode', - model_missing_values=False, - datetime_format='%M-%d-%Y', - ) - - # Asserts - assert transformer.missing_value_replacement == 'mode' - assert transformer.missing_value_generation == 'random' - assert transformer.datetime_format == '%M-%d-%Y' - def test__convert_to_datetime(self): """Test the ``_convert_to_datetime`` method. @@ -230,12 +216,12 @@ def test__reverse_transform_helper_nulls(self): datetimes = transformer.null_transformer.reverse_transform.mock_calls[0][1][0] np.testing.assert_array_equal(data.to_numpy(), datetimes) - def test__reverse_transform_helper_model_missing_values_true(self): + def test__reverse_transform_helper_missing_value_generation_from_column(self): """Test the ``_reverse_transform_helper`` with null values. Setup: - Mock the ``instance.null_transformer``. - - Set the ``model_missing_values``. + - Set the ``missing_value_generation`` to ``from_column``. Input: - a pandas series. @@ -248,7 +234,7 @@ def test__reverse_transform_helper_model_missing_values_true(self): """ # Setup data = pd.to_datetime(['2020-01-01', '2020-02-01', '2020-03-01']) - transformer = UnixTimestampEncoder(model_missing_values=True) + transformer = UnixTimestampEncoder(missing_value_generation='from_column') transformer.null_transformer = Mock() transformer.null_transformer.reverse_transform.return_value = pd.Series([1, 2, 3]) From 2863388385f35a2604b222fbf1cae3a62f1669a6 Mon Sep 17 00:00:00 2001 From: R-Palazzo Date: Thu, 20 Aug 2026 10:48:37 +0100 Subject: [PATCH 03/13] remove BaseTransformer.get_input_sdtype() --- rdt/transformers/base.py | 14 -------------- tests/unit/test__deprecations.py | 18 ++++++++++++++++++ tests/unit/transformers/test_base.py | 20 -------------------- 3 files changed, 18 insertions(+), 34 deletions(-) diff --git a/rdt/transformers/base.py b/rdt/transformers/base.py index 518e21005..b623f4dc8 100644 --- a/rdt/transformers/base.py +++ b/rdt/transformers/base.py @@ -147,20 +147,6 @@ def get_subclasses(cls): return subclasses - @classmethod - def get_input_sdtype(cls): - """Return the input sdtype supported by the transformer. - - Returns: - string: - Accepted input sdtype of the transformer. - """ - warnings.warn( - '`get_input_sdtype` is deprecated. Please use `get_supported_sdtypes` instead.', - FutureWarning, - ) - return cls.get_supported_sdtypes()[0] - @classmethod def get_supported_sdtypes(cls): """Return the supported sdtypes by the transformer. diff --git a/tests/unit/test__deprecations.py b/tests/unit/test__deprecations.py index f96007911..92452e4e4 100644 --- a/tests/unit/test__deprecations.py +++ b/tests/unit/test__deprecations.py @@ -4,6 +4,7 @@ from rdt.hyper_transformer import HyperTransformer from rdt.transformers import ( + BaseTransformer, BinaryEncoder, ClusterBasedNormalizer, FloatFormatter, @@ -43,3 +44,20 @@ def test_deprecated_parameters(class_, method, parameter): # Run and Assert with pytest.raises(TypeError, match=re.escape(expected_message)): getattr(instance, method)(**{parameter: 'value'}) + + +@pytest.mark.parametrize( + 'class_, method, parameter', + [ + (BaseTransformer, 'get_input_sdtype', None), + ], +) +def test_deprecated_methods(class_, method, parameter): + """Test that deprecated methods raise an error.""" + # Setup + instance = class_() + expected_message = f"'{class_.__name__}' object has no attribute '{method}'" + + # Run and Assert + with pytest.raises(AttributeError, match=re.escape(expected_message)): + getattr(instance, method)(**{parameter: 'value'} if parameter else {}) diff --git a/tests/unit/transformers/test_base.py b/tests/unit/transformers/test_base.py index 3445f7e12..0fd56c9c5 100644 --- a/tests/unit/transformers/test_base.py +++ b/tests/unit/transformers/test_base.py @@ -159,26 +159,6 @@ class Child(Parent): assert Child in subclasses assert Parent not in subclasses - @patch('rdt.transformers.base.BaseTransformer.get_supported_sdtypes') - def test_get_input_sdtype_raises_warning(self, mock_get_supported_sdtypes): - """Test the ``get_input_sdtype`` method. - - This method should raise a FutureWarning and then call ``get_supported_sdtypes_`` method. - """ - # Setup - mock_get_supported_sdtypes.return_value = ['categorical'] - - # Run - expected_message = ( - '`get_input_sdtype` is deprecated. Please use `get_supported_sdtypes` instead.' - ) - with pytest.warns(FutureWarning, match=expected_message): - input_sdtype = BaseTransformer.get_input_sdtype() - - # Assert - assert input_sdtype == 'categorical' - mock_get_supported_sdtypes.assert_called_once() - def test_get_supported_sdtypes_supported_sdtypes(self): """Test the ``get_supported_sdtypes`` method. From d485fafe458662d00c39380c125c7d77ad6dcace Mon Sep 17 00:00:00 2001 From: R-Palazzo Date: Thu, 20 Aug 2026 11:23:59 +0100 Subject: [PATCH 04/13] remove FrequencyEncoder --- README.md | 2 +- rdt/transformers/__init__.py | 2 - rdt/transformers/categorical.py | 247 ------- tests/integration/test_hyper_transformer.py | 69 +- .../transformers/test_categorical.py | 272 +------ tests/unit/test__deprecations.py | 20 + tests/unit/test_hyper_transformer.py | 37 +- tests/unit/transformers/test_categorical.py | 693 +----------------- 8 files changed, 69 insertions(+), 1273 deletions(-) diff --git a/README.md b/README.md index 3d7ef0f6d..c4d2a29ab 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ Config: "transformers": { "last_login": "UnixTimestampEncoder()", "email_optin": "BinaryEncoder()", - "credit_card": "FrequencyEncoder()", + "credit_card": "UniformEncoder()", "age": "FloatFormatter()", "dollars_spent": "FloatFormatter()" } diff --git a/rdt/transformers/__init__.py b/rdt/transformers/__init__.py index 49daa5e8e..62727bcec 100644 --- a/rdt/transformers/__init__.py +++ b/rdt/transformers/__init__.py @@ -10,7 +10,6 @@ from rdt.transformers.base import BaseMultiColumnTransformer, BaseTransformer from rdt.transformers.boolean import BinaryEncoder from rdt.transformers.categorical import ( - FrequencyEncoder, LabelEncoder, OneHotEncoder, UniformEncoder, @@ -45,7 +44,6 @@ 'ClusterBasedNormalizer', 'OrderedLabelEncoder', 'FloatFormatter', - 'FrequencyEncoder', 'GaussianNormalizer', 'LabelEncoder', 'LogScaler', diff --git a/rdt/transformers/categorical.py b/rdt/transformers/categorical.py index 36d715138..026dafbfe 100644 --- a/rdt/transformers/categorical.py +++ b/rdt/transformers/categorical.py @@ -5,7 +5,6 @@ import numpy as np import pandas as pd -from scipy.stats import norm from rdt.errors import TransformerInputError from rdt.transformers.base import BaseTransformer @@ -226,252 +225,6 @@ def _reverse_transform(self, data): return result -class FrequencyEncoder(BaseTransformer): - """Transformer for categorical data. - - This transformer computes a float representative for each one of the categories - found in the fit data, and then replaces the instances of these categories with - the corresponding representative. - - The representatives are decided by sorting the categorical values by their relative - frequency, then dividing the ``[0, 1]`` interval by these relative frequencies, and - finally assigning the middle point of each interval to the corresponding category. - - When the transformation is reverted, each value is assigned the category that - corresponds to the interval it falls in. - - Null values are considered just another category. - - Args: - add_noise (bool): - Whether to generate gaussian noise around the class representative of each interval - or just use the mean for all the replaced values. Defaults to ``False``. - """ - - INPUT_SDTYPE = 'categorical' - SUPPORTED_SDTYPES = ['categorical', 'boolean'] - mapping = None - intervals = None - starts = None - means = None - dtype = None - - def __setstate__(self, state): - """Replace any ``null`` key by the actual ``np.nan`` instance.""" - intervals = state.get('intervals') - if intervals: - for key in list(intervals): - if pd.isna(key): - intervals[np.nan] = intervals.pop(key) - - self.__dict__ = state - - def __init__(self, add_noise=False): - warnings.warn( - "The 'FrequencyEncoder' transformer will no longer be supported in future versions " - "of the RDT library. Please use the 'UniformEncoder' transformer instead.", - FutureWarning, - ) - super().__init__() - self.add_noise = add_noise - self._is_integer = None - - @staticmethod - def _get_intervals(data): - """Compute intervals for each categorical value. - - Args: - data (pandas.Series): - Data to analyze. - - Returns: - dict: - intervals for each categorical value (start, end). - """ - data = data.infer_objects().fillna(np.nan) - frequencies = data.value_counts(dropna=False) - augmented_frequencies = frequencies.to_frame() - sortable_column_name = f'sortable_{frequencies.name}' - column_name = frequencies.name or 0 - data_with_new_index = data.reset_index(drop=True) - data_is_na = data_with_new_index.isna() - - def tie_breaker(element): - if pd.isna(element): - return data_is_na.loc[data_is_na == 1].index[0] - - return data_with_new_index.loc[data_with_new_index == element].index[0] - - augmented_frequencies[sortable_column_name] = frequencies.index.map(tie_breaker) - augmented_frequencies = augmented_frequencies.sort_values( - [column_name, sortable_column_name], ascending=[False, True] - ) - sorted_frequencies = augmented_frequencies[column_name] - - start = 0 - end = 0 - elements = len(data) - - intervals = {} - means = [] - starts = [] - for value, frequency in sorted_frequencies.items(): - prob = frequency / elements - end = start + prob - mean = start + prob / 2 - std = prob / 6 - if pd.isna(value): - value = np.nan - - intervals[value] = (start, end, mean, std) - means.append(mean) - starts.append((value, start)) - start = end - - means = pd.Series(means, index=list(frequencies.keys())) - starts = pd.DataFrame(starts, columns=['category', 'start']).set_index('start') - - return intervals, means, starts - - def _fit(self, data): - """Fit the transformer to the data. - - Compute the intervals for each categorical value. - - Args: - data (pandas.Series): - Data to fit the transformer to. - """ - self.dtype = data.dtype - self.intervals, self.means, self.starts = self._get_intervals(data) - - @staticmethod - def _clip_noised_transform(result, start, end): - """Clip transformed values. - - Used to ensure the noise added to transformed values doesn't make it - go out of the bounds of a given category. - - The upper bound must be slightly lower than ``end`` - so it doesn't get treated as the next category. - """ - return np.clip(result, start, end - 1e-9) - - def _transform_by_category(self, data): - """Transform the data by iterating over the different categories.""" - result = np.empty(shape=(len(data),), dtype=float) - - # loop over categories - for category, values in self.intervals.items(): - start, end, mean, std = values - if category is np.nan: - mask = data.isna() - else: - mask = data.to_numpy() == category - - if self.add_noise: - result[mask] = norm.rvs( - mean, - std, - size=mask.sum(), - random_state=self.random_states['transform'], - ) - result[mask] = self._clip_noised_transform(result[mask], start, end) - else: - result[mask] = mean - - return result - - def _get_value(self, category): - """Get the value that represents this category.""" - if pd.isna(category): - category = np.nan - - start, end, mean, std = self.intervals[category] - - if self.add_noise: - result = norm.rvs(mean, std, random_state=self.random_states['transform']) - return self._clip_noised_transform(result, start, end) - - return mean - - def _transform_by_row(self, data): - """Transform the data row by row.""" - data = data.infer_objects().fillna(np.nan).apply(self._get_value).to_numpy() - - return data - - def _transform(self, data): - """Transform the categorical values to float representatives. - - Args: - data (pandas.Series): - Data to transform. - - Returns: - numpy.ndarray - """ - fit_categories = pd.Series(self.intervals.keys()) - has_nan = pd.isna(fit_categories).any() - unseen_indexes = ~(data.isin(fit_categories) | (pd.isna(data) & has_nan)) - if unseen_indexes.any(): - # Select only the first 5 unseen categories to avoid flooding the console. - unseen_categories = set(data[unseen_indexes][:5]) - warnings.warn( - f'The data contains {unseen_indexes.sum()} new categories that were not ' - f'seen in the original data (examples: {unseen_categories}). Assigning ' - 'them random values. If you want to model new categories, ' - 'please fit the transformer again with the new data.' - ) - - data[unseen_indexes] = np.random.choice(fit_categories, size=unseen_indexes.size) - if len(self.means) < len(data): - return self._transform_by_category(data) - - return self._transform_by_row(data) - - def _reverse_transform_by_category(self, data): - """Reverse transform the data by iterating over all the categories.""" - result = np.empty(shape=(len(data),), dtype=self.dtype) - - # loop over categories - for category, values in self.intervals.items(): - start = values[0] - mask = start <= data.to_numpy() - result[mask] = category - - return pd.Series(result, index=data.index, dtype=self.dtype) - - def _get_category_from_start(self, value): - lower = self.starts.loc[:value] - return lower.iloc[-1].category - - def _reverse_transform_by_row(self, data): - """Reverse transform the data by iterating over each row.""" - return data.apply(self._get_category_from_start).astype(self.dtype) - - def _reverse_transform(self, data): - """Convert float values back to the original categorical values. - - Args: - data (pd.Series): - Data to revert. - - Returns: - pandas.Series - """ - check_nan_in_transform(data, self.dtype) - data = data.clip(0, 1) - num_rows = len(data) - num_categories = len(self.means) - - if num_rows > num_categories: - return self._reverse_transform_by_category(data) - - # loop over rows - return self._reverse_transform_by_row(data) - - class OneHotEncoder(BaseTransformer): """OneHotEncoding for categorical data. diff --git a/tests/integration/test_hyper_transformer.py b/tests/integration/test_hyper_transformer.py index 4603ec4f9..a4b67b949 100644 --- a/tests/integration/test_hyper_transformer.py +++ b/tests/integration/test_hyper_transformer.py @@ -22,7 +22,6 @@ BinaryEncoder, ClusterBasedNormalizer, FloatFormatter, - FrequencyEncoder, LabelEncoder, OneHotEncoder, RegexGenerator, @@ -57,7 +56,7 @@ def __init__(self): self.output_properties = { None: { 'sdtype': 'datetime', - 'next_transformer': FrequencyEncoder(), + 'next_transformer': UniformEncoder(), } } @@ -416,14 +415,14 @@ def test_field_transformers(self): # Assert expected_transformed = get_transformed_data() transformed_datetimes = [ - 0.8125, - 0.8125, - 0.3125, - 0.3125, - 0.3125, - 0.8125, - 0.3125, - 0.3125, + 0.07685450842732083, + 0.05690074383292415, + 0.6128118364691126, + 0.5407800903723341, + 0.5126664459072409, + 0.1272438262115942, + 0.8958119139449103, + 0.644322077440101, ] expected_transformed['datetime'] = transformed_datetimes pd.testing.assert_frame_equal(transformed, expected_transformed) @@ -457,7 +456,7 @@ def test_categorical_encoders_with_booleans(self): 'paid': 'boolean', }, 'transformers': { - 'email_confirmed': FrequencyEncoder(), + 'email_confirmed': UniformEncoder(), 'subscribed': OneHotEncoder(), 'paid': LabelEncoder(), }, @@ -582,7 +581,7 @@ def test_multiple_fits_with_set_config(self): ht.set_config( config={ 'sdtypes': {'integer': 'categorical'}, - 'transformers': {'integer': FrequencyEncoder()}, + 'transformers': {'integer': UniformEncoder()}, } ) ht.fit(data) @@ -615,7 +614,7 @@ def test_multiple_detect_configs_with_set_config(self): ht.set_config( config={ 'sdtypes': {'integers': 'categorical'}, - 'transformers': {'integers': FrequencyEncoder()}, + 'transformers': {'integers': UniformEncoder()}, } ) @@ -860,19 +859,19 @@ def test_with_multiple_supported_sdtypes(self): """Test the ``HyperTransformer`` supports multiple ``sdtypes`` for a ``Transformer``. Test that the ``HyperTransformer`` works with ``get_supported_sdtypes`` allowing us - to asign different transformer to a ``sdtype``. For example, a ``FrequencyEncoder`` to + to asign different transformer to a ``sdtype``. For example, a ``OneHotEncoder`` to a ``boolean`` sdtype. Setup: - Dataframe with multiple datatypes. - Instance of ``HyperTransformer``. - - Update the transformer for ``boolean`` sdtype to ``FrequencyEncoder()``. + - Update the transformer for ``boolean`` sdtype to ``OneHotEncoder()``. Run: - Run end to end the ``hypertransformer``. Assert: - - Assert that the ``FerquencyEncoder`` is used for the ``boolean`` data. + - Assert that the ``OneHotEncoder`` is used for the ``boolean`` data. """ # Setup data = pd.DataFrame({ @@ -885,8 +884,7 @@ def test_with_multiple_supported_sdtypes(self): ht.detect_initial_config(data) ht.update_transformers_by_sdtype( sdtype='boolean', - transformer_name='FrequencyEncoder', - transformer_parameters={'add_noise': True}, + transformer_name='OneHotEncoder', ) # Run @@ -951,7 +949,7 @@ def test_set_config_with_supported_sdtypes(self): # Setup config = { 'transformers': { - 'boolean_col': FrequencyEncoder(add_noise=True), + 'boolean_col': OneHotEncoder(), }, 'sdtypes': {'boolean_col': 'boolean'}, } @@ -1159,24 +1157,24 @@ def test_field_transformers_correctly_set(self): assert ht.get_config()['transformers']['col'].new_attribute2 == '123' # if a transformer was set, it should use the provided instance - fe = FrequencyEncoder() + uniform_encoder = UniformEncoder() ht.set_config({ 'sdtypes': {'col': 'categorical'}, - 'transformers': {'col': fe}, + 'transformers': {'col': uniform_encoder}, }) ht.fit(data) transformer = ht.get_config()['transformers']['col'] - assert transformer is fe + assert transformer is uniform_encoder # the three cases below make sure any form of acess to the field_transformers # correctly accesses and stores the actual transformers - fe = FrequencyEncoder() - ht.update_transformers({'col': fe}) + uniform_encoder = UniformEncoder() + ht.update_transformers({'col': uniform_encoder}) ht.fit(data) transformer = ht.get_config()['transformers']['col'] - assert transformer is fe + assert transformer is uniform_encoder - ht.update_transformers_by_sdtype('categorical', transformer_name='FrequencyEncoder') + ht.update_transformers_by_sdtype('categorical', transformer_name='UniformEncoder') transformer = ht.get_config()['transformers']['col'] transformer.new_attribute3 = 'abc' ht.fit(data) @@ -1203,8 +1201,7 @@ def _get_hyper_transformer_with_random_transformers(self, data): }) ht.update_transformers_by_sdtype( 'categorical', - transformer_name='FrequencyEncoder', - transformer_parameters={'add_noise': True}, + transformer_name='LabelEncoder', ) return ht @@ -1261,13 +1258,7 @@ def test_reset_randomization(self): 1.286486e-01, ], 'balance.component': [0.0, 0, 0, 0, 0], - 'card_type': [ - 0.17901105796558806, - 0.3582933494588839, - 0.6532481234958804, - 0.8859678246550227, - 0.4245315684590038, - ], + 'card_type': [0, 0, 1, 2, 0], }) expected_second_transformed = pd.DataFrame({ 'age': [18.0, 25.0, 54.0, 60.0, 31.0], @@ -1286,13 +1277,7 @@ def test_reset_randomization(self): 1.286486e-01, ], 'balance.component': [0.0, 0, 0, 0, 0], - 'card_type': [ - 0.3012879880691509, - 0.2678513907358402, - 0.7060422948755574, - 0.9270899473086737, - 0.3107417744890652, - ], + 'card_type': [0, 0, 1, 2, 0], }) ht1.fit(data) diff --git a/tests/integration/transformers/test_categorical.py b/tests/integration/transformers/test_categorical.py index 1471933bc..4d1aa95e2 100644 --- a/tests/integration/transformers/test_categorical.py +++ b/tests/integration/transformers/test_categorical.py @@ -1,7 +1,6 @@ import pickle import re import warnings -from io import BytesIO import numpy as np import pandas as pd @@ -10,7 +9,6 @@ from rdt.errors import InvalidConfigError from rdt.hyper_transformer import HyperTransformer from rdt.transformers import ( - FrequencyEncoder, LabelEncoder, OneHotEncoder, UniformEncoder, @@ -219,271 +217,6 @@ def test_fit_transform_random_seeds(self): pd.testing.assert_frame_equal(transform1, transform3) -def test_frequency_encoder_numerical_nans(): - """Ensure FrequencyEncoder works on numerical + nan only columns.""" - - data = pd.DataFrame([1, 2, float('nan'), np.nan], columns=['column_name']) - column = 'column_name' - - transformer = FrequencyEncoder() - transformer.fit(data, column) - transformed = transformer.transform(data) - reverse = transformer.reverse_transform(transformed) - - pd.testing.assert_frame_equal(reverse, data) - - -def test_frequency_encoder_numerical_nans_no_warning(): - """Ensure FrequencyEncoder does not emit FutureWarning with nan values. - - Related to Issue #793 (https://github.com/sdv-dev/RDT/issues/793) - """ - # Setup - data = pd.DataFrame({'column_name': pd.Series([1, 2, float('nan'), np.nan], dtype='object')}) - column = 'column_name' - - # Run and Assert - transformer = FrequencyEncoder() - with warnings.catch_warnings(): - warnings.simplefilter('error', FutureWarning) - transformer.fit(data, column) - transformed = transformer.transform(data) - reverse = transformer.reverse_transform(transformed) - - pd.testing.assert_frame_equal(reverse, data) - - -def test_frequency_encoder_unseen_transform_data(): - """Ensure FrequencyEncoder works when data to transform wasn't seen during fit.""" - - fit_data = pd.DataFrame([1, 2, float('nan'), np.nan], columns=['column_name']) - transform_data = pd.DataFrame([1, 2, np.nan, 3], columns=['column_name']) - column = 'column_name' - - transformer = FrequencyEncoder() - transformer.fit(fit_data, column) - transformed = transformer.transform(transform_data) - reverse = transformer.reverse_transform(transformed) - - pd.testing.assert_frame_equal(reverse[:3], transform_data[:3]) - assert reverse.iloc[3][0] in {1, 2} or pd.isna(reverse.iloc[3])[0] - - -def test_frequency_encoder_unseen_transform_nan(): - """Ensure FrequencyEncoder works when np.nan to transform wasn't seen during fit.""" - - fit_data = pd.DataFrame([1.0, 2.0, 3.0], columns=['column_name']) - transform_data = pd.DataFrame([1, 2, 3, np.nan], columns=['column_name']) - column = 'column_name' - - transformer = FrequencyEncoder() - transformer.fit(fit_data, column) - transformed = transformer.transform(transform_data) - reverse = transformer.reverse_transform(transformed) - pd.testing.assert_frame_equal(reverse[:3], transform_data[:3]) - assert reverse.iloc[3][0] in {1, 2, 3} - - -def test_frequency_encoder_pickle_nans(): - """Ensure that FrequencyEncoder can be pickled and loaded with nan value.""" - # setup - data = pd.DataFrame([1, 2, float('nan'), np.nan], columns=['column_name']) - column = 'column_name' - - transformer = FrequencyEncoder() - transformer.fit(data, column) - transformed = transformer.transform(data) - - # create pickle file on memory - bytes_io = BytesIO() - pickle.dump(transformer, bytes_io) - # rewind - bytes_io.seek(0) - - # run - pickled_transformer = pickle.load(bytes_io) - - # assert - pickle_transformed = pickled_transformer.transform(data) - pd.testing.assert_frame_equal(pickle_transformed, transformed) - - -def test_frequency_encoder_strings(): - """Test the FrequencyEncoder on string data. - - Ensure that the FrequencyEncoder can fit, transform, and reverse - transform on string data. Expect that the reverse transformed data - is the same as the input. - - Input: - - 4 rows of string data - Output: - - The reverse transformed data - """ - # setup - data = pd.DataFrame(['a', 'b', 'a', 'c'], columns=['column_name']) - column = 'column_name' - transformer = FrequencyEncoder() - - # run - transformer.fit(data, column) - reverse = transformer.reverse_transform(transformer.transform(data)) - - # assert - pd.testing.assert_frame_equal(data, reverse) - - -def test_frequency_encoder_strings_2_categories(): - """Test the FrequencyEncoder on string data. - - Ensure that the FrequencyEncoder can fit, transform, and reverse - transform on string data, when there are 2 categories of strings with - the same value counts. Expect that the reverse transformed data is the - same as the input. - - Input: - - 4 rows of string data - Output: - - The reverse transformed data - """ - # setup - data = pd.DataFrame(['a', 'b', 'a', 'b'], columns=['column_name']) - column = 'column_name' - transformer = FrequencyEncoder() - - transformer.fit(data, column) - reverse = transformer.reverse_transform(transformer.transform(data)) - - # assert - pd.testing.assert_frame_equal(data, reverse) - - -def test_frequency_encoder_integers(): - """Test the FrequencyEncoder on integer data. - - Ensure that the FrequencyEncoder can fit, transform, and reverse - transform on integer data. Expect that the reverse transformed data is the - same as the input. - - Input: - - 4 rows of int data - Output: - - The reverse transformed data - """ - # setup - data = pd.DataFrame([1, 2, 3, 2], columns=['column_name']) - column = 'column_name' - transformer = FrequencyEncoder() - - # run - transformer.fit(data, column) - reverse = transformer.reverse_transform(transformer.transform(data)) - - # assert - pd.testing.assert_frame_equal(data, reverse) - - -def test_frequency_encoder_bool(): - """Test the FrequencyEncoder on boolean data. - - Ensure that the FrequencyEncoder can fit, transform, and reverse - transform on boolean data. Expect that the reverse transformed data is the - same as the input. - - Input: - - 4 rows of bool data - Output: - - The reverse transformed data - """ - # setup - data = pd.DataFrame([True, False, True, False], columns=['column_name']) - column = 'column_name' - transformer = FrequencyEncoder() - - # run - transformer.fit(data, column) - reverse = transformer.reverse_transform(transformer.transform(data)) - - # assert - pd.testing.assert_frame_equal(data, reverse) - - -def test_frequency_encoder_mixed(): - """Test the FrequencyEncoder on mixed type data. - - Ensure that the FrequencyEncoder can fit, transform, and reverse - transform on mixed type data. Expect that the reverse transformed data is - the same as the input. - - Input: - - 4 rows of mixed data - Output: - - The reverse transformed data - """ - # setup - data = pd.DataFrame([True, 'a', 1, None], columns=['column_name']) - column = 'column_name' - transformer = FrequencyEncoder() - - # run - transformer.fit(data, column) - reverse = transformer.reverse_transform(transformer.transform(data)) - - # assert - pd.testing.assert_frame_equal(data, reverse) - - -def test_frequency_encoder_mixed_more_rows(): - """Test the FrequencyEncoder on mixed type data. - - Ensure that the FrequencyEncoder can fit, transform, and reverse - transform on mixed type data, when there is a larger number of rows. - Expect that the reverse transformed data is the same as the input. - - Input: - - 4 rows of mixed data - Output: - - The reverse transformed data - """ - # setup - data = pd.DataFrame([True, 'a', 1, None], columns=['column_name']) - column = 'column_name' - transform_data = pd.DataFrame(['a', 1, None, 'a', True, 1], columns=['column_name']) - transformer = FrequencyEncoder() - - # run - transformer.fit(data, column) - transformed = transformer.transform(transform_data) - reverse = transformer.reverse_transform(transformed) - - # assert - pd.testing.assert_frame_equal(transform_data, reverse) - - -def test_frequency_encoder_noise(): - """Test the FrequencyEncoder with ``add_noise``. - - Ensure that the FrequencyEncoder can fit, transform, and reverse - transform when ``add_noise = True``. - - Input: - - Many rows of int data - Output: - - The reverse transformed data - """ - # setup - data = pd.DataFrame(np.random.choice(a=range(100), size=10000), columns=['column_name']) - column = 'column_name' - transformer = FrequencyEncoder(add_noise=True) - - # run - transformer.fit(data, column) - reverse = transformer.reverse_transform(transformer.transform(data)) - - # assert - pd.testing.assert_frame_equal(data, reverse) - - def test_one_hot_numerical_nans(): """Ensure OneHotEncoder works on numerical + nan only columns.""" @@ -825,11 +558,10 @@ def test_categorical_transformers_with_id_sdtype(sdtype, transformer): @pytest.mark.parametrize('sdtype', ['id', 'text']) -@pytest.mark.parametrize('transformer', [FrequencyEncoder(), OneHotEncoder()]) -def test_unsupported_categorical_transformers_with_id_sdtype(sdtype, transformer): +def test_unsupported_categorical_transformers_with_id_sdtype(sdtype): # Setup hyper_transformer = HyperTransformer() - config = {'sdtypes': {'col': sdtype}, 'transformers': {'col': transformer}} + config = {'sdtypes': {'col': sdtype}, 'transformers': {'col': OneHotEncoder()}} expected_invalid_error = re.escape( "Some transformers you've assigned are not compatible with the sdtypes. " "Please change the following columns: ['col']" diff --git a/tests/unit/test__deprecations.py b/tests/unit/test__deprecations.py index 92452e4e4..475b537c1 100644 --- a/tests/unit/test__deprecations.py +++ b/tests/unit/test__deprecations.py @@ -1,3 +1,4 @@ +import importlib import re import pytest @@ -61,3 +62,22 @@ def test_deprecated_methods(class_, method, parameter): # Run and Assert with pytest.raises(AttributeError, match=re.escape(expected_message)): getattr(instance, method)(**{parameter: 'value'} if parameter else {}) + + +@pytest.mark.parametrize( + 'class_path', + [ + 'rdt.transformers.FrequencyEncoder', + 'rdt.transformers.categorical.FrequencyEncoder', + ], +) +def test_deprecated_classes(class_path): + """Test that deprecated classes can no longer be imported.""" + # Setup + module_path, class_name = class_path.rsplit('.', 1) + module = importlib.import_module(module_path) + expected_message = f"module '{module_path}' has no attribute '{class_name}'" + + # Run and Assert + with pytest.raises(AttributeError, match=re.escape(expected_message)): + getattr(module, class_name) diff --git a/tests/unit/test_hyper_transformer.py b/tests/unit/test_hyper_transformer.py index 2294ab8a3..8037a909f 100644 --- a/tests/unit/test_hyper_transformer.py +++ b/tests/unit/test_hyper_transformer.py @@ -20,7 +20,6 @@ BaseMultiColumnTransformer, BinaryEncoder, FloatFormatter, - FrequencyEncoder, LabelEncoder, RegexGenerator, UniformEncoder, @@ -523,7 +522,7 @@ def test__validate_config(self): # Setup transformers = { 'column1': FloatFormatter(), - 'column2': FrequencyEncoder(), + 'column2': UniformEncoder(), } sdtypes = {'column1': 'numerical', 'column2': 'numerical'} config = {'sdtypes': sdtypes, 'transformers': transformers} @@ -541,7 +540,7 @@ def test_validate_config_not_unique_field(self): # Setup transformers = { 'column1': FloatFormatter(), - 'column2': FrequencyEncoder(), + 'column2': UniformEncoder(), ('column2', 'column3'): None, } sdtypes = { @@ -577,7 +576,7 @@ def test__validate_config_no_warning(self, warnings_mock): # Setup transformers = { 'column1': FloatFormatter(), - 'column2': FrequencyEncoder(), + 'column2': UniformEncoder(), 'column3': None, } sdtypes = { @@ -607,7 +606,7 @@ def test__validate_config_invalid_key(self): # Setup transformers = { 'column1': FloatFormatter(), - 'column2': FrequencyEncoder(), + 'column2': UniformEncoder(), } sdtypes = {'column1': 'numerical', 'column2': 'numerical'} config = { @@ -638,7 +637,7 @@ def test__validate_config_missing_sdtypes(self): # Setup transformers = { 'column1': FloatFormatter(), - 'column2': FrequencyEncoder(), + 'column2': UniformEncoder(), } config = { 'transformers': transformers, @@ -667,7 +666,7 @@ def test__validate_config_mismatched_columns(self): sdtypes = {'column1': 'numerical', 'column2': 'numerical'} transformers = { 'column1': FloatFormatter(), - 'column3': FrequencyEncoder(), + 'column3': UniformEncoder(), } config = { 'sdtypes': sdtypes, @@ -697,7 +696,7 @@ def test__validate_config_invalid_sdtype(self): sdtypes = {'column1': 'numerical', 'column2': 'unexpected'} transformers = { 'column1': FloatFormatter(), - 'column2': FrequencyEncoder(), + 'column2': UniformEncoder(), } config = { 'sdtypes': sdtypes, @@ -758,7 +757,7 @@ def test_get_config(self): ht = HyperTransformer() ht.field_transformers = { 'column1': FloatFormatter(), - 'column2': FrequencyEncoder(), + 'column2': UniformEncoder(), } ht.field_sdtypes = {'column1': 'numerical', 'column2': 'categorical'} @@ -814,7 +813,7 @@ def test_set_config(self): # Setup transformers = { 'column1': FloatFormatter(), - 'column2': FrequencyEncoder(), + 'column2': UniformEncoder(), } sdtypes = {'column1': 'numerical', 'column2': 'categorical'} config = {'sdtypes': sdtypes, 'transformers': transformers} @@ -2075,7 +2074,7 @@ def test_update_transformers_by_sdtype_field_sdtypes_not_fitted(self): # Setup ht = HyperTransformer() ht.field_transformers = { - 'categorical_column': FrequencyEncoder(), + 'categorical_column': UniformEncoder(), 'numerical_column': FloatFormatter(), } ht.field_sdtypes = { @@ -2110,7 +2109,7 @@ def test_update_transformers_by_sdtype_field_sdtypes_fitted(self, mock_warnings) # Setup ht = HyperTransformer() ht._fitted = True - ht.field_transformers = {'categorical_column': FrequencyEncoder()} + ht.field_transformers = {'categorical_column': UniformEncoder()} ht.field_sdtypes = {'categorical_column': 'categorical'} # Run @@ -2240,7 +2239,7 @@ def test_update_transformers_by_sdtype_transformer_name(self): ht = HyperTransformer() ff = FloatFormatter() ht.field_transformers = { - 'categorical_column': FrequencyEncoder(), + 'categorical_column': UniformEncoder(), 'numerical_column': ff, } ht.field_sdtypes = { @@ -2268,7 +2267,7 @@ def test_update_transformers_by_sdtype_with_transformer_name_transformer_paramet ht = HyperTransformer() ff = FloatFormatter() ht.field_transformers = { - 'categorical_column': FrequencyEncoder(), + 'categorical_column': UniformEncoder(), 'numerical_column': ff, } ht.field_sdtypes = { @@ -2419,7 +2418,7 @@ def test_update_transformers_fitted(self, mock_warnings): instance.field_sdtypes = {'my_column': 'categorical'} instance.field_transformers = {'my_column': object()} instance._validate_transformers = Mock() - transformer = FrequencyEncoder() + transformer = UniformEncoder() column_name_to_transformer = {'my_column': transformer} # Run @@ -2531,7 +2530,7 @@ def _validate_sdtypes(cls, columns_to_sdtypes): def test__validate_updated_transformer_unique(self): """Test method errors if an updated transformer already exists in the config.""" # Setup - transformer = FrequencyEncoder() + transformer = UniformEncoder() instance = HyperTransformer() instance.field_transformers = {'existing_column': transformer, 'my_column': object()} @@ -2841,7 +2840,7 @@ def test_update_sdtypes_fitted(self, mock_warnings, mock_logger): # Setup instance = HyperTransformer() instance.field_transformers = { - 'a': FrequencyEncoder, + 'a': UniformEncoder, 'b': FloatFormatter, } instance.field_sdtypes = {'my_column': 'categorical'} @@ -3260,7 +3259,7 @@ def test__validate_transformers(self): # Setup instance = HyperTransformer() column_name_to_transformer = { - 'col1': FrequencyEncoder(), + 'col1': UniformEncoder(), 'col2': 'Unexpected', 'col3': None, } @@ -3277,7 +3276,7 @@ def test__validate_transformers_reused_instances(self): """Test ``_validate_transformers`` with reused transformer isntances.""" # Setup instance = HyperTransformer() - duplicated_transformer = FrequencyEncoder() + duplicated_transformer = UniformEncoder() column_name_to_transformer = { 'col1': duplicated_transformer, 'col2': duplicated_transformer, diff --git a/tests/unit/transformers/test_categorical.py b/tests/unit/transformers/test_categorical.py index bc1aa02bd..667d4509f 100644 --- a/tests/unit/transformers/test_categorical.py +++ b/tests/unit/transformers/test_categorical.py @@ -1,6 +1,6 @@ import logging import re -from unittest.mock import Mock, call, patch +from unittest.mock import patch import numpy as np import pandas as pd @@ -8,7 +8,6 @@ from rdt.errors import TransformerInputError from rdt.transformers.categorical import ( - FrequencyEncoder, LabelEncoder, OneHotEncoder, UniformEncoder, @@ -336,696 +335,6 @@ def _setup_caplog(caplog): caplog.set_level(logging.INFO) -class TestFrequencyEncoder: - def test___setstate__(self): - """Test the ``__set_state__`` method. - - Validate that the ``__dict__`` attribute is correctly udpdated when - - Setup: - - create an instance of a ``FrequencyEncoder``. - - Side effect: - - it updates the ``__dict__`` attribute of the object. - """ - # Setup - transformer = FrequencyEncoder() - - # Run - transformer.__setstate__({'intervals': {None: 'abc'}}) - - # Assert - assert transformer.__dict__['intervals'][np.nan] == 'abc' - - def test___init__(self): - """Passed arguments must be stored as attributes.""" - # Run - warn_message = ( - "The 'FrequencyEncoder' transformer will no longer be supported in future " - "versions of the RDT library. Please use the 'UniformEncoder' transformer instead." - ) - with pytest.warns(FutureWarning, match=warn_message): - transformer = FrequencyEncoder(add_noise='add_noise_value') - - # Asserts - assert transformer.add_noise == 'add_noise_value' - - def test__get_intervals(self): - """Test the ``_get_intervals`` method. - - Validate that the intervals for each categorical value are correct. - - Input: - - a pandas series containing categorical values. - - Output: - - a tuple, where the first element describes the intervals for each - categorical value (start, end). - """ - # Run - data = pd.Series(['foo', 'bar', 'bar', 'foo', 'foo', 'tar']) - result = FrequencyEncoder._get_intervals(data) - - # Asserts - expected_intervals = { - 'foo': (0, 0.5, 0.25, 0.5 / 6), - 'bar': ( - 0.5, - 0.8333333333333333, - 0.6666666666666666, - 0.05555555555555555, - ), - 'tar': ( - 0.8333333333333333, - 0.9999999999999999, - 0.9166666666666666, - 0.027777777777777776, - ), - } - expected_means = pd.Series({ - 'foo': 0.25, - 'bar': 0.6666666666666666, - 'tar': 0.9166666666666666, - }) - expected_starts = pd.DataFrame({ - 'category': ['foo', 'bar', 'tar'], - 'start': [0, 0.5, 0.8333333333333333], - }).set_index('start') - - assert result[0] == expected_intervals - pd.testing.assert_series_equal(result[1], expected_means) - pd.testing.assert_frame_equal(result[2], expected_starts) - - def test__get_intervals_nans(self): - """Test the ``_get_intervals`` method when data contains nan's. - - Validate that the intervals for each categorical value are correct, when passed - data containing nan values. - - Input: - - a pandas series cotaining nan values and categorical values. - - Output: - - a tuple, where the first element describes the intervals for each - categorical value (start, end). - """ - # Setup - data = pd.Series(['foo', np.nan, None, 'foo', 'foo', 'tar']) - - # Run - result = FrequencyEncoder._get_intervals(data) - - # Assert - expected_intervals = { - 'foo': (0, 0.5, 0.25, 0.5 / 6), - np.nan: ( - 0.5, - 0.8333333333333333, - 0.6666666666666666, - 0.05555555555555555, - ), - 'tar': ( - 0.8333333333333333, - 0.9999999999999999, - 0.9166666666666666, - 0.027777777777777776, - ), - } - expected_means = pd.Series({ - 'foo': 0.25, - np.nan: 0.6666666666666666, - 'tar': 0.9166666666666666, - }) - expected_starts = pd.DataFrame({ - 'category': ['foo', np.nan, 'tar'], - 'start': [0, 0.5, 0.8333333333333333], - }).set_index('start') - - assert result[0] == expected_intervals - pd.testing.assert_series_equal(result[1], expected_means) - pd.testing.assert_frame_equal(result[2], expected_starts) - - def test__fit_intervals(self): - # Setup - transformer = FrequencyEncoder() - - # Run - data = pd.Series(['foo', 'bar', 'bar', 'foo', 'foo', 'tar']) - transformer._fit(data) - - # Asserts - expected_intervals = { - 'foo': (0, 0.5, 0.25, 0.5 / 6), - 'bar': ( - 0.5, - 0.8333333333333333, - 0.6666666666666666, - 0.05555555555555555, - ), - 'tar': ( - 0.8333333333333333, - 0.9999999999999999, - 0.9166666666666666, - 0.027777777777777776, - ), - } - expected_means = pd.Series({ - 'foo': 0.25, - 'bar': 0.6666666666666666, - 'tar': 0.9166666666666666, - }) - expected_starts = pd.DataFrame({ - 'category': ['foo', 'bar', 'tar'], - 'start': [0, 0.5, 0.8333333333333333], - }).set_index('start') - - assert transformer.intervals == expected_intervals - pd.testing.assert_series_equal(transformer.means, expected_means) - pd.testing.assert_frame_equal(transformer.starts, expected_starts) - assert transformer.output_properties == { - None: {'sdtype': 'float', 'next_transformer': None}, - } - - def test__get_value_add_noise_false(self): - # Setup - transformer = FrequencyEncoder(add_noise=False) - transformer.intervals = { - 'foo': (0, 0.5, 0.25, 0.5 / 6), - np.nan: (0.5, 1.0, 0.75, 0.5 / 6), - } - - # Run - result_foo = transformer._get_value('foo') - result_nan = transformer._get_value(np.nan) - - # Asserts - assert result_foo == 0.25 - assert result_nan == 0.75 - - @patch('rdt.transformers.categorical.norm') - def test__get_value_add_noise_true(self, norm_mock): - # setup - norm_mock.rvs.return_value = 0.2745 - - transformer = FrequencyEncoder(add_noise=True) - transformer.intervals = { - 'foo': (0, 0.5, 0.25, 0.5 / 6), - } - - # Run - result = transformer._get_value('foo') - - # Asserts - assert result == 0.2745 - - @patch('rdt.transformers.categorical.check_nan_in_transform') - def test__reverse_transform_series(self, mock_check_nan): - """Test reverse_transform a pandas Series""" - # Setup - data = pd.Series(['foo', 'bar', 'bar', 'foo', 'foo', 'tar']) - rt_data = pd.Series([-0.6, 0.5, 0.6, 0.2, 0.1, -0.2]) - transformer = FrequencyEncoder() - - # Run - transformer._fit(data) - result = transformer._reverse_transform(rt_data) - - # Asserts - mock_input_data = mock_check_nan.call_args.args[0] - mock_input_dtype = mock_check_nan.call_args.args[1] - pd.testing.assert_series_equal(mock_input_data, rt_data) - assert mock_input_dtype == transformer.dtype - expected_intervals = { - 'foo': (0, 0.5, 0.25, 0.5 / 6), - 'bar': ( - 0.5, - 0.8333333333333333, - 0.6666666666666666, - 0.05555555555555555, - ), - 'tar': ( - 0.8333333333333333, - 0.9999999999999999, - 0.9166666666666666, - 0.027777777777777776, - ), - } - - assert transformer.intervals == expected_intervals - - expect = pd.Series(['foo', 'bar', 'bar', 'foo', 'foo', 'foo']) - pd.testing.assert_series_equal(result, expect) - - def test__transform_user_warning(self): - """Test the ``_transform`` method generates the correct user warning. - - When asked to transform data not seen during the fit, a UserWarning should be raised. - - Setup: - - create an instance of the ``FrequencyEncoder``, where ``means`` is a list - of floats and ``intervals`` is the appropriate dictionary. - - Input: - - a pandas series containing a np.nan. - - Output: - - a numpy array containing the transformed data. - - Raises: - - a UserWarning with the correct message. - """ - # Setup - data = pd.Series([1, 2, 3, 4, np.nan]) - transformer = FrequencyEncoder() - transformer.means = [0.125, 0.375, 0.625, 0.875] - transformer.intervals = { - 4: (0, 0.25, 0.125, 0.041666666666666664), - 3: (0.25, 0.5, 0.375, 0.041666666666666664), - 2: (0.5, 0.75, 0.625, 0.041666666666666664), - 1: (0.75, 1.0, 0.875, 0.041666666666666664), - } - - # Run - warning_msg = re.escape( - 'The data contains 1 new categories that were not ' - 'seen in the original data (examples: {nan}). Assigning ' - 'them random values. If you want to model new categories, ' - 'please fit the transformer again with the new data.' - ) - with pytest.warns(UserWarning, match=warning_msg): - transformed = transformer._transform(data) - - # Assert - expected = pd.Series([0.875, 0.625, 0.375, 0.125]) - np.testing.assert_array_equal(transformed[:4], expected) - - assert transformed[4] in transformer.means - - def test__transform_by_category_called(self): - """Test that the `_transform_by_category` method is called. - - When the number of rows is greater than the number of categories, expect - that the `_transform_by_category` method is called. - - Setup: - The categorical transformer is instantiated with 4 categories. - - Input: - - data with 5 rows. - - Output: - - the output of `_transform_by_category`. - - Side effects: - - `_transform_by_category` will be called once. - """ - # Setup - data = pd.Series([1, 3, 3, 2, 1]) - - categorical_transformer_mock = Mock() - categorical_transformer_mock.means = pd.Series([ - 0.125, - 0.375, - 0.625, - 0.875, - ]) - - # Run - transformed = FrequencyEncoder._transform(categorical_transformer_mock, data) - - # Asserts - categorical_transformer_mock._transform_by_category.assert_called_once_with(data) - assert transformed == categorical_transformer_mock._transform_by_category.return_value - - def test__transform_by_category(self): - """Test the `_transform_by_category` method with numerical data. - - Expect that the correct transformed data is returned. - - Setup: - The categorical transformer is instantiated with 4 categories and intervals. - - Input: - - data with 5 rows. - - Ouptut: - - the transformed data. - """ - # Setup - data = pd.Series([1, 3, 3, 2, 1]) - transformer = FrequencyEncoder() - transformer.intervals = { - 4: (0, 0.25, 0.125, 0.041666666666666664), - 3: (0.25, 0.5, 0.375, 0.041666666666666664), - 2: (0.5, 0.75, 0.625, 0.041666666666666664), - 1: (0.75, 1.0, 0.875, 0.041666666666666664), - } - - # Run - transformed = transformer._transform_by_category(data) - - # Asserts - expected = np.array([0.875, 0.375, 0.375, 0.625, 0.875]) - assert (transformed == expected).all() - - def test__transform_by_category_nans(self): - """Test the ``_transform_by_category`` method with data containing nans. - - Validate that the data is transformed correctly when it contains nan's. - - Setup: - - the categorical transformer is instantiated, and the appropriate ``intervals`` - attribute is set. - - Input: - - a pandas series containing nan's. - - Output: - - a numpy array containing the transformed data. - """ - # Setup - data = pd.Series([np.nan, 3, 3, 2, np.nan]) - transformer = FrequencyEncoder() - transformer.intervals = { - 4: (0, 0.25, 0.125, 0.041666666666666664), - 3: (0.25, 0.5, 0.375, 0.041666666666666664), - 2: (0.5, 0.75, 0.625, 0.041666666666666664), - np.nan: (0.75, 1.0, 0.875, 0.041666666666666664), - } - - # Run - transformed = transformer._transform_by_category(data) - - # Asserts - expected = np.array([0.875, 0.375, 0.375, 0.625, 0.875]) - assert (transformed == expected).all() - - @patch('rdt.transformers.categorical.norm') - def test__transform_by_category_add_noise_true(self, norm_mock): - """Test the ``_transform_by_category`` method when ``add_noise`` is True. - - Validate that the data is transformed correctly when ``add_noise`` is True. - - Setup: - - the categorical transformer is instantiated with ``add_noise`` as True, - and the appropriate ``intervals`` attribute is set. - - the ``intervals`` attribute is set to a a dictionary of intervals corresponding - to the elements of the passed data. - - set the ``side_effect`` of the ``rvs_mock`` to the appropriate function. - - Input: - - a pandas series. - - Output: - - a numpy array containing the transformed data. - - Side effect: - - ``rvs_mock`` should be called four times, one for each element of the - intervals dictionary. - """ - - # Setup - def rvs_mock_func(loc, scale, **kwargs): - return loc - - norm_mock.rvs.side_effect = rvs_mock_func - - data = pd.Series([1, 3, 3, 2, 1]) - transformer = FrequencyEncoder(add_noise=True) - transformer.intervals = { - 4: (0, 0.25, 0.125, 0.041666666666666664), - 3: (0.25, 0.5, 0.375, 0.041666666666666664), - 2: (0.5, 0.75, 0.625, 0.041666666666666664), - 1: (0.75, 1.0, 0.875, 0.041666666666666664), - } - transform_random_state_mock = Mock() - transformer.random_states['transform'] = transform_random_state_mock - - # Run - transformed = transformer._transform_by_category(data) - - # Assert - expected = np.array([0.875, 0.375, 0.375, 0.625, 0.875]) - assert (transformed == expected).all() - norm_mock.rvs.assert_has_calls([ - call( - 0.125, - 0.041666666666666664, - size=0, - random_state=transform_random_state_mock, - ), - call( - 0.375, - 0.041666666666666664, - size=2, - random_state=transform_random_state_mock, - ), - call( - 0.625, - 0.041666666666666664, - size=1, - random_state=transform_random_state_mock, - ), - call( - 0.875, - 0.041666666666666664, - size=2, - random_state=transform_random_state_mock, - ), - ]) - - def test__transform_by_row_called(self): - """Test that the `_transform_by_row` method is called. - - When the number of rows is less than or equal to the number of categories, - expect that the `_transform_by_row` method is called. - - Setup: - The categorical transformer is instantiated with 4 categories. - Input: - - data with 4 rows - Output: - - the output of `_transform_by_row` - Side effects: - - `_transform_by_row` will be called once - """ - # Setup - data = pd.Series([1, 2, 3, 4]) - - categorical_transformer_mock = Mock() - categorical_transformer_mock.means = pd.Series([ - 0.125, - 0.375, - 0.625, - 0.875, - ]) - - # Run - transformed = FrequencyEncoder._transform(categorical_transformer_mock, data) - - # Asserts - categorical_transformer_mock._transform_by_row.assert_called_once_with(data) - assert transformed == categorical_transformer_mock._transform_by_row.return_value - - def test__transform_by_row(self): - """Test the `_transform_by_row` method with numerical data. - - Expect that the correct transformed data is returned. - - Setup: - The categorical transformer is instantiated with 4 categories and intervals. - Input: - - data with 4 rows - Ouptut: - - the transformed data - """ - # Setup - data = pd.Series([1, 2, 3, 4]) - transformer = FrequencyEncoder() - transformer.intervals = { - 4: (0, 0.25, 0.125, 0.041666666666666664), - 3: (0.25, 0.5, 0.375, 0.041666666666666664), - 2: (0.5, 0.75, 0.625, 0.041666666666666664), - 1: (0.75, 1.0, 0.875, 0.041666666666666664), - } - - # Run - transformed = transformer._transform_by_row(data) - - # Asserts - expected = np.array([0.875, 0.625, 0.375, 0.125]) - assert (transformed == expected).all() - - def test__reverse_transform_by_category_called(self): - """Test that the `_reverse_transform_by_category` method is called. - - When the number of rows is greater than the number of categories, expect - that the `_reverse_transform_by_category` method is called. - - Setup: - The categorical transformer is instantiated with 4 categories. - Input: - - numerical data with 5 rows - Output: - - the output of `_reverse_transform_by_category` - Side effects: - - `_reverse_transform_by_category` will be called once - """ - # Setup - transform_data = pd.Series([1, 3, 3, 2, 1]) - - categorical_transformer_mock = Mock() - categorical_transformer_mock.means = pd.Series([ - 0.125, - 0.375, - 0.625, - 0.875, - ]) - - # Run - reverse = FrequencyEncoder._reverse_transform(categorical_transformer_mock, transform_data) - - # Asserts - reverse_arg = categorical_transformer_mock._reverse_transform_by_category.call_args[0][0] - np.testing.assert_array_equal(reverse_arg, transform_data.clip(0, 1)) - assert reverse == categorical_transformer_mock._reverse_transform_by_category.return_value - - def test__reverse_transform_by_category(self): - """Test the _reverse_transform_by_category method with numerical data. - - Expect that the transformed data is correctly reverse transformed. - - Setup: - The categorical transformer is instantiated with 4 categories, and the means - and intervals are set for those categories. - Input: - - transformed data with 5 rows - Ouptut: - - the original data - """ - data = pd.Series([1, 3, 3, 2, 1]) - transformed = pd.Series([0.875, 0.375, 0.375, 0.625, 0.875]) - - transformer = FrequencyEncoder() - transformer.means = pd.Series([0.125, 0.375, 0.625, 0.875], index=[4, 3, 2, 1]) - transformer.intervals = { - 4: (0, 0.25, 0.125, 0.041666666666666664), - 3: (0.25, 0.5, 0.375, 0.041666666666666664), - 2: (0.5, 0.75, 0.625, 0.041666666666666664), - 1: (0.75, 1.0, 0.875, 0.041666666666666664), - } - transformer.dtype = data.dtype - - reverse = transformer._reverse_transform_by_category(transformed) - - pd.testing.assert_series_equal(data, reverse) - - def test__get_category_from_start(self): - """Test the ``_get_category_from_start`` method. - - Setup: - - instantiate a ``FrequencyEncoder``, and set the attribute ``starts`` - to a pandas dataframe with ``set_index`` as ``'start'``. - - Input: - - an integer, an index from data. - - Output: - - a category from the data. - """ - # Setup - transformer = FrequencyEncoder() - transformer.starts = pd.DataFrame({ - 'start': [0.0, 0.5, 0.7], - 'category': ['a', 'b', 'c'], - }).set_index('start') - - # Run - category = transformer._get_category_from_start(2) - - # Assert - assert category == 'c' - - def test__reverse_transform_by_row_called(self): - """Test that the `_reverse_transform_by_row` method is called. - - When the number of rows is less than or equal to the number of categories, - expect that the `_reverse_transform_by_row` method is called. - - Setup: - The categorical transformer is instantiated with 4 categories. - Input: - - numerical data with 4 rows - Output: - - the output of `_reverse_transform_by_row` - Side effects: - - `_reverse_transform_by_row` will be called once - """ - # Setup - data = pd.Series([1, 2, 3, 4]) - - categorical_transformer_mock = Mock() - categorical_transformer_mock.means = pd.Series([ - 0.125, - 0.375, - 0.625, - 0.875, - ]) - categorical_transformer_mock.starts = pd.DataFrame( - [0.0, 0.25, 0.5, 0.75], index=[4, 3, 2, 1], columns=['category'] - ) - categorical_transformer_mock._normalize.return_value = data - - # Run - reverse = FrequencyEncoder._reverse_transform(categorical_transformer_mock, data) - - # Asserts - reverse_arg = categorical_transformer_mock._reverse_transform_by_row.call_args[0][0] - np.testing.assert_array_equal(reverse_arg, data.clip(0, 1)) - assert reverse == categorical_transformer_mock._reverse_transform_by_row.return_value - - @patch('rdt.transformers.categorical.check_nan_in_transform') - def test__reverse_transform_by_row(self, mock_check_nan): - """Test the _reverse_transform_by_row method with numerical data. - - Expect that the transformed data is correctly reverse transformed. - - Setup: - The categorical transformer is instantiated with 4 categories, and the means, starts, - and intervals are set for those categories. - Input: - - transformed data with 4 rows - Ouptut: - - the original data - """ - # Setup - data = pd.Series([1, 2, 3, 4]) - transformed = pd.Series([0.875, 0.625, 0.375, 0.125]) - - transformer = FrequencyEncoder() - transformer.means = pd.Series([0.125, 0.375, 0.625, 0.875], index=[4, 3, 2, 1]) - transformer.starts = pd.DataFrame( - [4, 3, 2, 1], index=[0.0, 0.25, 0.5, 0.75], columns=['category'] - ) - transformer.intervals = { - 4: (0, 0.25, 0.125, 0.041666666666666664), - 3: (0.25, 0.5, 0.375, 0.041666666666666664), - 2: (0.5, 0.75, 0.625, 0.041666666666666664), - 1: (0.75, 1.0, 0.875, 0.041666666666666664), - } - transformer.dtype = data.dtype - - # Run - reverse = transformer._reverse_transform(transformed) - - # Assert - mock_input_data = mock_check_nan.call_args.args[0] - mock_input_dtype = mock_check_nan.call_args.args[1] - pd.testing.assert_series_equal(mock_input_data, transformed) - assert mock_input_dtype == data.dtype - pd.testing.assert_series_equal(data, reverse) - - class TestOneHotEncoder: def test__prepare_data_empty_lists(self): # Setup From a0873d62018905d5f3b0c22cda1dc5632ae4da53 Mon Sep 17 00:00:00 2001 From: R-Palazzo Date: Thu, 20 Aug 2026 11:26:51 +0100 Subject: [PATCH 05/13] remove CustomLabelEncoder --- tests/unit/test__deprecations.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/test__deprecations.py b/tests/unit/test__deprecations.py index 475b537c1..0295ab0f1 100644 --- a/tests/unit/test__deprecations.py +++ b/tests/unit/test__deprecations.py @@ -69,6 +69,7 @@ def test_deprecated_methods(class_, method, parameter): [ 'rdt.transformers.FrequencyEncoder', 'rdt.transformers.categorical.FrequencyEncoder', + 'rdt.transformers.categorical.CustomLabelEncoder', ], ) def test_deprecated_classes(class_path): From d6a29f9aa6e998771cb109d35bab3d8ec1417698 Mon Sep 17 00:00:00 2001 From: R-Palazzo Date: Thu, 20 Aug 2026 11:31:02 +0100 Subject: [PATCH 06/13] remove IDGenerator --- rdt/transformers/__init__.py | 3 +-- rdt/transformers/id.py | 15 --------------- rdt/transformers/text.py | 4 ++-- tests/unit/test__deprecations.py | 1 + tests/unit/transformers/test_id.py | 13 +------------ tests/unit/transformers/test_text.py | 4 ++-- 6 files changed, 7 insertions(+), 33 deletions(-) diff --git a/rdt/transformers/__init__.py b/rdt/transformers/__init__.py index 62727bcec..12e096432 100644 --- a/rdt/transformers/__init__.py +++ b/rdt/transformers/__init__.py @@ -18,7 +18,7 @@ OptimizedTimestampEncoder, UnixTimestampEncoder, ) -from rdt.transformers.id import IDGenerator, IndexGenerator, RegexGenerator +from rdt.transformers.id import IndexGenerator, RegexGenerator from rdt.transformers.null import NullTransformer from rdt.transformers.numerical import ( ClusterBasedNormalizer, @@ -54,7 +54,6 @@ 'RegexGenerator', 'AnonymizedFaker', 'PseudoAnonymizedFaker', - 'IDGenerator', 'IndexGenerator', 'get_transformer_name', 'get_transformer_class', diff --git a/rdt/transformers/id.py b/rdt/transformers/id.py index b8c6515c4..d5642f22c 100644 --- a/rdt/transformers/id.py +++ b/rdt/transformers/id.py @@ -79,21 +79,6 @@ def _reverse_transform(self, data): return pd.Series(values) -class IDGenerator(IndexGenerator): - """Deprecated class name for ``IndexGenerator``. - - Class to ensure backwards compatibility with previous versions of RDT. - """ - - def __init__(self, prefix=None, starting_value=0, suffix=None): - warnings.warn( - "The 'IDGenerator' has been renamed to 'IndexGenerator'. Please update the" - 'name to ensure compatibility with future versions of RDT.', - FutureWarning, - ) - super().__init__(prefix, starting_value, suffix) - - class RegexGenerator(BaseTransformer): """RegexGenerator transformer. diff --git a/rdt/transformers/text.py b/rdt/transformers/text.py index 2de3d0eb1..a3f69b7c9 100644 --- a/rdt/transformers/text.py +++ b/rdt/transformers/text.py @@ -2,10 +2,10 @@ import warnings -from rdt.transformers.id import IDGenerator, RegexGenerator # noqa: F401 +from rdt.transformers.id import RegexGenerator # noqa: F401 warnings.warn( - "Importing 'IDGenerator' or 'RegexGenerator' for ID columns from 'rdt.transformers.text' " + "Importing 'RegexGenerator' for ID columns from 'rdt.transformers.text' " "is deprecated. Please use 'rdt.transformers.id' instead.", DeprecationWarning, stacklevel=2, diff --git a/tests/unit/test__deprecations.py b/tests/unit/test__deprecations.py index 0295ab0f1..8602e30c8 100644 --- a/tests/unit/test__deprecations.py +++ b/tests/unit/test__deprecations.py @@ -70,6 +70,7 @@ def test_deprecated_methods(class_, method, parameter): 'rdt.transformers.FrequencyEncoder', 'rdt.transformers.categorical.FrequencyEncoder', 'rdt.transformers.categorical.CustomLabelEncoder', + 'rdt.transformers.id.IDGenerator', ], ) def test_deprecated_classes(class_path): diff --git a/tests/unit/transformers/test_id.py b/tests/unit/transformers/test_id.py index 156db006c..54feff618 100644 --- a/tests/unit/transformers/test_id.py +++ b/tests/unit/transformers/test_id.py @@ -10,7 +10,7 @@ import pandas as pd import pytest -from rdt.transformers.id import IDGenerator, IndexGenerator, RegexGenerator +from rdt.transformers.id import IndexGenerator, RegexGenerator class AsciiGenerator: @@ -145,17 +145,6 @@ def test__reverse_transform_with_everything(self): assert transformer._counter == 3 -class TestIDGenerator: - def test___init__(self): - """Test the warning message for ``IDGenerator``.""" - msg = re.escape( - "The 'IDGenerator' has been renamed to 'IndexGenerator'. Please update the" - 'name to ensure compatibility with future versions of RDT.' - ) - with pytest.warns(FutureWarning, match=msg): - IDGenerator(prefix='prefix_', starting_value=100, suffix='_suffix') - - class TestRegexGenerator: """Test class for ``RegexGenerator``.""" diff --git a/tests/unit/transformers/test_text.py b/tests/unit/transformers/test_text.py index 56cd528f5..0c549e0ae 100644 --- a/tests/unit/transformers/test_text.py +++ b/tests/unit/transformers/test_text.py @@ -7,8 +7,8 @@ def test_deprecation_warning_is_raised(): """Test that a deprecation warning is raised when importing from this module.""" # Run and Assert expected_message = ( - "Importing 'IDGenerator' or 'RegexGenerator' for ID columns from 'rdt.transformers.text' " + "Importing 'RegexGenerator' for ID columns from 'rdt.transformers.text' " "is deprecated. Please use 'rdt.transformers.id' instead." ) with pytest.warns(DeprecationWarning, match=expected_message): - from rdt.transformers.text import IDGenerator, RegexGenerator # noqa: F401 + from rdt.transformers.text import RegexGenerator # noqa: F401 From 4253c8e1bbac101883b92478d3fc281ab8428e5a Mon Sep 17 00:00:00 2001 From: R-Palazzo Date: Thu, 20 Aug 2026 12:04:48 +0100 Subject: [PATCH 07/13] remove enforce_uniqueness --- rdt/transformers/id.py | 14 +++------- rdt/transformers/pii/anonymizer.py | 16 +++--------- rdt/transformers/utils.py | 18 ------------- ...tions.py => test__rdt_2_0_deprecations.py} | 9 +++++++ .../unit/transformers/pii/test_anonymizer.py | 18 ++----------- tests/unit/transformers/test_id.py | 26 ------------------- tests/unit/transformers/test_utils.py | 23 ---------------- 7 files changed, 17 insertions(+), 107 deletions(-) rename tests/unit/{test__deprecations.py => test__rdt_2_0_deprecations.py} (91%) diff --git a/rdt/transformers/id.py b/rdt/transformers/id.py index d5642f22c..6794ffac1 100644 --- a/rdt/transformers/id.py +++ b/rdt/transformers/id.py @@ -10,7 +10,6 @@ from rdt.transformers.base import BaseTransformer from rdt.transformers.utils import ( _get_cardinality_frequency, - _handle_enforce_uniqueness_and_cardinality_rule, _sample_repetitions, fill_nan_with_none, strings_from_regex, @@ -88,10 +87,6 @@ class RegexGenerator(BaseTransformer): Args: regex_format (str): String representing the regex function. - enforce_uniqueness (bool): - **DEPRECATED** Whether or not to ensure that the new generated data is all unique. - If it isn't possible to create the requested number of rows, then an error will - be raised. Defaults to ``None``. cardinality_rule (str): Rule that the generated data must follow. - If set to 'unique', the generated data must be unique. @@ -164,14 +159,11 @@ def __init__( regex_format='[A-Za-z]{5}', cardinality_rule=None, generation_order='alphanumeric', - enforce_uniqueness=None, ): super().__init__() self.output_properties = {None: {'next_transformer': None}} self.regex_format = regex_format - self.cardinality_rule = _handle_enforce_uniqueness_and_cardinality_rule( - enforce_uniqueness, cardinality_rule - ) + self.cardinality_rule = cardinality_rule self.data_length = None self.generator = None if generation_order not in ['alphanumeric', 'scrambled']: @@ -208,7 +200,7 @@ def _sample_fallback(self, num_samples, template_samples): unique_condition = ( self.cardinality_rule == 'unique' if hasattr(self, 'cardinality_rule') - else self.enforce_uniqueness + else getattr(self, 'enforce_uniqueness', False) ) if unique_condition: if not self._num_fallback_samples_generated: @@ -470,7 +462,7 @@ def _reverse_transform(self, data): if match_cardinality and self._unique_regex_values is None: self._unique_regex_values = self._generate_unique_regexes() else: - unique_condition = self.enforce_uniqueness + unique_condition = getattr(self, 'enforce_uniqueness', False) match_cardinality = False num_samples = len(data) if (data is not None and len(data)) else self.data_length diff --git a/rdt/transformers/pii/anonymizer.py b/rdt/transformers/pii/anonymizer.py index f6ab18aa7..7b6781e4d 100644 --- a/rdt/transformers/pii/anonymizer.py +++ b/rdt/transformers/pii/anonymizer.py @@ -17,7 +17,6 @@ from rdt.transformers.categorical import LabelEncoder from rdt.transformers.utils import ( _get_cardinality_frequency, - _handle_enforce_uniqueness_and_cardinality_rule, _sample_repetitions, ) @@ -46,11 +45,6 @@ class AnonymizedFaker(BaseTransformer): each value is allowed to have. If ``None`` do not consider cardinality. Defaults to ``None``. - enforce_uniqueness (bool): - **DEPRECATED** Whether or not to ensure that the new anonymized data is all unique. - If it isn't possible to create the requested number of rows, then an error will be - raised. - Defaults to ``False``. missing_value_generation (str or None): The way missing values are being handled. There are two strategies: @@ -120,7 +114,6 @@ def __init__( function_kwargs=None, locales=None, cardinality_rule=None, - enforce_uniqueness=None, missing_value_generation='random', ): super().__init__() @@ -128,11 +121,7 @@ def __init__( self._remaining_samples = {'value': None, 'repetitions': 0} self._data_cardinality = None self.data_length = None - self.enforce_uniqueness = enforce_uniqueness self.cardinality_rule = cardinality_rule.lower() if cardinality_rule else None - self.cardinality_rule = _handle_enforce_uniqueness_and_cardinality_rule( - enforce_uniqueness, cardinality_rule - ) self.provider_name = provider_name if provider_name else 'BaseProvider' if self.provider_name != 'BaseProvider' and function_name is None: @@ -210,7 +199,9 @@ def _function(self): faker_attr = self.faker except AttributeError: - faker_attr = self.faker.unique if self.enforce_uniqueness else self.faker + faker_attr = ( + self.faker.unique if getattr(self, 'enforce_uniqueness', False) else self.faker + ) result = getattr(faker_attr, self.function_name)(**self.function_kwargs) if isinstance(result, Iterable) and not isinstance(result, str): @@ -427,7 +418,6 @@ def __repr__(self): args = inspect.getfullargspec(self.__init__) keys = args.args[1:] defaults = dict(zip(keys, args.defaults)) - keys.remove('enforce_uniqueness') instanced = {key: getattr(self, key) for key in keys} defaults['function_name'] = None diff --git a/rdt/transformers/utils.py b/rdt/transformers/utils.py index 646adbb6f..4f8d8b3be 100644 --- a/rdt/transformers/utils.py +++ b/rdt/transformers/utils.py @@ -400,24 +400,6 @@ def __getitem__(self, sdtype): return self.get(sdtype) -def _handle_enforce_uniqueness_and_cardinality_rule(enforce_uniqueness, cardinality_rule): - if enforce_uniqueness is not None: - warnings.warn( - "The 'enforce_uniqueness' parameter is no longer supported. " - "Please use the 'cardinality_rule' parameter instead.", - FutureWarning, - ) - if enforce_uniqueness and cardinality_rule is None: - return 'unique' - - if cardinality_rule not in ['unique', 'match', 'scale', None]: - raise ValueError( - "The 'cardinality_rule' parameter must be one of 'unique', 'match', 'scale', or None." - ) - - return cardinality_rule - - def _extract_timezone_from_a_string(dt_str): if not isinstance(dt_str, str): dt_str = str(dt_str) diff --git a/tests/unit/test__deprecations.py b/tests/unit/test__rdt_2_0_deprecations.py similarity index 91% rename from tests/unit/test__deprecations.py rename to tests/unit/test__rdt_2_0_deprecations.py index 8602e30c8..41165cf53 100644 --- a/tests/unit/test__deprecations.py +++ b/tests/unit/test__rdt_2_0_deprecations.py @@ -5,11 +5,13 @@ from rdt.hyper_transformer import HyperTransformer from rdt.transformers import ( + AnonymizedFaker, BaseTransformer, BinaryEncoder, ClusterBasedNormalizer, FloatFormatter, GaussianNormalizer, + RegexGenerator, UnixTimestampEncoder, ) @@ -32,6 +34,13 @@ ClusterBasedNormalizer, ) ], + *[ + (class_, '__init__', 'enforce_uniqueness') + for class_ in ( + RegexGenerator, + AnonymizedFaker, + ) + ], ], ) def test_deprecated_parameters(class_, method, parameter): diff --git a/tests/unit/transformers/pii/test_anonymizer.py b/tests/unit/transformers/pii/test_anonymizer.py index b656be06c..07e454d2f 100644 --- a/tests/unit/transformers/pii/test_anonymizer.py +++ b/tests/unit/transformers/pii/test_anonymizer.py @@ -121,14 +121,6 @@ def test__function_cardinality_rule_none(self): function.assert_called_once_with(type='int') assert result == 1 - def test___init___enforce_uniqueness_exists(self): - """Test `enforce_uniqueness` attribute exists.""" - # Run - instance = AnonymizedFaker() - - # Assert - assert instance.enforce_uniqueness is None - def test__function_cardinality_rule_unique(self): """Test that ``_function`` uses the ``faker.unique``. @@ -198,7 +190,6 @@ def test__function_cardinality_rule_missing_attribute(self): unique_function.return_value = 1 delattr(instance, 'cardinality_rule') - instance.enforce_uniqueness = True instance.faker.unique.number = unique_function instance.faker.number = function instance.function_name = 'number' @@ -394,8 +385,7 @@ def test___init__error_missing_value_generation(self): @patch('rdt.transformers.pii.anonymizer.faker') @patch('rdt.transformers.pii.anonymizer.AnonymizedFaker.check_provider_function') - @patch('rdt.transformers.pii.anonymizer._handle_enforce_uniqueness_and_cardinality_rule') - def test___init__custom(self, mock__handle, mock_check_provider_function, mock_faker): + def test___init__custom(self, mock_check_provider_function, mock_faker): """Test the instantiation of the transformer with custom parameters. Test that the transformer can be instantiated with a custom provider and function, and @@ -417,16 +407,13 @@ def test___init__custom(self, mock__handle, mock_check_provider_function, mock_f ``credit_card_full``. - the ``instance._function`` is ``instance.faker.credit_card_full``. """ - # Setup - mock__handle.return_value = 'unique' - # Run instance = AnonymizedFaker( provider_name='credit_card', function_name='credit_card_full', function_kwargs={'type': 'visa'}, locales=['en_US', 'fr_FR'], - enforce_uniqueness=True, + cardinality_rule='unique', ) # Assert @@ -437,7 +424,6 @@ def test___init__custom(self, mock__handle, mock_check_provider_function, mock_f assert instance.locales == ['en_US', 'fr_FR'] mock_faker.Faker.assert_called_once_with(['en_US', 'fr_FR']) assert instance.cardinality_rule == 'unique' - mock__handle.assert_called_once_with(True, None) def test___init__no_function_name(self): """Test the instantiation of the transformer with custom parameters. diff --git a/tests/unit/transformers/test_id.py b/tests/unit/transformers/test_id.py index 54feff618..15880010c 100644 --- a/tests/unit/transformers/test_id.py +++ b/tests/unit/transformers/test_id.py @@ -299,32 +299,6 @@ def test___init__bad_value_generation_order(self): with pytest.raises(ValueError, match=error_message): RegexGenerator(generation_order='afdsfd') - def test__init__with_enforce_uniqueness(self): - """Test that the ``enforce_uniqueness`` parameter is deprecated.""" - # Setup - expected_message = re.escape( - "The 'enforce_uniqueness' parameter is no longer supported. " - "Please use the 'cardinality_rule' parameter instead." - ) - - # Run - with pytest.warns(FutureWarning, match=expected_message): - instance_1 = RegexGenerator(enforce_uniqueness=True, cardinality_rule='unique') - - with pytest.warns(FutureWarning, match=expected_message): - RegexGenerator('A-Za-z', None, 'alphanumeric', True) - - with pytest.warns(FutureWarning, match=expected_message): - instance_2 = RegexGenerator(enforce_uniqueness=True) - - with pytest.warns(FutureWarning, match=expected_message): - instance_3 = RegexGenerator(enforce_uniqueness=False) - - # Assert - assert instance_1.cardinality_rule == 'unique' - assert instance_2.cardinality_rule == 'unique' - assert instance_3.cardinality_rule is None - @patch('rdt.transformers.id.BaseTransformer.reset_randomization') @patch('rdt.transformers.id.strings_from_regex') def test_reset_randomization(self, mock_strings_from_regex, mock_base_reset): diff --git a/tests/unit/transformers/test_utils.py b/tests/unit/transformers/test_utils.py index 39fda870f..ff43b14ac 100644 --- a/tests/unit/transformers/test_utils.py +++ b/tests/unit/transformers/test_utils.py @@ -18,7 +18,6 @@ _extract_timezone_from_a_string, _fill_nan_with_none_series, _get_utc_offset, - _handle_enforce_uniqueness_and_cardinality_rule, _max_repeat, _parser, _safe_parse_datetime, @@ -589,28 +588,6 @@ def test_warn_dict_get(): assert result_get_no_warn == 'text_transformer' -def test__handle_enforce_uniqueness_and_cardinality_rule(): - """Test that ``_handle_enforce_uniqueness_and_cardinality_rule`` works as expected.""" - # Run and Assert - assert _handle_enforce_uniqueness_and_cardinality_rule(None, None) is None - - expected_message = re.escape( - "The 'enforce_uniqueness' parameter is no longer supported. " - "Please use the 'cardinality_rule' parameter instead." - ) - with pytest.warns(FutureWarning, match=expected_message): - assert _handle_enforce_uniqueness_and_cardinality_rule(True, None) == 'unique' - - err_msg = "The 'cardinality_rule' parameter must be one of 'unique', 'match', 'scale', or None." - with pytest.raises(ValueError, match=err_msg): - _handle_enforce_uniqueness_and_cardinality_rule(None, 'invalid') - - assert _handle_enforce_uniqueness_and_cardinality_rule(None, 'unique') == 'unique' - assert _handle_enforce_uniqueness_and_cardinality_rule(None, 'match') == 'match' - assert _handle_enforce_uniqueness_and_cardinality_rule(None, 'scale') == 'scale' - assert _handle_enforce_uniqueness_and_cardinality_rule(None, None) is None - - def test__extract_timezone_from_a_string_with_valid_timezone(): """Test that `_extract_timezone_from_a_string` extracts a valid timezone from a string.""" # Setup From c947df26fbbb90512e12f30c6a385389491e044c Mon Sep 17 00:00:00 2001 From: R-Palazzo Date: Thu, 20 Aug 2026 12:21:00 +0100 Subject: [PATCH 08/13] remove support for distributions non consistent with scipy --- rdt/transformers/numerical.py | 14 ------ tests/unit/test__rdt_2_0_deprecations.py | 25 +++++++++++ tests/unit/transformers/test_numerical.py | 53 ----------------------- 3 files changed, 25 insertions(+), 67 deletions(-) diff --git a/rdt/transformers/numerical.py b/rdt/transformers/numerical.py index d5067eb79..8fceef205 100644 --- a/rdt/transformers/numerical.py +++ b/rdt/transformers/numerical.py @@ -291,11 +291,6 @@ class GaussianNormalizer(FloatFormatter): _univariate = None _fallback_distribution = 'norm' _learned_distribution_name = None - _DEPRECATED_DISTRIBUTIONS_MAPPING = { - 'gaussian': 'norm', - 'student_t': 't', - 'truncated_gaussian': 'truncnorm', - } @staticmethod def _get_distributions(): @@ -339,15 +334,6 @@ def __init__( self._distributions = self._get_distributions() if isinstance(distribution, str): - if distribution in {'gaussian', 'student_t', 'truncated_gaussian'}: - warnings.warn( - f"Future versions of RDT will not support '{distribution}' as an option. " - f"Please use '{self._DEPRECATED_DISTRIBUTIONS_MAPPING[distribution]}' " - 'instead.', - FutureWarning, - ) - distribution = self._DEPRECATED_DISTRIBUTIONS_MAPPING[distribution] - self._learned_distribution_name = distribution distribution = self._distributions[distribution] diff --git a/tests/unit/test__rdt_2_0_deprecations.py b/tests/unit/test__rdt_2_0_deprecations.py index 41165cf53..b8fec90a8 100644 --- a/tests/unit/test__rdt_2_0_deprecations.py +++ b/tests/unit/test__rdt_2_0_deprecations.py @@ -56,6 +56,31 @@ def test_deprecated_parameters(class_, method, parameter): getattr(instance, method)(**{parameter: 'value'}) +@pytest.mark.parametrize( + 'class_, method, parameter, value, error', + [ + *[ + ( + GaussianNormalizer, + '__init__', + 'distribution', + value, + KeyError, + ) + for value in ('gaussian', 'student_t', 'truncated_gaussian') + ], + ], +) +def test_deprecated_parameters_with_value(class_, method, parameter, value, error): + """Test that deprecated parameters raise an error.""" + # Setup + instance = class_() + + # Run and Assert + with pytest.raises(error): + getattr(instance, method)(**{parameter: value}) + + @pytest.mark.parametrize( 'class_, method, parameter', [ diff --git a/tests/unit/transformers/test_numerical.py b/tests/unit/transformers/test_numerical.py index 96c4e57d1..bf4202a02 100644 --- a/tests/unit/transformers/test_numerical.py +++ b/tests/unit/transformers/test_numerical.py @@ -815,21 +815,6 @@ def test___init__non_distr(self): assert ct._distribution is univariate - def test___init__deprecated_distributions_warning(self): - """Test it warns when using deprecated distributions.""" - # Run and Assert - dists = zip( - ['gaussian', 'student_t', 'truncated_gaussian'], - ['norm', 't', 'truncnorm'], - ) - for deprecated, distribution in dists: - err_msg = re.escape( - f"Future versions of RDT will not support '{deprecated}' as an option. " - f"Please use '{distribution}' instead." - ) - with pytest.warns(FutureWarning, match=err_msg): - GaussianNormalizer(distribution=deprecated) - def test__get_distributions_copulas_not_installed(self): """Test the ``_get_distributions`` method when copulas is not installed. @@ -1251,28 +1236,6 @@ def test_fallback_with_string_distribution(self): assert transformer._learned_distribution_name == 'norm' - def test_fallback_with_deprecated_distribution(self): - """Test fallback with deprecated distribution names.""" - # Setup - transformer = GaussianNormalizer(distribution='gaussian') - data = pd.DataFrame({'test_column': [1, 2, 3, 4, 5]}) - - # Run - with patch.object(transformer, '_get_univariate') as mock_get_univariate: - mock_univariate = Mock() - mock_univariate.fit.side_effect = Exception('Fitting failed') - mock_get_univariate.return_value = mock_univariate - - with patch('rdt.transformers.numerical.LOGGER.info') as mock_logger: - transformer.fit(data, 'test_column') - - # Assert - mock_logger.assert_called_once_with( - "Unable to fit the distribution 'norm'. Falling back to 'norm'." - ) - - assert transformer._learned_distribution_name == 'norm' - def test_fallback_with_class_distribution(self): """Test fallback with class distribution.""" # Setup @@ -1462,22 +1425,6 @@ def test_learned_distribution_with_truncnorm(self): assert learned_dist['distribution'] == 'truncnorm' assert isinstance(learned_dist['parameters'], dict) - def test_learned_distribution_with_deprecated_distribution(self): - """Test learned_distribution with deprecated distribution names.""" - # Setup - with pytest.warns(FutureWarning): - transformer = GaussianNormalizer(distribution='gaussian') - - data = pd.DataFrame({'test_column': [1, 2, 3, 4, 5]}) - - # Run - transformer.fit(data, 'test_column') - learned_dist = transformer.learned_distribution - - # Assert - assert learned_dist['distribution'] == 'norm' - assert isinstance(learned_dist['parameters'], dict) - class TestClusterBasedNormalizer(TestCase): def test__get_current_random_seed_random_states_is_none(self): From dbdb02b67a09ade004c1459ae6fc36dddca7b3b8 Mon Sep 17 00:00:00 2001 From: R-Palazzo Date: Thu, 20 Aug 2026 12:50:08 +0100 Subject: [PATCH 09/13] remove support text sdtype --- rdt/performance/datasets/__init__.py | 2 - rdt/performance/datasets/text.py | 57 ------------------- rdt/transformers/__init__.py | 18 +++--- rdt/transformers/categorical.py | 4 +- rdt/transformers/id.py | 4 +- rdt/transformers/text.py | 12 ---- rdt/transformers/utils.py | 35 ------------ tests/integration/test_hyper_transformer.py | 10 ++-- tests/integration/test_transformers.py | 1 - .../transformers/pii/test_anonymizer.py | 2 +- .../transformers/test_categorical.py | 10 ++-- tests/unit/test__rdt_2_0_deprecations.py | 13 +++++ tests/unit/test_hyper_transformer.py | 4 -- .../unit/transformers/pii/test_anonymizer.py | 4 -- tests/unit/transformers/test_text.py | 14 ----- tests/unit/transformers/test_utils.py | 48 ---------------- 16 files changed, 35 insertions(+), 203 deletions(-) delete mode 100644 rdt/performance/datasets/text.py delete mode 100644 rdt/transformers/text.py delete mode 100644 tests/unit/transformers/test_text.py diff --git a/rdt/performance/datasets/__init__.py b/rdt/performance/datasets/__init__.py index 2dcfdc407..e0e280365 100644 --- a/rdt/performance/datasets/__init__.py +++ b/rdt/performance/datasets/__init__.py @@ -9,7 +9,6 @@ id, numerical, pii, - text, ) from rdt.performance.datasets.base import BaseDatasetGenerator @@ -20,7 +19,6 @@ 'id', 'numerical', 'pii', - 'text', 'BaseDatasetGenerator', ] diff --git a/rdt/performance/datasets/text.py b/rdt/performance/datasets/text.py deleted file mode 100644 index 055ddfdd2..000000000 --- a/rdt/performance/datasets/text.py +++ /dev/null @@ -1,57 +0,0 @@ -"""Dataset Generators for 'text' transformers.""" - -from abc import ABC - -import numpy as np - -from rdt.performance.datasets.base import BaseDatasetGenerator -from rdt.performance.datasets.utils import add_nans - - -class RegexGeneratorGenerator(BaseDatasetGenerator, ABC): - """Base class for generators that generate ID data.""" - - SDTYPE = 'text' - - -class RandomStringGenerator(RegexGeneratorGenerator): - """Generator that creates an array of random strings.""" - - @staticmethod - def generate(num_rows): - """Generate a ``num_rows`` number of rows.""" - categories = ['Alice', 'Bob', 'Charlie', 'Dave', 'Eve'] - return np.random.choice(a=categories, size=num_rows) - - @staticmethod - def get_performance_thresholds(): - """Return the expected thresholds.""" - return { - 'fit': {'time': 1e-05, 'memory': 500.0}, - 'transform': {'time': 1e-05, 'memory': 500.0}, - 'reverse_transform': { - 'time': 2e-05, - 'memory': 1000.0, - }, - } - - -class RandomStringNaNsGenerator(RegexGeneratorGenerator): - """Generator that creates an array of random strings with nans.""" - - @staticmethod - def generate(num_rows): - """Generate a ``num_rows`` number of rows.""" - return add_nans(RandomStringGenerator.generate(num_rows).astype('O')) - - @staticmethod - def get_performance_thresholds(): - """Return the expected thresholds.""" - return { - 'fit': {'time': 1e-05, 'memory': 400.0}, - 'transform': {'time': 1e-05, 'memory': 1000.0}, - 'reverse_transform': { - 'time': 2e-05, - 'memory': 1000.0, - }, - } diff --git a/rdt/transformers/__init__.py b/rdt/transformers/__init__.py index 12e096432..09c01d25a 100644 --- a/rdt/transformers/__init__.py +++ b/rdt/transformers/__init__.py @@ -35,7 +35,6 @@ AnonymizedFaker, PseudoAnonymizedFaker, ) -from rdt.transformers.utils import WarnDict __all__ = [ 'BaseTransformer', @@ -91,15 +90,14 @@ def get_transformer_name(transformer): for transformer in BaseTransformer.get_subclasses() } -DEFAULT_TRANSFORMERS = WarnDict( - boolean=UniformEncoder(), - categorical=UniformEncoder(), - datetime=UnixTimestampEncoder(), - id=RegexGenerator(), - numerical=FloatFormatter(), - pii=AnonymizedFaker(), - text=RegexGenerator(), -) +DEFAULT_TRANSFORMERS = { + 'boolean': UniformEncoder(), + 'categorical': UniformEncoder(), + 'datetime': UnixTimestampEncoder(), + 'id': RegexGenerator(), + 'numerical': FloatFormatter(), + 'pii': AnonymizedFaker(), +} @lru_cache() diff --git a/rdt/transformers/categorical.py b/rdt/transformers/categorical.py index 026dafbfe..5760b1b48 100644 --- a/rdt/transformers/categorical.py +++ b/rdt/transformers/categorical.py @@ -47,7 +47,7 @@ class UniformEncoder(BaseTransformer): """ INPUT_SDTYPE = 'categorical' - SUPPORTED_SDTYPES = ['categorical', 'boolean', 'id', 'text'] + SUPPORTED_SDTYPES = ['categorical', 'boolean', 'id'] frequencies = None intervals = None dtype = None @@ -397,7 +397,7 @@ class LabelEncoder(BaseTransformer): """ INPUT_SDTYPE = 'categorical' - SUPPORTED_SDTYPES = ['categorical', 'boolean', 'id', 'text'] + SUPPORTED_SDTYPES = ['categorical', 'boolean', 'id'] values_to_categories = None categories_to_values = None dtype = 'O' diff --git a/rdt/transformers/id.py b/rdt/transformers/id.py index 6794ffac1..53b7af629 100644 --- a/rdt/transformers/id.py +++ b/rdt/transformers/id.py @@ -37,7 +37,7 @@ class IndexGenerator(BaseTransformer): IS_GENERATOR = True INPUT_SDTYPE = 'id' - SUPPORTED_SDTYPES = ['id', 'text'] + SUPPORTED_SDTYPES = ['id'] def __init__(self, prefix=None, starting_value=0, suffix=None): super().__init__() @@ -105,7 +105,7 @@ class RegexGenerator(BaseTransformer): IS_GENERATOR = True INPUT_SDTYPE = 'id' - SUPPORTED_SDTYPES = ['id', 'text'] + SUPPORTED_SDTYPES = ['id'] def __getstate__(self): """Remove the generator when pickling.""" diff --git a/rdt/transformers/text.py b/rdt/transformers/text.py deleted file mode 100644 index a3f69b7c9..000000000 --- a/rdt/transformers/text.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Transformers for text data.""" - -import warnings - -from rdt.transformers.id import RegexGenerator # noqa: F401 - -warnings.warn( - "Importing 'RegexGenerator' for ID columns from 'rdt.transformers.text' " - "is deprecated. Please use 'rdt.transformers.id' instead.", - DeprecationWarning, - stacklevel=2, -) diff --git a/rdt/transformers/utils.py b/rdt/transformers/utils.py index 4f8d8b3be..eaec54e48 100644 --- a/rdt/transformers/utils.py +++ b/rdt/transformers/utils.py @@ -6,7 +6,6 @@ import string import sys import warnings -from collections import defaultdict from decimal import Decimal import numpy as np @@ -23,7 +22,6 @@ LOGGER = logging.getLogger(__name__) MAX_DECIMALS = sys.float_info.dig -DEPRECATED_SDTYPES_MAPPING = {'text': 'id'} def _literal(character, max_repeat): @@ -367,39 +365,6 @@ def sigmoid(data, low, high): return data -class WarnDict(dict): - """Custom dictionary to raise a deprecation warning.""" - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self._warned = defaultdict() - - def get(self, sdtype): - """Return the value for sdtype if sdtype is in the dictionary, else default. - - If the sdtype is `text` raises a `DeprecationWarning` stating that it will be - phased out. - """ - if sdtype in DEPRECATED_SDTYPES_MAPPING and not self._warned.get(sdtype): - new_sdtype = DEPRECATED_SDTYPES_MAPPING.get(sdtype) - warnings.warn( - f"The sdtype '{sdtype}' is deprecated and will be phased out. " - f"Please use '{new_sdtype}' instead.", - DeprecationWarning, - ) - self._warned[sdtype] = True - - return super().get(sdtype) - - def __getitem__(self, sdtype): - """Return the value for sdtype if sdtype is in the dictionary. - - If the sdtype is `text` raises a `DeprecationWarning` stating that it will be - phased out. - """ - return self.get(sdtype) - - def _extract_timezone_from_a_string(dt_str): if not isinstance(dt_str, str): dt_str = str(dt_str) diff --git a/tests/integration/test_hyper_transformer.py b/tests/integration/test_hyper_transformer.py index a4b67b949..6086e19fb 100644 --- a/tests/integration/test_hyper_transformer.py +++ b/tests/integration/test_hyper_transformer.py @@ -921,8 +921,8 @@ def test_reverse_transform_subset_and_generators(self): ht = HyperTransformer() ht.detect_initial_config(customers) - # credit_card and id are pii and text columns - ht.update_sdtypes({'credit_card': 'pii', 'id': 'text'}) + # credit_card and id are pii and id columns + ht.update_sdtypes({'credit_card': 'pii', 'id': 'id'}) ht.update_transformers({ 'credit_card': AnonymizedFaker(), @@ -1180,7 +1180,7 @@ def test_field_transformers_correctly_set(self): ht.fit(data) assert ht.get_config()['transformers']['col'].new_attribute3 == 'abc' - ht.update_sdtypes({'col': 'text'}) + ht.update_sdtypes({'col': 'id'}) transformer = ht.get_config()['transformers']['col'] transformer.new_attribute3 = 'abc' ht.fit(data) @@ -1191,7 +1191,7 @@ def _get_hyper_transformer_with_random_transformers(self, data): ht.detect_initial_config(data) ht.update_sdtypes({ 'credit_card': 'pii', - 'name': 'text', + 'name': 'id', 'signup_day': 'datetime', }) ht.update_transformers({ @@ -1417,7 +1417,7 @@ def test_anonymized_faker_text(self): # Run - simple run ht.detect_initial_config(data) - ht.update_sdtypes({'id1': 'pii', 'info': 'text'}) + ht.update_sdtypes({'id1': 'pii', 'info': 'id'}) ht.update_transformers({ 'id1': AnonymizedFaker(), 'info': AnonymizedFaker(), diff --git a/tests/integration/test_transformers.py b/tests/integration/test_transformers.py index d16ba6b76..2f399d6d7 100644 --- a/tests/integration/test_transformers.py +++ b/tests/integration/test_transformers.py @@ -45,7 +45,6 @@ 'integer': ['i'], 'numerical': ['f', 'i'], 'pii': ['O', 'i', 'f'], - 'text': ['O', 'i', 'f'], } diff --git a/tests/integration/transformers/pii/test_anonymizer.py b/tests/integration/transformers/pii/test_anonymizer.py index 314c072d8..0f7182d11 100644 --- a/tests/integration/transformers/pii/test_anonymizer.py +++ b/tests/integration/transformers/pii/test_anonymizer.py @@ -46,7 +46,7 @@ def test_get_supported_sdtypes(self): supported_sdtypes = AnonymizedFaker.get_supported_sdtypes() # Assert - assert sorted(supported_sdtypes) == sorted(['pii', 'text', 'id']) + assert sorted(supported_sdtypes) == sorted(['pii', 'id']) def test_custom_provider(self): """End to end test with a custom provider and function for the ``AnonymizedFaker``.""" diff --git a/tests/integration/transformers/test_categorical.py b/tests/integration/transformers/test_categorical.py index 4d1aa95e2..6193ab865 100644 --- a/tests/integration/transformers/test_categorical.py +++ b/tests/integration/transformers/test_categorical.py @@ -537,15 +537,14 @@ def test_categorical_transformers_default_missing_value_encoding_new_category(tr ) -@pytest.mark.parametrize('sdtype', ['id', 'text']) @pytest.mark.parametrize('transformer', categorical_transformers) -def test_categorical_transformers_with_id_sdtype(sdtype, transformer): +def test_categorical_transformers_with_id_sdtype(transformer): # Setup data = pd.DataFrame({ 'col': [1, 'two', 3, 'four', None], }) hyper_transformer = HyperTransformer() - config = {'sdtypes': {'col': sdtype}, 'transformers': {'col': transformer}} + config = {'sdtypes': {'col': 'id'}, 'transformers': {'col': transformer}} # Run hyper_transformer.set_config(config) @@ -557,11 +556,10 @@ def test_categorical_transformers_with_id_sdtype(sdtype, transformer): pd.testing.assert_frame_equal(data, reverse_transformed) -@pytest.mark.parametrize('sdtype', ['id', 'text']) -def test_unsupported_categorical_transformers_with_id_sdtype(sdtype): +def test_unsupported_categorical_transformers_with_id_sdtype(): # Setup hyper_transformer = HyperTransformer() - config = {'sdtypes': {'col': sdtype}, 'transformers': {'col': OneHotEncoder()}} + config = {'sdtypes': {'col': 'id'}, 'transformers': {'col': OneHotEncoder()}} expected_invalid_error = re.escape( "Some transformers you've assigned are not compatible with the sdtypes. " "Please change the following columns: ['col']" diff --git a/tests/unit/test__rdt_2_0_deprecations.py b/tests/unit/test__rdt_2_0_deprecations.py index b8fec90a8..ab4b9050b 100644 --- a/tests/unit/test__rdt_2_0_deprecations.py +++ b/tests/unit/test__rdt_2_0_deprecations.py @@ -117,3 +117,16 @@ def test_deprecated_classes(class_path): # Run and Assert with pytest.raises(AttributeError, match=re.escape(expected_message)): getattr(module, class_name) + + +def test_text_sdtype(): + """Test that the text sdtype is no longer supported.""" + # Setup + ht = HyperTransformer() + ht.field_sdtypes = {'col_text': 'text'} + + # Run + supported_sdtypes = ht._get_supported_sdtypes() + + # Assert + assert 'text' not in supported_sdtypes diff --git a/tests/unit/test_hyper_transformer.py b/tests/unit/test_hyper_transformer.py index 8037a909f..7fc5c7950 100644 --- a/tests/unit/test_hyper_transformer.py +++ b/tests/unit/test_hyper_transformer.py @@ -252,7 +252,6 @@ def test__learn_config(self, get_default_transformer_mock): # Setup data = self.get_data() data['pii'] = ['a', 'b', 'c', 'd'] - data['text'] = ['e', 'f', 'g', 'h'] field_transformers = { 'integer': FloatFormatter(), 'float': ClusterBasedNormalizer(), @@ -269,7 +268,6 @@ def test__learn_config(self, get_default_transformer_mock): ht.field_sdtypes = { 'datetime': 'datetime', 'pii': 'pii', - 'text': 'text', } ht._unfit = Mock() @@ -284,7 +282,6 @@ def test__learn_config(self, get_default_transformer_mock): 'categorical': 'categorical', 'datetime': 'datetime', 'pii': 'pii', - 'text': 'text', } assert isinstance(ht.field_transformers['integer'], FloatFormatter) @@ -293,7 +290,6 @@ def test__learn_config(self, get_default_transformer_mock): assert isinstance(ht.field_transformers['bool'], LabelEncoder) assert isinstance(ht.field_transformers['datetime'], UnixTimestampEncoder) assert isinstance(ht.field_transformers['pii'], AnonymizedFaker) - assert isinstance(ht.field_transformers['text'], RegexGenerator) ht._unfit.assert_called_once() @patch('rdt.hyper_transformer.LOGGER') diff --git a/tests/unit/transformers/pii/test_anonymizer.py b/tests/unit/transformers/pii/test_anonymizer.py index 07e454d2f..518b4ed13 100644 --- a/tests/unit/transformers/pii/test_anonymizer.py +++ b/tests/unit/transformers/pii/test_anonymizer.py @@ -457,8 +457,6 @@ def test_get_supported_sdtypes(self, base_mock, issubclass_mock): 'boolean', 'categorical', ] - text_mock = Mock() - text_mock.get_supported_sdtypes.return_value = ['text'] phone_mock = Mock() phone_mock.get_supported_sdtypes.return_value = ['phone_number'] pii_mock = Mock() @@ -468,7 +466,6 @@ def test_get_supported_sdtypes(self, base_mock, issubclass_mock): categorical_mock, datetime_mock, boolean_mock, - text_mock, phone_mock, pii_mock, ] @@ -480,7 +477,6 @@ def test_get_supported_sdtypes(self, base_mock, issubclass_mock): assert sorted(supported_sdtypes) == sorted([ 'phone_number', 'pii', - 'text', ]) @patch('rdt.transformers.pii.anonymizer.BaseTransformer.reset_randomization') diff --git a/tests/unit/transformers/test_text.py b/tests/unit/transformers/test_text.py deleted file mode 100644 index 0c549e0ae..000000000 --- a/tests/unit/transformers/test_text.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Test Text Transformers.""" - -import pytest - - -def test_deprecation_warning_is_raised(): - """Test that a deprecation warning is raised when importing from this module.""" - # Run and Assert - expected_message = ( - "Importing 'RegexGenerator' for ID columns from 'rdt.transformers.text' " - "is deprecated. Please use 'rdt.transformers.id' instead." - ) - with pytest.warns(DeprecationWarning, match=expected_message): - from rdt.transformers.text import RegexGenerator # noqa: F401 diff --git a/tests/unit/transformers/test_utils.py b/tests/unit/transformers/test_utils.py index ff43b14ac..5db088866 100644 --- a/tests/unit/transformers/test_utils.py +++ b/tests/unit/transformers/test_utils.py @@ -1,7 +1,6 @@ import datetime import re import sys -import warnings from decimal import Decimal from unittest.mock import Mock, patch @@ -12,7 +11,6 @@ import rdt.transformers.utils as utils_module from rdt.transformers.utils import ( - WarnDict, _any, _cast_to_type, _extract_timezone_from_a_string, @@ -542,52 +540,6 @@ def test_sigmoid(): assert res == expected_res -def test_warn_dict(): - """Test that ``WarnDict`` will raise a warning when called with `text`.""" - # Setup - instance = WarnDict() - instance['text'] = 'text_transformer' - - # Run - warning_msg = "The sdtype 'text' is deprecated and will be phased out. Please use 'id' instead." - with pytest.warns(DeprecationWarning, match=warning_msg): - result_access = instance['text'] - - # Run second time and no warning gets shown - with warnings.catch_warnings(record=True) as record: - result_access_no_warn = instance['text'] - result_get_no_warn = instance.get('text') - - # Assert - assert len(record) == 0 - assert result_access == 'text_transformer' - assert result_access_no_warn == 'text_transformer' - assert result_get_no_warn == 'text_transformer' - - -def test_warn_dict_get(): - """Test that ``WarnDict`` will raise a warning when called with `text`.""" - # Setup - instance = WarnDict() - instance['text'] = 'text_transformer' - - # Run - warning_msg = "The sdtype 'text' is deprecated and will be phased out. Please use 'id' instead." - with pytest.warns(DeprecationWarning, match=warning_msg): - result_access = instance.get('text') - - # Run second time and no warning gets shown - with warnings.catch_warnings(record=True) as record: - result_access_no_warn = instance['text'] - result_get_no_warn = instance.get('text') - - # Assert - assert len(record) == 0 - assert result_access == 'text_transformer' - assert result_access_no_warn == 'text_transformer' - assert result_get_no_warn == 'text_transformer' - - def test__extract_timezone_from_a_string_with_valid_timezone(): """Test that `_extract_timezone_from_a_string` extracts a valid timezone from a string.""" # Setup From 1732865df4d987f95d0a893f419f598fa989cd0d Mon Sep 17 00:00:00 2001 From: R-Palazzo Date: Thu, 20 Aug 2026 14:38:52 +0100 Subject: [PATCH 10/13] fix unit test python 3.9 --- tests/unit/test__rdt_2_0_deprecations.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/unit/test__rdt_2_0_deprecations.py b/tests/unit/test__rdt_2_0_deprecations.py index ab4b9050b..984bba3e6 100644 --- a/tests/unit/test__rdt_2_0_deprecations.py +++ b/tests/unit/test__rdt_2_0_deprecations.py @@ -47,9 +47,7 @@ def test_deprecated_parameters(class_, method, parameter): """Test that deprecated parameters raise an error.""" # Setup instance = class_() - expected_message = ( - f"{class_.__name__}.{method}() got an unexpected keyword argument '{parameter}'" - ) + expected_message = f"{method}() got an unexpected keyword argument '{parameter}'" # Run and Assert with pytest.raises(TypeError, match=re.escape(expected_message)): From e87bd3549430ebca51d3441befe2c44547524526 Mon Sep 17 00:00:00 2001 From: R-Palazzo Date: Thu, 20 Aug 2026 20:43:41 +0100 Subject: [PATCH 11/13] keep WarnDict --- rdt/transformers/utils.py | 35 +++++++++++++++++++ tests/unit/transformers/test_utils.py | 50 +++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) diff --git a/rdt/transformers/utils.py b/rdt/transformers/utils.py index eaec54e48..69ac8a798 100644 --- a/rdt/transformers/utils.py +++ b/rdt/transformers/utils.py @@ -6,6 +6,7 @@ import string import sys import warnings +from collections import defaultdict from decimal import Decimal import numpy as np @@ -22,6 +23,7 @@ LOGGER = logging.getLogger(__name__) MAX_DECIMALS = sys.float_info.dig +DEPRECATED_SDTYPES_MAPPING = {} def _literal(character, max_repeat): @@ -365,6 +367,39 @@ def sigmoid(data, low, high): return data +class WarnDict(dict): + """Custom dictionary to raise a deprecation warning.""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._warned = defaultdict() + + def get(self, sdtype): + """Return the value for sdtype if sdtype is in the dictionary, else default. + + If the sdtype is `text` raises a `DeprecationWarning` stating that it will be + phased out. + """ + if sdtype in DEPRECATED_SDTYPES_MAPPING and not self._warned.get(sdtype): + new_sdtype = DEPRECATED_SDTYPES_MAPPING.get(sdtype) + warnings.warn( + f"The sdtype '{sdtype}' is deprecated and will be phased out. " + f"Please use '{new_sdtype}' instead.", + DeprecationWarning, + ) + self._warned[sdtype] = True + + return super().get(sdtype) + + def __getitem__(self, sdtype): + """Return the value for sdtype if sdtype is in the dictionary. + + If the sdtype is `text` raises a `DeprecationWarning` stating that it will be + phased out. + """ + return self.get(sdtype) + + def _extract_timezone_from_a_string(dt_str): if not isinstance(dt_str, str): dt_str = str(dt_str) diff --git a/tests/unit/transformers/test_utils.py b/tests/unit/transformers/test_utils.py index 5db088866..dae279080 100644 --- a/tests/unit/transformers/test_utils.py +++ b/tests/unit/transformers/test_utils.py @@ -1,6 +1,7 @@ import datetime import re import sys +import warnings from decimal import Decimal from unittest.mock import Mock, patch @@ -11,6 +12,7 @@ import rdt.transformers.utils as utils_module from rdt.transformers.utils import ( + WarnDict, _any, _cast_to_type, _extract_timezone_from_a_string, @@ -540,6 +542,54 @@ def test_sigmoid(): assert res == expected_res +@patch('rdt.transformers.utils.DEPRECATED_SDTYPES_MAPPING', new={'text': 'id'}) +def test_warn_dict(): + """Test that ``WarnDict`` will raise a warning when called with `text`.""" + # Setup + instance = WarnDict() + instance['text'] = 'text_transformer' + + # Run + warning_msg = "The sdtype 'text' is deprecated and will be phased out. Please use 'id' instead." + with pytest.warns(DeprecationWarning, match=warning_msg): + result_access = instance['text'] + + # Run second time and no warning gets shown + with warnings.catch_warnings(record=True) as record: + result_access_no_warn = instance['text'] + result_get_no_warn = instance.get('text') + + # Assert + assert len(record) == 0 + assert result_access == 'text_transformer' + assert result_access_no_warn == 'text_transformer' + assert result_get_no_warn == 'text_transformer' + + +@patch('rdt.transformers.utils.DEPRECATED_SDTYPES_MAPPING', new={'text': 'id'}) +def test_warn_dict_get(): + """Test that ``WarnDict`` will raise a warning when called with `text`.""" + # Setup + instance = WarnDict() + instance['text'] = 'text_transformer' + + # Run + warning_msg = "The sdtype 'text' is deprecated and will be phased out. Please use 'id' instead." + with pytest.warns(DeprecationWarning, match=warning_msg): + result_access = instance.get('text') + + # Run second time and no warning gets shown + with warnings.catch_warnings(record=True) as record: + result_access_no_warn = instance['text'] + result_get_no_warn = instance.get('text') + + # Assert + assert len(record) == 0 + assert result_access == 'text_transformer' + assert result_access_no_warn == 'text_transformer' + assert result_get_no_warn == 'text_transformer' + + def test__extract_timezone_from_a_string_with_valid_timezone(): """Test that `_extract_timezone_from_a_string` extracts a valid timezone from a string.""" # Setup From 1dffdc3e117e1b26ae4e1f54b4b6fa699de21fcd Mon Sep 17 00:00:00 2001 From: R-Palazzo Date: Fri, 21 Aug 2026 15:48:50 +0100 Subject: [PATCH 12/13] remove all 'enforce_uniqueness' --- rdt/transformers/id.py | 18 +++------- rdt/transformers/pii/anonymizer.py | 14 +++----- .../transformers/pii/test_anonymizer.py | 31 ----------------- .../unit/transformers/pii/test_anonymizer.py | 22 ------------ tests/unit/transformers/test_id.py | 34 ------------------- 5 files changed, 9 insertions(+), 110 deletions(-) diff --git a/rdt/transformers/id.py b/rdt/transformers/id.py index 53b7af629..5a2dd4ac3 100644 --- a/rdt/transformers/id.py +++ b/rdt/transformers/id.py @@ -197,11 +197,7 @@ def reset_randomization(self): def _sample_fallback(self, num_samples, template_samples): """Sample num_samples values such that they are all unique, disregarding the regex.""" - unique_condition = ( - self.cardinality_rule == 'unique' - if hasattr(self, 'cardinality_rule') - else getattr(self, 'enforce_uniqueness', False) - ) + unique_condition = self.cardinality_rule == 'unique' if unique_condition: if not self._num_fallback_samples_generated: self.generator = self._create_numerical_fallback_generator() @@ -456,14 +452,10 @@ def _reverse_transform(self, data): Returns: pandas.Series """ - if hasattr(self, 'cardinality_rule'): - unique_condition = self.cardinality_rule == 'unique' - match_cardinality = self.cardinality_rule == 'match' - if match_cardinality and self._unique_regex_values is None: - self._unique_regex_values = self._generate_unique_regexes() - else: - unique_condition = getattr(self, 'enforce_uniqueness', False) - match_cardinality = False + unique_condition = self.cardinality_rule == 'unique' + match_cardinality = self.cardinality_rule == 'match' + if match_cardinality and self._unique_regex_values is None: + self._unique_regex_values = self._generate_unique_regexes() num_samples = len(data) if (data is not None and len(data)) else self.data_length self._warn_not_enough_unique_values(num_samples, unique_condition, match_cardinality) diff --git a/rdt/transformers/pii/anonymizer.py b/rdt/transformers/pii/anonymizer.py index 7b6781e4d..32d888705 100644 --- a/rdt/transformers/pii/anonymizer.py +++ b/rdt/transformers/pii/anonymizer.py @@ -192,16 +192,10 @@ def _function(self): # so accessing it can raise runtime errors. setattr(self.faker.unique, '_excluded_types', ()) - try: - if self.cardinality_rule in {'unique', 'match', 'scale'}: - faker_attr = self.faker.unique - else: - faker_attr = self.faker - - except AttributeError: - faker_attr = ( - self.faker.unique if getattr(self, 'enforce_uniqueness', False) else self.faker - ) + if self.cardinality_rule in {'unique', 'match', 'scale'}: + faker_attr = self.faker.unique + else: + faker_attr = self.faker result = getattr(faker_attr, self.function_name)(**self.function_kwargs) if isinstance(result, Iterable) and not isinstance(result, str): diff --git a/tests/integration/transformers/pii/test_anonymizer.py b/tests/integration/transformers/pii/test_anonymizer.py index 0f7182d11..96196846c 100644 --- a/tests/integration/transformers/pii/test_anonymizer.py +++ b/tests/integration/transformers/pii/test_anonymizer.py @@ -242,37 +242,6 @@ def test_cardinality_rule_match_too_many_nans(self): assert len(reverse_transform['col'].unique()) == 3 assert reverse_transform['col'].isna().sum() == 2 - def test_enforce_uniqueness_backwards_compatability(self): - """Test that ``AnonymizedFaker`` is backwards compatible with ``enforce_uniqueness``. - - Checks that transformers without the ``cardinality_rule`` attribute still function as - expected (can happen when previous transformer version is loaded from a pkl file). - """ - # Setup - data = pd.DataFrame({'job': np.arange(500)}) - - instance = AnonymizedFaker('job', 'job', cardinality_rule='match') - instance.enforce_uniqueness = True - - transformed = instance.fit_transform(data, 'job') - delattr(instance, 'cardinality_rule') - - # Run - reverse_transform = instance.reverse_transform(transformed) - - # Assert - assert len(reverse_transform['job'].unique()) == 500 - - warning_msg = re.escape( - "Unable to generate enough unique values for column 'job' in " - 'a human-readable format. Additional values may be created randomly.' - ) - with pytest.warns(UserWarning, match=warning_msg): - instance.reverse_transform(transformed) - - instance.reset_randomization() - instance.reverse_transform(transformed) - def test__reverse_transform_from_manually_set_parameters(self): """Test the ``reverse_transform`` after manually setting parameters.""" # Setup diff --git a/tests/unit/transformers/pii/test_anonymizer.py b/tests/unit/transformers/pii/test_anonymizer.py index 518b4ed13..eb6015f21 100644 --- a/tests/unit/transformers/pii/test_anonymizer.py +++ b/tests/unit/transformers/pii/test_anonymizer.py @@ -181,28 +181,6 @@ def test__function_cardinality_rule_match(self): unique_function.assert_called_once_with(type='int') assert result == 1 - def test__function_cardinality_rule_missing_attribute(self): - """Test it when ``cardinality_rule`` attribute is missing.""" - # Setup - instance = Mock() - function = Mock() - unique_function = Mock() - unique_function.return_value = 1 - - delattr(instance, 'cardinality_rule') - instance.faker.unique.number = unique_function - instance.faker.number = function - instance.function_name = 'number' - instance.function_kwargs = {'type': 'int'} - - # Run - result = AnonymizedFaker._function(instance) - - # Assert - function.assert_not_called() - unique_function.assert_called_once_with(type='int') - assert result == 1 - def test__function_with_iterables_return(self): """Test that ``_function`` returns the values of the iterable.""" # Setup diff --git a/tests/unit/transformers/test_id.py b/tests/unit/transformers/test_id.py index 15880010c..c956d8f77 100644 --- a/tests/unit/transformers/test_id.py +++ b/tests/unit/transformers/test_id.py @@ -783,40 +783,6 @@ def test__reverse_transform_unique_not_enough_remaining(self, mock_warnings): ) np.testing.assert_array_equal(out, np.array(['A', 'B', 'C', 'D', 'E', 'F'])) - @patch('rdt.transformers.id.LOGGER') - def test__reverse_transform_info_message(self, mock_logger): - """Test the ``_reverse_transform`` method. - - Validate that the ``_reverse_transform`` method logs an info message when - ``enforce_uniqueness`` is ``False`` and the ``instance.data_length`` is bigger than - ``instance.generator_size``. - - In this test we also test the backward compatibility, so when the transformer - does not have the ``cardinality_rule`` attribute, it should use the ``enforce_uniqueness`` - attribute. This is necessary to keep a coverage of 100%. - """ - # Setup - instance = RegexGenerator('[A-Z]', cardinality_rule=None) - del instance.cardinality_rule - instance.enforce_uniqueness = False - instance.data_length = 6 - instance.generator_size = 5 - instance.generated = 0 - instance.columns = ['a'] - columns_data = pd.Series() - - # Run - instance._reverse_transform(columns_data) - - # Assert - expected_format = ( - "The data has %s rows but the regex for '%s' can only create %s unique values. Some " - "values in '%s' may be repeated." - ) - expected_args = (6, 'a', 5, 'a') - - mock_logger.info.assert_called_once_with(expected_format, *expected_args) - def test__reverse_transform_match_not_enough_values(self): """Test the case when there are not enough values to match the cardinality rule.""" # Setup From 9d1fbcb7c8dfd054aa23956692f824e49bc33a27 Mon Sep 17 00:00:00 2001 From: R-Palazzo Date: Mon, 24 Aug 2026 17:56:58 +0100 Subject: [PATCH 13/13] rebase: remove 'text' from supported sdtype for ordinal transformers --- rdt/transformers/ordinal.py | 4 ++-- tests/integration/transformers/test_ordinal.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/rdt/transformers/ordinal.py b/rdt/transformers/ordinal.py index 18c02f13d..e88a1c0d1 100644 --- a/rdt/transformers/ordinal.py +++ b/rdt/transformers/ordinal.py @@ -38,7 +38,7 @@ class OrderedUniformEncoder(UniformEncoder): """ INPUT_SDTYPE = 'ordinal' - SUPPORTED_SDTYPES = ['ordinal', 'categorical', 'boolean', 'id', 'text'] + SUPPORTED_SDTYPES = ['ordinal', 'categorical', 'boolean', 'id'] def __init__(self, order=None, missing_value_encoding='new_category'): _validate_missing_value_encoding(missing_value_encoding) @@ -185,7 +185,7 @@ class OrderedLabelEncoder(LabelEncoder): """ INPUT_SDTYPE = 'ordinal' - SUPPORTED_SDTYPES = ['ordinal', 'categorical', 'boolean', 'id', 'text'] + SUPPORTED_SDTYPES = ['ordinal', 'categorical', 'boolean', 'id'] def __init__(self, order=None, add_noise=False, missing_value_encoding='new_category'): _validate_missing_value_encoding(missing_value_encoding) diff --git a/tests/integration/transformers/test_ordinal.py b/tests/integration/transformers/test_ordinal.py index c3111bf75..8a0229bc9 100644 --- a/tests/integration/transformers/test_ordinal.py +++ b/tests/integration/transformers/test_ordinal.py @@ -303,10 +303,10 @@ def test_ordinal_transformers_default_missing_value_encoding_new_category(transf ) -@pytest.mark.parametrize('sdtype', ['id', 'text']) @pytest.mark.parametrize('transformer', ordinal_transformers) -def test_ordinal_transformers_with_id_sdtype(sdtype, transformer): +def test_ordinal_transformers_with_id_sdtype(transformer): # Setup + sdtype = 'id' data = pd.DataFrame({ 'col': [1, 'two', 3, 'four', None], })