diff --git a/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_column_transformer.py b/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_column_transformer.py index f4cb94bdf1..7be6d321a6 100644 --- a/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_column_transformer.py +++ b/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_column_transformer.py @@ -37,6 +37,7 @@ import cuml from cuml.internals.global_settings import _global_settings_data from cuml.internals.mixins import DeprecatedGetFeatureNamesMixin +from cuml.internals.outputs import _is_object_dtype from cuml.internals.validation import ( check_is_fitted, check_features, @@ -300,7 +301,8 @@ def _list_indexing(X, key, key_dtype): def _transform_one(transformer, X, y, weight, **fit_params): - with cuml.using_output_type("cupy"): + output_type = "numpy" if _is_object_dtype(X) else "cupy" + with cuml.using_output_type(output_type): res = transformer.transform(X) # if we have a weight for this transformer, multiply output @@ -322,7 +324,8 @@ def _fit_transform_one(transformer, be multiplied by ``weight``. """ with _print_elapsed_time(message_clsname, message): - with cuml.using_output_type("cupy"): + output_type = "numpy" if _is_object_dtype(X) else "cupy" + with cuml.using_output_type(output_type): transformer.accept_sparse = True if hasattr(transformer, 'fit_transform'): res = transformer.fit_transform(X, y, **fit_params) @@ -1014,6 +1017,12 @@ def _hstack(self, Xs): return cu_sparse.hstack(converted_Xs).tocsr() else: Xs = [f.toarray() if issparse(f) else f for f in Xs] + if any(_is_object_dtype(X) for X in Xs): + # Object dtype (e.g. string columns from a categorical + # SimpleImputer) has no device representation - cupy has no + # way to store it. Stack on host instead. + Xs = [X.get() if isinstance(X, np.ndarray) else X for X in Xs] + return cpu_np.hstack(Xs) return np.hstack(Xs) diff --git a/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_imputation.py b/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_imputation.py index 7fef551763..cd12440120 100644 --- a/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_imputation.py +++ b/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_imputation.py @@ -20,6 +20,7 @@ import cupy as np import numpy as cpu_np +import pandas as pd from cupyx.scipy import sparse import cuml @@ -29,11 +30,11 @@ StringInputTagMixin, _ensure_transformer_tags, ) -from cuml.internals.outputs import mlfunc, ReflectedAttr +from cuml.internals.outputs import ReflectedAttr, _is_object_dtype, mlfunc from cuml.internals.validation import ( - check_is_fitted, - check_inputs, check_input_features, + check_inputs, + check_is_fitted, ) from ....thirdparty_adapters import ( @@ -144,7 +145,6 @@ def _concatenate_indicator(self, X_imputed, X_indicator): if not self.add_indicator: return X_imputed - hstack = sparse.hstack if sparse.issparse(X_imputed) else np.hstack if X_indicator is None: raise ValueError( "Data from the missing indicator are not provided. Call " @@ -152,7 +152,17 @@ def _concatenate_indicator(self, X_imputed, X_indicator): "implementation." ) - return hstack((X_imputed, X_indicator)) + if sparse.issparse(X_imputed): + return sparse.hstack((X_imputed, X_indicator)) + + if _is_object_dtype(X_imputed) or _is_object_dtype(X_indicator): + arrays = [ + array.get() if isinstance(array, np.ndarray) else array + for array in (X_imputed, X_indicator) + ] + return cpu_np.hstack(arrays) + + return np.hstack((X_imputed, X_indicator)) def __sklearn_tags__(self): tags = super().__sklearn_tags__() @@ -264,10 +274,12 @@ def __init__(self, *, missing_values=np.nan, strategy="mean", @classmethod def _get_param_names(cls): return super()._get_param_names() + [ + "missing_values", "strategy", "fill_value", "verbose", - "copy" + "copy", + "add_indicator", ] def _validate_input(self, X, in_fit): @@ -288,6 +300,7 @@ def _validate_input(self, X, in_fit): ensure_all_finite = "allow-nan" try: + mem_type = "host" if _is_object_dtype(X) else "device" X = check_inputs( self, X, @@ -295,6 +308,7 @@ def _validate_input(self, X, in_fit): dtype=dtype, ensure_all_finite=ensure_all_finite, copy=self.copy, + mem_type=mem_type, reset=in_fit, ) except ValueError as ve: @@ -305,6 +319,18 @@ def _validate_input(self, X, in_fit): else: raise ve + # Object nulls reach this host path as NaN. Restore pd.NA for its + # identity-based path, but reject NaN when None was requested, + # matching scikit-learn's validation semantics. + if _is_object_dtype(X): + if hasattr(X, "flags") and not X.flags.writeable: + X = X.copy() + nan_mask = _get_mask(X, np.nan) + if self.missing_values is pd.NA: + X[nan_mask] = pd.NA + elif self.missing_values is None and nan_mask.any(): + raise ValueError("Input contains NaN") + _check_inputs_dtype(X, self.missing_values) if X.dtype.kind not in ("i", "u", "f", "O"): raise ValueError("SimpleImputer does not support data with dtype " @@ -427,7 +453,8 @@ def _dense_fit(self, X, strategy, missing_values, fill_value): # Constant elif strategy == "constant": - return np.full(X.shape[1], fill_value, dtype=X.dtype) + xp = np.get_array_module(X) + return xp.full(X.shape[1], fill_value, dtype=X.dtype) @mlfunc def transform(self, X): @@ -443,24 +470,27 @@ def transform(self, X): X = self._validate_input(X, in_fit=False) X_indicator = super()._transform_indicator(X) - statistics = self.statistics_ + # Use the stored value for internal computation. ColumnTransformer + # may request cupy output, which cannot reflect object statistics. + statistics = type(self).statistics_.get_raw(self) if X.shape[1] != statistics.shape[0]: raise ValueError("X has %d features per sample, expected %d" - % (X.shape[1], self.statistics_.shape[0])) + % (X.shape[1], statistics.shape[0])) # Delete the invalid columns if strategy is not constant if self.strategy == "constant": valid_statistics = statistics else: + xp = np.get_array_module(statistics) # same as np.isnan but also works for object dtypes invalid_mask = _get_mask(statistics, np.nan) - valid_mask = np.logical_not(invalid_mask) + valid_mask = xp.logical_not(invalid_mask) valid_statistics = statistics[valid_mask] - valid_statistics_indexes = np.flatnonzero(valid_mask) + valid_statistics_indexes = xp.flatnonzero(valid_mask) if invalid_mask.any(): - missing = np.arange(X.shape[1])[invalid_mask] + missing = xp.arange(X.shape[1])[invalid_mask] if self.verbose: warnings.warn("Deleting features without " "observed values: %s" % missing) @@ -485,9 +515,10 @@ def transform(self, X): if self.strategy == "constant": X[mask] = valid_statistics[0] else: - for i, vi in enumerate(valid_statistics_indexes): - feature_idxs = np.flatnonzero(mask[:, vi]) - X[feature_idxs, vi] = valid_statistics[i] + xp = np.get_array_module(mask) + for i in range(valid_statistics.shape[0]): + feature_idxs = xp.flatnonzero(mask[:, i]) + X[feature_idxs, i] = valid_statistics[i] X = super()._concatenate_indicator(X, X_indicator) return X @@ -508,7 +539,11 @@ def get_feature_names_out(self, input_features=None): """ check_is_fitted(self) input_features = check_input_features(self, input_features) - non_missing_mask = np.logical_not(_get_mask(self.statistics_, np.nan)).get() + statistics = type(self).statistics_.get_raw(self) + xp = np.get_array_module(statistics) + non_missing_mask = xp.logical_not(_get_mask(statistics, np.nan)) + if isinstance(non_missing_mask, np.ndarray): + non_missing_mask = non_missing_mask.get() names = input_features[non_missing_mask] if self.add_indicator: indicator_names = self.indicator_.get_feature_names_out(input_features) @@ -657,9 +692,11 @@ def _get_missing_features_info(self, X): imputer_mask = sparse.csc_matrix(imputer_mask) if self.features == 'all': - features_indices = np.arange(X.shape[1]) + xp = np.get_array_module(imputer_mask) + features_indices = xp.arange(X.shape[1]) else: - features_indices = np.flatnonzero(n_missing) + xp = np.get_array_module(n_missing) + features_indices = xp.flatnonzero(n_missing) return imputer_mask, features_indices @@ -668,11 +705,13 @@ def _validate_input(self, X, in_fit): ensure_all_finite = True else: ensure_all_finite = "allow-nan" + mem_type = "host" if _is_object_dtype(X) else "device" X = check_inputs( self, X, accept_sparse=('csc', 'csr'), ensure_all_finite=ensure_all_finite, + mem_type=mem_type, reset=in_fit, ) _check_inputs_dtype(X, self.missing_values) @@ -802,10 +841,13 @@ def get_feature_names_out(self, input_features=None): check_is_fitted(self) input_features = check_input_features(self, input_features) prefix = self.__class__.__name__.lower() + features = type(self).features_.get_raw(self) + if isinstance(features, np.ndarray): + features = features.get() return cpu_np.asarray( [ f"{prefix}_{feature_name}" - for feature_name in input_features[self.features_.get()] + for feature_name in input_features[features] ], dtype=object, ) diff --git a/python/cuml/cuml/internals/outputs.py b/python/cuml/cuml/internals/outputs.py index 69a4b44251..44066e9d08 100644 --- a/python/cuml/cuml/internals/outputs.py +++ b/python/cuml/cuml/internals/outputs.py @@ -284,6 +284,31 @@ def infer_output_type(array, array_like="numpy"): return None +def _is_object_dtype(res): + """Check for object or string dtype on an array/dataframe-like.""" + + def is_object_or_string(dtype): + try: + if np.dtype(dtype).kind == "O": + return True + except TypeError: + pass + return pd.api.types.is_string_dtype(dtype) + + dtype = getattr(res, "dtype", None) + if dtype is not None: + return is_object_or_string(dtype) + + dtypes = getattr(res, "dtypes", None) + if dtypes is not None: + try: + return any(is_object_or_string(dt) for dt in dtypes) + except TypeError: + return False + + return False + + class ArrayIndexPair: """An array paired with an aligned index. @@ -521,6 +546,40 @@ def convert_arrays( return pd.Series(obj.flatten(), index=index) return pd.DataFrame(obj, index=index) return pd.Series(obj, index=index) + elif _is_object_dtype(obj): + # NumPy object arrays have no device representation. Preserve + # host arrays for internal outputs, or wrap them in a + # dataframe-like type. + if output_type == "cuml": + return obj + elif output_type == "cupy": + raise TypeError( + f"{output_type=!r} doesn't support outputs of dtype " + f"object and shape {obj.shape}" + ) + + if hasattr(index, "to_pandas"): + index = index.to_pandas() + + if obj.ndim == 2: + if one_col_2d_as_series and obj.shape[1] == 1: + host_df = pd.Series( + obj.flatten(), index=index + ).infer_objects() + else: + host_df = pd.DataFrame(obj, index=index).infer_objects() + else: + host_df = pd.Series(obj, index=index).infer_objects() + + try: + return cudf.from_pandas(host_df) + except (TypeError, ValueError, NotImplementedError) as exc: + raise TypeError( + "Cannot convert an object-dtype output with shape " + f"{obj.shape} to output_type='cudf'. Use " + "output_type='pandas' or output_type='numpy' for this " + "object layout." + ) from exc else: # Other output types use device memory, coerce to cupy and take # cupy code path. @@ -639,6 +698,16 @@ def __reduce__(self): def __set_name__(self, owner, name): self.name = name + def get_raw(self, instance): + """Return the value so internal work can dispatch on its stored type.""" + cache = instance.__dict__.get(self.name) + if cache is None: + raise AttributeError( + f"{type(instance).__name__!r} object has no attribute " + f"{self.name!r}" + ) + return cache.value + def __get__(self, instance, owner): if instance is None: return self diff --git a/python/cuml/cuml/thirdparty_adapters/adapters.py b/python/cuml/cuml/thirdparty_adapters/adapters.py index c799301495..fc2d4375bb 100644 --- a/python/cuml/cuml/thirdparty_adapters/adapters.py +++ b/python/cuml/cuml/thirdparty_adapters/adapters.py @@ -1,17 +1,43 @@ # -# SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # import cupy as cp import numpy as np +import pandas as pd + + +def _is_nan(value): + """Return whether a scalar is a floating-point NaN.""" + return isinstance(value, (float, np.floating)) and np.isnan(value) + + +def _is_nan_sentinel(value): + """Return whether a scalar selects floating-point NaN values.""" + return (isinstance(value, str) and value == "NaN") or _is_nan(value) + + +def _get_host_mask(X, predicate): + return np.fromiter( + (predicate(value) for value in X.flat), dtype=bool, count=X.size + ).reshape(X.shape) def _get_mask(X, value_to_mask): """Compute the boolean mask X == missing_values.""" - if value_to_mask == "NaN" or cp.isnan(value_to_mask): - return cp.isnan(X) - else: - return X == value_to_mask + if value_to_mask is pd.NA: + if isinstance(X, cp.ndarray): + return cp.zeros(X.shape, dtype=bool) + return _get_host_mask(X, lambda value: value is pd.NA) + if value_to_mask is None: + if isinstance(X, cp.ndarray): + return cp.zeros(X.shape, dtype=bool) + return _get_host_mask(X, lambda value: value is None) + if _is_nan_sentinel(value_to_mask): + if isinstance(X, cp.ndarray): + return cp.isnan(X) + return _get_host_mask(X, _is_nan) + return X == value_to_mask def _masked_column_median(arr, masked_value): @@ -20,7 +46,7 @@ def _masked_column_median(arr, masked_value): mask = _get_mask(arr, masked_value) if arr.size == 0: return cp.full(arr.shape[1], cp.nan) - if not cp.isnan(masked_value): + if not _is_nan_sentinel(masked_value): arr_sorted = arr.copy() # If nan is not the missing value, any column with nans should # have a median of nan @@ -59,7 +85,7 @@ def _masked_column_mean(arr, masked_value): count_missing_values = mask.sum(axis=0) n_elems = arr.shape[0] - count_missing_values mean = cp.nansum(arr, axis=0) - if not cp.isnan(masked_value): + if not _is_nan_sentinel(masked_value): mean -= count_missing_values * masked_value mean /= n_elems return mean @@ -68,18 +94,27 @@ def _masked_column_mean(arr, masked_value): def _masked_column_mode(arr, masked_value): """Determine the most frequently appearing element in each column in the 2D array arr, ignoring any instances of masked_value""" + xp = cp.get_array_module(arr) mask = _get_mask(arr, masked_value) + if arr.dtype.kind == "O": + na_mask = pd.isna(arr) + # Never pass NA-like values into object sorting. The strict sentinel + # mask is still used later to select values for imputation. + count_mask = xp.logical_or(mask, na_mask) + else: + count_mask = mask n_features = arr.shape[1] - most_frequent = np.empty(n_features, dtype=arr.dtype) + result_dtype = object if arr.dtype.kind == "O" else arr.dtype + most_frequent = np.empty(n_features, dtype=result_dtype) for i in range(n_features): - feature_mask_idxs = cp.where(~mask[:, i])[0] - values, counts = cp.unique( + feature_mask_idxs = xp.where(~count_mask[:, i])[0] + values, counts = xp.unique( arr[feature_mask_idxs, i], return_counts=True ) - count_max = counts.max() - if count_max > 0: - value = values[counts == count_max].min() + if counts.size == 0: + value = xp.nan else: - value = cp.nan + count_max = counts.max() + value = values[counts == count_max].min() most_frequent[i] = value - return cp.array(most_frequent) + return xp.array(most_frequent) diff --git a/python/cuml/tests/test_adapters.py b/python/cuml/tests/test_adapters.py index b59542c473..14fba82ebe 100644 --- a/python/cuml/tests/test_adapters.py +++ b/python/cuml/tests/test_adapters.py @@ -1,5 +1,5 @@ # -# SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # @@ -8,6 +8,7 @@ import cupy as cp import cupyx as cpx import numpy as np +import pandas as pd import pytest from scipy import stats from sklearn.utils._mask import _get_mask as sk_get_mask @@ -110,6 +111,44 @@ def test_get_mask(failure_logger, mask_dataset): assert_allclose(cu_mask, sk_mask) +def test_get_mask_nan_string_on_host_object_array(): + X = np.array([1.0, np.nan, "present"], dtype=object) + + np.testing.assert_array_equal( + cu_get_mask(X, value_to_mask="NaN"), [False, True, False] + ) + + +def test_get_mask_distinguishes_missing_sentinels(): + X = np.array([None, np.nan, pd.NA, "present"], dtype=object) + masks = { + None: cu_get_mask(X, None), + "nan": cu_get_mask(X, np.nan), + "pd.NA": cu_get_mask(X, pd.NA), + } + + np.testing.assert_array_equal(masks[None], [True, False, False, False]) + np.testing.assert_array_equal(masks["nan"], [False, True, False, False]) + np.testing.assert_array_equal(masks["pd.NA"], [False, False, True, False]) + np.testing.assert_array_equal(sum(masks.values()), [1, 1, 1, 0]) + + +@pytest.mark.parametrize( + ("function", "expected"), + [ + (_masked_column_mean, [3.0, 5.0]), + (_masked_column_median, [3.0, 5.0]), + ], +) +@pytest.mark.parametrize("missing_value", [np.nan, "NaN"]) +def test_masked_column_numeric_na_sentinel(function, expected, missing_value): + X = cp.array([[1.0, cp.nan], [3.0, 4.0], [5.0, 6.0]]) + + result = function(X, missing_value) + + np.testing.assert_allclose(result.get(), expected) + + def test_masked_column_median(failure_logger, mask_dataset): mask_value, X_np, X = mask_dataset median = _masked_column_median(X, mask_value).get() @@ -141,3 +180,11 @@ def test_masked_column_mode(failure_logger, mask_dataset): column_mask = mask[:, i] column_mode = stats.mode(X_np[:, i][column_mask], keepdims=True)[0][0] assert column_mode == mode[i] + + +def test_masked_column_mode_numeric_nan_regression_guard(): + X = np.array([[0.0], [np.nan], [np.nan], [1.0]]) + + result = _masked_column_mode(X, 0) + + assert np.isnan(result[0]) diff --git a/python/cuml/tests/test_compose.py b/python/cuml/tests/test_compose.py index 665ad12b8a..2a063c93e4 100644 --- a/python/cuml/tests/test_compose.py +++ b/python/cuml/tests/test_compose.py @@ -13,6 +13,7 @@ from sklearn.compose import ( make_column_transformer as sk_make_column_transformer, ) +from sklearn.impute import SimpleImputer as skSimpleImputer from sklearn.preprocessing import Normalizer as skNormalizer from sklearn.preprocessing import OneHotEncoder as skOneHotEncoder from sklearn.preprocessing import PolynomialFeatures as skPolynomialFeatures @@ -24,6 +25,7 @@ from cuml.preprocessing import Normalizer as cuNormalizer from cuml.preprocessing import OneHotEncoder as cuOneHotEncoder from cuml.preprocessing import PolynomialFeatures as cuPolynomialFeatures +from cuml.preprocessing import SimpleImputer as cuSimpleImputer from cuml.preprocessing import StandardScaler as cuStandardScaler from cuml.testing.test_preproc_utils import ( # noqa: F401 assert_allclose, @@ -383,3 +385,239 @@ def test_column_transform_properly_handles_sub_output_type(): ] ).fit(df) transformer.transform(df) + + +def test_column_transformer_simple_imputer_categorical_cudf(): + """Regression test for https://github.com/rapidsai/cuml/issues/6183 + + ColumnTransformer + SimpleImputer on a native cuDF DataFrame with + categorical/string columns used to raise (KeyError / AttributeError on + ``.dtype``, later ``ValueError: Unsupported dtype object`` after + unrelated refactors). impute-then-transform is one of the most common + sklearn pipeline shapes, so this must work end to end. + """ + df = cudf.DataFrame( + { + "num1": [1.0, np.nan, 3.0], + "num2": [4.0, 5.0, np.nan], + "cat1": ["a", None, "c"], + "cat2": ["x", "y", None], + } + ) + df_np = df.to_pandas() + + num_cols = ["num1", "num2"] + cat_cols = ["cat1"] + mode_cols = ["cat2"] + + cu_transformer = cuColumnTransformer( + transformers=[ + ( + "num", + cuSimpleImputer(strategy="constant", fill_value=0), + num_cols, + ), + ( + "cat", + cuSimpleImputer( + strategy="constant", + fill_value="missing", + missing_values=pd.NA, + ), + cat_cols, + ), + ( + "mod", + cuSimpleImputer( + strategy="most_frequent", missing_values=pd.NA + ), + mode_cols, + ), + ] + ) + cu_result = cu_transformer.fit_transform(df) + + sk_transformer = skColumnTransformer( + transformers=[ + ( + "num", + skSimpleImputer(strategy="constant", fill_value=0), + num_cols, + ), + ( + "cat", + skSimpleImputer( + strategy="constant", + fill_value="missing", + missing_values=pd.NA, + ), + cat_cols, + ), + ( + "mod", + skSimpleImputer( + strategy="most_frequent", missing_values=pd.NA + ), + mode_cols, + ), + ] + ) + sk_result = sk_transformer.fit_transform(df_np) + + np.testing.assert_array_equal(cu_result.to_numpy(), sk_result) + + +def test_simple_imputer_add_indicator_object_cudf(): + """Regression: SimpleImputer(add_indicator=True) on string/object columns + must stack the imputed (host object) data with the indicator mask on host + instead of routing the host array through cupy.hstack (issue #6183 follow-up). + """ + df = cudf.DataFrame( + { + "cat1": ["a", None, "c", "a"], + "cat2": ["x", "y", None, "x"], + } + ) + df_np = df.to_pandas() + cu_imp = cuSimpleImputer( + strategy="most_frequent", missing_values=pd.NA, add_indicator=True + ) + sk_imp = skSimpleImputer( + strategy="most_frequent", missing_values=pd.NA, add_indicator=True + ) + + cu_result = cu_imp.fit_transform(df) + sk_result = sk_imp.fit_transform(df_np) + + np.testing.assert_array_equal(cu_result.to_numpy(), np.asarray(sk_result)) + + +def test_simple_imputer_drops_fully_missing_object_column_cudf(): + df = cudf.DataFrame( + { + "all_missing": [None, None, None], + "observed": ["a", "b", "a"], + } + ) + df_np = df.to_pandas() + cu_imp = cuSimpleImputer(strategy="most_frequent", missing_values=pd.NA) + sk_imp = skSimpleImputer(strategy="most_frequent", missing_values=pd.NA) + + cu_result = cu_imp.fit_transform(df) + sk_result = sk_imp.fit_transform(df_np) + + assert cu_result.shape == (3, 1) + np.testing.assert_array_equal(cu_result.to_numpy(), sk_result) + + +def test_simple_imputer_add_indicator_clone_params(): + imputer = cuSimpleImputer(add_indicator=True, missing_values=pd.NA) + + cloned = sk_clone(imputer) + + params = cloned.get_params() + assert params["add_indicator"] is True + assert params["missing_values"] is pd.NA + + +@pytest.mark.parametrize("input_type", ["pandas", "numpy"]) +@pytest.mark.parametrize("missing_values", [pd.NA, None, np.nan]) +def test_simple_imputer_missing_sentinel_parity(input_type, missing_values): + if input_type == "pandas": + X = pd.DataFrame({"cat": ["x", missing_values, "x", "y"]}) + else: + X = np.array([["x"], [np.nan], ["x"], ["y"]], dtype=object) + + cu_imputer = cuSimpleImputer( + strategy="most_frequent", missing_values=missing_values + ) + sk_imputer = skSimpleImputer( + strategy="most_frequent", missing_values=missing_values + ) + + if missing_values is None: + with pytest.raises(ValueError) as sk_error: + sk_imputer.fit_transform(X) + with pytest.raises(ValueError) as cu_error: + cu_imputer.fit_transform(X) + + assert str(cu_error.value) == str(sk_error.value) + else: + cu_result = cu_imputer.fit_transform(X) + sk_result = sk_imputer.fit_transform(X) + + np.testing.assert_array_equal(np.asarray(cu_result), sk_result) + + +@pytest.mark.parametrize("input_type", ["pandas", "readonly-numpy"]) +@pytest.mark.parametrize("copy", [False, True]) +@pytest.mark.parametrize("missing_values", [pd.NA, np.nan]) +def test_simple_imputer_copy_object_parity(input_type, copy, missing_values): + def make_input(): + if input_type == "pandas": + return pd.DataFrame( + { + "cat": pd.Series( + ["x", missing_values, "x", "y"], dtype=object + ) + } + ) + + X = np.array([["x"], [np.nan], ["x"], ["y"]], dtype=object) + X.flags.writeable = False + return X + + cu_imputer = cuSimpleImputer( + strategy="most_frequent", missing_values=missing_values, copy=copy + ) + sk_imputer = skSimpleImputer( + strategy="most_frequent", missing_values=missing_values, copy=copy + ) + + cu_result = cu_imputer.fit_transform(make_input()) + sk_result = sk_imputer.fit_transform(make_input()) + + np.testing.assert_array_equal(np.asarray(cu_result), sk_result) + + +@pytest.mark.parametrize("add_indicator", [False, True]) +def test_simple_imputer_string_get_feature_names_out(add_indicator): + df = pd.DataFrame( + { + "cat": pd.Series(["x", pd.NA, "x"], dtype=object), + "other": pd.Series(["a", "b", "a"], dtype=object), + } + ) + cu_imputer = cuSimpleImputer( + strategy="most_frequent", + missing_values=pd.NA, + add_indicator=add_indicator, + ).fit(df) + sk_imputer = skSimpleImputer( + strategy="most_frequent", + missing_values=pd.NA, + add_indicator=add_indicator, + ).fit(df) + + np.testing.assert_array_equal( + cu_imputer.get_feature_names_out(), + sk_imputer.get_feature_names_out(), + ) + + +def test_is_object_dtype_handles_series_and_extension_dtypes(): + from cuml.internals.outputs import _is_object_dtype + + pa = pytest.importorskip("pyarrow") + + assert _is_object_dtype(pd.Series(["a", "b"])) is True + assert _is_object_dtype(pd.Series([1, 2, 3])) is False + assert _is_object_dtype(pd.Series(pd.Categorical(["a", "b"]))) is False + assert _is_object_dtype(pd.Series(["a"], dtype="string")) is True + assert ( + _is_object_dtype(pd.Series(["a"], dtype=pd.ArrowDtype(pa.string()))) + is True + ) + assert _is_object_dtype(pd.DataFrame({"a": ["x"], "b": [1]})) is True + assert _is_object_dtype(np.array(["a", "b"], dtype=object)) is True + assert _is_object_dtype(np.array([1, 2, 3])) is False diff --git a/python/cuml/tests/test_reflection.py b/python/cuml/tests/test_reflection.py index 84725699f2..c53948c520 100644 --- a/python/cuml/tests/test_reflection.py +++ b/python/cuml/tests/test_reflection.py @@ -113,6 +113,16 @@ def check_descriptor(self): assert_output_type(self.X_, "numpy") +def test_reflected_attr_get_raw(): + estimator = DummyEstimator() + value = cp.arange(3) + estimator.X_ = value + + with cuml.using_output_type("numpy"): + assert isinstance(estimator.X_, np.ndarray) + assert DummyEstimator.X_.get_raw(estimator) is value + + @mlfunc def returns_cupy(X): return cp.asarray(X) @@ -393,6 +403,56 @@ def test_convert_arrays_dataframe_with_index( cudf.testing.assert_series_equal(cudf.Series(res), sol) +@pytest.mark.parametrize( + ("output_type", "expected_type"), + [ + ("pandas", pd.DataFrame), + ("cudf", cudf.DataFrame), + ], +) +def test_convert_arrays_object_array_dataframe_output_with_index( + output_type, expected_type +): + arr = np.array([["a", "x"], ["b", "y"]], dtype=object) + index = pd.Index(["first", "second"]) + + result = convert_arrays(arr, output_type, index=index) + + assert isinstance(result, expected_type) + cudf.testing.assert_frame_equal( + cudf.DataFrame(result), + cudf.from_pandas(pd.DataFrame(arr, index=index)), + ) + + +@pytest.mark.parametrize("output_type", ["cupy"]) +def test_convert_arrays_object_array_device_output_error(output_type): + arr = np.array(["a", "b"], dtype=object) + + with pytest.raises( + TypeError, + match=f"output_type={output_type!r} doesn't support outputs of dtype", + ): + convert_arrays(arr, output_type) + + +@pytest.mark.parametrize("output_type", ["cuml"]) +def test_convert_arrays_object_array_array_output(output_type): + arr = np.array(["a", "b"], dtype=object) + + assert convert_arrays(arr, output_type) is arr + + +def test_convert_arrays_unsupported_object_layout_cudf_error(): + arr = np.array([[object()], [object()]], dtype=object) + + with pytest.raises( + TypeError, + match="Use output_type='pandas' or output_type='numpy'", + ): + convert_arrays(arr, "cudf") + + @pytest.mark.parametrize( "construct", [