Backport PR #3571 on branch 1.11.x (normalize_total with numba)

[ilan-gold]

• Closes #
• Tests included or not required because:

• Release notes not necessary because:

[codecov]

❌ 85 Tests Failed:

Tests completed	Failed	Passed	Skipped
1993	85	1908	93

View the top 3 failed test(s) by shortest run time

tests/test_scrublet.py::test_scrublet_params[log_transform-True]

Stack Traces | 0.001s run time

@pytest.fixture(scope="module")
    def scrub_small_sess() -> AnnData:
        # Reduce size of input for faster test
        adata = pbmc200()
        sc.pp.filter_genes(adata, min_counts=100)
    
>       sc.pp.scrublet(adata, use_approx_neighbors=False)

#x1B[1m#x1B[31mtests/test_scrublet.py#x1B[0m:175: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
#x1B[1m#x1B[.../hostedtoolcache/Python/3.10.17......................................./x64/lib/python3.10.../site-packages/legacy_api_wrap/__init__.py#x1B[0m:82: in fn_compatible
    return fn(*args_all, **kw)
#x1B[1m#x1B[.../hostedtoolcache/Python/3.10.17......................................./x64/lib/python3.10.../preprocessing/_scrublet/__init__.py#x1B[0m:291: in scrublet
    scrubbed = _run_scrublet(adata_obs, adata_sim)
#x1B[1m#x1B[.../hostedtoolcache/Python/3.10.17......................................./x64/lib/python3.10.../preprocessing/_scrublet/__init__.py#x1B[0m:209: in _run_scrublet
    pp.log1p(ad_obs, layer="log1p")
#x1B[1m#x1B[.../hostedtoolcache/Python/3.10.17......................................./x64/lib/python3.10.../scanpy/_utils/__init__.py#x1B[0m:151: in wrapper
    return func(*args, **kwargs)
#x1B[1m#x1B[.../hostedtoolcache/Python/3.10.17......................................./x64/lib/python3.10/functools.py#x1B[0m:889: in wrapper
    return dispatch(args[0].__class__)(*args, **kw)
#x1B[1m#x1B[.../hostedtoolcache/Python/3.10.17......................................./x64/lib/python3.10.../scanpy/preprocessing/_simple.py#x1B[0m:435: in log1p_anndata
    X = log1p(X, copy=False, base=base)
#x1B[1m#x1B[.../hostedtoolcache/Python/3.10.17......................................./x64/lib/python3.10.../scanpy/_utils/__init__.py#x1B[0m:151: in wrapper
    return func(*args, **kwargs)
#x1B[1m#x1B[.../hostedtoolcache/Python/3.10.17......................................./x64/lib/python3.10/functools.py#x1B[0m:889: in wrapper
    return dispatch(args[0].__class__)(*args, **kw)
#x1B[1m#x1B[.../hostedtoolcache/Python/3.10.17......................................./x64/lib/python3.10.../scanpy/preprocessing/_simple.py#x1B[0m:377: in log1p_sparse
    X = check_array(
#x1B[1m#x1B[.../hostedtoolcache/Python/3.10.17......................................./x64/lib/python3.10.../sklearn/utils/validation.py#x1B[0m:1014: in check_array
    array = _ensure_sparse_format(
#x1B[1m#x1B[.../hostedtoolcache/Python/3.10.17......................................./x64/lib/python3.10.../sklearn/utils/validation.py#x1B[0m:649: in _ensure_sparse_format
    _assert_all_finite(
#x1B[1m#x1B[.../hostedtoolcache/Python/3.10.17......................................./x64/lib/python3.10.../sklearn/utils/validation.py#x1B[0m:120: in _assert_all_finite
    _assert_all_finite_element_wise(
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

X = array([0., 0., 0., ..., 0., 0., 0.], shape=(60031,))

    def _assert_all_finite_element_wise(
        X, *, xp, allow_nan, msg_dtype=None, estimator_name=None, input_name=""
    ):
        # Cython implementation doesn't support FP16 or complex numbers
        use_cython = (
            xp is np and X.data.contiguous and X.dtype.type in {np.float32, np.float64}
        )
        if use_cython:
            out = cy_isfinite(X.reshape(-1), allow_nan=allow_nan)
            has_nan_error = False if allow_nan else out == FiniteStatus.has_nan
            has_inf = out == FiniteStatus.has_infinite
        else:
            has_inf = xp.any(xp.isinf(X))
            has_nan_error = False if allow_nan else xp.any(xp.isnan(X))
        if has_inf or has_nan_error:
            if has_nan_error:
                type_err = "NaN"
            else:
                msg_dtype = msg_dtype if msg_dtype is not None else X.dtype
                type_err = f"infinity or a value too large for {msg_dtype!r}"
            padded_input_name = input_name + " " if input_name else ""
            msg_err = f"Input {padded_input_name}contains {type_err}."
            if estimator_name and input_name == "X" and has_nan_error:
                # Improve the error message on how to handle missing values in
                # scikit-learn.
                msg_err += (
                    f"\n{estimator_name} does not accept missing values"
                    " encoded as NaN natively. For supervised learning, you might want"
                    " to consider sklearn.ensemble.HistGradientBoostingClassifier and"
                    " Regressor which accept missing values encoded as NaNs natively."
                    " Alternatively, it is possible to preprocess the data, for"
                    " instance by using an imputer transformer in a pipeline or drop"
                    " samples with missing values. See"
                    " https://scikit-learn..../stable/modules/impute.html"
                    " You can find a list of all estimators that handle NaN values"
                    " at the following page:"
                    " https://scikit-learn..../stable/modules/impute.html"
                    "#estimators-that-handle-nan-values"
                )
>           raise ValueError(msg_err)
#x1B[1m#x1B[31mE           ValueError: Input contains NaN.#x1B[0m

#x1B[1m#x1B[.../hostedtoolcache/Python/3.10.17......................................./x64/lib/python3.10.../sklearn/utils/validation.py#x1B[0m:169: ValueError

tests/test_scrublet.py::test_scrublet_params[n_prin_comps-10]

Stack Traces | 0.001s run time

@pytest.fixture(scope="module")
    def scrub_small_sess() -> AnnData:
        # Reduce size of input for faster test
        adata = pbmc200()
        sc.pp.filter_genes(adata, min_counts=100)
    
>       sc.pp.scrublet(adata, use_approx_neighbors=False)

#x1B[1m#x1B[31mtests/test_scrublet.py#x1B[0m:175: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
#x1B[1m#x1B[.../hostedtoolcache/Python/3.10.17......................................./x64/lib/python3.10.../site-packages/legacy_api_wrap/__init__.py#x1B[0m:82: in fn_compatible
    return fn(*args_all, **kw)
#x1B[1m#x1B[.../hostedtoolcache/Python/3.10.17......................................./x64/lib/python3.10.../preprocessing/_scrublet/__init__.py#x1B[0m:291: in scrublet
    scrubbed = _run_scrublet(adata_obs, adata_sim)
#x1B[1m#x1B[.../hostedtoolcache/Python/3.10.17......................................./x64/lib/python3.10.../preprocessing/_scrublet/__init__.py#x1B[0m:209: in _run_scrublet
    pp.log1p(ad_obs, layer="log1p")
#x1B[1m#x1B[.../hostedtoolcache/Python/3.10.17......................................./x64/lib/python3.10.../scanpy/_utils/__init__.py#x1B[0m:151: in wrapper
    return func(*args, **kwargs)
#x1B[1m#x1B[.../hostedtoolcache/Python/3.10.17......................................./x64/lib/python3.10/functools.py#x1B[0m:889: in wrapper
    return dispatch(args[0].__class__)(*args, **kw)
#x1B[1m#x1B[.../hostedtoolcache/Python/3.10.17......................................./x64/lib/python3.10.../scanpy/preprocessing/_simple.py#x1B[0m:435: in log1p_anndata
    X = log1p(X, copy=False, base=base)
#x1B[1m#x1B[.../hostedtoolcache/Python/3.10.17......................................./x64/lib/python3.10.../scanpy/_utils/__init__.py#x1B[0m:151: in wrapper
    return func(*args, **kwargs)
#x1B[1m#x1B[.../hostedtoolcache/Python/3.10.17......................................./x64/lib/python3.10/functools.py#x1B[0m:889: in wrapper
    return dispatch(args[0].__class__)(*args, **kw)
#x1B[1m#x1B[.../hostedtoolcache/Python/3.10.17......................................./x64/lib/python3.10.../scanpy/preprocessing/_simple.py#x1B[0m:377: in log1p_sparse
    X = check_array(
#x1B[1m#x1B[.../hostedtoolcache/Python/3.10.17......................................./x64/lib/python3.10.../sklearn/utils/validation.py#x1B[0m:1014: in check_array
    array = _ensure_sparse_format(
#x1B[1m#x1B[.../hostedtoolcache/Python/3.10.17......................................./x64/lib/python3.10.../sklearn/utils/validation.py#x1B[0m:649: in _ensure_sparse_format
    _assert_all_finite(
#x1B[1m#x1B[.../hostedtoolcache/Python/3.10.17......................................./x64/lib/python3.10.../sklearn/utils/validation.py#x1B[0m:120: in _assert_all_finite
    _assert_all_finite_element_wise(
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

X = array([0., 0., 0., ..., 0., 0., 0.], shape=(60031,))

    def _assert_all_finite_element_wise(
        X, *, xp, allow_nan, msg_dtype=None, estimator_name=None, input_name=""
    ):
        # Cython implementation doesn't support FP16 or complex numbers
        use_cython = (
            xp is np and X.data.contiguous and X.dtype.type in {np.float32, np.float64}
        )
        if use_cython:
            out = cy_isfinite(X.reshape(-1), allow_nan=allow_nan)
            has_nan_error = False if allow_nan else out == FiniteStatus.has_nan
            has_inf = out == FiniteStatus.has_infinite
        else:
            has_inf = xp.any(xp.isinf(X))
            has_nan_error = False if allow_nan else xp.any(xp.isnan(X))
        if has_inf or has_nan_error:
            if has_nan_error:
                type_err = "NaN"
            else:
                msg_dtype = msg_dtype if msg_dtype is not None else X.dtype
                type_err = f"infinity or a value too large for {msg_dtype!r}"
            padded_input_name = input_name + " " if input_name else ""
            msg_err = f"Input {padded_input_name}contains {type_err}."
            if estimator_name and input_name == "X" and has_nan_error:
                # Improve the error message on how to handle missing values in
                # scikit-learn.
                msg_err += (
                    f"\n{estimator_name} does not accept missing values"
                    " encoded as NaN natively. For supervised learning, you might want"
                    " to consider sklearn.ensemble.HistGradientBoostingClassifier and"
                    " Regressor which accept missing values encoded as NaNs natively."
                    " Alternatively, it is possible to preprocess the data, for"
                    " instance by using an imputer transformer in a pipeline or drop"
                    " samples with missing values. See"
                    " https://scikit-learn..../stable/modules/impute.html"
                    " You can find a list of all estimators that handle NaN values"
                    " at the following page:"
                    " https://scikit-learn..../stable/modules/impute.html"
                    "#estimators-that-handle-nan-values"
                )
>           raise ValueError(msg_err)
#x1B[1m#x1B[31mE           ValueError: Input contains NaN.#x1B[0m

#x1B[1m#x1B[.../hostedtoolcache/Python/3.10.17......................................./x64/lib/python3.10.../sklearn/utils/validation.py#x1B[0m:169: ValueError

tests/test_scrublet.py::test_scrublet_params[normalize_variance-False]

Stack Traces | 0.001s run time

@pytest.fixture(scope="module")
    def scrub_small_sess() -> AnnData:
        # Reduce size of input for faster test
        adata = pbmc200()
        sc.pp.filter_genes(adata, min_counts=100)
    
>       sc.pp.scrublet(adata, use_approx_neighbors=False)

#x1B[1m#x1B[31mtests/test_scrublet.py#x1B[0m:175: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
#x1B[1m#x1B[.../hostedtoolcache/Python/3.10.17......................................./x64/lib/python3.10.../site-packages/legacy_api_wrap/__init__.py#x1B[0m:82: in fn_compatible
    return fn(*args_all, **kw)
#x1B[1m#x1B[.../hostedtoolcache/Python/3.10.17......................................./x64/lib/python3.10.../preprocessing/_scrublet/__init__.py#x1B[0m:291: in scrublet
    scrubbed = _run_scrublet(adata_obs, adata_sim)
#x1B[1m#x1B[.../hostedtoolcache/Python/3.10.17......................................./x64/lib/python3.10.../preprocessing/_scrublet/__init__.py#x1B[0m:209: in _run_scrublet
    pp.log1p(ad_obs, layer="log1p")
#x1B[1m#x1B[.../hostedtoolcache/Python/3.10.17......................................./x64/lib/python3.10.../scanpy/_utils/__init__.py#x1B[0m:151: in wrapper
    return func(*args, **kwargs)
#x1B[1m#x1B[.../hostedtoolcache/Python/3.10.17......................................./x64/lib/python3.10/functools.py#x1B[0m:889: in wrapper
    return dispatch(args[0].__class__)(*args, **kw)
#x1B[1m#x1B[.../hostedtoolcache/Python/3.10.17......................................./x64/lib/python3.10.../scanpy/preprocessing/_simple.py#x1B[0m:435: in log1p_anndata
    X = log1p(X, copy=False, base=base)
#x1B[1m#x1B[.../hostedtoolcache/Python/3.10.17......................................./x64/lib/python3.10.../scanpy/_utils/__init__.py#x1B[0m:151: in wrapper
    return func(*args, **kwargs)
#x1B[1m#x1B[.../hostedtoolcache/Python/3.10.17......................................./x64/lib/python3.10/functools.py#x1B[0m:889: in wrapper
    return dispatch(args[0].__class__)(*args, **kw)
#x1B[1m#x1B[.../hostedtoolcache/Python/3.10.17......................................./x64/lib/python3.10.../scanpy/preprocessing/_simple.py#x1B[0m:377: in log1p_sparse
    X = check_array(
#x1B[1m#x1B[.../hostedtoolcache/Python/3.10.17......................................./x64/lib/python3.10.../sklearn/utils/validation.py#x1B[0m:1014: in check_array
    array = _ensure_sparse_format(
#x1B[1m#x1B[.../hostedtoolcache/Python/3.10.17......................................./x64/lib/python3.10.../sklearn/utils/validation.py#x1B[0m:649: in _ensure_sparse_format
    _assert_all_finite(
#x1B[1m#x1B[.../hostedtoolcache/Python/3.10.17......................................./x64/lib/python3.10.../sklearn/utils/validation.py#x1B[0m:120: in _assert_all_finite
    _assert_all_finite_element_wise(
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

X = array([0., 0., 0., ..., 0., 0., 0.], shape=(60031,))

    def _assert_all_finite_element_wise(
        X, *, xp, allow_nan, msg_dtype=None, estimator_name=None, input_name=""
    ):
        # Cython implementation doesn't support FP16 or complex numbers
        use_cython = (
            xp is np and X.data.contiguous and X.dtype.type in {np.float32, np.float64}
        )
        if use_cython:
            out = cy_isfinite(X.reshape(-1), allow_nan=allow_nan)
            has_nan_error = False if allow_nan else out == FiniteStatus.has_nan
            has_inf = out == FiniteStatus.has_infinite
        else:
            has_inf = xp.any(xp.isinf(X))
            has_nan_error = False if allow_nan else xp.any(xp.isnan(X))
        if has_inf or has_nan_error:
            if has_nan_error:
                type_err = "NaN"
            else:
                msg_dtype = msg_dtype if msg_dtype is not None else X.dtype
                type_err = f"infinity or a value too large for {msg_dtype!r}"
            padded_input_name = input_name + " " if input_name else ""
            msg_err = f"Input {padded_input_name}contains {type_err}."
            if estimator_name and input_name == "X" and has_nan_error:
                # Improve the error message on how to handle missing values in
                # scikit-learn.
                msg_err += (
                    f"\n{estimator_name} does not accept missing values"
                    " encoded as NaN natively. For supervised learning, you might want"
                    " to consider sklearn.ensemble.HistGradientBoostingClassifier and"
                    " Regressor which accept missing values encoded as NaNs natively."
                    " Alternatively, it is possible to preprocess the data, for"
                    " instance by using an imputer transformer in a pipeline or drop"
                    " samples with missing values. See"
                    " https://scikit-learn..../stable/modules/impute.html"
                    " You can find a list of all estimators that handle NaN values"
                    " at the following page:"
                    " https://scikit-learn..../stable/modules/impute.html"
                    "#estimators-that-handle-nan-values"
                )
>           raise ValueError(msg_err)
#x1B[1m#x1B[31mE           ValueError: Input contains NaN.#x1B[0m

#x1B[1m#x1B[.../hostedtoolcache/Python/3.10.17......................................./x64/lib/python3.10.../sklearn/utils/validation.py#x1B[0m:169: ValueError

To view more test analytics, go to the Test Analytics Dashboard
_{📋 Got 3 mins? Take this short survey to help us improve Test Analytics.}