From d53a37af8a1757ec6875889c236c266e87988665 Mon Sep 17 00:00:00 2001 From: Patricia Reina Date: Thu, 11 Aug 2022 11:59:41 +0200 Subject: [PATCH 1/9] initial ordinal encoder --- src/skprometheus/preprocessing.py | 33 +++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/src/skprometheus/preprocessing.py b/src/skprometheus/preprocessing.py index 1dd7711..171f105 100644 --- a/src/skprometheus/preprocessing.py +++ b/src/skprometheus/preprocessing.py @@ -36,3 +36,36 @@ def transform(self, X): MetricRegistry.model_categorical(feature=str(features[idx]), category=str(category)).inc() return transformed_X + + +class OrdinalEncoder(preprocessing.OrdinalEncoder): + """ + OrdinalEncoder that adds metrics to the prometheus metric registry. + """ + @wraps(preprocessing.OneHotEncoder.__init__, assigned=["__signature__"]) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + MetricRegistry.add_counter( + "model_categorical", + "Counts category occurrence for each categorical feature.", + additional_labels=("feature", "category"), + ) + + def transform(self, X): + """ + Transform method that adds the count for each category in each feature to the prometheus + metric registry. + """ + transformed_X = super().transform(X) + features = get_feature_names(X) + + # Use inverse method on transformed_X to get all missing values back as 'None' + categories = self.inverse_transform(transformed_X) + + for idx, row in enumerate(categories.T): + for category in row: + if not category: + category = "missing" + MetricRegistry.model_categorical(feature=str(features[idx]), category=str(category)).inc() + + return transformed_X From 84a6c43df08c6fec8c4a15112beb4e2932ab2b67 Mon Sep 17 00:00:00 2001 From: Patricia Reina Date: Thu, 11 Aug 2022 15:06:22 +0200 Subject: [PATCH 2/9] ordinal enc and utils refactor --- src/skprometheus/preprocessing.py | 31 ++++++++++++++++++------------- src/skprometheus/utils.py | 2 +- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/src/skprometheus/preprocessing.py b/src/skprometheus/preprocessing.py index 171f105..a6cf881 100644 --- a/src/skprometheus/preprocessing.py +++ b/src/skprometheus/preprocessing.py @@ -1,3 +1,4 @@ +import numpy as np from functools import wraps from sklearn import preprocessing @@ -5,6 +6,20 @@ from skprometheus.utils import get_feature_names +def feature_category_count(X, categories): + + features = get_feature_names(X) + + for idx, row in enumerate(categories.T): + for category in row: + if category is None: + category = "missing" + MetricRegistry.model_categorical(feature=str(features[idx]), category=category).inc() + MetricRegistry.model_categorical(feature=str(features[idx]), category=str(category)).inc() + + + + class OneHotEncoder(preprocessing.OneHotEncoder): """ OneHotEncoder that adds metrics to the prometheus metric registry. @@ -24,16 +39,11 @@ def transform(self, X): metric registry. """ transformed_X = super().transform(X) - features = get_feature_names(X) # Use inverse method on transformed_X to get all missing values back as 'None' categories = self.inverse_transform(transformed_X) - for idx, row in enumerate(categories.T): - for category in row: - if not category: - category = "missing" - MetricRegistry.model_categorical(feature=str(features[idx]), category=str(category)).inc() + feature_category_count(X, categories) return transformed_X @@ -42,7 +52,7 @@ class OrdinalEncoder(preprocessing.OrdinalEncoder): """ OrdinalEncoder that adds metrics to the prometheus metric registry. """ - @wraps(preprocessing.OneHotEncoder.__init__, assigned=["__signature__"]) + @wraps(preprocessing.OrdinalEncoder.__init__, assigned=["__signature__"]) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) MetricRegistry.add_counter( @@ -57,15 +67,10 @@ def transform(self, X): metric registry. """ transformed_X = super().transform(X) - features = get_feature_names(X) # Use inverse method on transformed_X to get all missing values back as 'None' categories = self.inverse_transform(transformed_X) - for idx, row in enumerate(categories.T): - for category in row: - if not category: - category = "missing" - MetricRegistry.model_categorical(feature=str(features[idx]), category=str(category)).inc() + feature_category_count(X, categories) return transformed_X diff --git a/src/skprometheus/utils.py b/src/skprometheus/utils.py index 3f189bb..3f77616 100644 --- a/src/skprometheus/utils.py +++ b/src/skprometheus/utils.py @@ -32,5 +32,5 @@ def get_feature_names(X): if isinstance(X, pd.DataFrame): return X.columns else: - X = check_array(X, force_all_finite=False) + X = check_array(X, dtype = None, force_all_finite=False) return list(range(X.shape[1])) From 92c6f182d7ae15ebc3e6fb8e0bc7e67901ceb95f Mon Sep 17 00:00:00 2001 From: Patricia Reina Date: Thu, 11 Aug 2022 15:12:58 +0200 Subject: [PATCH 3/9] feature_category_count refactor --- src/skprometheus/preprocessing.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/skprometheus/preprocessing.py b/src/skprometheus/preprocessing.py index a6cf881..9ebfd30 100644 --- a/src/skprometheus/preprocessing.py +++ b/src/skprometheus/preprocessing.py @@ -14,12 +14,9 @@ def feature_category_count(X, categories): for category in row: if category is None: category = "missing" - MetricRegistry.model_categorical(feature=str(features[idx]), category=category).inc() MetricRegistry.model_categorical(feature=str(features[idx]), category=str(category)).inc() - - class OneHotEncoder(preprocessing.OneHotEncoder): """ OneHotEncoder that adds metrics to the prometheus metric registry. From 976dfabc7752d78a8f51e878ae33d3446f8af0b5 Mon Sep 17 00:00:00 2001 From: Patricia Reina Date: Thu, 11 Aug 2022 15:33:03 +0200 Subject: [PATCH 4/9] added tests for ordinal encoder --- testing.ipynb | 193 ++++++++++++++++++++++++++++++++++++ tests/test_preprocessing.py | 48 ++++++++- 2 files changed, 240 insertions(+), 1 deletion(-) create mode 100644 testing.ipynb diff --git a/testing.ipynb b/testing.ipynb new file mode 100644 index 0000000..3c6bd95 --- /dev/null +++ b/testing.ipynb @@ -0,0 +1,193 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "\n", + "from sklearn.utils import check_array\n", + "\n", + "from src.skprometheus.utils import get_feature_names\n", + "from src.skprometheus.preprocessing import OrdinalEncoder, OneHotEncoder" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "x = np.array([['ndhbfg', 'akshf'],\n", + " ['abhvg', 'likrghfb'],\n", + " ['bksdh', 'lsbvjl']], dtype=np.str_)\n", + "#df = pd.DataFrame.from_records(x, columns=['X', 'Y'])" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "x_test = np.array([['aaaa', 'bbb'],\n", + " ['abhvg', 'likrghfb'],\n", + " ['bksdh', 'lsbvjl']], dtype=np.str_)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
OrdinalEncoder(handle_unknown='use_encoded_value', unknown_value=nan)
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
" + ], + "text/plain": [ + "OrdinalEncoder(handle_unknown='use_encoded_value', unknown_value=nan)" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ordinal = OrdinalEncoder(handle_unknown =\"use_encoded_value\", unknown_value = np.nan)\n", + "ordinal.fit(x)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Category : None\n", + "Missing category: None\n", + "feature=0, category=missing\n", + "feature=0, category=missing\n", + "Category : abhvg\n", + "feature=0, category=abhvg\n", + "Category : bksdh\n", + "feature=0, category=bksdh\n", + "Category : None\n", + "Missing category: None\n", + "feature=1, category=missing\n", + "feature=1, category=missing\n", + "Category : likrghfb\n", + "feature=1, category=likrghfb\n", + "Category : lsbvjl\n", + "feature=1, category=lsbvjl\n" + ] + } + ], + "source": [ + "x_transformed = ordinal.transform(x_test)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[nan, nan],\n", + " [ 0., 1.],\n", + " [ 1., 2.]])" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "x_transformed" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "one_hot = OneHotEncoder(handle_unknown='ignore')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "x_one_hot = np.array([\n", + " [1, 3, 4, 6],\n", + " [2, 3, 4, 5],\n", + " [4, 5, 6, 6],\n", + " [0, 0, 0, 0],\n", + " [6, 7, 8, 9]\n", + " ])\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "one_hot.fit(X)\n", + "x_one_hot_transformed = one_hot.transform(X)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "interpreter": { + "hash": "5b8c8a0feddb0b1a2050ab082eb17cd5cd66c83cf07524bb459aafa5dd354693" + }, + "kernelspec": { + "display_name": "Python 3.9.12 ('venv': venv)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.12" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/test_preprocessing.py b/tests/test_preprocessing.py index 864b5d7..c073459 100644 --- a/tests/test_preprocessing.py +++ b/tests/test_preprocessing.py @@ -1,6 +1,6 @@ import pytest -from skprometheus.preprocessing import OneHotEncoder +from skprometheus.preprocessing import OneHotEncoder, OrdinalEncoder import numpy as np from prometheus_client import REGISTRY import pandas as pd @@ -77,3 +77,49 @@ def test_OneHotEncoder_missing(): one_hot.transform(X_test) assert REGISTRY.get_sample_value('skprom_model_categorical_total', {'feature': '1', 'category': 'missing'}) == 2 + + +def test_OrdinalEncoder(): + ordinal = OrdinalEncoder(handle_unknown ="use_encoded_value", unknown_value = np.nan) + x = np.array([['ndhbfg', 'akshf'], + ['abhvg', 'likrghfb'], + ['ndhbfg', 'lsbvjl']], dtype=np.str_) + + ordinal.fit(x) + ordinal.transform(x) + + assert 'skprom_model_categorical' in [m.name for m in REGISTRY.collect()] + + assert REGISTRY.get_sample_value('skprom_model_categorical_total', {'feature': '0', 'category': 'ndhbfg'}) == 2 + assert REGISTRY.get_sample_value('skprom_model_categorical_total', {'feature': '1', 'category': 'likrghfb'}) == 1 + + +def test_OrdinalEncoder_pandas(): + ordinal_pd = OrdinalEncoder(handle_unknown ="use_encoded_value", unknown_value = np.nan) + x = np.array([['ndhbfg', 'akshf'], + ['abhvg', 'likrghfb'], + ['ndhbfg', 'lsbvjl']], dtype=np.str_) + + df = pd.DataFrame.from_records(x, columns=['X', 'Y']) + ordinal_pd.fit(df) + ordinal_pd.transform(df) + + assert REGISTRY.get_sample_value('skprom_model_categorical_total', {'feature': 'X', 'category': 'ndhbfg'}) == 2 + + +def test_OrdinalEncoder_missing(): + ordinal = OrdinalEncoder(handle_unknown ="use_encoded_value", unknown_value = np.nan) + x = np.array([['ndhbfg', 'akshf'], + ['abhvg', 'likrghfb'], + ['ndhbfg', 'lsbvjl']], dtype=np.str_) + + ordinal.fit(x) + + x_test = np.array([['aaaa', 'bbb'], + ['abhvg', 'likrghfb'], + ['ndhbfg', 'lsbvjl']], dtype=np.str_) + + ordinal.transform(x_test) + + assert REGISTRY.get_sample_value('skprom_model_categorical_total', {'feature': '0', 'category': 'missing'}) == 1 + assert REGISTRY.get_sample_value('skprom_model_categorical_total', {'feature': '1', 'category': 'missing'}) == 1 \ No newline at end of file From 679df2eba86b7e88349faccb9a52bd6ab67f7f26 Mon Sep 17 00:00:00 2001 From: Patricia Reina Date: Thu, 11 Aug 2022 15:54:33 +0200 Subject: [PATCH 5/9] deleted notebook for testing new functionalities --- testing.ipynb | 193 -------------------------------------------------- 1 file changed, 193 deletions(-) delete mode 100644 testing.ipynb diff --git a/testing.ipynb b/testing.ipynb deleted file mode 100644 index 3c6bd95..0000000 --- a/testing.ipynb +++ /dev/null @@ -1,193 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "import pandas as pd\n", - "\n", - "from sklearn.utils import check_array\n", - "\n", - "from src.skprometheus.utils import get_feature_names\n", - "from src.skprometheus.preprocessing import OrdinalEncoder, OneHotEncoder" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "x = np.array([['ndhbfg', 'akshf'],\n", - " ['abhvg', 'likrghfb'],\n", - " ['bksdh', 'lsbvjl']], dtype=np.str_)\n", - "#df = pd.DataFrame.from_records(x, columns=['X', 'Y'])" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "x_test = np.array([['aaaa', 'bbb'],\n", - " ['abhvg', 'likrghfb'],\n", - " ['bksdh', 'lsbvjl']], dtype=np.str_)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
OrdinalEncoder(handle_unknown='use_encoded_value', unknown_value=nan)
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
" - ], - "text/plain": [ - "OrdinalEncoder(handle_unknown='use_encoded_value', unknown_value=nan)" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ordinal = OrdinalEncoder(handle_unknown =\"use_encoded_value\", unknown_value = np.nan)\n", - "ordinal.fit(x)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Category : None\n", - "Missing category: None\n", - "feature=0, category=missing\n", - "feature=0, category=missing\n", - "Category : abhvg\n", - "feature=0, category=abhvg\n", - "Category : bksdh\n", - "feature=0, category=bksdh\n", - "Category : None\n", - "Missing category: None\n", - "feature=1, category=missing\n", - "feature=1, category=missing\n", - "Category : likrghfb\n", - "feature=1, category=likrghfb\n", - "Category : lsbvjl\n", - "feature=1, category=lsbvjl\n" - ] - } - ], - "source": [ - "x_transformed = ordinal.transform(x_test)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[nan, nan],\n", - " [ 0., 1.],\n", - " [ 1., 2.]])" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "x_transformed" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "one_hot = OneHotEncoder(handle_unknown='ignore')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "x_one_hot = np.array([\n", - " [1, 3, 4, 6],\n", - " [2, 3, 4, 5],\n", - " [4, 5, 6, 6],\n", - " [0, 0, 0, 0],\n", - " [6, 7, 8, 9]\n", - " ])\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "one_hot.fit(X)\n", - "x_one_hot_transformed = one_hot.transform(X)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "interpreter": { - "hash": "5b8c8a0feddb0b1a2050ab082eb17cd5cd66c83cf07524bb459aafa5dd354693" - }, - "kernelspec": { - "display_name": "Python 3.9.12 ('venv': venv)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.12" - }, - "orig_nbformat": 4 - }, - "nbformat": 4, - "nbformat_minor": 2 -} From ed843f274bda7a4f73eb57875b8c982b04f9b020 Mon Sep 17 00:00:00 2001 From: Patricia Reina Date: Thu, 11 Aug 2022 16:55:23 +0200 Subject: [PATCH 6/9] added label encoder --- src/skprometheus/preprocessing.py | 39 ++++++++++++++++++++++++++++++- tests/test_preprocessing.py | 16 +++++++++++-- 2 files changed, 52 insertions(+), 3 deletions(-) diff --git a/src/skprometheus/preprocessing.py b/src/skprometheus/preprocessing.py index 9ebfd30..e6b6e05 100644 --- a/src/skprometheus/preprocessing.py +++ b/src/skprometheus/preprocessing.py @@ -1,3 +1,4 @@ +from tkinter import Y import numpy as np from functools import wraps @@ -15,7 +16,15 @@ def feature_category_count(X, categories): if category is None: category = "missing" MetricRegistry.model_categorical(feature=str(features[idx]), category=str(category)).inc() - + + +def label_count(labels): + + for label in labels: + if label[0] is None: + label[0] = "missing" + MetricRegistry.label_categorical(Y=str(label[0])).inc() + class OneHotEncoder(preprocessing.OneHotEncoder): """ @@ -71,3 +80,31 @@ def transform(self, X): feature_category_count(X, categories) return transformed_X + + +class LabelEncoder(preprocessing.OrdinalEncoder): + """ + LabelEncoder that adds metrics to the prometheus metric registry. + """ + @wraps(preprocessing.LabelEncoder.__init__, assigned=["__signature__"]) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + MetricRegistry.add_counter( + "label_categorical", + "Counts category occurrence for each target label.", + additional_labels=tuple("Y"), + ) + + def transform(self, Y): + """ + Transform method that adds the count for each label to the prometheus + metric registry. + """ + transformed_Y = super().transform(Y) + + # Use inverse method on transformed_X to get all missing values back as 'None' + labels = self.inverse_transform(transformed_Y) + + label_count(labels) + + return transformed_Y \ No newline at end of file diff --git a/tests/test_preprocessing.py b/tests/test_preprocessing.py index c073459..5180f3c 100644 --- a/tests/test_preprocessing.py +++ b/tests/test_preprocessing.py @@ -1,6 +1,6 @@ import pytest -from skprometheus.preprocessing import OneHotEncoder, OrdinalEncoder +from skprometheus.preprocessing import OneHotEncoder, OrdinalEncoder, LabelEncoder import numpy as np from prometheus_client import REGISTRY import pandas as pd @@ -122,4 +122,16 @@ def test_OrdinalEncoder_missing(): ordinal.transform(x_test) assert REGISTRY.get_sample_value('skprom_model_categorical_total', {'feature': '0', 'category': 'missing'}) == 1 - assert REGISTRY.get_sample_value('skprom_model_categorical_total', {'feature': '1', 'category': 'missing'}) == 1 \ No newline at end of file + assert REGISTRY.get_sample_value('skprom_model_categorical_total', {'feature': '1', 'category': 'missing'}) == 1 + +def test_LabelEncoder(): + label_enc = LabelEncoder() + Y = np.array(['A', 'B', 'C', 'B', 'E', 'D', 'E', 'E'], dtype = np.str_). reshape((-1, 1)) + + label_enc.fit(Y) + label_enc.transform(Y) + + assert 'skprom_label_categorical' in [m.name for m in REGISTRY.collect()] + + assert REGISTRY.get_sample_value('skprom_label_categorical_total', {'Y': 'A'}) == 1 + assert REGISTRY.get_sample_value('skprom_label_categorical_total', {'Y': 'E'}) == 3 \ No newline at end of file From b9c266452a142e1640625204399967a168de8b3e Mon Sep 17 00:00:00 2001 From: Patricia Reina Date: Thu, 18 Aug 2022 10:07:46 +0200 Subject: [PATCH 7/9] removed spaces for flake8 --- src/skprometheus/preprocessing.py | 2 +- src/skprometheus/utils.py | 2 +- tests/test_preprocessing.py | 24 ++++++++++++------------ 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/skprometheus/preprocessing.py b/src/skprometheus/preprocessing.py index e6b6e05..2f34be6 100644 --- a/src/skprometheus/preprocessing.py +++ b/src/skprometheus/preprocessing.py @@ -1,5 +1,4 @@ from tkinter import Y -import numpy as np from functools import wraps from sklearn import preprocessing @@ -15,6 +14,7 @@ def feature_category_count(X, categories): for category in row: if category is None: category = "missing" + MetricRegistry.model_categorical(feature=str(features[idx]), category=str(category)).inc() diff --git a/src/skprometheus/utils.py b/src/skprometheus/utils.py index 3f77616..a9c5424 100644 --- a/src/skprometheus/utils.py +++ b/src/skprometheus/utils.py @@ -32,5 +32,5 @@ def get_feature_names(X): if isinstance(X, pd.DataFrame): return X.columns else: - X = check_array(X, dtype = None, force_all_finite=False) + X = check_array(X, dtype=None, force_all_finite=False) return list(range(X.shape[1])) diff --git a/tests/test_preprocessing.py b/tests/test_preprocessing.py index 5180f3c..a3a1fb8 100644 --- a/tests/test_preprocessing.py +++ b/tests/test_preprocessing.py @@ -80,10 +80,10 @@ def test_OneHotEncoder_missing(): def test_OrdinalEncoder(): - ordinal = OrdinalEncoder(handle_unknown ="use_encoded_value", unknown_value = np.nan) + ordinal = OrdinalEncoder(handle_unknown="use_encoded_value", unknown_value=np.nan) x = np.array([['ndhbfg', 'akshf'], - ['abhvg', 'likrghfb'], - ['ndhbfg', 'lsbvjl']], dtype=np.str_) + ['abhvg', 'likrghfb'], + ['ndhbfg', 'lsbvjl']], dtype=np.str_) ordinal.fit(x) ordinal.transform(x) @@ -95,10 +95,10 @@ def test_OrdinalEncoder(): def test_OrdinalEncoder_pandas(): - ordinal_pd = OrdinalEncoder(handle_unknown ="use_encoded_value", unknown_value = np.nan) + ordinal_pd = OrdinalEncoder(handle_unknown="use_encoded_value", unknown_value=np.nan) x = np.array([['ndhbfg', 'akshf'], - ['abhvg', 'likrghfb'], - ['ndhbfg', 'lsbvjl']], dtype=np.str_) + ['abhvg', 'likrghfb'], + ['ndhbfg', 'lsbvjl']], dtype=np.str_) df = pd.DataFrame.from_records(x, columns=['X', 'Y']) ordinal_pd.fit(df) @@ -108,16 +108,16 @@ def test_OrdinalEncoder_pandas(): def test_OrdinalEncoder_missing(): - ordinal = OrdinalEncoder(handle_unknown ="use_encoded_value", unknown_value = np.nan) + ordinal = OrdinalEncoder(handle_unknown="use_encoded_value", unknown_value=np.nan) x = np.array([['ndhbfg', 'akshf'], - ['abhvg', 'likrghfb'], - ['ndhbfg', 'lsbvjl']], dtype=np.str_) + ['abhvg', 'likrghfb'], + ['ndhbfg', 'lsbvjl']], dtype=np.str_) ordinal.fit(x) - x_test = np.array([['aaaa', 'bbb'], - ['abhvg', 'likrghfb'], - ['ndhbfg', 'lsbvjl']], dtype=np.str_) + x_test = np.array([['aaaa', 'bbbvbg'], + ['abhvg', 'likrghfb'], + ['ndhbfg', 'lsbvjl']], dtype=np.str_) ordinal.transform(x_test) From 186233ca3a6c0fd0423c88cae3ac2a893f85ea48 Mon Sep 17 00:00:00 2001 From: Patricia Reina Date: Thu, 18 Aug 2022 10:11:15 +0200 Subject: [PATCH 8/9] added ipynb extension to gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index b6e4761..959c297 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,9 @@ __pycache__/ # C extensions *.so +# Python notebooks +*.ipynb + # Distribution / packaging .Python build/ From 55bb5c5f8d69b5cc6239c44b6357d31f013f2a14 Mon Sep 17 00:00:00 2001 From: Patricia Reina Date: Thu, 18 Aug 2022 11:50:27 +0200 Subject: [PATCH 9/9] std checks to ordinalEnc missing test for LabelEnc --- tests/test_preprocessing.py | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/tests/test_preprocessing.py b/tests/test_preprocessing.py index a3a1fb8..04755d3 100644 --- a/tests/test_preprocessing.py +++ b/tests/test_preprocessing.py @@ -16,11 +16,25 @@ exclude=["check_fit2d_predict1d"], ) ) -def test_standard_checks(test_fn): + +def test_standard_checks_OneHot(test_fn): trf = OneHotEncoder() test_fn(OneHotEncoder.__name__, trf) +@pytest.mark.parametrize( + "test_func", + select_tests( + flatten([general_checks, transformer_checks]), + exclude=["check_fit2d_predict1d"], + ) +) + +def test_standard_checks_Ordinal(test_func): + trf = OrdinalEncoder() + test_func(OrdinalEncoder.__name__, trf) + + def test_OneHotEncoder(): one_hot = OneHotEncoder() X = np.array([ @@ -134,4 +148,19 @@ def test_LabelEncoder(): assert 'skprom_label_categorical' in [m.name for m in REGISTRY.collect()] assert REGISTRY.get_sample_value('skprom_label_categorical_total', {'Y': 'A'}) == 1 - assert REGISTRY.get_sample_value('skprom_label_categorical_total', {'Y': 'E'}) == 3 \ No newline at end of file + assert REGISTRY.get_sample_value('skprom_label_categorical_total', {'Y': 'E'}) == 3 + + +def test_LabelEncoder_missing(): + label_enc = LabelEncoder(handle_unknown="use_encoded_value", unknown_value=np.nan) + Y = np.array(['A', 'B', 'C', 'B', 'E', 'D', 'E', 'E'], dtype = np.str_). reshape((-1, 1)) + + Y_test = np.array(['A', 'B', 'C', 'B', 'E', 'D', 'F', 'E'], dtype = np.str_). reshape((-1, 1)) + + label_enc.fit(Y) + label_enc.transform(Y_test) + + assert 'skprom_label_categorical' in [m.name for m in REGISTRY.collect()] + + assert REGISTRY.get_sample_value('skprom_label_categorical_total', {'Y': 'A'}) == 1 + assert REGISTRY.get_sample_value('skprom_label_categorical_total', {'Y': 'missing'}) == 1 \ No newline at end of file