From 2796b9a2133136816ce3447bac280b21b2f0b2e1 Mon Sep 17 00:00:00 2001 From: neeratyoy Date: Tue, 15 Oct 2019 20:10:06 +0200 Subject: [PATCH 1/6] Adding Perrone example for building surrogate --- .../40_paper/2018_neurips_perrone_example.py | 169 +++++++++++++++++- 1 file changed, 168 insertions(+), 1 deletion(-) diff --git a/examples/40_paper/2018_neurips_perrone_example.py b/examples/40_paper/2018_neurips_perrone_example.py index 3262ee4a1..1b5ea181d 100644 --- a/examples/40_paper/2018_neurips_perrone_example.py +++ b/examples/40_paper/2018_neurips_perrone_example.py @@ -13,5 +13,172 @@ | In *Advances in Neural Information Processing Systems 31*, 2018 | Available at http://papers.nips.cc/paper/7917-scalable-hyperparameter-transfer-learning.pdf -This is currently a placeholder. +This example demonstrates how OpenML runs can be used to construct a surrogate model. + +In the following section, we shall do the following: + +* Retrieve tasks and flows as used in the experiments by Perrone et al. +* Build a tabular data by fetching the evaluations uploaded to OpenML +* Impute missing values and handle categorical data before building a Random Forest model that + maps hyperparameter values to the area under curve score +""" + +############################################################################ +import openml +import numpy as np +import pandas as pd +from sklearn.impute import SimpleImputer +from sklearn.preprocessing import OneHotEncoder +from sklearn.ensemble import RandomForestRegressor + +user_id = 2702 +############################################################################ + +""" +The subsequent functions are defined to fetch tasks, flows, evaluations and preprocess them into +a tabular format that can be used to build models. """ + +def fetch_evaluations(run_full=False, flow_type='svm', metric = 'area_under_roc_curve'): + ''' + Fetch a list of evaluations based on the flows and tasks used in the experiments. + + Parameters + ---------- + run_full : boolean + If True, use the full list of tasks used in the paper + If False, use 5 tasks with the smallest number of evaluations available + flow_type : str, {'svm', 'xgboost'} + To select whether svm or xgboost experiments are to be run + metric : str + The evaluation measure that is passed to openml.evaluations.list_evaluations + + Returns + ------- + eval_df : dataframe + task_ids : list + flow_id : int + ''' + # Collecting task IDs as used by the experiments from the paper + if flow_type == 'svm' and run_full: + task_ids = [10101, 145878, 146064, 14951, 34537, 3485, 3492, 3493, 3494, 37, 3889, 3891, + 3899, 3902, 3903, 3913, 3918, 3950, 9889, 9914, 9946, 9952, 9967, 9971, 9976, + 9978, 9980, 9983] + elif flow_type == 'svm' and not run_full: + task_ids = [9983, 3485, 3902, 3903, 145878] + elif flow_type == 'xgboost' and run_full: + task_ids = [10093, 10101, 125923, 145847, 145857, 145862, 145872, 145878, 145953, 145972, + 145976, 145979, 146064, 14951, 31, 3485, 3492, 3493, 37, 3896, 3903, 3913, + 3917, 3918, 3, 49, 9914, 9946, 9952, 9967] + else: #flow_type == 'xgboost' and not run_full: + task_ids = [3903, 37, 3485, 49, 3913] + + # Fetching the relevant flow + flow_id = 5891 if flow_type == 'svm' else 6767 + + # Fetching evaluations + eval_df = openml.evaluations.list_evaluations(function=metric, task=task_ids, flow=[flow_id], + uploader=[2702], output_format='dataframe') + return eval_df, task_ids, flow_id + + +def create_table_from_evaluations(eval_df, flow_type='svm', run_count=np.iinfo(np.int64).max, + metric = 'area_under_roc_curve', task_ids=None): + ''' + Create a tabular data with its ground truth from a dataframe of evaluations. + Optionally, can filter out records based on task ids. + + Parameters + ---------- + eval_df : dataframe + Containing list of runs as obtained from list_evaluations() + flow_type : str, {'svm', 'xgboost'} + To select whether svm or xgboost experiments are to be run + run_count : int + Maximum size of the table created, or number of runs included in the table + metric : str + The evaluation measure that is passed to openml.evaluations.list_evaluations + task_ids : list, (optional) + List of integers specifying the tasks to be retained from the evaluations dataframe + + Returns + ------- + eval_table : dataframe + values : list + ''' + if task_ids is not None: + eval_df = eval_df.loc[eval_df.task_id.isin(task_ids)] + ncols = 4 if flow_type == 'svm' else 10 # ncols determine the number of hyperparameters + if flow_type == 'svm': + ncols = 4 + colnames = ['cost', 'degree', 'gamma', 'kernel'] + else: + ncols = 10 + colnames = ['alpha', 'booster', 'colsample_bylevel', 'colsample_bytree', 'eta', 'lambda', + 'max_depth', 'min_child_weight', 'nrounds', 'subsample'] + eval_df = eval_df.sample(frac=1) # shuffling rows + run_ids = eval_df.run_id[:run_count] + eval_table = pd.DataFrame(np.nan, index=run_ids, columns=colnames) + values = [] + for run_id in run_ids: + r = openml.runs.get_run(run_id) + params = r.parameter_settings + for p in params: + name, value = p['oml:name'], p['oml:value'] + if name in colnames: + eval_table.loc[run_id, name] = value + values.append(r.evaluations[metric]) + return eval_table, values + + +def impute_missing_values(eval_table, flow_type='svm'): + # Replacing NaNs with fixed values outside the range of the parameters + # given in the supplement material of the paper + if flow_type == 'svm': + eval_table.kernel.fillna("None", inplace=True) + eval_table.fillna(-1, inplace=True) + else: + eval_table.booster.fillna("None", inplace=True) + eval_table.fillna(-1, inplace=True) + return eval_table + + +def preprocess(eval_table, flow_type='svm'): + eval_table = impute_missing_values(eval_table, flow_type) + # Encode categorical variables as one-hot vectors + enc = OneHotEncoder(handle_unknown='ignore') + enc.fit(eval_table.kernel.to_numpy().reshape(-1, 1)) + one_hots = enc.transform(eval_table.kernel.to_numpy().reshape(-1, 1)).toarray() + if flow_type == 'svm': + eval_table = np.hstack((eval_table.drop('kernel', 1), one_hots)).astype(float) + else: + eval_table = np.hstack((eval_table.drop('booster', 1), one_hots)).astype(float) + return eval_table + + +############################################################################# +# Fetching the tasks and evaluations +# ================================== +# To read all the tasks and evaluations for them and collate into a table. Here, we are reading +# all the tasks and evaluations for the SVM flow and preprocessing all retrieved evaluations. + +eval_df, task_ids, flow_id = fetch_evaluations(run_full=False) +X, y = create_table_from_evaluations(eval_df, run_count=1000) +X = preprocess(X) + + +############################################################################# +# Building a surrogate model on a task's evaluation +# ================================================= +# The same set of functions can be used for a single task to retrieve a singular table which can +# be used for the surrogate model construction. We shall use the SVM flow here to keep execution +# time simple and quick. + +# Selecting a task +task_id = task_ids[-1] +X, y = create_table_from_evaluations(eval_df, run_count=1000, task_ids=[task_id], flow_type='svm') +X = preprocess(X, flow_type='svm') + +# Surrogate model +clf = RandomForestRegressor(n_estimators=50, max_depth=3) +clf.fit(X, y) From 1a3f456dfd04f3095c3ae945c08222923111ce4d Mon Sep 17 00:00:00 2001 From: neeratyoy Date: Wed, 16 Oct 2019 11:40:20 +0200 Subject: [PATCH 2/6] Intermediate changes; pipeline additions remain --- .../40_paper/2018_neurips_perrone_example.py | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/examples/40_paper/2018_neurips_perrone_example.py b/examples/40_paper/2018_neurips_perrone_example.py index 1b5ea181d..85e436d0f 100644 --- a/examples/40_paper/2018_neurips_perrone_example.py +++ b/examples/40_paper/2018_neurips_perrone_example.py @@ -39,7 +39,9 @@ a tabular format that can be used to build models. """ -def fetch_evaluations(run_full=False, flow_type='svm', metric = 'area_under_roc_curve'): +def fetch_evaluations(run_full=False, + flow_type='svm', + metric='area_under_roc_curve'): ''' Fetch a list of evaluations based on the flows and tasks used in the experiments. @@ -77,13 +79,19 @@ def fetch_evaluations(run_full=False, flow_type='svm', metric = 'area_under_roc_ flow_id = 5891 if flow_type == 'svm' else 6767 # Fetching evaluations - eval_df = openml.evaluations.list_evaluations(function=metric, task=task_ids, flow=[flow_id], - uploader=[2702], output_format='dataframe') + eval_df = openml.evaluations.list_evaluations(function=metric, + task=task_ids, + flow=[flow_id], + uploader=[2702], + output_format='dataframe') return eval_df, task_ids, flow_id -def create_table_from_evaluations(eval_df, flow_type='svm', run_count=np.iinfo(np.int64).max, - metric = 'area_under_roc_curve', task_ids=None): +def create_table_from_evaluations(eval_df, + flow_type='svm', + run_count=np.iinfo(np.int64).max, + metric = 'area_under_roc_curve', + task_ids=None): ''' Create a tabular data with its ground truth from a dataframe of evaluations. Optionally, can filter out records based on task ids. @@ -108,7 +116,6 @@ def create_table_from_evaluations(eval_df, flow_type='svm', run_count=np.iinfo(n ''' if task_ids is not None: eval_df = eval_df.loc[eval_df.task_id.isin(task_ids)] - ncols = 4 if flow_type == 'svm' else 10 # ncols determine the number of hyperparameters if flow_type == 'svm': ncols = 4 colnames = ['cost', 'degree', 'gamma', 'kernel'] @@ -165,6 +172,8 @@ def preprocess(eval_table, flow_type='svm'): eval_df, task_ids, flow_id = fetch_evaluations(run_full=False) X, y = create_table_from_evaluations(eval_df, run_count=1000) X = preprocess(X) +print("Type: {}; Shape: {}".format(type(X), X.shape)) +print(X[:5]) ############################################################################# From cfba39d56043ed89e3e4c774de434565842c9457 Mon Sep 17 00:00:00 2001 From: neeratyoy Date: Thu, 17 Oct 2019 16:29:21 +0200 Subject: [PATCH 3/6] Finishing the whole example design --- .../40_paper/2018_neurips_perrone_example.py | 113 +++++++++++++++--- 1 file changed, 99 insertions(+), 14 deletions(-) diff --git a/examples/40_paper/2018_neurips_perrone_example.py b/examples/40_paper/2018_neurips_perrone_example.py index 85e436d0f..e33cdc048 100644 --- a/examples/40_paper/2018_neurips_perrone_example.py +++ b/examples/40_paper/2018_neurips_perrone_example.py @@ -27,11 +27,17 @@ import openml import numpy as np import pandas as pd +from matplotlib import pyplot as plt +from sklearn.pipeline import Pipeline from sklearn.impute import SimpleImputer +from sklearn.compose import ColumnTransformer +from sklearn.metrics import mean_squared_error from sklearn.preprocessing import OneHotEncoder from sklearn.ensemble import RandomForestRegressor + user_id = 2702 +flow_type = 'svm' # this example will use the smaller svm flow evaluations ############################################################################ """ @@ -138,6 +144,12 @@ def create_table_from_evaluations(eval_df, return eval_table, values +def list_categorical_attributes(flow_type='svm'): + if flow_type == 'svm': + return ['kernel'] + return ['booster'] + + def impute_missing_values(eval_table, flow_type='svm'): # Replacing NaNs with fixed values outside the range of the parameters # given in the supplement material of the paper @@ -164,30 +176,103 @@ def preprocess(eval_table, flow_type='svm'): ############################################################################# -# Fetching the tasks and evaluations -# ================================== +# Fetching the data from OpenML +# ***************************** # To read all the tasks and evaluations for them and collate into a table. Here, we are reading -# all the tasks and evaluations for the SVM flow and preprocessing all retrieved evaluations. +# all the tasks and evaluations for the SVM flow and pre-processing all retrieved evaluations. + +eval_df, task_ids, flow_id = fetch_evaluations(run_full=False, flow_type=flow_type) +# run_count can not be passed if all the results are required +# it is set to 1000 here arbitrarily to get results quickly +X, y = create_table_from_evaluations(eval_df, run_count=1000, flow_type=flow_type) +print(X.head()) +print("Y : ", y[:5]) + +############################################################################# +# Creating pre-processing and modelling pipelines +# *********************************************** +# The two primary tasks are to impute the missing values, that is, account for the hyperparameters +# that are not available with the runs from OpenML. And secondly, to handle categorical variables +# using One-hot encoding prior to modelling. + +# Separating data into categorical and non-categorical (numeric for this example) columns +cat_cols = list_categorical_attributes(flow_type=flow_type) +num_cols = list(set(X.columns) - set(cat_cols)) +X_cat = X.loc[:, cat_cols] +X_num = X.loc[:, num_cols] + +# Missing value imputers +cat_imputer = SimpleImputer(missing_values=np.nan, strategy='constant', fill_value='None') +num_imputer = SimpleImputer(missing_values=np.nan, strategy='constant', fill_value=-1) + +# Creating the one-hot encoder +enc = OneHotEncoder(handle_unknown='ignore') -eval_df, task_ids, flow_id = fetch_evaluations(run_full=False) -X, y = create_table_from_evaluations(eval_df, run_count=1000) -X = preprocess(X) -print("Type: {}; Shape: {}".format(type(X), X.shape)) -print(X[:5]) +# Pipeline to handle categorical column transformations +cat_transforms = Pipeline([('impute', cat_imputer), ('encode', enc)]) + +# Combining column transformers +ct = ColumnTransformer([('cat', cat_transforms, cat_cols), ('num', num_imputer, num_cols)]) + +# Creating the full pipeline with the surrogate model +clf = RandomForestRegressor(n_estimators=50) +model = Pipeline(steps=[('preprocess', ct), ('surrogate', clf)]) ############################################################################# # Building a surrogate model on a task's evaluation -# ================================================= +# ************************************************* # The same set of functions can be used for a single task to retrieve a singular table which can # be used for the surrogate model construction. We shall use the SVM flow here to keep execution # time simple and quick. -# Selecting a task +# Selecting a task for the surrogate task_id = task_ids[-1] +print("Task ID : ", task_id) X, y = create_table_from_evaluations(eval_df, run_count=1000, task_ids=[task_id], flow_type='svm') -X = preprocess(X, flow_type='svm') -# Surrogate model -clf = RandomForestRegressor(n_estimators=50, max_depth=3) -clf.fit(X, y) +model.fit(X, y) +y_pred = model.predict(X) + +print("Training RMSE : {:.5}".format(mean_squared_error(y, y_pred))) + + +############################################################################# +# Evaluating the surrogate model +# ****************************** +# The surrogate model built from a task's evaluations fetched from OpenML will be put into +# trivial action here, where we shall randomly sample configurations and observe the trajectory +# of the area under curve (auc) we can obtain from the surrogate we've built. +# NOTE: This section is written exclusively for the SVM flow + +# Sampling random configurations +def random_sample_configurations(num_samples=100): + colnames = ['cost', 'degree', 'gamma', 'kernel'] + ranges = [(0.000986, 998.492437), + (2.0, 5.0), + (0.000988, 913.373845), + (['linear', 'polynomial', 'radial', 'sigmoid'])] + X = pd.DataFrame(np.nan, index=range(num_samples), columns=colnames) + for i in range(len(colnames)): + if len(ranges[i]) == 2: + col_val = np.random.uniform(low=ranges[i][0], high=ranges[i][1], size=num_samples) + else: + col_val = np.random.choice(ranges[i], size=num_samples) + X.iloc[:, i] = col_val + return X + +configs = random_sample_configurations(num_samples=1000) +preds = model.predict(configs) + +# tracking the maximum AUC obtained over the functions evaluations +preds = np.maximum.accumulate(preds) +# computing regret (1 - predicted_auc) +regret = 1 - preds + +# plotting the regret curve +plt.plot(regret) +# plt.yscale('log') +plt.title('AUC regret for Random Search on surrogate') +plt.xlabel('Numbe of function evaluations') +plt.ylabel('Regret') +plt.show() From 9ca9d8783ee2abdb17a457ccfdc9a848a8cadc3a Mon Sep 17 00:00:00 2001 From: neeratyoy Date: Thu, 17 Oct 2019 16:44:36 +0200 Subject: [PATCH 4/6] Making pandas related changes suggested by Matthias --- .../40_paper/2018_neurips_perrone_example.py | 29 ++----------------- 1 file changed, 2 insertions(+), 27 deletions(-) diff --git a/examples/40_paper/2018_neurips_perrone_example.py b/examples/40_paper/2018_neurips_perrone_example.py index e33cdc048..f4594ae1d 100644 --- a/examples/40_paper/2018_neurips_perrone_example.py +++ b/examples/40_paper/2018_neurips_perrone_example.py @@ -121,7 +121,7 @@ def create_table_from_evaluations(eval_df, values : list ''' if task_ids is not None: - eval_df = eval_df.loc[eval_df.task_id.isin(task_ids)] + eval_df = eval_df[eval_df['task_id'].isin(task_ids)] if flow_type == 'svm': ncols = 4 colnames = ['cost', 'degree', 'gamma', 'kernel'] @@ -130,7 +130,7 @@ def create_table_from_evaluations(eval_df, colnames = ['alpha', 'booster', 'colsample_bylevel', 'colsample_bytree', 'eta', 'lambda', 'max_depth', 'min_child_weight', 'nrounds', 'subsample'] eval_df = eval_df.sample(frac=1) # shuffling rows - run_ids = eval_df.run_id[:run_count] + run_ids = eval_df.loc[:,"run_id"][:run_count] eval_table = pd.DataFrame(np.nan, index=run_ids, columns=colnames) values = [] for run_id in run_ids: @@ -150,31 +150,6 @@ def list_categorical_attributes(flow_type='svm'): return ['booster'] -def impute_missing_values(eval_table, flow_type='svm'): - # Replacing NaNs with fixed values outside the range of the parameters - # given in the supplement material of the paper - if flow_type == 'svm': - eval_table.kernel.fillna("None", inplace=True) - eval_table.fillna(-1, inplace=True) - else: - eval_table.booster.fillna("None", inplace=True) - eval_table.fillna(-1, inplace=True) - return eval_table - - -def preprocess(eval_table, flow_type='svm'): - eval_table = impute_missing_values(eval_table, flow_type) - # Encode categorical variables as one-hot vectors - enc = OneHotEncoder(handle_unknown='ignore') - enc.fit(eval_table.kernel.to_numpy().reshape(-1, 1)) - one_hots = enc.transform(eval_table.kernel.to_numpy().reshape(-1, 1)).toarray() - if flow_type == 'svm': - eval_table = np.hstack((eval_table.drop('kernel', 1), one_hots)).astype(float) - else: - eval_table = np.hstack((eval_table.drop('booster', 1), one_hots)).astype(float) - return eval_table - - ############################################################################# # Fetching the data from OpenML # ***************************** From cd3ba2991e0ed3c2a34f6d2a8a30a37e2c9286d7 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Thu, 17 Oct 2019 19:56:01 +0200 Subject: [PATCH 5/6] minor reformatting --- .../40_paper/2018_neurips_perrone_example.py | 62 ++++++++++--------- 1 file changed, 32 insertions(+), 30 deletions(-) diff --git a/examples/40_paper/2018_neurips_perrone_example.py b/examples/40_paper/2018_neurips_perrone_example.py index f4594ae1d..17595922a 100644 --- a/examples/40_paper/2018_neurips_perrone_example.py +++ b/examples/40_paper/2018_neurips_perrone_example.py @@ -17,10 +17,10 @@ In the following section, we shall do the following: -* Retrieve tasks and flows as used in the experiments by Perrone et al. -* Build a tabular data by fetching the evaluations uploaded to OpenML +* Retrieve tasks and flows as used in the experiments by Perrone et al. (2018). +* Build a tabular data by fetching the evaluations uploaded to OpenML. * Impute missing values and handle categorical data before building a Random Forest model that - maps hyperparameter values to the area under curve score + maps hyperparameter values to the area under curve score. """ ############################################################################ @@ -35,15 +35,11 @@ from sklearn.preprocessing import OneHotEncoder from sklearn.ensemble import RandomForestRegressor - -user_id = 2702 flow_type = 'svm' # this example will use the smaller svm flow evaluations ############################################################################ - -""" -The subsequent functions are defined to fetch tasks, flows, evaluations and preprocess them into -a tabular format that can be used to build models. -""" +# The subsequent functions are defined to fetch tasks, flows, evaluations and preprocess them into +# a tabular format that can be used to build models. +# def fetch_evaluations(run_full=False, flow_type='svm', @@ -69,15 +65,20 @@ def fetch_evaluations(run_full=False, ''' # Collecting task IDs as used by the experiments from the paper if flow_type == 'svm' and run_full: - task_ids = [10101, 145878, 146064, 14951, 34537, 3485, 3492, 3493, 3494, 37, 3889, 3891, - 3899, 3902, 3903, 3913, 3918, 3950, 9889, 9914, 9946, 9952, 9967, 9971, 9976, - 9978, 9980, 9983] + task_ids = [ + 10101, 145878, 146064, 14951, 34537, 3485, 3492, 3493, 3494, + 37, 3889, 3891, 3899, 3902, 3903, 3913, 3918, 3950, 9889, + 9914, 9946, 9952, 9967, 9971, 9976, 9978, 9980, 9983, + ] elif flow_type == 'svm' and not run_full: task_ids = [9983, 3485, 3902, 3903, 145878] elif flow_type == 'xgboost' and run_full: - task_ids = [10093, 10101, 125923, 145847, 145857, 145862, 145872, 145878, 145953, 145972, - 145976, 145979, 146064, 14951, 31, 3485, 3492, 3493, 37, 3896, 3903, 3913, - 3917, 3918, 3, 49, 9914, 9946, 9952, 9967] + task_ids = [ + 10093, 10101, 125923, 145847, 145857, 145862, 145872, 145878, + 145953, 145972, 145976, 145979, 146064, 14951, 31, 3485, + 3492, 3493, 37, 3896, 3903, 3913, 3917, 3918, 3, 49, 9914, + 9946, 9952, 9967, + ] else: #flow_type == 'xgboost' and not run_full: task_ids = [3903, 37, 3485, 49, 3913] @@ -123,23 +124,24 @@ def create_table_from_evaluations(eval_df, if task_ids is not None: eval_df = eval_df[eval_df['task_id'].isin(task_ids)] if flow_type == 'svm': - ncols = 4 colnames = ['cost', 'degree', 'gamma', 'kernel'] else: - ncols = 10 - colnames = ['alpha', 'booster', 'colsample_bylevel', 'colsample_bytree', 'eta', 'lambda', - 'max_depth', 'min_child_weight', 'nrounds', 'subsample'] + colnames = [ + 'alpha', 'booster', 'colsample_bylevel', 'colsample_bytree', + 'eta', 'lambda', 'max_depth', 'min_child_weight', 'nrounds', + 'subsample', + ] eval_df = eval_df.sample(frac=1) # shuffling rows - run_ids = eval_df.loc[:,"run_id"][:run_count] + run_ids = eval_df["run_id"][:run_count] eval_table = pd.DataFrame(np.nan, index=run_ids, columns=colnames) values = [] - for run_id in run_ids: - r = openml.runs.get_run(run_id) + runs = openml.runs.get_runs(run_ids) + for r in runs: params = r.parameter_settings for p in params: name, value = p['oml:name'], p['oml:value'] if name in colnames: - eval_table.loc[run_id, name] = value + eval_table.loc[r.run_id, name] = value values.append(r.evaluations[metric]) return eval_table, values @@ -153,13 +155,14 @@ def list_categorical_attributes(flow_type='svm'): ############################################################################# # Fetching the data from OpenML # ***************************** -# To read all the tasks and evaluations for them and collate into a table. Here, we are reading -# all the tasks and evaluations for the SVM flow and pre-processing all retrieved evaluations. +# Now, we read all the tasks and evaluations for them and collate into a table. +# Here, we are reading all the tasks and evaluations for the SVM flow and +# pre-processing all retrieved evaluations. eval_df, task_ids, flow_id = fetch_evaluations(run_full=False, flow_type=flow_type) # run_count can not be passed if all the results are required -# it is set to 1000 here arbitrarily to get results quickly -X, y = create_table_from_evaluations(eval_df, run_count=1000, flow_type=flow_type) +# it is set to 500 here arbitrarily to get results quickly +X, y = create_table_from_evaluations(eval_df, run_count=500, flow_type=flow_type) print(X.head()) print("Y : ", y[:5]) @@ -218,6 +221,7 @@ def list_categorical_attributes(flow_type='svm'): # The surrogate model built from a task's evaluations fetched from OpenML will be put into # trivial action here, where we shall randomly sample configurations and observe the trajectory # of the area under curve (auc) we can obtain from the surrogate we've built. +# # NOTE: This section is written exclusively for the SVM flow # Sampling random configurations @@ -246,8 +250,6 @@ def random_sample_configurations(num_samples=100): # plotting the regret curve plt.plot(regret) -# plt.yscale('log') plt.title('AUC regret for Random Search on surrogate') plt.xlabel('Numbe of function evaluations') plt.ylabel('Regret') -plt.show() From f6a2a958f57c28f0b06cebe3707b8c55ffb5ad49 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Thu, 17 Oct 2019 19:58:30 +0200 Subject: [PATCH 6/6] add a print statement --- examples/40_paper/2018_neurips_perrone_example.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/examples/40_paper/2018_neurips_perrone_example.py b/examples/40_paper/2018_neurips_perrone_example.py index 17595922a..6c8207153 100644 --- a/examples/40_paper/2018_neurips_perrone_example.py +++ b/examples/40_paper/2018_neurips_perrone_example.py @@ -241,6 +241,9 @@ def random_sample_configurations(num_samples=100): return X configs = random_sample_configurations(num_samples=1000) +print(configs) + +############################################################################# preds = model.predict(configs) # tracking the maximum AUC obtained over the functions evaluations