Issue 540

.travis.yml

@@ -25,6 +25,11 @@ env:
  - DISTRIB="conda" PYTHON_VERSION="3.6" SKLEARN_VERSION="0.19.2"
  - DISTRIB="conda" PYTHON_VERSION="3.6" SKLEARN_VERSION="0.18.2"
 
+# Travis issue
+# https://github.com/travis-ci/travis-ci/issues/8920
+before_install:
+ - python -c "import fcntl; fcntl.fcntl(1, fcntl.F_SETFL, 0)"
+
 install: source ci_scripts/install.sh
 script: bash ci_scripts/test.sh
 after_success: source ci_scripts/success.sh && source ci_scripts/create_doc.sh $TRAVIS_BRANCH "doc_result"

ci_scripts/flake8_diff.sh

@@ -125,7 +125,7 @@ check_files() {
  if [ -n "$files" ]; then
  # Conservative approach: diff without context (--unified=0) so that code
  # that was not changed does not create failures
- git diff --unified=0 $COMMIT_RANGE -- $files | flake8 --diff --show-source $options
+ git diff --unified=0 $COMMIT_RANGE -- $files | flake8 --ignore E402 --diff --show-source $options
  fi
 }
 

examples/create_upload_tutorial.py

@@ -5,53 +5,87 @@
 A tutorial on how to create and upload a dataset to OpenML.
 """
 import numpy as np
-import openml
 import sklearn.datasets
+from scipy.sparse import coo_matrix
+
+import openml
+from openml.datasets.functions import create_dataset
 
 ############################################################################
-# For this example we will upload to the test server to not pollute the live server with countless copies of the same dataset.
+# For this tutorial we will upload to the test server to not pollute the live
+# server with countless copies of the same dataset.
 openml.config.server = 'https://test.openml.org/api/v1/xml'
 
 ############################################################################
-# Prepare the data
-# ^^^^^^^^^^^^^^^^
-# Load an example dataset from scikit-learn which we will upload to OpenML.org via the API.
-breast_cancer = sklearn.datasets.load_breast_cancer()
-name = 'BreastCancer(scikit-learn)'
-X = breast_cancer.data
-y = breast_cancer.target
-attribute_names = breast_cancer.feature_names
-targets = breast_cancer.target_names
-description = breast_cancer.DESCR
+# Below we will cover the following cases of the
+# dataset object:
+#
+# * A numpy array
+# * A list
+# * A sparse matrix
 
 ############################################################################
-# OpenML does not distinguish between the attributes and targets on the data level and stores all data in a
-# single matrix. The target feature is indicated as meta-data of the dataset (and tasks on that data).
+# Dataset is a numpy array
+# ========================
+# A numpy array can contain lists in the case of dense data
+# or it can contain OrderedDicts in the case of sparse data.
+#
+# Prepare dataset
+# ^^^^^^^^^^^^^^^
+# Load an example dataset from scikit-learn which we
+# will upload to OpenML.org via the API.
+
+diabetes = sklearn.datasets.load_diabetes()
+name = 'Diabetes(scikit-learn)'
+X = diabetes.data
+y = diabetes.target
+attribute_names = diabetes.feature_names
+description = diabetes.DESCR
+
+############################################################################
+# OpenML does not distinguish between the attributes and
+# targets on the data level and stores all data in a single matrix.
+#
+# The target feature is indicated as meta-data of the
+# dataset (and tasks on that data).
+
 data = np.concatenate((X, y.reshape((-1, 1))), axis=1)
 attribute_names = list(attribute_names)
 attributes = [
  (attribute_name, 'REAL') for attribute_name in attribute_names
-] + [('class', 'REAL')]
+] + [('class', 'INTEGER')]
+citation = (
+ "Bradley Efron, Trevor Hastie, Iain Johnstone and "
+ "Robert Tibshirani (2004) (Least Angle Regression) "
+ "Annals of Statistics (with discussion), 407-499"
+)
+paper_url = (
+ 'http://web.stanford.edu/~hastie/Papers/'
+ 'LARS/LeastAngle_2002.pdf'
+)
 
 ############################################################################
 # Create the dataset object
 # ^^^^^^^^^^^^^^^^^^^^^^^^^
-# The definition of all fields can be found in the XSD files describing the expected format:
+# The definition of all fields can be found in the
+# XSD files describing the expected format:
 #
 # https://github.com/openml/OpenML/blob/master/openml_OS/views/pages/api_new/v1/xsd/openml.data.upload.xsd
-dataset = openml.datasets.functions.create_dataset(
+
+diabetes_dataset = create_dataset(
  # The name of the dataset (needs to be unique). 
  # Must not be longer than 128 characters and only contain
  # a-z, A-Z, 0-9 and the following special characters: _\-\.(),
  name=name,
  # Textual description of the dataset.
  description=description,
  # The person who created the dataset.
- creator='Dr. William H. Wolberg, W. Nick Street, Olvi L. Mangasarian',
+ creator="Bradley Efron, Trevor Hastie, "
+ "Iain Johnstone and Robert Tibshirani",
  # People who contributed to the current version of the dataset.
  contributor=None,
  # The date the data was originally collected, given by the uploader.
- collection_date='01-11-1995',
+ collection_date='09-01-2012',
  # Language in which the data is represented.
  # Starts with 1 upper case letter, rest lower case, e.g. 'English'.
  language='English',
@@ -64,26 +98,129 @@
  # Attributes that should be excluded in modelling, such as identifiers and indexes.
  ignore_attribute=None,
  # How to cite the paper.
- citation=(
- "W.N. Street, W.H. Wolberg and O.L. Mangasarian. "
- "Nuclear feature extraction for breast tumor diagnosis. "
- "IS&T/SPIE 1993 International Symposium on Electronic Imaging: Science and Technology, "
- "volume 1905, pages 861-870, San Jose, CA, 1993."
- ),
+ citation=citation,
  # Attributes of the data
  attributes=attributes,
  data=data,
- # Format of the dataset. Only 'arff' for now.
- format='arff',
  # A version label which is provided by the user.
  version_label='test',
- original_data_url='https://archive.ics.uci.edu/ml/datasets/Breast+Cancer+Wisconsin+(Diagnostic)',
- paper_url='https://www.spiedigitallibrary.org/conference-proceedings-of-spie/1905/0000/Nuclear-feature-extraction-for-breast-tumor-diagnosis/10.1117/12.148698.short?SSO=1'
+ original_data_url=(
+ 'http://www4.stat.ncsu.edu/~boos/var.select/diabetes.html'
+ ),
+ paper_url=paper_url,
 )
 
 ############################################################################
-try:
- upload_id = dataset.publish()
- print('URL for dataset: %s/data/%d' % (openml.config.server, upload_id))
-except openml.exceptions.PyOpenMLError as err:
- print("OpenML: {0}".format(err))
+
+upload_did = diabetes_dataset.publish()
+print('URL for dataset: %s/data/%d' % (openml.config.server, upload_did))
+
+############################################################################
+# Dataset is a list
+# =================
+# A list can contain lists in the case of dense data
+# or it can contain OrderedDicts in the case of sparse data.
+#
+# Weather dataset:
+# http://storm.cis.fordham.edu/~gweiss/data-mining/datasets.html
+
+data = [
+ ['sunny', 85, 85, 'FALSE', 'no'],
+ ['sunny', 80, 90, 'TRUE', 'no'],
+ ['overcast', 83, 86, 'FALSE', 'yes'],
+ ['rainy', 70, 96, 'FALSE', 'yes'],
+ ['rainy', 68, 80, 'FALSE', 'yes'],
+ ['rainy', 65, 70, 'TRUE', 'no'],
+ ['overcast', 64, 65, 'TRUE', 'yes'],
+ ['sunny', 72, 95, 'FALSE', 'no'],
+ ['sunny', 69, 70, 'FALSE', 'yes'],
+ ['rainy', 75, 80, 'FALSE', 'yes'],
+ ['sunny', 75, 70, 'TRUE', 'yes'],
+ ['overcast', 72, 90, 'TRUE', 'yes'],
+ ['overcast', 81, 75, 'FALSE', 'yes'],
+ ['rainy', 71, 91, 'TRUE', 'no'],
+]
+
+attribute_names = [
+ ('outlook', ['sunny', 'overcast', 'rainy']),
+ ('temperature', 'REAL'),
+ ('humidity', 'REAL'),
+ ('windy', ['TRUE', 'FALSE']),
+ ('play', ['yes', 'no']),
+]
+
+description = (
+ 'The weather problem is a tiny dataset that we will use repeatedly'
+ ' to illustrate machine learning methods. Entirely fictitious, it '
+ 'supposedly concerns the conditions that are suitable for playing '
+ 'some unspecified game. In general, instances in a dataset are '
+ 'characterized by the values of features, or attributes, that measure '
+ 'different aspects of the instance. In this case there are four '
+ 'attributes: outlook, temperature, humidity, and windy. '
+ 'The outcome is whether to play or not.'
+)
+
+citation = (
+ 'I. H. Witten, E. Frank, M. A. Hall, and ITPro,'
+ 'Data mining practical machine learning tools and techniques, '
+ 'third edition. Burlington, Mass.: Morgan Kaufmann Publishers, 2011'
+)
+
+weather_dataset = create_dataset(
+ name="Weather",
+ description=description,
+ creator='I. H. Witten, E. Frank, M. A. Hall, and ITPro',
+ contributor=None,
+ collection_date='01-01-2011',
+ language='English',
+ licence=None,
+ default_target_attribute='play',
+ row_id_attribute=None,
+ ignore_attribute=None,
+ citation=citation,
+ attributes=attribute_names,
+ data=data,
+ version_label='example',
+)
+
+############################################################################
+
+upload_did = weather_dataset.publish()
+print('URL for dataset: %s/data/%d' % (openml.config.server, upload_did))
+
+############################################################################
+# Dataset is a sparse matrix
+# ==========================
+
+sparse_data = coo_matrix((
+ [0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0],
+ ([0, 1, 1, 2, 2, 3, 3], [0, 1, 2, 0, 2, 0, 1]),
+))
+
+column_names = [
+ ('input1', 'REAL'),
+ ('input2', 'REAL'),
+ ('y', 'REAL'),
+]
+
+xor_dataset = create_dataset(
+ name="XOR",
+ description='Dataset representing the XOR operation',
+ creator=None,
+ contributor=None,
+ collection_date=None,
+ language='English',
+ licence=None,
+ default_target_attribute='y',
+ row_id_attribute=None,
+ ignore_attribute=None,
+ citation=None,
+ attributes=column_names,
+ data=sparse_data,
+ version_label='example',
+)
+
+############################################################################
+
+upload_did = xor_dataset.publish()
+print('URL for dataset: %s/data/%d' % (openml.config.server, upload_did))

openml/datasets/__init__.py

@@ -1,8 +1,21 @@
-from .functions import (list_datasets, check_datasets_active,
- get_datasets, get_dataset, status_update)
+from .functions import (
+ check_datasets_active,
+ create_dataset,
+ get_dataset,
+ get_datasets,
+ list_datasets,
+ status_update,
+)
 from .dataset import OpenMLDataset
 from .data_feature import OpenMLDataFeature
 
-__all__ = ['check_datasets_active', 'get_dataset', 'get_datasets',
- 'OpenMLDataset', 'OpenMLDataFeature', 'list_datasets',
- 'status_update']
+__all__ = [
+ 'check_datasets_active',
+ 'create_dataset',
+ 'get_dataset',
+ 'get_datasets',
+ 'list_datasets',
+ 'OpenMLDataset',
+ 'OpenMLDataFeature',
+ 'status_update',
+]