added propper support for data qualities

Author: janvanrijn

openml/datasets/dataset.py

@@ -39,7 +39,7 @@ def __init__(self, dataset_id=None, name=None, version=None, description=None,
                  row_id_attribute=None, ignore_attribute=None,
                  version_label=None, citation=None, tag=None, visibility=None,
                  original_data_url=None, paper_url=None, update_comment=None,
-                 md5_checksum=None, data_file=None, features=None):
+                 md5_checksum=None, data_file=None, features=None, qualities=None):
         # Attributes received by querying the RESTful API
         self.dataset_id = int(dataset_id) if dataset_id is not None else None
         self.name = name
@@ -74,6 +74,7 @@ def __init__(self, dataset_id=None, name=None, version=None, description=None,
         self.md5_cheksum = md5_checksum
         self.data_file = data_file
         self.features = None
+        self.qualities = None
 
         if features is not None:
             self.features = {}
@@ -87,6 +88,12 @@ def __init__(self, dataset_id=None, name=None, version=None, description=None,
                     raise ValueError('Data features not provided in right order')
                 self.features[feature.index] = feature
 
+        if qualities is not None:
+            self.qualities = {}
+            for idx, xmlquality in enumerate(qualities['oml:quality']):
+                name = xmlquality['oml:name']
+                value = xmlquality['oml:value']
+                self.qualities[name] = value
 
         if data_file is not None:
             if self._data_features_supported():

openml/datasets/functions.py

@@ -75,7 +75,8 @@ def _get_cached_dataset(dataset_id):
     description = _get_cached_dataset_description(dataset_id)
     arff_file = _get_cached_dataset_arff(dataset_id)
     features = _get_cached_dataset_features(dataset_id)
-    dataset = _create_dataset_from_description(description, features, arff_file)
+    qualities = _get_cached_dataset_qualities(dataset_id)
+    dataset = _create_dataset_from_description(description, features, qualities, arff_file)
 
     return dataset
 
@@ -107,6 +108,19 @@ def _get_cached_dataset_features(dataset_id):
                                    "cached" % dataset_id)
 
 
+def _get_cached_dataset_qualities(dataset_id):
+    cache_dir = config.get_cache_directory()
+    did_cache_dir = os.path.join(cache_dir, "datasets", str(dataset_id))
+    qualities_file = os.path.join(did_cache_dir, "qualities.xml")
+    try:
+        with io.open(qualities_file, encoding='utf8') as fh:
+            qualities_xml = fh.read()
+            return xmltodict.parse(qualities_xml)["oml:data_qualities"]
+    except (IOError, OSError):
+        raise OpenMLCacheException("Dataset qualities for dataset id %d not "
+                                   "cached" % dataset_id)
+
+
 def _get_cached_dataset_arff(dataset_id):
     cache_dir = config.get_cache_directory()
     did_cache_dir = os.path.join(cache_dir, "datasets", str(dataset_id))
@@ -272,7 +286,7 @@ def get_dataset(dataset_id):
         _remove_dataset_cache_dir(did_cache_dir)
         raise e
 
-    dataset = _create_dataset_from_description(description, features, arff_file)
+    dataset = _create_dataset_from_description(description, features, qualities, arff_file)
     return dataset
 
 
@@ -470,7 +484,7 @@ def _remove_dataset_cache_dir(did_cache_dir):
                              'Please do this manually!' % did_cache_dir)
 
 
-def _create_dataset_from_description(description, features, arff_file):
+def _create_dataset_from_description(description, features, qualities, arff_file):
     """Create a dataset object from a description dict.
 
     Parameters
@@ -510,5 +524,6 @@ def _create_dataset_from_description(description, features, arff_file):
         description.get("oml:update_comment"),
         description.get("oml:md5_checksum"),
         data_file=arff_file,
-        features=features)
+        features=features,
+        qualities=qualities)
     return dataset

tests/test_datasets/test_dataset_functions.py

@@ -169,6 +169,9 @@ def test_get_dataset(self):
         self.assertTrue(os.path.exists(os.path.join(
             openml.config.get_cache_directory(), "datasets", "1", "qualities.xml")))
 
+        self.assertGreater(len(dataset.features), 1)
+        self.assertGreater(len(dataset.qualities), 4)
+
     def test_get_dataset_with_string(self):
         dataset = openml.datasets.get_dataset(101)
         self.assertRaises(PyOpenMLError, dataset._get_arff, 'arff')