Merge pull request #412 from bsc-wdc/Simulation

Added Simulation Grid Search and its tests

Author: FernandoVN98

dislib/classification/csvm/base.py

@@ -443,7 +443,7 @@ def save_model(self, filepath, overwrite=True, save_format="json"):
 
         sync_obj(self.__dict__)
         model_metadata = self.__dict__
-        model_metadata["model_name"] = "kmeans"
+        model_metadata["model_name"] = "csvm"
 
         # Save model
         if save_format == "json":

dislib/classification/knn/base.py

@@ -7,9 +7,17 @@
 from dislib.data.array import Array
 from dislib.neighbors import NearestNeighbors
 from sklearn.metrics import accuracy_score
+from sklearn.neighbors import NearestNeighbors as SKNeighbors
+from sklearn.neighbors import KDTree
 
 from collections import defaultdict
 
+import os
+import json
+import dislib.data.util.model as utilmodel
+import pickle
+from dislib.data.util import sync_obj, decoder_helper, encoder_helper
+
 
 class KNeighborsClassifier(BaseEstimator):
     """Classifier implementing the k-nearest neighbors vote.
@@ -136,6 +144,104 @@ def score(self, q: Array, y: Array, collect=False):
 
         return compss_wait_on(score) if collect else score
 
+    def save_model(self, filepath, overwrite=True, save_format="json"):
+        """Saves a model to a file.
+        The model is synchronized before saving and can be reinstantiated
+        in the exact same state, without any of the code used for model
+        definition or fitting.
+        Parameters
+        ----------
+        filepath : str
+            Path where to save the model
+        overwrite : bool, optional (default=True)
+            Whether any existing model at the target
+            location should be overwritten.
+        save_format : str, optional (default='json)
+            Format used to save the models.
+        Examples
+        --------
+        >>> from dislib.classification import KNeighborsClassifier
+        >>> import numpy as np
+        >>> import dislib as ds
+        >>>  data = np.array([[0, 0, 5], [3, 0, 5], [3, 1, 2]])
+        >>> y_data = np.array([2, 1, 1, 2, 0])
+        >>> train = ds.array(x=ratings, block_size=(1, 1))
+        >>> knn = KNeighborsClassifier()
+        >>> knn.fit(train)
+        >>> knn.save_model("./model_KNN")
+        """
+
+        # Check overwrite
+        if not overwrite and os.path.isfile(filepath):
+            return
+
+        sync_obj(self.__dict__)
+        model_metadata = self.__dict__
+        model_metadata["model_name"] = "knn"
+
+        # Save model
+        if save_format == "json":
+            with open(filepath, "w") as f:
+                json.dump(model_metadata, f, default=_encode_helper)
+        elif save_format == "cbor":
+            if utilmodel.cbor2 is None:
+                raise ModuleNotFoundError("No module named 'cbor2'")
+            with open(filepath, "wb") as f:
+                utilmodel.cbor2.dump(model_metadata, f,
+                                     default=_encode_helper_cbor)
+        elif save_format == "pickle":
+            with open(filepath, "wb") as f:
+                pickle.dump(model_metadata, f)
+        else:
+            raise ValueError("Wrong save format.")
+
+    def load_model(self, filepath, load_format="json"):
+        """Loads a model from a file.
+        The model is reinstantiated in the exact same state in which it was
+        saved, without any of the code used for model definition or fitting.
+        Parameters
+        ----------
+        filepath : str
+            Path of the saved the model
+        load_format : str, optional (default='json')
+            Format used to load the model.
+        Examples
+        --------
+        >>> from dislib.clasiffication import KNeighborsClassifier
+        >>> import numpy as np
+        >>> import dislib as ds
+        >>> x_data = np.array([[1, 2], [2, 0], [3, 1], [4, 4], [5, 3]])
+        >>> y_data = np.array([2, 1, 1, 2, 0])
+        >>> x_test_m = np.array([[3, 2], [4, 4], [1, 3]])
+        >>> bn, bm = 2, 2
+        >>> x = ds.array(x=x_data, block_size=(bn, bm))
+        >>> y = ds.array(x=y_data, block_size=(bn, 1))
+        >>> test_data_m = ds.array(x=x_test_m, block_size=(bn, bm))
+        >>> knn = KNeighborsClassifier()
+        >>> knn.fit(x, y)
+        >>> knn.save_model("./model_KNN")
+        >>> knn_loaded = KNeighborsClassifier()
+        >>> knn_loaded.load_model("./model_KNN")
+        >>> pred = knn_loaded.predict(test_data).collect()
+        """
+        # Load model
+        if load_format == "json":
+            with open(filepath, "r") as f:
+                model_metadata = json.load(f, object_hook=_decode_helper)
+        elif load_format == "cbor":
+            if utilmodel.cbor2 is None:
+                raise ModuleNotFoundError("No module named 'cbor2'")
+            with open(filepath, "rb") as f:
+                model_metadata = utilmodel.cbor2. \
+                    load(f, object_hook=_decode_helper_cbor)
+        elif load_format == "pickle":
+            with open(filepath, "rb") as f:
+                model_metadata = pickle.load(f)
+        else:
+            raise ValueError("Wrong load format.")
+        for key, val in model_metadata.items():
+            setattr(self, key, val)
+
 
 @constraint(computing_units="${ComputingUnits}")
 @task(ind_blocks={Type: COLLECTION_IN, Depth: 2},
@@ -180,3 +286,59 @@ def _get_score(y_blocks, ypred_blocks):
     y_pred = Array._merge_blocks(ypred_blocks).flatten()
 
     return accuracy_score(y, y_pred)
+
+
+def _decode_helper_cbor(decoder, obj):
+    """Special decoder wrapper for dislib using cbor2."""
+    return _decode_helper(obj)
+
+
+def _decode_helper(obj):
+    if isinstance(obj, dict) and "class_name" in obj:
+        class_name = obj["class_name"]
+        if class_name == "NearestNeighbors":
+            nn = NearestNeighbors(obj["n_neighbors"])
+            nn.__setstate__(_decode_helper(obj["items"]))
+            return nn
+        elif class_name == "SKNeighbors":
+            dict_ = _decode_helper(obj["items"])
+            model = SKNeighbors()
+            model.__setstate__(dict_)
+            return model
+        elif class_name == "KDTree":
+            dict_ = _decode_helper(obj["items"])
+            model = KDTree(dict_[0])
+            return model
+        else:
+            decoded = decoder_helper(class_name, obj)
+            if decoded is not None:
+                return decoded
+    return obj
+
+
+def _encode_helper_cbor(encoder, obj):
+    encoder.encode(_encode_helper(obj))
+
+
+def _encode_helper(obj):
+    encoded = encoder_helper(obj)
+    if encoded is not None:
+        return encoded
+    elif isinstance(obj, SKNeighbors):
+        return {
+            "class_name": "SKNeighbors",
+            "n_neighbors": obj.n_neighbors,
+            "radius": obj.radius,
+            "items": obj.__getstate__(),
+        }
+    elif isinstance(obj, KDTree):
+        return {
+            "class_name": "KDTree",
+            "items": obj.__getstate__(),
+        }
+    elif isinstance(obj, NearestNeighbors):
+        return {
+            "class_name": obj.__class__.__name__,
+            "n_neighbors": obj.n_neighbors,
+            "items": obj.__getstate__(),
+        }

dislib/cluster/gm/base.py

@@ -589,7 +589,7 @@ def save_model(self, filepath, overwrite=True, save_format="json"):
 
         sync_obj(self.__dict__)
         model_metadata = self.__dict__
-        model_metadata["model_name"] = "kmeans"
+        model_metadata["model_name"] = "gm"
 
         # Save model
         if save_format == "json":

dislib/model_selection/__init__.py

@@ -1,4 +1,6 @@
 from dislib.model_selection._search import GridSearchCV, RandomizedSearchCV
+from dislib.model_selection._simulation import SimulationGridSearch
 from dislib.model_selection._split import KFold
 
-__all__ = ['GridSearchCV', 'RandomizedSearchCV', 'KFold']
+__all__ = ['GridSearchCV', 'RandomizedSearchCV', 'KFold',
+           'SimulationGridSearch']

dislib/model_selection/_simulation.py

@@ -0,0 +1,118 @@
+from pycompss.api.api import compss_wait_on
+from sklearn.model_selection import ParameterGrid
+from dislib.model_selection._validation import simulation_execution
+from collections import defaultdict
+from functools import partial
+from numpy.ma import MaskedArray
+from scipy.stats import rankdata
+import numpy as np
+
+
+class SimulationGridSearch():
+    def __init__(self, estimator, param_grid,
+                 sim_number=1, order="max"):
+        self.estimator = estimator
+        self.param_grid = param_grid
+        self.sim_number = sim_number
+        self.order = order
+        self.raw_results = None
+        self.cv_results_ = None
+
+    def _run_search(self, evaluate_candidates):
+        """Abstract method to perform the search. The parameter
+        `evaluate_candidates` is a function that evaluates a ParameterGrid at a
+        time """
+        evaluate_candidates(ParameterGrid(self.param_grid))
+
+    def fit(self, x, y=None, **fit_params):
+        """Run fit with all sets of parameters.
+
+        Parameters
+        ----------
+        x : ds-array
+            Training data samples.
+        y : ds-array, optional (default = None)
+            Training data labels or values.
+        **fit_params : dict of string -> object
+            Parameters passed to the ``fit`` method of the estimator
+        """
+        estimator = self.estimator
+        all_candidate_params = []
+        all_out = []
+
+        def evaluate_candidates_simulation(candidate_params):
+            candidate_params = list(candidate_params)
+            fits = []
+            for parameters in candidate_params:
+                fits.append(simulation_execution(
+                    estimator, parameters=parameters,
+                    simulation_params=fit_params,
+                    number_simulations=self.sim_number))
+            out = [simulation_results for simulation_results in fits]
+            out = compss_wait_on(out)
+            all_candidate_params.extend(candidate_params)
+            all_out.extend(out)
+
+        if callable(estimator):
+            self._run_search(evaluate_candidates_simulation)
+        else:
+            raise NotImplementedError("The simulation needs to "
+                                      "be contained on a function")
+
+        self.raw_results = all_out
+        results = self._format_results(all_candidate_params, all_out,
+                                       order=self.order,
+                                       sim_number=self.sim_number)
+
+        self.best_index_ = results["rank_test_simulation"].argmin()
+        self.best_score_ = results["mean_test_simulation"][self.best_index_]
+        self.best_params_ = results["params"][self.best_index_]
+
+        self.cv_results_ = results
+
+        return self
+
+    @staticmethod
+    def _format_results(candidate_params, out, order="max", sim_number=1):
+        n_candidates = len(candidate_params)
+        test_scores = out
+        results = {}
+
+        def _store(key_name, array, rank=False):
+            """A small helper to store the scores/times to
+            the cv_results_"""
+            array = np.array(array, dtype=np.float64).reshape(
+                n_candidates, sim_number)
+            if sim_number > 0:
+                for i in range(sim_number):
+                    results["results_%d_%s" % (i, key_name)] = array[:, i]
+            array_means = np.mean(array, axis=1)
+            array_stds = np.std(array, axis=1)
+            results['mean_%s' % key_name] = array_means
+            results['std_%s' % key_name] = array_stds
+            if rank:
+                if order == "max":
+                    results["rank_%s" % key_name] = np.asarray(
+                        rankdata(-array_means, method='min'), dtype=np.int32)
+                else:
+                    results["rank_%s" % key_name] = np.asarray(
+                        rankdata(array_means, method='min'), dtype=np.int32)
+        # Use one MaskedArray and mask all the places where the param is not
+        # applicable for that candidate. Use defaultdict as each candidate may
+        # not contain all the params
+        param_results = defaultdict(partial(MaskedArray,
+                                            np.empty(n_candidates, ),
+                                            mask=True,
+                                            dtype=object))
+        for cand_i, params in enumerate(candidate_params):
+            for name, value in params.items():
+                # An all masked empty array gets created for the key
+                # `"param_%s" % name` at the first occurrence of `name`.
+                # Setting the value at an index also unmasks that index
+                param_results["param_%s" % name][cand_i] = value
+
+        results.update(param_results)
+        # Store a list of param dicts at the key 'params'
+        results['params'] = candidate_params
+        _store('test_simulation', test_scores, rank=True)
+        return results

dislib/model_selection/_validation.py

@@ -3,7 +3,7 @@
 from dislib.data.array import Array
 from pycompss.api.task import task
 from pycompss.api.parameter import INOUT, Depth, Type, COLLECTION_IN
-
+import sys
 import numpy as np
 
 
@@ -39,6 +39,24 @@ def score_sklearn_estimator(est, scorer,  blocks_x, blocks_y):
     return _score(est, x, y, scorer)
 
 
+def execute_simulation(simulation, **parameters):
+    sys.stdout.write("PARAMETERS")
+    for param in parameters:
+        sys.stdout.write(str(param))
+    return simulation(**parameters)
+
+
+def simulation_execution(simulation, parameters,
+                         simulation_params, number_simulations):
+    simulations_result = []
+    if parameters is not None:
+        for _ in range(number_simulations):
+            simulations_result.append(execute_simulation(simulation,
+                                                         **parameters,
+                                                         **simulation_params))
+    return simulations_result
+
+
 def sklearn_fit(estimator, train_ds,
                 parameters, fit_params):
     if parameters is not None:

dislib/neighbors/base.py

@@ -51,7 +51,7 @@ def fit(self, x):
         for row in x._iterator(axis=0):
             sknnstruct = _compute_fit(row._blocks)
             n_samples = row.shape[0]
-            self._fit_data.append((sknnstruct, n_samples))
+            self._fit_data.append([sknnstruct, n_samples])
 
         return self
 

dislib/recommendation/als/base.py

@@ -252,7 +252,7 @@ def save_model(self, filepath, overwrite=True, save_format="json"):
 
         sync_obj(self.__dict__)
         model_metadata = self.__dict__
-        model_metadata["model_name"] = "kmeans"
+        model_metadata["model_name"] = "als"
 
         # Save model
         if save_format == "json":

dislib/regression/lasso/base.py

@@ -165,7 +165,7 @@ def save_model(self, filepath, overwrite=True, save_format="json"):
 
         sync_obj(self.__dict__)
         model_metadata = self.__dict__
-        model_metadata["model_name"] = "kmeans"
+        model_metadata["model_name"] = "lasso"
 
         # Save model
         if save_format == "json":

dislib/regression/linear/base.py

@@ -168,7 +168,7 @@ def save_model(self, filepath, overwrite=True, save_format="json"):
 
         sync_obj(self.__dict__)
         model_metadata = self.__dict__
-        model_metadata["model_name"] = "kmeans"
+        model_metadata["model_name"] = "linear"
 
         # Save model
         if save_format == "json":