Fixed subspace, more Doc (#31)

* fixed subspace plus doc

* format

Author: pierrenodet

core/src/main/scala/org/apache/spark/ml/ensemble/HasSubBag.scala

@@ -19,6 +19,7 @@ import java.util.UUID
 
 import org.apache.spark.SparkException
 import org.apache.spark.ml.ensemble.HasSubBag.SubSpace
+import org.apache.spark.ml.feature.VectorSlicer
 import org.apache.spark.ml.linalg.{DenseVector, SparseVector, Vector, Vectors}
 import org.apache.spark.ml.param._
 import org.apache.spark.ml.param.shared.HasSeed
@@ -112,12 +113,15 @@ private[ml] trait HasSubBag extends Params with HasSeed {
       .withColumn(tmpColName, replicate_row(element_at(col(bagColName), index + 1)))
       .drop(col(tmpColName))
 
-    val slicerUDF = udf { slicer(subspace) }
+    val tmpSubSpaceColName = "bag$tmp" + UUID.randomUUID().toString
+    val vs = new VectorSlicer()
+      .setInputCol(featuresColName)
+      .setOutputCol(tmpSubSpaceColName)
+      .setIndices(subspace)
 
-    replicated.withColumn(
-      featuresColName,
-      slicerUDF(col(featuresColName)),
-      df.schema(df.schema.fieldIndex(featuresColName)).metadata)
+    vs.transform(replicated)
+      .withColumn(featuresColName, col(tmpSubSpaceColName))
+      .drop(tmpSubSpaceColName)
 
   }
 

core/src/test/scala/org/apache/spark/ml/classification/GBMClassifierSuite.scala

@@ -35,7 +35,7 @@ class GBMClassifierSuite extends FunSuite with DatasetSuiteBase {
         .addGrid(gbmc.validationIndicatorCol, Array("val"))
         .addGrid(gbmc.sampleRatio, Array(0.8))
         .addGrid(gbmc.replacement, Array(true))
-        .addGrid(gbmc.subspaceRatio, Array(1.0))
+        .addGrid(gbmc.subspaceRatio, Array(0.8))
         .addGrid(gbmc.optimizedWeights, Array(false,true))
         .addGrid(gbmc.loss, Array("divergence"))
         .addGrid(dr.maxDepth, Array(10))

core/src/test/scala/org/apache/spark/ml/regression/GBMRegressorSuite.scala

@@ -30,14 +30,14 @@ class GBMRegressorSuite extends FunSuite with DatasetSuiteBase {
 
     time {
       val gbmrParamGrid = new ParamGridBuilder()
-        .addGrid(gmbr.learningRate, Array(1.0))
+        .addGrid(gmbr.learningRate, Array(0.1))
         .addGrid(gmbr.numBaseLearners, Array(30))
         .addGrid(gmbr.validationIndicatorCol, Array("val"))
         .addGrid(gmbr.tol, Array(1E-3))
         .addGrid(gmbr.numRound, Array(8))
         .addGrid(gmbr.sampleRatio, Array(0.8))
         .addGrid(gmbr.replacement, Array(true))
-        .addGrid(gmbr.subspaceRatio, Array(1.0))
+        .addGrid(gmbr.subspaceRatio, Array(0.8))
         .addGrid(gmbr.optimizedWeights, Array(false,true))
         .addGrid(gmbr.loss, Array("squared"))
         .addGrid(gmbr.alpha, Array(0.5))
@@ -73,7 +73,7 @@ class GBMRegressorSuite extends FunSuite with DatasetSuiteBase {
 
     time {
       val paramGrid = new ParamGridBuilder()
-        .addGrid(gbt.stepSize, Array(1.0))
+        .addGrid(gbt.stepSize, Array(0.1))
         .addGrid(gbt.maxDepth, Array(10))
         .addGrid(gbt.maxIter, Array(30))
         .addGrid(gbt.subsamplingRate, Array(0.8))

docs/boosting.md

@@ -3,4 +3,32 @@ id: boosting
 title: Boosting
 ---
 
-Documentation Coming Soon.
+The old Boosting, à la papa, from Freund and Schapire [[1](#references)].
+
+For classification, SAMME (Multi-class AdaBoost) [[2](#references)] from Ji Zhu is implemented.
+
+For regression, R2 (Improving Regressors using Boosting Techniques) [[3](#references)] from H. Drucker has been chosen.
+
+For convenience, a N Round early stop is available.
+
+## Parameters
+
+The parameters available for Boosting are related to early stop and the loss function for weight computation.
+
+```scala
+import org.apache.spark.ml.classification.{BoostingClassifier, DecisionTreeClassifier}
+        
+new BoostingClassifier()
+        .setBaseLearner(new DecisionTreeClassifier()) //Base learner used by the meta-estimator.
+        .setNumBaseLearners(10) //Number of base learners.
+        .setLoss("exponential") //Loss function used for weight computation.
+        .setValidationIndicatorCol("val") //Column name that contains true or false for the early stop data set.
+        .setTol(1E-3) //Tolerance for optimized step size and gain in loss on early stop set.
+        .setNumRound(8) //Number of rounds to wait for the loss on early stop set to decrease.               
+```
+
+## References
+
+ * [[1](https://pdf.sciencedirectassets.com/272574/1-s2.0-S0022000000X00384/1-s2.0-S002200009791504X/main.pdf?X-Amz-Security-Token=AgoJb3JpZ2luX2VjEKP%2F%2F%2F%2F%2F%2F%2F%2F%2F%2FwEaCXVzLWVhc3QtMSJHMEUCID7xdi4RVOyBg2JyJaKf%2Bo1b0VB2lLHjOH7N4qiYc1zjAiEA1jYOFh6CjJQZmZ3QcaiDrtaD1P9fYiZkPZM6gR%2B%2FKF8q2gMIXBACGgwwNTkwMDM1NDY4NjUiDAVj5twFPKo1W86Ylyq3A1nD9LiuPFB7iWNzcbJyfjY0ZQjoHwoUo4yrPs9kyH3qntJCFhwM8v1I3278TKFu%2BAtZU%2BJP3OxpJeeXYZ5MPe5g8eYKuwDpdT9mubV3aWr2Vw3EjEkHrVBFE1%2B%2B8Ds3dc9mYqcV87AJCns5uL9mQbh3JTFGuuubYMLkQssmVky%2B3SUvOpW%2Bnl5BTq%2FPqaShPUVW7ky1CLk8%2B3INirdGvWsTeU5GZJRiJqpWYpAS9Qa0Km5BkIPDSHKh5u53tTIUXPqBW6P3MXr2k0XLqpFEIi0%2F8BHDaP%2FcI5EsMvsCFZDsZZAlXZn2Vm8MNZUOnRhOC%2BE2Q1R11o3hly2LGDfz74IihRvXDY40kHvfEwNmeK8y9p7j2NTVUeiNvdjdXpByoEJkmJduPiBVpsQ3SMM3Q6dIm%2BNVzJwMJyQioLcKI7kyC%2FvG6hF9z%2FGRAu7K7hRcbdW5XX2pTES9A5AK9LdeGxvhFThiGfODJaCTPwccTn%2Fw2gDP23uETJ0ldmaqRUJo6TB5LqgeoE6Ll0BwWRJSeUUHybTcVfbFmf6S0ItX42eM0%2Fv7qMsIIWU%2FUV6QRLpth7InXxm7KbUwqurI6wU6tAGcV%2FHkqjW5CYxpXREYK2hHWz12ZKxPV13aBjDyEjTvd85BW0VPpmOixpSlBdV67AnrWSBo1Coo0DNkscAwWepWNDTbZfwHaCd6q7pAyb0RvuD4URqwi2WDTahX9bRK%2BNTAA7vpnfSmv0qLqir02wSLYIP%2Fzf%2FXlhAKyb%2BPTaPhY1Y2JGWkqlykiOMsG2oP42c9LKEVDYkn7y%2Bv62TwYYGQylb%2By1xVnCY3cm9rAFgEGO%2BSWk4%3D&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Date=20190906T114916Z&X-Amz-SignedHeaders=host&X-Amz-Expires=300&X-Amz-Credential=ASIAQ3PHCVTY4UOJFHA3%2F20190906%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Signature=8bfc5bbd29b405a4b4d174cb515bdc2621ac4246c4e30f9a108b143169c0c6c2&hash=d85f9856ba204a5941472cb769535e404f48935a697c8f25a9d241024018527f&host=68042c943591013ac2b2430a89b270f6af2c76d8dfd086a07176afe7c76c2c61&pii=S002200009791504X&tid=spdf-9461bd1f-a6a2-4a1d-9bb1-a6ada11fbdc2&sid=f588c56f21cb124fa58b550402b796a79eb7gxrqb&type=client)] Freund, Y., & Schapire, R. E. (1997). A decision-theoretic generalization of on-line learning and an application to boosting. Journal of computer and system sciences, 55(1), 119-139.
+ * [[2](https://web.stanford.edu/~hastie/Papers/samme.pdf)] Hastie, T., Rosset, S., Zhu, J., & Zou, H. (2009). Multi-class adaboost. Statistics and its Interface, 2(3), 349-360.
+ * [[3](https://pdfs.semanticscholar.org/8d49/e2dedb817f2c3330e74b63c5fc86d2399ce3.pdf)] Drucker, H. (1997, July). Improving regressors using boosting techniques. In ICML (Vol. 97, pp. 107-115).

docs/gbm.md

@@ -3,4 +3,38 @@ id: gbm
 title: GBM
 ---
 
-Documentation Coming Soon.
+God Jerome H. Friedman enlightened mankind with GBM (Gradient Boosting Machines) in the beginning of the third millennium.
+
+The first of his ten commandments was named : Greedy Function Approximation: A Gradient Boosting Machine [[1](#references)], introducing a meta algorithm that was aimed to do gradient descent in function space. In the end you were kinda doing it in error space, but the heuristic was god sent.
+
+The second commandment was : Stochastic Gradient Boosting [[2](#references)] introducing randomness in each iteration by using SubBags.
+
+PS : It works for multi-class Classification.
+PPS : Early stop is implemented with a N Round variant.
+
+## Parameters
+
+The parameters available for GBM are related to the base framework, the stochastic version and early stop.
+
+```scala
+import org.apache.spark.ml.classification.{GBMClassifier, DecisionTreeClassifier}
+
+new GBMClassifier()
+        .setBaseLearner(new DecisionTreeClassifier()) //Base learner used by the meta-estimator.
+        .setNumBaseLearners(10) //Number of base learners.
+        .setLearningRate(0.1) //Shrinkage parameter.
+        .setSampleRatio(0.8) //Ratio sampling of exemples.
+        .setReplacement(true) //Exemples drawn with replacement or not.
+        .setSubspaceRatio(0.8) //Ratio sampling of features.
+        .setOptimizedWeights(true) //Line search the best step size or use 1 instead.
+        .setLoss("squared") //Loss function used for residuals and optimized step size.
+        .setAlpha(0.5) //Extra parameter for certain loss functions as quantile or huber.
+        .setValidationIndicatorCol("val") //Column name that contains true or false for the early stop data set.
+        .setTol(1E-3) //Tolerance for optimized step size and gain in loss on early stop set.
+        .setNumRound(8) //Number of rounds to wait for the loss on early stop set to decrease.               
+```
+
+## References
+
+ * [[1](https://statweb.stanford.edu/~jhf/ftp/trebst.pdf)] Friedman, J. H. (2001). Greedy function approximation: a gradient boosting machine. Annals of statistics, 1189-1232.
+ * [[2](https://astro.temple.edu/~msobel/courses_files/StochasticBoosting(gradient).pdf)] Friedman, J. H. (2002). Stochastic gradient boosting. Computational statistics & data analysis, 38(4), 367-378.