From 7a9684ecbd22b2324e68f7672a45158c347da8f0 Mon Sep 17 00:00:00 2001 From: Austin Gibbons Date: Tue, 5 Nov 2013 22:55:07 -0800 Subject: [PATCH 1/2] call sc.stop() in accordance with spark 0.8 --- src/main/scala/util/MLLRDemo.scala | 4 +++- src/test/scala/ml/RegressionTests.scala | 2 ++ .../scala/ml/classification/LogisticRegressionSuite.scala | 2 ++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/scala/util/MLLRDemo.scala b/src/main/scala/util/MLLRDemo.scala index 01e96e2..50308a6 100644 --- a/src/main/scala/util/MLLRDemo.scala +++ b/src/main/scala/util/MLLRDemo.scala @@ -7,7 +7,8 @@ import org.apache.spark.SparkContext object MLILRDemo { def main(args: Array[String]) = { - val mc = new MLContext(new SparkContext("local[4]", "MLILRtest")) + val sc = new SparkContext("local[4]", "MLIRtest") + val mc = new MLContext(sc) val data = mc.loadCsvFile(args(0)) val d2 = data.map((x: MLRow) => x.drop(0).+:(if(x(0).toString == "n07760859") MLValue(1.0) else MLValue(0.0))).cache() @@ -16,5 +17,6 @@ object MLILRDemo { //val model = SVMAlgorithm.train(d2) println("Time to train: " + model.trainingTime) + sc.stop() } } \ No newline at end of file diff --git a/src/test/scala/ml/RegressionTests.scala b/src/test/scala/ml/RegressionTests.scala index b2af9ed..e8229e7 100644 --- a/src/test/scala/ml/RegressionTests.scala +++ b/src/test/scala/ml/RegressionTests.scala @@ -39,6 +39,8 @@ class RegressionTests extends FunSuite with BeforeAndAfter with LocalSparkContex println("\t Learned model: ") println(model.wOpt) */ + + sc.stop() } diff --git a/src/test/scala/ml/classification/LogisticRegressionSuite.scala b/src/test/scala/ml/classification/LogisticRegressionSuite.scala index dc09f3a..cc05406 100644 --- a/src/test/scala/ml/classification/LogisticRegressionSuite.scala +++ b/src/test/scala/ml/classification/LogisticRegressionSuite.scala @@ -27,6 +27,7 @@ class LogisticRegressionSuite extends FunSuite with LocalSparkContext { x = model.predict(MLVector(Array(0.0,0.0))).toNumber println("Model prediction for (-1.0,0.0): " + x) assert(x <= 0.5) + sc.stop() } test("Basic test of logistic regression via Parallel Gradient") { @@ -39,6 +40,7 @@ class LogisticRegressionSuite extends FunSuite with LocalSparkContext { println(model.explain) assert(true) + sc.stop() } } From bc8fab3e9c20fd43653229c5db11ea6833b29a85 Mon Sep 17 00:00:00 2001 From: Austin Gibbons Date: Thu, 7 Nov 2013 09:40:38 -0800 Subject: [PATCH 2/2] Remove sc.stop from tests that mixin LocalSparkContext --- src/test/scala/ml/RegressionTests.scala | 2 -- src/test/scala/ml/classification/LogisticRegressionSuite.scala | 2 -- 2 files changed, 4 deletions(-) diff --git a/src/test/scala/ml/RegressionTests.scala b/src/test/scala/ml/RegressionTests.scala index e8229e7..b2af9ed 100644 --- a/src/test/scala/ml/RegressionTests.scala +++ b/src/test/scala/ml/RegressionTests.scala @@ -39,8 +39,6 @@ class RegressionTests extends FunSuite with BeforeAndAfter with LocalSparkContex println("\t Learned model: ") println(model.wOpt) */ - - sc.stop() } diff --git a/src/test/scala/ml/classification/LogisticRegressionSuite.scala b/src/test/scala/ml/classification/LogisticRegressionSuite.scala index cc05406..dc09f3a 100644 --- a/src/test/scala/ml/classification/LogisticRegressionSuite.scala +++ b/src/test/scala/ml/classification/LogisticRegressionSuite.scala @@ -27,7 +27,6 @@ class LogisticRegressionSuite extends FunSuite with LocalSparkContext { x = model.predict(MLVector(Array(0.0,0.0))).toNumber println("Model prediction for (-1.0,0.0): " + x) assert(x <= 0.5) - sc.stop() } test("Basic test of logistic regression via Parallel Gradient") { @@ -40,7 +39,6 @@ class LogisticRegressionSuite extends FunSuite with LocalSparkContext { println(model.explain) assert(true) - sc.stop() } }