From 292745b5ea8fc7815db0bcc1be995cae0a33b0ec Mon Sep 17 00:00:00 2001 From: Xiangrui Meng Date: Mon, 12 Dec 2016 22:55:41 -0800 Subject: [PATCH 1/4] mention spark.randomForest in vignettes --- R/pkg/vignettes/sparkr-vignettes.Rmd | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/R/pkg/vignettes/sparkr-vignettes.Rmd b/R/pkg/vignettes/sparkr-vignettes.Rmd index 625b759626f36..1325b152858b9 100644 --- a/R/pkg/vignettes/sparkr-vignettes.Rmd +++ b/R/pkg/vignettes/sparkr-vignettes.Rmd @@ -449,6 +449,8 @@ SparkR supports the following machine learning models and algorithms. * Generalized Linear Model (GLM) +* Random Forest + * Naive Bayes Model * $k$-means Clustering @@ -526,6 +528,20 @@ gaussianFitted <- predict(gaussianGLM, carsDF) head(select(gaussianFitted, "model", "prediction", "mpg", "wt", "hp")) ``` +#### Random Forest + +`spark.randomForest` fits a [random forest](https://en.wikipedia.org/wiki/Random_forest) classification or regression model on a `SparkDataFrame`. +Users can use `summary` to get a summary of the fitted model, `predict` to make predictions on new data, and `write.ml`/`read.ml` to save/load fitted models. + +In the following example, we use the `longley` dataset to train a random forest and make predictions: + +```{r} +df <- createDataFrame(longley) +rfModel <- spark.randomForest(df, Employed ~ ., type = "regression", numTrees = 5) +summary(rfModel) +predictions <- predict(rfModel, df) +``` + #### Naive Bayes Model Naive Bayes model assumes independence among the features. `spark.naiveBayes` fits a [Bernoulli naive Bayes model](https://en.wikipedia.org/wiki/Naive_Bayes_classifier#Bernoulli_naive_Bayes) against a SparkDataFrame. The data should be all categorical. These models are often used for document classification. From fb1933d22bfd8294b38c7f0e712c0f753ce56ba6 Mon Sep 17 00:00:00 2001 From: Xiangrui Meng Date: Mon, 12 Dec 2016 23:07:51 -0800 Subject: [PATCH 2/4] add spark.gbt as well --- R/pkg/vignettes/sparkr-vignettes.Rmd | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/R/pkg/vignettes/sparkr-vignettes.Rmd b/R/pkg/vignettes/sparkr-vignettes.Rmd index 1325b152858b9..3662820f62cb1 100644 --- a/R/pkg/vignettes/sparkr-vignettes.Rmd +++ b/R/pkg/vignettes/sparkr-vignettes.Rmd @@ -451,6 +451,8 @@ SparkR supports the following machine learning models and algorithms. * Random Forest +* Gradient-Boosted Trees (GBT) + * Naive Bayes Model * $k$-means Clustering @@ -531,7 +533,7 @@ head(select(gaussianFitted, "model", "prediction", "mpg", "wt", "hp")) #### Random Forest `spark.randomForest` fits a [random forest](https://en.wikipedia.org/wiki/Random_forest) classification or regression model on a `SparkDataFrame`. -Users can use `summary` to get a summary of the fitted model, `predict` to make predictions on new data, and `write.ml`/`read.ml` to save/load fitted models. +Users can call `summary` to get a summary of the fitted model, `predict` to make predictions, and `write.ml`/`read.ml` to save/load fitted models. In the following example, we use the `longley` dataset to train a random forest and make predictions: @@ -542,6 +544,20 @@ summary(rfModel) predictions <- predict(rfModel, df) ``` +#### Gradient-Boosted Trees + +`spark.gbt` fits a [gradient-boosted tree](https://en.wikipedia.org/wiki/Gradient_boosting) classification or regression model on a `SparkDataFrame`. +Users can call `summary` to get a summary of the fitted model, `predict` to make predictions, and `write.ml`/`read.ml` to save/load fitted models. + +Similar to the random forest example above, we use the `longley` dataset to train a gradient-boosted tree and make predictions: + +```{r} +df <- createDataFrame(longley) +gbtModel <- spark.gbt(df, Employed ~ ., type = "regression", maxIter = 5) +summary(gbtModel) +predictions <- predict(gbtModel, df) +``` + #### Naive Bayes Model Naive Bayes model assumes independence among the features. `spark.naiveBayes` fits a [Bernoulli naive Bayes model](https://en.wikipedia.org/wiki/Naive_Bayes_classifier#Bernoulli_naive_Bayes) against a SparkDataFrame. The data should be all categorical. These models are often used for document classification. From be9c8466209a0d656ce9b7bb08ee94014391ea0d Mon Sep 17 00:00:00 2001 From: Xiangrui Meng Date: Mon, 12 Dec 2016 23:13:41 -0800 Subject: [PATCH 3/4] update params so the output is shorter --- R/pkg/vignettes/sparkr-vignettes.Rmd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/R/pkg/vignettes/sparkr-vignettes.Rmd b/R/pkg/vignettes/sparkr-vignettes.Rmd index 3662820f62cb1..93ffccf23d21c 100644 --- a/R/pkg/vignettes/sparkr-vignettes.Rmd +++ b/R/pkg/vignettes/sparkr-vignettes.Rmd @@ -539,7 +539,7 @@ In the following example, we use the `longley` dataset to train a random forest ```{r} df <- createDataFrame(longley) -rfModel <- spark.randomForest(df, Employed ~ ., type = "regression", numTrees = 5) +rfModel <- spark.randomForest(df, Employed ~ ., type = "regression", maxDepth = 2, numTrees = 2) summary(rfModel) predictions <- predict(rfModel, df) ``` @@ -553,7 +553,7 @@ Similar to the random forest example above, we use the `longley` dataset to trai ```{r} df <- createDataFrame(longley) -gbtModel <- spark.gbt(df, Employed ~ ., type = "regression", maxIter = 5) +gbtModel <- spark.gbt(df, Employed ~ ., type = "regression", maxDepth = 2, maxIter = 2) summary(gbtModel) predictions <- predict(gbtModel, df) ``` From b3bf19fef03f22a5d92b167fa6058a07d85294e0 Mon Sep 17 00:00:00 2001 From: Xiangrui Meng Date: Tue, 13 Dec 2016 12:18:49 -0800 Subject: [PATCH 4/4] address comments --- R/pkg/vignettes/sparkr-vignettes.Rmd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/R/pkg/vignettes/sparkr-vignettes.Rmd b/R/pkg/vignettes/sparkr-vignettes.Rmd index 93ffccf23d21c..334daa51f019d 100644 --- a/R/pkg/vignettes/sparkr-vignettes.Rmd +++ b/R/pkg/vignettes/sparkr-vignettes.Rmd @@ -537,7 +537,7 @@ Users can call `summary` to get a summary of the fitted model, `predict` to make In the following example, we use the `longley` dataset to train a random forest and make predictions: -```{r} +```{r, warning=FALSE} df <- createDataFrame(longley) rfModel <- spark.randomForest(df, Employed ~ ., type = "regression", maxDepth = 2, numTrees = 2) summary(rfModel) @@ -551,7 +551,7 @@ Users can call `summary` to get a summary of the fitted model, `predict` to make Similar to the random forest example above, we use the `longley` dataset to train a gradient-boosted tree and make predictions: -```{r} +```{r, warning=FALSE} df <- createDataFrame(longley) gbtModel <- spark.gbt(df, Employed ~ ., type = "regression", maxDepth = 2, maxIter = 2) summary(gbtModel)