From b2831be902e0d4b4d824a4ace25e2eb4cf7f4334 Mon Sep 17 00:00:00 2001 From: madhuka Date: Sun, 12 Jul 2015 23:05:17 +0530 Subject: [PATCH 1/6] Adding leaflet for map --- zeppelin-web/bower.json | 3 ++- zeppelin-web/src/app/notebook/paragraph/paragraph.html | 4 ++++ zeppelin-web/src/index.html | 6 +++++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/zeppelin-web/bower.json b/zeppelin-web/bower.json index b5b439ae7f7..23616a54dc0 100644 --- a/zeppelin-web/bower.json +++ b/zeppelin-web/bower.json @@ -25,7 +25,8 @@ "angular-xeditable": "0.1.8", "highlightjs": "~8.4.0", "lodash": "~3.9.3", - "angular-filter": "~0.5.4" + "angular-filter": "~0.5.4", + "leaflet": "~0.7.3" }, "devDependencies": { "angular-mocks": "1.3.8" diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.html b/zeppelin-web/src/app/notebook/paragraph/paragraph.html index 4d0f4510fc7..b0d28a860fb 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.html +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.html @@ -129,6 +129,10 @@ ng-class="{'active': isGraphMode('scatterChart')}" ng-click="setGraphMode('scatterChart', true)"> + diff --git a/zeppelin-web/src/index.html b/zeppelin-web/src/index.html index 6fdc2f2f4ca..f456ea874cc 100644 --- a/zeppelin-web/src/index.html +++ b/zeppelin-web/src/index.html @@ -42,6 +42,7 @@ + @@ -109,7 +110,10 @@ - + + + + From a000cbb4001aa3fe066bd47a76fe8fb802f56164 Mon Sep 17 00:00:00 2001 From: madhuka Date: Sun, 12 Jul 2015 23:38:08 +0530 Subject: [PATCH 2/6] Adding mapping for UI --- .../src/app/notebook/paragraph/paragraph.controller.js | 7 +++++++ zeppelin-web/src/app/notebook/paragraph/paragraph.html | 5 +++++ 2 files changed, 12 insertions(+) diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js index 3102869f2a0..d5923d31a14 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js @@ -721,6 +721,10 @@ angular.module('zeppelinWebApp') } }; + $scope.defaults= { + scrollWheelZoom: false + }; + $scope.setGraphMode = function(type, emit, refresh) { if (emit) { setNewMode(type); @@ -733,6 +737,9 @@ angular.module('zeppelinWebApp') if (!type || type === 'table') { setTable($scope.paragraph.result, refresh); } + if (!type || type === 'mapChart') { + //setting the map + } else { setD3Chart(type, $scope.paragraph.result, refresh); } diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.html b/zeppelin-web/src/app/notebook/paragraph/paragraph.html index b0d28a860fb..305964617cf 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.html +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.html @@ -337,6 +337,11 @@ id="p{{paragraph.id}}_scatterChart"> + +
+ +
Date: Mon, 13 Jul 2015 00:25:02 +0530 Subject: [PATCH 3/6] Adding map chart for zeppelin --- zeppelin-web/bower.json | 3 ++- zeppelin-web/src/app/app.js | 3 ++- .../src/app/notebook/paragraph/paragraph.controller.js | 7 ++++++- zeppelin-web/src/app/notebook/paragraph/paragraph.html | 4 ++-- zeppelin-web/src/index.html | 3 ++- 5 files changed, 14 insertions(+), 6 deletions(-) diff --git a/zeppelin-web/bower.json b/zeppelin-web/bower.json index 23616a54dc0..511bf75b744 100644 --- a/zeppelin-web/bower.json +++ b/zeppelin-web/bower.json @@ -26,7 +26,8 @@ "highlightjs": "~8.4.0", "lodash": "~3.9.3", "angular-filter": "~0.5.4", - "leaflet": "~0.7.3" + "leaflet": "~0.7.3", + "angular-leaflet-directive": "~0.8.5" }, "devDependencies": { "angular-mocks": "1.3.8" diff --git a/zeppelin-web/src/app/app.js b/zeppelin-web/src/app/app.js index 6dfa8e5d911..de93a9d591c 100644 --- a/zeppelin-web/src/app/app.js +++ b/zeppelin-web/src/app/app.js @@ -30,7 +30,8 @@ angular.module('zeppelinWebApp', [ 'angular.filter', 'monospaced.elastic', 'puElasticInput', - 'xeditable' + 'xeditable', + 'leaflet-directive' ]) .filter('breakFilter', function() { return function (text) { diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js index d5923d31a14..d0884607744 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js @@ -738,7 +738,12 @@ angular.module('zeppelinWebApp') setTable($scope.paragraph.result, refresh); } if (!type || type === 'mapChart') { - //setting the map + //setting the map : To-DO Moving the map for separate file +/* angular.extend($scope, { + defaults: { + scrollWheelZoom: false + } + });*/ } else { setD3Chart(type, $scope.paragraph.result, refresh); diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.html b/zeppelin-web/src/app/notebook/paragraph/paragraph.html index 305964617cf..b7878ef1c8e 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.html +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.html @@ -134,7 +134,7 @@ ng-click="setGraphMode('mapChart', true)">
- @@ -340,7 +340,7 @@
- +
diff --git a/zeppelin-web/src/index.html b/zeppelin-web/src/index.html index f456ea874cc..9a946c473d5 100644 --- a/zeppelin-web/src/index.html +++ b/zeppelin-web/src/index.html @@ -43,6 +43,7 @@ + @@ -113,7 +114,7 @@ - + From e10931c4ccf76b43a83db013b5a8ee403cfe2dd0 Mon Sep 17 00:00:00 2001 From: madhuka Date: Sun, 19 Jul 2015 20:29:04 +0530 Subject: [PATCH 4/6] adding sample tutorial for notbook on map --- notebook/2AU6K8KS2/note.json | 136 +++++++++++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 notebook/2AU6K8KS2/note.json diff --git a/notebook/2AU6K8KS2/note.json b/notebook/2AU6K8KS2/note.json new file mode 100644 index 00000000000..a6068cff2c6 --- /dev/null +++ b/notebook/2AU6K8KS2/note.json @@ -0,0 +1,136 @@ +{ + "paragraphs": [ + { + "text": "%md\n## Welcome to Zeppelin.\n##### This is a quickly start on zeppelin map visualization tutorial. It is using Leaflet mapping basics, including setting up a Leaflet map, working with markers, polylines and popups, and dealing with events.you can run the code yourself. (Shift-Enter to Run)", + "config": { + "colWidth": 12.0, + "graph": { + "mode": "table", + "height": 300.0, + "optionOpen": false, + "keys": [], + "values": [], + "groups": [], + "scatter": {} + }, + "editorHide": true, + "tableHide": false + }, + "settings": { + "params": {}, + "forms": {} + }, + "jobName": "paragraph_1437313582566_-967040274", + "id": "20150719-191622_9501079", + "result": { + "code": "SUCCESS", + "type": "HTML", + "msg": "\u003ch2\u003eWelcome to Zeppelin.\u003c/h2\u003e\n\u003ch5\u003eThis is a quickly start on zeppelin map visualization tutorial. It is using Leaflet mapping basics, including setting up a Leaflet map, working with markers, polylines and popups, and dealing with events.you can run the code yourself. (Shift-Enter to Run)\u003c/h5\u003e\n" + }, + "dateCreated": "Jul 19, 2015 7:16:22 PM", + "dateStarted": "Jul 19, 2015 8:20:26 PM", + "dateFinished": "Jul 19, 2015 8:20:26 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "text": "import org.apache.commons.io.IOUtils\nimport java.net.URL\nimport java.nio.charset.Charset\n\n// Zeppelin creates and injects sc (SparkContext) and sqlContext (HiveContext or SqlContext)\n// So you don\u0027t need create them manually\n\n// load bank data\nval myMapText \u003d sc.parallelize(\n IOUtils.toString(\n new URL(\"https://gist.githubusercontent.com/Madhuka/74cb9a6577c87aa7d2fd/raw/2f758d33d28ddc01c162293ad45dc16be2806a6b/data.csv\"),\n Charset.forName(\"utf8\")).split(\"\\n\"))\n\n\ncase class Map(Country:String, Name:String, lat : Float, lan : Float, Altitude : Float)\n\nval myMap \u003d myMapText.map(s\u003d\u003es.split(\",\")).filter(s\u003d\u003es(0)!\u003d\"Country\").map(\n s\u003d\u003eMap(s(0), \n s(1),\n s(2).toFloat,\n s(3).toFloat,\n s(5).toFloat\n )\n)\n\n// Below line works only in spark 1.3.0.\n// For spark 1.1.x and spark 1.2.x,\n// use bank.registerTempTable(\"bank\") instead.\nmyMap.toDF().registerTempTable(\"myMap\")\n", + "config": { + "colWidth": 12.0, + "graph": { + "mode": "table", + "height": 300.0, + "optionOpen": false, + "keys": [], + "values": [], + "groups": [], + "scatter": {} + } + }, + "settings": { + "params": {}, + "forms": {} + }, + "jobName": "paragraph_1437314214306_141157507", + "id": "20150719-192654_12434929", + "result": { + "code": "SUCCESS", + "type": "TEXT", + "msg": "import org.apache.commons.io.IOUtils\nimport java.net.URL\nimport java.nio.charset.Charset\nmyMapText: org.apache.spark.rdd.RDD[String] \u003d ParallelCollectionRDD[14] at parallelize at \u003cconsole\u003e:39\ndefined class Map\nmyMap: org.apache.spark.rdd.RDD[Map] \u003d MapPartitionsRDD[17] at map at \u003cconsole\u003e:39\n" + }, + "dateCreated": "Jul 19, 2015 7:26:54 PM", + "dateStarted": "Jul 19, 2015 8:20:26 PM", + "dateFinished": "Jul 19, 2015 8:20:32 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "text": "%sql \nselect * from myMap", + "config": { + "colWidth": 12.0, + "graph": { + "mode": "table", + "height": 300.0, + "optionOpen": false, + "keys": [], + "values": [], + "groups": [], + "scatter": {} + } + }, + "settings": { + "params": {}, + "forms": {} + }, + "jobName": "paragraph_1437316940442_271264979", + "id": "20150719-201220_10279868", + "result": { + "code": "ERROR", + "type": "TEXT", + "msg": "org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 4.0 failed 1 times, most recent failure: Lost task 0.0 in stage 4.0 (TID 4, localhost): java.lang.ArrayIndexOutOfBoundsException: 5\n\tat $line41.$read$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$anonfun$3.apply(\u003cconsole\u003e:44)\n\tat $line41.$read$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$anonfun$3.apply(\u003cconsole\u003e:40)\n\tat scala.collection.Iterator$$anon$11.next(Iterator.scala:328)\n\tat scala.collection.Iterator$$anon$1.next(Iterator.scala:853)\n\tat scala.collection.Iterator$$anon$1.head(Iterator.scala:840)\n\tat org.apache.spark.sql.execution.RDDConversions$$anonfun$productToRowRdd$1.apply(ExistingRDD.scala:42)\n\tat org.apache.spark.sql.execution.RDDConversions$$anonfun$productToRowRdd$1.apply(ExistingRDD.scala:37)\n\tat org.apache.spark.rdd.RDD$$anonfun$14.apply(RDD.scala:634)\n\tat org.apache.spark.rdd.RDD$$anonfun$14.apply(RDD.scala:634)\n\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:35)\n\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:277)\n\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:244)\n\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:35)\n\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:277)\n\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:244)\n\tat org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:61)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:64)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:203)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)\n\tat java.lang.Thread.run(Thread.java:745)\n\nDriver stacktrace:\n\tat org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1204)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1193)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1192)\n\tat scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)\n\tat scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47)\n\tat org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1192)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:693)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:693)\n\tat scala.Option.foreach(Option.scala:236)\n\tat org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:693)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1393)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1354)\n\tat org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48)\n" + }, + "dateCreated": "Jul 19, 2015 8:12:20 PM", + "dateStarted": "Jul 19, 2015 8:20:26 PM", + "dateFinished": "Jul 19, 2015 8:20:32 PM", + "status": "ERROR", + "progressUpdateIntervalMs": 500 + }, + { + "text": "", + "config": { + "colWidth": 12.0, + "graph": { + "mode": "table", + "height": 300.0, + "optionOpen": false, + "keys": [], + "values": [], + "groups": [], + "scatter": {} + } + }, + "settings": { + "params": {}, + "forms": {} + }, + "jobName": "paragraph_1437317266677_-261952669", + "id": "20150719-201746_23407076", + "result": { + "code": "SUCCESS", + "type": "TEXT" + }, + "dateCreated": "Jul 19, 2015 8:17:46 PM", + "dateStarted": "Jul 19, 2015 8:20:32 PM", + "dateFinished": "Jul 19, 2015 8:20:32 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + } + ], + "name": "Map Visualization Tutorial", + "id": "2AU6K8KS2", + "angularObjects": {}, + "config": { + "looknfeel": "default" + }, + "info": {} +} \ No newline at end of file From 7225d0bddc50d6781742e2d3f277698e1b47f550 Mon Sep 17 00:00:00 2001 From: madhuka Date: Sun, 19 Jul 2015 20:32:15 +0530 Subject: [PATCH 5/6] adding sql --- notebook/2AU6K8KS2/note.json | 51 ++++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 14 deletions(-) diff --git a/notebook/2AU6K8KS2/note.json b/notebook/2AU6K8KS2/note.json index a6068cff2c6..ece8c513e91 100644 --- a/notebook/2AU6K8KS2/note.json +++ b/notebook/2AU6K8KS2/note.json @@ -34,7 +34,7 @@ "progressUpdateIntervalMs": 500 }, { - "text": "import org.apache.commons.io.IOUtils\nimport java.net.URL\nimport java.nio.charset.Charset\n\n// Zeppelin creates and injects sc (SparkContext) and sqlContext (HiveContext or SqlContext)\n// So you don\u0027t need create them manually\n\n// load bank data\nval myMapText \u003d sc.parallelize(\n IOUtils.toString(\n new URL(\"https://gist.githubusercontent.com/Madhuka/74cb9a6577c87aa7d2fd/raw/2f758d33d28ddc01c162293ad45dc16be2806a6b/data.csv\"),\n Charset.forName(\"utf8\")).split(\"\\n\"))\n\n\ncase class Map(Country:String, Name:String, lat : Float, lan : Float, Altitude : Float)\n\nval myMap \u003d myMapText.map(s\u003d\u003es.split(\",\")).filter(s\u003d\u003es(0)!\u003d\"Country\").map(\n s\u003d\u003eMap(s(0), \n s(1),\n s(2).toFloat,\n s(3).toFloat,\n s(5).toFloat\n )\n)\n\n// Below line works only in spark 1.3.0.\n// For spark 1.1.x and spark 1.2.x,\n// use bank.registerTempTable(\"bank\") instead.\nmyMap.toDF().registerTempTable(\"myMap\")\n", + "text": "import org.apache.commons.io.IOUtils\nimport java.net.URL\nimport java.nio.charset.Charset\n\n// Zeppelin creates and injects sc (SparkContext) and sqlContext (HiveContext or SqlContext)\n// So you don\u0027t need create them manually\n\n// load bank data\nval myMapText \u003d sc.parallelize(\n IOUtils.toString(\n new URL(\"https://gist.githubusercontent.com/Madhuka/74cb9a6577c87aa7d2fd/raw/2f758d33d28ddc01c162293ad45dc16be2806a6b/data.csv\"),\n Charset.forName(\"utf8\")).split(\"\\n\"))\n\n\ncase class Map(Country:String, Name:String, lat : Float, lan : Float, Altitude : Float)\n\nval myMap \u003d myMapText.map(s\u003d\u003es.split(\",\")).filter(s\u003d\u003es(0)!\u003d\"Country\").map(\n s\u003d\u003eMap(s(0), \n s(1),\n s(2).toFloat,\n s(3).toFloat,\n s(4).toFloat\n )\n)\n\n// Below line works only in spark 1.3.0.\n// For spark 1.1.x and spark 1.2.x,\n// use bank.registerTempTable(\"bank\") instead.\nmyMap.toDF().registerTempTable(\"myMap\")\n", "config": { "colWidth": 12.0, "graph": { @@ -56,11 +56,11 @@ "result": { "code": "SUCCESS", "type": "TEXT", - "msg": "import org.apache.commons.io.IOUtils\nimport java.net.URL\nimport java.nio.charset.Charset\nmyMapText: org.apache.spark.rdd.RDD[String] \u003d ParallelCollectionRDD[14] at parallelize at \u003cconsole\u003e:39\ndefined class Map\nmyMap: org.apache.spark.rdd.RDD[Map] \u003d MapPartitionsRDD[17] at map at \u003cconsole\u003e:39\n" + "msg": "import org.apache.commons.io.IOUtils\nimport java.net.URL\nimport java.nio.charset.Charset\nmyMapText: org.apache.spark.rdd.RDD[String] \u003d ParallelCollectionRDD[37] at parallelize at \u003cconsole\u003e:47\ndefined class Map\nmyMap: org.apache.spark.rdd.RDD[Map] \u003d MapPartitionsRDD[40] at map at \u003cconsole\u003e:47\n" }, "dateCreated": "Jul 19, 2015 7:26:54 PM", - "dateStarted": "Jul 19, 2015 8:20:26 PM", - "dateFinished": "Jul 19, 2015 8:20:32 PM", + "dateStarted": "Jul 19, 2015 8:31:04 PM", + "dateFinished": "Jul 19, 2015 8:31:07 PM", "status": "FINISHED", "progressUpdateIntervalMs": 500 }, @@ -69,13 +69,36 @@ "config": { "colWidth": 12.0, "graph": { - "mode": "table", + "mode": "mapChart", "height": 300.0, "optionOpen": false, - "keys": [], - "values": [], + "keys": [ + { + "name": "Country", + "index": 0.0, + "aggr": "sum" + } + ], + "values": [ + { + "name": "Name", + "index": 1.0, + "aggr": "sum" + } + ], "groups": [], - "scatter": {} + "scatter": { + "xAxis": { + "name": "Country", + "index": 0.0, + "aggr": "sum" + }, + "yAxis": { + "name": "Name", + "index": 1.0, + "aggr": "sum" + } + } } }, "settings": { @@ -85,14 +108,14 @@ "jobName": "paragraph_1437316940442_271264979", "id": "20150719-201220_10279868", "result": { - "code": "ERROR", - "type": "TEXT", - "msg": "org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 4.0 failed 1 times, most recent failure: Lost task 0.0 in stage 4.0 (TID 4, localhost): java.lang.ArrayIndexOutOfBoundsException: 5\n\tat $line41.$read$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$anonfun$3.apply(\u003cconsole\u003e:44)\n\tat $line41.$read$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$anonfun$3.apply(\u003cconsole\u003e:40)\n\tat scala.collection.Iterator$$anon$11.next(Iterator.scala:328)\n\tat scala.collection.Iterator$$anon$1.next(Iterator.scala:853)\n\tat scala.collection.Iterator$$anon$1.head(Iterator.scala:840)\n\tat org.apache.spark.sql.execution.RDDConversions$$anonfun$productToRowRdd$1.apply(ExistingRDD.scala:42)\n\tat org.apache.spark.sql.execution.RDDConversions$$anonfun$productToRowRdd$1.apply(ExistingRDD.scala:37)\n\tat org.apache.spark.rdd.RDD$$anonfun$14.apply(RDD.scala:634)\n\tat org.apache.spark.rdd.RDD$$anonfun$14.apply(RDD.scala:634)\n\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:35)\n\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:277)\n\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:244)\n\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:35)\n\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:277)\n\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:244)\n\tat org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:61)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:64)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:203)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)\n\tat java.lang.Thread.run(Thread.java:745)\n\nDriver stacktrace:\n\tat org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1204)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1193)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1192)\n\tat scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)\n\tat scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47)\n\tat org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1192)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:693)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:693)\n\tat scala.Option.foreach(Option.scala:236)\n\tat org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:693)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1393)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1354)\n\tat org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48)\n" + "code": "SUCCESS", + "type": "TABLE", + "msg": "Country\tName\tlat\tlan\tAltitude\nUnited States\tNew York City\t40.714268\t-74.005974\t2.0\nUnited States\tLos Angeles\t34.052235\t-118.24368\t115.0\nUnited States\tChicago\t41.850033\t-87.650055\t181.0\nUnited States\tHouston\t29.763283\t-95.36327\t15.0\nUnited States\tPhiladelphia\t39.952335\t-75.16379\t8.0\nUnited States\tPhoenix\t33.448376\t-112.074036\t0.0\nUnited States\tSan Antonio\t29.424122\t-98.49363\t196.0\nUnited States\tSan Diego\t32.71533\t-117.15726\t26.0\nUnited States\tDallas\t32.783054\t-96.80666\t137.0\nUnited States\tDetroit\t42.33143\t-83.04575\t181.0\nUnited States\tSan Jose\t37.339386\t-121.89496\t24.0\nUnited States\tJacksonville\t30.332184\t-81.65565\t7.0\nUnited States\tIndianapolis\t39.768375\t-86.15804\t222.0\nUnited States\tColumbus\t39.961174\t-82.998795\t237.0\nUnited States\tSan Francisco\t37.77493\t-122.41942\t60.0\nUnited States\tAustin\t30.267153\t-97.74306\t163.0\nUnited States\tMemphis\t35.149536\t-90.04898\t75.0\nUnited States\tNew South Memphis\t35.086758\t-90.056755\t74.0\nUnited States\tFort Worth\t32.72541\t-97.32085\t199.0\nUnited States\tBaltimore\t39.290386\t-76.61219\t2.0\nUnited States\tCharlotte\t35.227085\t-80.843124\t229.0\nUnited States\tBoston\t42.35843\t-71.05978\t11.0\nUnited States\tMilwaukee\t43.038902\t-87.90647\t195.0\nUnited States\tSouth Boston\t42.33343\t-71.04949\t10.0\nUnited States\tSeattle\t47.60621\t-122.33207\t60.0\nUnited States\tEl Paso\t31.75872\t-106.48693\t1147.0\nUnited States\tDenver\t39.739155\t-104.9847\t1598.0\nUnited States\tWashington\t38.89511\t-77.03637\t11.0\nUnited States\tPortland\t45.523453\t-122.67621\t61.0\nUnited States\tOklahoma City\t35.46756\t-97.516426\t364.0\nUnited States\tNashville\t36.16589\t-86.78445\t151.0\nUnited States\tTucson\t32.221745\t-110.926476\t751.0\nUnited States\tAlbuquerque\t35.08449\t-106.65114\t1510.0\nUnited States\tNew Orleans\t29.954649\t-90.07507\t0.0\nUnited States\tLong Beach\t33.766964\t-118.18923\t1.0\nUnited States\tLas Vegas\t36.17497\t-115.13722\t609.0\nUnited States\tSacramento\t38.581573\t-121.4944\t5.0\nUnited States\tFresno\t36.747726\t-119.77237\t91.0\nUnited States\tCleveland\t41.499496\t-81.69541\t185.0\nUnited States\tNorth Kansas City\t39.130005\t-94.56218\t223.0\nUnited States\tKansas City\t39.099728\t-94.57857\t271.0\nUnited States\tVirginia Beach\t36.85293\t-75.97798\t3.0\nUnited States\tAtlanta\t33.748997\t-84.387985\t305.0\nUnited States\tOakland\t37.80437\t-122.270805\t0.0\nUnited States\tMesa\t33.422268\t-111.82264\t386.0\nUnited States\tTulsa\t36.15398\t-95.992775\t213.0\nUnited States\tOmaha\t41.25861\t-95.93779\t0.0\nUnited States\tMiami\t25.774265\t-80.19366\t2.0\nUnited States\tHonolulu\t21.306944\t-157.85834\t0.0\nUnited States\tMinneapolis\t44.979965\t-93.26384\t259.0\nUnited States\tColorado Springs\t38.83388\t-104.821365\t1829.0\nUnited States\tWichita\t37.692238\t-97.33755\t402.0\nUnited States\tSanta Ana\t33.745575\t-117.867836\t30.0\nUnited States\tWest Raleigh\t35.78682\t-78.663895\t122.0\nUnited States\tArlington\t32.735687\t-97.10806\t184.0\nUnited States\tAnaheim\t33.835293\t-117.914505\t40.0\nUnited States\tTampa\t27.947521\t-82.45843\t1.0\nUnited States\tSaint Louis\t38.627274\t-90.19789\t135.0\nUnited States\tPittsburgh\t40.440624\t-79.99589\t235.0\nUnited States\tToledo\t41.663937\t-83.555214\t191.0\nUnited States\tCincinnati\t39.162003\t-84.45689\t203.0\nUnited States\tRiverside\t33.95335\t-117.39616\t262.0\nUnited States\tBakersfield\t35.37329\t-119.018715\t122.0\nUnited States\tStockton\t37.957703\t-121.29078\t4.0\nUnited States\tNewark\t40.735657\t-74.17236\t16.0\nUnited States\tBuffalo\t42.886448\t-78.87837\t183.0\nUnited States\tCorpus Christi\t27.800583\t-97.39638\t6.0\nUnited States\tAurora\t39.72943\t-104.83192\t1647.0\nUnited States\tRaleigh\t35.772095\t-78.63861\t91.0\nUnited States\tSaint Paul\t44.94441\t-93.09328\t235.0\nUnited States\tLexington-Fayette\t38.0498\t-84.45855\t312.0\nUnited States\tAnchorage\t61.218056\t-149.90028\t0.0\nUnited States\tPlano\t33.019844\t-96.69888\t200.0\nUnited States\tSaint Petersburg\t27.77086\t-82.67927\t14.0\nUnited States\tLouisville\t38.25424\t-85.75941\t128.0\nUnited States\tLincoln\t40.8\t-96.66696\t382.0\nUnited States\tGlendale\t33.53865\t-112.18599\t336.0\nUnited States\tHenderson\t36.0397\t-114.98193\t580.0\nUnited States\tJersey City\t40.728157\t-74.077644\t14.0\nUnited States\tNorfolk\t36.846813\t-76.28522\t3.0\nUnited States\tChandler\t33.30616\t-111.84125\t380.0\nUnited States\tGreensboro\t36.072636\t-79.79198\t244.0\nUnited States\tBirmingham\t33.52066\t-86.80249\t173.0\nUnited States\tFort Wayne\t41.130604\t-85.12886\t243.0\nUnited States\tLexington\t37.98869\t-84.477715\t298.0\nUnited States\tHialeah\t25.857595\t-80.27811\t2.0\nUnited States\tMadison\t43.07305\t-89.40123\t270.0\nUnited States\tBaton Rouge\t30.450747\t-91.15455\t0.0\nUnited States\tGarland\t32.912624\t-96.638885\t165.0\nUnited States\tModesto\t37.639095\t-120.99688\t12.0\nUnited States\tParadise\t36.097195\t-115.14667\t626.0\nUnited States\tChula Vista\t32.640053\t-117.0842\t23.0\nUnited States\tLubbock\t33.57786\t-101.85516\t976.0\nUnited States\tRochester\t43.154785\t-77.615555\t155.0\nUnited States\tLaredo\t27.506407\t-99.507545\t115.0\nUnited States\tAkron\t41.081444\t-81.519005\t290.0\nUnited States\tOrlando\t28.538336\t-81.379234\t34.0\nUnited States\tDurham\t35.994034\t-78.89862\t122.0\nUnited States\tNorth Glendale\t34.160564\t-118.26452\t174.0\nUnited States\tScottsdale\t33.50921\t-111.89903\t386.0\n" }, "dateCreated": "Jul 19, 2015 8:12:20 PM", - "dateStarted": "Jul 19, 2015 8:20:26 PM", - "dateFinished": "Jul 19, 2015 8:20:32 PM", - "status": "ERROR", + "dateStarted": "Jul 19, 2015 8:31:10 PM", + "dateFinished": "Jul 19, 2015 8:31:10 PM", + "status": "FINISHED", "progressUpdateIntervalMs": 500 }, { From 68332fbed69cc1357f87485d1c5a8911c766ce8c Mon Sep 17 00:00:00 2001 From: madhuka Date: Fri, 21 Aug 2015 20:34:03 +0530 Subject: [PATCH 6/6] adding data validator with test --- .../paragraph/paragraph.controller.js | 14 ++- .../chart-data-validator-factory.js | 84 ++++++++++++++ .../data-validator/data-validator-factory.js | 104 ++++++++++++++++++ .../data-validator/data-validator-schema.js | 40 +++++++ .../data-validator/data-validator-service.js | 53 +++++++++ .../map-data-validator-factory.js | 61 ++++++++++ .../scatter-data-validator-factory.js | 22 ++++ zeppelin-web/src/index.html | 8 +- zeppelin-web/test/karma.conf.js | 7 +- .../chart-data-validator-test.js | 67 +++++++++++ .../data-validator/data-validator-service.js | 35 ++++++ .../data-validator/map-data-validator-test.js | 47 ++++++++ .../scatter-data-validator-test.js | 63 +++++++++++ 13 files changed, 602 insertions(+), 3 deletions(-) create mode 100644 zeppelin-web/src/components/data-validator/chart-data-validator-factory.js create mode 100644 zeppelin-web/src/components/data-validator/data-validator-factory.js create mode 100644 zeppelin-web/src/components/data-validator/data-validator-schema.js create mode 100644 zeppelin-web/src/components/data-validator/data-validator-service.js create mode 100644 zeppelin-web/src/components/data-validator/map-data-validator-factory.js create mode 100644 zeppelin-web/src/components/data-validator/scatter-data-validator-factory.js create mode 100644 zeppelin-web/test/spec/components/data-validator/chart-data-validator-test.js create mode 100644 zeppelin-web/test/spec/components/data-validator/data-validator-service.js create mode 100644 zeppelin-web/test/spec/components/data-validator/map-data-validator-test.js create mode 100644 zeppelin-web/test/spec/components/data-validator/scatter-data-validator-test.js diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js index d0884607744..0ce3c6da533 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js @@ -17,7 +17,7 @@ angular.module('zeppelinWebApp') .controller('ParagraphCtrl', function($scope,$rootScope, $route, $window, $element, $routeParams, $location, - $timeout, $compile, websocketMsgSrv) { + $timeout, $compile, websocketMsgSrv, dataValidatorSrv) { $scope.paragraph = null; $scope.editor = null; @@ -1351,6 +1351,12 @@ angular.module('zeppelinWebApp') }); } + var msg = dataValidatorSrv.validateChartData(data); + //Only console print error found + if(msg.error){ + console.log(msg.msg); + } + // clear aggregation name, if possible var namesWithoutAggr = {}; var colName; @@ -1486,6 +1492,12 @@ angular.module('zeppelinWebApp') }; } + var msg = dataValidatorSrv.validateScatterData(data); + //TODO- warning need to error model or services will needed + if(msg.error){ + console.log(msg.msg); + } + for (var i = 0; i < data.rows.length; i++) { row = data.rows[i]; if (xAxis) { diff --git a/zeppelin-web/src/components/data-validator/chart-data-validator-factory.js b/zeppelin-web/src/components/data-validator/chart-data-validator-factory.js new file mode 100644 index 00000000000..ffc2f7c9b53 --- /dev/null +++ b/zeppelin-web/src/components/data-validator/chart-data-validator-factory.js @@ -0,0 +1,84 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +'use strict'; + +angular.module('zeppelinWebApp').factory('chartdataValidator', function( + $rootScope, DataValidator, dataModelSchemas) { + + var simpleChartSchema = dataModelSchemas.D3ChartDataSchema; + var chartdataValidator = new DataValidator(simpleChartSchema); + + //overriding the check data + chartdataValidator.checkData = function() { + basicCheck(); + }; + + function rowCheck(dataRows) { + if(dataRows instanceof Object) { + for(var key in dataRows) { + schemaChecker(key, 0); + if(dataRows.hasOwnProperty(key)) { + var obj = dataRows[key]; + rowValueCheck(obj); + } + } + } else { + markValidationError('dataRows is not a Object | '); + } + } + + function rowValueCheck(record) { + if(record instanceof Object) { + for(var key in record) { + var recordValues = record[key]; + var countKey = 1; + for(var recordKey in recordValues) { + + if(recordValues.hasOwnProperty(recordKey)) { + var values = recordValues[recordKey]; + schemaChecker(values, countKey); + countKey++; + } + } + } + } else { + markValidationError('record is not a Object | '); + } + } + + function basicCheck() { + var data = chartdataValidator.data; + if(data.schema && data.rows) { + rowCheck(data.rows); + return true; + } else { + markValidationError('data rows does not exisiting | '); + } + } + + function schemaChecker(record, index) { + if(isNaN(record) !== (simpleChartSchema.type[index] === 'string')) { + markValidationError('data record ' + record + + ' is not matching for schema | '); + } + //record is passed + } + + function markValidationError(msg) { + chartdataValidator.setMsg(msg); + chartdataValidator.setError(); + } + + return chartdataValidator; +}); \ No newline at end of file diff --git a/zeppelin-web/src/components/data-validator/data-validator-factory.js b/zeppelin-web/src/components/data-validator/data-validator-factory.js new file mode 100644 index 00000000000..6024f6913de --- /dev/null +++ b/zeppelin-web/src/components/data-validator/data-validator-factory.js @@ -0,0 +1,104 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +'use strict'; + +angular.module('zeppelinWebApp').factory('DataValidator', function($rootScope) { + + var msg = ''; + var errorStatus = false; + var DataValidator = function(schema) { + this.schema = schema; + this.error = getErrorStatus; + this.setError = setError; + this.checkData = checkData; + this.getMsg = getMsg; + this.setMsg = setMsg; + this.clearMsg = clearMsg; + this.data = null; + }; + + function checkData() { + if(basicCheck(this.data, this.schema)) { // jshint ignore:line + msg += 'data is exisiting | '; + } else { + msg += 'data does not exisiting | '; + } + } + + function getMsg() { + return msg; + } + + function setMsg(newMsg) { + msg += newMsg; + } + + function clearMsg() { + msg = ''; + } + + function getErrorStatus() { + return errorStatus; + } + + function setError() { + errorStatus = true; + } + + function basicCheck(data, schema) { + if(data.code && data.rows) { + rowCheck(data.rows, 3, schema); + return true; + } else { + msg += 'data rows does not exisiting | '; + setError(); + return false; + } + } + + function rowCheck(rowData, num, schema) { + if(rowData) { + for(var i = 0; i < rowData.length; i++) { + var row = rowData[i]; + if(dataCheckValidator(row, schema)) { + msg += 'data record does not mapping to data schema| '; + } + } + } else { + setError(); + msg += 'data row does not exisiting | '; + } + } + + function dataCheckValidator(record, schema) { + if(record) { + for(var i = 0; i < schema.type.length; i++) { + if(isNaN(record[i]) !== (schema.type[i] === 'string')) { + errorStatus = true; + msg += 'data record ' + (record[i]) + + ' is not matching for schema | '; + return true; + } + } //end validation on data record + errorStatus = false; + return false; + } else { + msg += 'data record does not exisiting | '; + setError(); + return true; + } + } + + return DataValidator; +}); \ No newline at end of file diff --git a/zeppelin-web/src/components/data-validator/data-validator-schema.js b/zeppelin-web/src/components/data-validator/data-validator-schema.js new file mode 100644 index 00000000000..d514a7e15bc --- /dev/null +++ b/zeppelin-web/src/components/data-validator/data-validator-schema.js @@ -0,0 +1,40 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +'use strict'; + +angular.module('zeppelinWebApp') + .constant('dataModelSchemas', { + 'MapSchema': { + type: ['string', 'string', 'number', 'number', 'number'], + range: { + latitude: { + low: -90, + high: 90 + }, + longitude: { + low: -180, + high: 180 + } + } + }, + 'D3ChartDataSchema': { + type: ['number', 'number', 'number'], + }, + 'PieChartSchema': { + type: ['number', 'number'], + }, + 'ScatterDataSchema': { + type: ['number', 'number'], + } + }); \ No newline at end of file diff --git a/zeppelin-web/src/components/data-validator/data-validator-service.js b/zeppelin-web/src/components/data-validator/data-validator-service.js new file mode 100644 index 00000000000..5c9a0843c83 --- /dev/null +++ b/zeppelin-web/src/components/data-validator/data-validator-service.js @@ -0,0 +1,53 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +'use strict'; + +angular.module('zeppelinWebApp').service('dataValidatorSrv', function( + $rootScope, mapdataValidator, scatterdataValidator, chartdataValidator, + dataModelSchemas) { + + this.validateMapData = function(data) { + var mapValidator = mapdataValidator; + doBasicCheck(mapValidator, data); + mapValidator.checkLatiLong(); + return buildMsg(mapValidator); + }; + + this.validateChartData = function(data) { + var chartValidator = chartdataValidator; + doBasicCheck(chartValidator, data); + return buildMsg(chartValidator); + }; + + this.validateScatterData = function(data) { + var scatterChartValidator = scatterdataValidator; + doBasicCheck(scatterChartValidator, data); + return buildMsg(scatterChartValidator); + }; + + function buildMsg(validator) { + var msg = { + 'error': validator.error(), + 'msg': validator.getMsg() + }; + return msg; + } + + function doBasicCheck(validator, data) { + validator.clearMsg(); + validator.data = data; + validator.checkData(); + } + +}); \ No newline at end of file diff --git a/zeppelin-web/src/components/data-validator/map-data-validator-factory.js b/zeppelin-web/src/components/data-validator/map-data-validator-factory.js new file mode 100644 index 00000000000..9b75b79763c --- /dev/null +++ b/zeppelin-web/src/components/data-validator/map-data-validator-factory.js @@ -0,0 +1,61 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +'use strict'; + +angular.module('zeppelinWebApp').factory('mapdataValidator', function( + $rootScope, DataValidator, dataModelSchemas) { + + var msg = ''; + var errorStatus = false; + var mapSchema = dataModelSchemas.MapSchema; + + var mapdataValidator = new DataValidator(mapSchema); + mapdataValidator.checkLatiLong = function() { + errorStatus = false; + var data = mapdataValidator.data.rows; + msg = ''; + for(var i = 0; i < data.length; i++) { + latitudeValidator(data[i][2], mapSchema.range); + longitudeValidator(data[i][3], mapSchema.range); + if(errorStatus) { + mapdataValidator.setError(); + } + } + mapdataValidator.setMsg(msg); + }; + + //Latitude measurements range from 0° to (+/–)90°. + function latitudeValidator(record, schema) { + var latitude = parseFloat(record); + if(schema.latitude.low < latitude && latitude < schema.latitude.high) { + msg += 'latitudes are ok | '; + } else { + msg += 'Latitude ' + record + ' is not in range | '; + errorStatus = true; + } + } + + //Longitude measurements range from 0° to (+/–)180°. + function longitudeValidator(record, schema) { + var longitude = parseFloat(record); + if(schema.longitude.low < longitude && longitude < schema.longitude.high) { + msg += 'longitude are ok | '; + } else { + msg += 'Longitude' + record + ' is not in range | '; + errorStatus = true; + } + } + + return mapdataValidator; +}); \ No newline at end of file diff --git a/zeppelin-web/src/components/data-validator/scatter-data-validator-factory.js b/zeppelin-web/src/components/data-validator/scatter-data-validator-factory.js new file mode 100644 index 00000000000..416dd3627da --- /dev/null +++ b/zeppelin-web/src/components/data-validator/scatter-data-validator-factory.js @@ -0,0 +1,22 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +'use strict'; + +angular.module('zeppelinWebApp').factory('scatterdataValidator', function( + $rootScope, DataValidator, dataModelSchemas) { + + var simpleChartSchema = dataModelSchemas.ScatterDataSchema; + var scatterdataValidator = new DataValidator(simpleChartSchema); + return scatterdataValidator; +}); \ No newline at end of file diff --git a/zeppelin-web/src/index.html b/zeppelin-web/src/index.html index 9a946c473d5..1541fe61945 100644 --- a/zeppelin-web/src/index.html +++ b/zeppelin-web/src/index.html @@ -111,7 +111,7 @@ - + @@ -136,6 +136,12 @@ + + + + + + diff --git a/zeppelin-web/test/karma.conf.js b/zeppelin-web/test/karma.conf.js index e85abdb0741..73b83dd4919 100644 --- a/zeppelin-web/test/karma.conf.js +++ b/zeppelin-web/test/karma.conf.js @@ -35,9 +35,10 @@ module.exports = function(config) { 'bower_components/ace-builds/src-noconflict/mode-scala.js', 'bower_components/ace-builds/src-noconflict/mode-sql.js', 'bower_components/ace-builds/src-noconflict/mode-markdown.js', + 'bower_components/ace-builds/src-noconflict/mode-sh.js', 'bower_components/ace-builds/src-noconflict/keybinding-emacs.js', 'bower_components/ace-builds/src-noconflict/ext-language_tools.js', - 'bower_components/ace-builds/src-noconflict/theme-github.js', + 'bower_components/ace-builds/src-noconflict/theme-chrome.js', 'bower_components/angular-ui-ace/ui-ace.js', 'bower_components/jquery.scrollTo/jquery.scrollTo.js', 'bower_components/d3/d3.js', @@ -52,10 +53,14 @@ module.exports = function(config) { 'bower_components/highlightjs/highlight.pack.js', 'bower_components/lodash/lodash.js', 'bower_components/angular-filter/dist/angular-filter.min.js', + 'bower_components/leaflet/dist/leaflet.js', + 'bower_components/leaflet/dist/leaflet-src.js', + 'bower_components/angular-leaflet-directive/dist/angular-leaflet-directive.js', 'bower_components/angular-mocks/angular-mocks.js', // endbower 'src/app/app.js', 'src/app/app.controller.js', + 'src/components/data-validator/*.js', 'src/app/**/*.js', 'test/spec/**/*.js' ], diff --git a/zeppelin-web/test/spec/components/data-validator/chart-data-validator-test.js b/zeppelin-web/test/spec/components/data-validator/chart-data-validator-test.js new file mode 100644 index 00000000000..fa3f080ecba --- /dev/null +++ b/zeppelin-web/test/spec/components/data-validator/chart-data-validator-test.js @@ -0,0 +1,67 @@ +'use strict'; + +describe('DataValidatorSrv ', function() { + beforeEach(module('zeppelinWebApp')); + + var chartDataValidatorSrv; + beforeEach(inject(function(dataValidatorSrv) { + chartDataValidatorSrv = dataValidatorSrv; + })); + + describe('validateChartData function', function() { + + var mockDataOne = { + 'schema': Object, + 'rows': Object + }; + it('testing with empty data', function() { + expect(chartDataValidatorSrv.validateChartData({}).error).toBe( + true); + expect(chartDataValidatorSrv.validateChartData({}).msg).toBe( + 'data rows does not exisiting | '); + }); + it('testing with data contain empty rows objects', function() { + expect(chartDataValidatorSrv.validateChartData({}).error).toBe( + true); + expect(chartDataValidatorSrv.validateChartData({}).msg).toBe( + 'data rows does not exisiting | '); + }); + //mock correct data set model + var mockRecord = { + 'value(sum)': { + 'count': 1, + 'value': '4' + } + }; + var mockRow = { + '19': mockRecord + }; + mockDataOne.rows = mockRow; + it('testing with validated data', function() { + expect(chartDataValidatorSrv.validateChartData(mockDataOne).error) + .toBe(false); + expect(chartDataValidatorSrv.validateChartData(mockDataOne).msg) + .toBe(''); + }); + //mock incorrect data set model + var incorrectMockRecord = { + 'value(sum)': { + 'count': 1, + 'value': 'wrong' + } + }; + var incorrectMockRow = { + '23': incorrectMockRecord + }; + var mockDataTwo = { + 'schema': Object, + 'rows': incorrectMockRow + }; + it('testing with invalidated data', function() { + expect(chartDataValidatorSrv.validateChartData(mockDataTwo).error) + .toBe(true); + expect(chartDataValidatorSrv.validateChartData(mockDataTwo).msg) + .toBe('data record wrong is not matching for schema | '); + }); + }); +}); \ No newline at end of file diff --git a/zeppelin-web/test/spec/components/data-validator/data-validator-service.js b/zeppelin-web/test/spec/components/data-validator/data-validator-service.js new file mode 100644 index 00000000000..b2a8ab5bd73 --- /dev/null +++ b/zeppelin-web/test/spec/components/data-validator/data-validator-service.js @@ -0,0 +1,35 @@ +'use strict'; + +describe('Service : DataValidatorSrv', function() { + beforeEach(module('zeppelinWebApp')); + + var chartDataValidatorSrv; + beforeEach(inject(function(dataValidatorSrv) { + chartDataValidatorSrv = dataValidatorSrv; + })); + + describe('Testing API', function() { + + it('to be define', function() { + expect(chartDataValidatorSrv).toBeDefined(); + }); + + it('Services to be define', function() { + expect(chartDataValidatorSrv.validateMapData).toBeDefined(); + expect(chartDataValidatorSrv.validateScatterData).toBeDefined(); + expect(chartDataValidatorSrv.validateChartData).toBeDefined(); + }); + //mock data for testing + var mockData = { + 'schema': Object, + 'rows': Object + }; + it('testing return message format', function() { + expect(chartDataValidatorSrv.validateChartData(mockData).msg) + .toBeDefined(); + expect(chartDataValidatorSrv.validateChartData(mockData).error) + .toBeDefined(); + }); + + }); +}); \ No newline at end of file diff --git a/zeppelin-web/test/spec/components/data-validator/map-data-validator-test.js b/zeppelin-web/test/spec/components/data-validator/map-data-validator-test.js new file mode 100644 index 00000000000..62659214160 --- /dev/null +++ b/zeppelin-web/test/spec/components/data-validator/map-data-validator-test.js @@ -0,0 +1,47 @@ +'use strict'; + +describe('DataValidatorSrv ', function() { + beforeEach(module('zeppelinWebApp')); + + var chartDataValidatorSrv; + beforeEach(inject(function(dataValidatorSrv) { + chartDataValidatorSrv = dataValidatorSrv; + })); + + describe('validateMapData function', function() { + + var mockDataOne = { + 'msg': 'mock data', + 'code': 'SUCCESS', + 'rows': [ + ['us', 'El paso', '31.758', '-106.4869', '1147'] + ] + }; + //test use case + //testing with correct validated data set then change data and test again + it('testing with validate GIS data and range check', function() { + expect(chartDataValidatorSrv.validateMapData(mockDataOne).error) + .toBe( + false); + expect(chartDataValidatorSrv.validateMapData(mockDataOne).msg) + .toBe( + 'data is exisiting | latitudes are ok | longitude are ok | '); + //change number to string and test + mockDataOne.rows[0][3] = 'text'; + expect(chartDataValidatorSrv.validateMapData(mockDataOne).error) + .toBe( + true); + expect(chartDataValidatorSrv.validateMapData(mockDataOne).msg) + .toContain('data record text is not matching for schema'); + //testing for Longitude range + mockDataOne.rows[0][3] = 250; + expect(chartDataValidatorSrv.validateMapData(mockDataOne).error) + .toBe( + true); + expect(chartDataValidatorSrv.validateMapData(mockDataOne).msg) + .toBe('data is exisiting | latitudes are ok | Longitude250 is not in range | '); + }); + + }); + +}); \ No newline at end of file diff --git a/zeppelin-web/test/spec/components/data-validator/scatter-data-validator-test.js b/zeppelin-web/test/spec/components/data-validator/scatter-data-validator-test.js new file mode 100644 index 00000000000..c8bf2d9acb3 --- /dev/null +++ b/zeppelin-web/test/spec/components/data-validator/scatter-data-validator-test.js @@ -0,0 +1,63 @@ +'use strict'; + +describe('DataValidatorSrv ', function() { + beforeEach(module('zeppelinWebApp')); + + var validateScatterDataSrv; + beforeEach(inject(function(dataValidatorSrv) { + validateScatterDataSrv = dataValidatorSrv; + })); + + describe('validateScatterData function', function() { + + var mockDataOne = { + 'code': 'SUCCESS', + 'rows': Object + }; + it('testing with empty data', function() { + expect(validateScatterDataSrv.validateScatterData({}).error).toBe( + true); + expect(validateScatterDataSrv.validateScatterData({}).msg).toBe( + 'data rows does not exisiting | data does not exisiting | ' + ); + }); + it('testing with data contain empty rows objects', function() { + //mockDataOne.rows = [[]]; + expect(validateScatterDataSrv.validateScatterData(mockDataOne) + .error).toBe( + true); + expect(validateScatterDataSrv.validateScatterData(mockDataOne) + .msg).toContain( + 'data is exisiting | '); + expect(validateScatterDataSrv.validateScatterData(mockDataOne) + .msg).toContain( + 'data record does not exisiting | '); + }); + + it('testing with valid data ', function() { + mockDataOne.rows = [ + [12, 34] + ]; + expect(validateScatterDataSrv.validateScatterData(mockDataOne) + .error).toBe( + false); + expect(validateScatterDataSrv.validateScatterData(mockDataOne) + .msg).toBe( + 'data is exisiting | '); + }); + + it('testing with string data model', function() { + mockDataOne.rows = [ + ['test', '34'] + ]; + expect(validateScatterDataSrv.validateScatterData(mockDataOne) + .error).toBe( + true); + expect(validateScatterDataSrv.validateScatterData(mockDataOne) + .msg).toBe( + 'data record test is not matching for schema | data record does not mapping to data schema| data is exisiting | ' + ); + }); + + }); +}); \ No newline at end of file