crossfilter.json

{"paragraphs":[{"text":"","dateUpdated":"2017-02-16T14:49:17+0530","config":{"colWidth":12,"editorHide":false,"results":[],"enabled":true,"editorSetting":{"language":"scala"},"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[]},"apps":[],"jobName":"paragraph_1487236757403_-1235301839","id":"20170210-120212_1097034046","dateCreated":"2017-02-16T14:49:17+0530","status":"READY","errorMessage":"","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:159"},{"text":"%spark\n\n \n import org.apache.spark.rdd.RDD\n import org.apache.spark.sql.{DataFrame, Row}\n import org.apache.spark.sql.types.StructType\n import org.apache.spark.sql.SparkSession\n import org.apache.spark.sql.types._\n import spark.implicits._\n \n  \n   def createRowRDD(stringRDD: RDD[String]): RDD[Row]  = {\n    stringRDD\n      .map(_.split(\",\"))\n      .map( a => Row(a:_*) )\n  }\n  \n  def sparkRDDFromFile(spark: SparkSession, uri: String): RDD[Row] ={\n    val stringRDD=spark.sparkContext.textFile(uri)\n    createRowRDD(stringRDD)\n  }\n\n\n  def sparkRDD(spark: SparkSession, seq: Seq[String]): RDD[Row]={\n    val stringRDD=spark.sparkContext.makeRDD(seq)\n    createRowRDD(stringRDD)\n  }\n\n \n  \n  def createSchema(schemaString: String): StructType = {\n    val fields = schemaString.split(\",\")\n      .map(fieldName => StructField(fieldName, StringType, nullable = true))\n     StructType(fields)\n  }\n\n  def createDataFrame(spark: SparkSession,rowRDD: RDD[Row],schema :StructType, name: String): DataFrame ={\n    val dataframe=spark.createDataFrame(rowRDD, schema)\n    dataframe.createOrReplaceTempView(name)\n    dataframe\n  }\n  \n  def createRDDView(spark: SparkSession, rowRDD: RDD[Row], schema: StructType, table: String): Unit={\n    val tableDF = spark.createDataFrame(rowRDD, schema)\n\n    // Creates a temporary view using the DataFrame\n    tableDF.createOrReplaceTempView(table)\n  }\n  \n  def createTable(spark: SparkSession, url: String,schema :String, table: String): Unit ={\n   val rowRDD=sparkRDDFromFile(spark,url)\n   val schemaStructure=createSchema(schema)\n   createDataFrame(spark,rowRDD,schemaStructure,table)\n  }\n \n  def runSparkQuery(spark: SparkSession,query: String):Unit={\n    val results = spark.sql(query)\n    print(\"[\")\n    \n    results.collect().zipWithIndex.foreach ({ (row) => {\n      if (row._2 == 0) {\n        print(row._1)\n      } else {\n        print(\",\")\n        print(row._1)\n      }\n    }\n    })\n    print(\"]\")\n  }","user":"anonymous","dateUpdated":"2017-02-20T12:03:53+0530","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":[{"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}}}],"enabled":true,"editorSetting":{"language":"scala"}},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"\nimport org.apache.spark.rdd.RDD\n\nimport org.apache.spark.sql.{DataFrame, Row}\n\nimport org.apache.spark.sql.types.StructType\n\nimport org.apache.spark.sql.SparkSession\n\nimport org.apache.spark.sql.types._\n\nimport spark.implicits._\n\ncreateRowRDD: (stringRDD: org.apache.spark.rdd.RDD[String])org.apache.spark.rdd.RDD[org.apache.spark.sql.Row]\n\nsparkRDDFromFile: (spark: org.apache.spark.sql.SparkSession, uri: String)org.apache.spark.rdd.RDD[org.apache.spark.sql.Row]\n\nsparkRDD: (spark: org.apache.spark.sql.SparkSession, seq: Seq[String])org.apache.spark.rdd.RDD[org.apache.spark.sql.Row]\n\ncreateSchema: (schemaString: String)org.apache.spark.sql.types.StructType\n\ncreateDataFrame: (spark: org.apache.spark.sql.SparkSession, rowRDD: org.apache.spark.rdd.RDD[org.apache.spark.sql.Row], schema: org.apache.spark.sql.types.StructType, name: String)org.apache.spark.sql.DataFrame\n\ncreateRDDView: (spark: org.apache.spark.sql.SparkSession, rowRDD: org.apache.spark.rdd.RDD[org.apache.spark.sql.Row], schema: org.apache.spark.sql.types.StructType, table: String)Unit\n\ncreateTable: (spark: org.apache.spark.sql.SparkSession, url: String, schema: String, table: String)Unit\n\nrunSparkQuery: (spark: org.apache.spark.sql.SparkSession, query: String)Unit\n"}]},"apps":[],"jobName":"paragraph_1487236757405_-1237610332","id":"20170210-120217_1521769425","dateCreated":"2017-02-16T14:49:17+0530","dateStarted":"2017-02-20T12:03:53+0530","dateFinished":"2017-02-20T12:04:04+0530","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:160"},{"text":"%spark\n\n\n\"\"\" create spark view \"\"\"\nval url:String = z.input(\"url\").toString\nval schema:String =z.input(\"schema\").toString\nval tableName:String = z.input(\"table\").toString\nif(url.length >0 && schema.length >0 && tableName.length >0 )\n  {\n      createTable(spark,url,schema,tableName)\n      \n  }\n\n","user":"anonymous","dateUpdated":"2017-02-20T11:59:30+0530","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":[{"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}}}],"enabled":true,"editorSetting":{"language":"scala"}},"settings":{"params":{"schema":"msisdn,val1,val2,val3","table":"test1","url":"/home/aslam/Downloads/map_105687"},"forms":{"url":{"name":"url","displayName":"url","type":"input","defaultValue":"","hidden":false,"$$hashKey":"object:4065"},"schema":{"name":"schema","displayName":"schema","type":"input","defaultValue":"","hidden":false,"$$hashKey":"object:4063"},"table":{"name":"table","displayName":"table","type":"input","defaultValue":"","hidden":false,"$$hashKey":"object:4064"}}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"\nres95: String = \" create spark view \"\n\nurl: String = /home/aslam/Downloads/map_105687\n\nschema: String = msisdn,val1,val2,val3\n\ntableName: String = test1\n"}]},"apps":[],"jobName":"paragraph_1487236757406_-1236456086","id":"20170210-124059_565804652","dateCreated":"2017-02-16T14:49:17+0530","dateStarted":"2017-02-20T11:59:30+0530","dateFinished":"2017-02-20T11:59:34+0530","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:161"},{"text":"%spark\n{\nval query =z.input(\"query\").toString\n\nif(query.length > 0)\n  runSparkQuery(spark,query)\n}\n","dateUpdated":"2017-02-16T16:56:08+0530","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":[{"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}}}],"enabled":true,"editorSetting":{"language":"scala"}},"settings":{"params":{"query":"select * from test1"},"forms":{}},"apps":[],"jobName":"paragraph_1487236757406_-1236456086","id":"20170214-125114_1890917933","dateCreated":"2017-02-16T14:49:17+0530","status":"FINISHED","errorMessage":"","progressUpdateIntervalMs":500,"$$hashKey":"object:162","user":"anonymous","dateFinished":"2017-02-20T11:59:46+0530","dateStarted":"2017-02-20T12:04:10+0530"},{"dateUpdated":"2017-02-16T14:49:17+0530","config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"results":{},"editorSetting":{"language":"scala"},"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"apps":[],"jobName":"paragraph_1487236757407_-1236840834","id":"20170214-125142_943281693","dateCreated":"2017-02-16T14:49:17+0530","status":"READY","errorMessage":"","progressUpdateIntervalMs":500,"$$hashKey":"object:163"}],"name":"crossfilter","id":"2C9TUFWXB","angularObjects":{"2CANAAPBW:shared_process":[]},"config":{"looknfeel":"default","personalizedMode":"false"},"info":{}}