-
Notifications
You must be signed in to change notification settings - Fork 0
/
trap2017.json
1 lines (1 loc) · 12.1 KB
/
trap2017.json
1
{"paragraphs":[{"text":"%spark\n\ndef readParquet(path: String) = sqlContext.read.parquet(path)","user":"anonymous","dateUpdated":"2017-12-03T11:23:48+0100","config":{"colWidth":12,"enabled":true,"results":{},"editorSetting":{"language":"scala"},"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"readParquet: (path: String)org.apache.spark.sql.DataFrame\n"}]},"apps":[],"jobName":"paragraph_1512153363032_170192954","id":"20171201-193603_2062862508","dateCreated":"2017-12-01T19:36:03+0100","dateStarted":"2017-12-03T11:23:48+0100","dateFinished":"2017-12-03T11:23:48+0100","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:2718"},{"text":"%spark\n\nval df = readParquet(\"/home/asolimando/git/trap2017spark/data/raw.parquet\")","user":"anonymous","dateUpdated":"2017-12-02T18:27:32+0100","config":{"colWidth":12,"enabled":true,"results":{},"editorSetting":{"language":"scala"},"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"df: org.apache.spark.sql.DataFrame = [plate: int, gate: int ... 3 more fields]\n"}]},"apps":[],"jobName":"paragraph_1512204017232_-1026976222","id":"20171202-094017_158548917","dateCreated":"2017-12-02T09:40:17+0100","dateStarted":"2017-12-02T18:27:32+0100","dateFinished":"2017-12-02T18:27:44+0100","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:2719"},{"text":"%spark\ndf.count\ndf.filter(col(\"plate\").isNull).count","user":"anonymous","dateUpdated":"2017-12-02T18:28:42+0100","config":{"colWidth":12,"enabled":true,"results":{},"editorSetting":{"language":"scala"},"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"res1: Long = 155368907\nres2: Long = 0\n"}]},"apps":[],"jobName":"paragraph_1512235686249_470263699","id":"20171202-182806_1037587074","dateCreated":"2017-12-02T18:28:06+0100","dateStarted":"2017-12-02T18:28:42+0100","dateFinished":"2017-12-02T18:28:43+0100","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:2720"},{"text":"%spark\n\nval conflicts = readParquet(\"/home/asolimando/git/trap2017spark/data/conflicts.parquet\")","user":"anonymous","dateUpdated":"2017-12-02T09:40:39+0100","config":{"colWidth":12,"enabled":true,"results":{},"editorSetting":{"language":"scala"},"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"conflicts: org.apache.spark.sql.DataFrame = [gate: int, lane: double ... 2 more fields]\n"}]},"apps":[],"jobName":"paragraph_1512204007000_-1851324549","id":"20171202-094007_1655834910","dateCreated":"2017-12-02T09:40:07+0100","dateStarted":"2017-12-02T09:40:39+0100","dateFinished":"2017-12-02T09:40:41+0100","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:2721"},{"text":"%spark\n\nconflicts.count\n\nconflicts.orderBy(desc(\"plates\")).limit(10).show(false)\n\nconflicts.groupBy(date_format(col(\"timestamp\"), \"yyyy-MM\").as(\"yearMonth\")).count.orderBy(\"yearMonth\").show(false)","user":"anonymous","dateUpdated":"2017-12-02T09:49:56+0100","config":{"colWidth":12,"enabled":true,"results":{},"editorSetting":{"language":"scala"},"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"res80: Long = 7178903\n+----+----+---------------------+------+\n|gate|lane|timestamp |plates|\n+----+----+---------------------+------+\n|18 |2.0 |2016-07-22 11:39:00.0|52 |\n|23 |2.0 |2016-07-22 11:38:00.0|52 |\n|23 |2.0 |2016-07-22 11:48:00.0|50 |\n|23 |2.0 |2016-07-22 11:11:00.0|49 |\n|18 |2.0 |2016-07-22 11:16:00.0|49 |\n|20 |2.0 |2016-07-22 10:43:00.0|48 |\n|23 |2.0 |2016-07-22 10:26:00.0|48 |\n|5 |2.0 |2016-07-22 10:39:00.0|48 |\n|5 |2.0 |2016-07-22 10:59:00.0|46 |\n|23 |2.0 |2016-07-22 10:40:00.0|46 |\n+----+----+---------------------+------+\n\n+---------+------+\n|yearMonth|count |\n+---------+------+\n|2016-01 |345539|\n|2016-02 |415999|\n|2016-03 |644539|\n|2016-04 |638525|\n|2016-05 |744058|\n|2016-06 |791434|\n|2016-07 |868423|\n|2016-08 |668125|\n|2016-09 |674792|\n|2016-10 |533220|\n|2016-11 |473790|\n|2016-12 |380459|\n+---------+------+\n\n"}]},"apps":[],"jobName":"paragraph_1512204130409_1550826084","id":"20171202-094210_365863190","dateCreated":"2017-12-02T09:42:10+0100","dateStarted":"2017-12-02T09:49:56+0100","dateFinished":"2017-12-02T09:50:00+0100","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:2722"},{"text":"%spark\n\nval multinat = readParquet(\"/home/asolimando/git/trap2017spark/data/multinat.parquet\")","user":"anonymous","dateUpdated":"2017-12-02T18:30:08+0100","config":{"colWidth":12,"enabled":true,"results":{},"editorSetting":{"language":"scala"},"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"multinat: org.apache.spark.sql.DataFrame = [plate: int, nats: array<string> ... 1 more field]\n"}]},"apps":[],"jobName":"paragraph_1512206555201_-342893798","id":"20171202-102235_1346236117","dateCreated":"2017-12-02T10:22:35+0100","dateStarted":"2017-12-02T18:30:08+0100","dateFinished":"2017-12-02T18:30:08+0100","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:2723"},{"text":"%spark\n\nmultinat.filter(col(\"plate\").isNull).count","user":"anonymous","dateUpdated":"2017-12-02T18:30:15+0100","config":{"colWidth":12,"enabled":true,"results":{},"editorSetting":{"language":"scala"},"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"res3: Long = 0\n"}]},"apps":[],"jobName":"paragraph_1512235810451_1919044167","id":"20171202-183010_1779701510","dateCreated":"2017-12-02T18:30:10+0100","dateStarted":"2017-12-02T18:30:15+0100","dateFinished":"2017-12-02T18:30:15+0100","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:2724"},{"text":"%spark\n\nmultinat.count\n\nmultinat.orderBy(desc(\"num_nat\")).limit(10).show(false)\n\nmultinat.groupBy(\"num_nat\").count.orderBy(\"num_nat\").show\n\nmultinat.groupBy(\"nats\").count.orderBy(desc(\"count\")).show\n\nmultinat.filter(col(\"num_nat\") === 2 && array_contains(col(\"nats\"),\"?\")).count","user":"anonymous","dateUpdated":"2017-12-02T10:30:42+0100","config":{"colWidth":12,"enabled":true,"results":{},"editorSetting":{"language":"scala"},"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"res126: Long = 117717\n+-------+----------+-------+\n|plate |nats |num_nat|\n+-------+----------+-------+\n|132449 |[?, HR, I]|3 |\n|8515350|[?, HR, I]|3 |\n|1363101|[?, HR, I]|3 |\n|1574581|[?, HR, I]|3 |\n|1770385|[?, HR, I]|3 |\n|8029721|[?, HR, I]|3 |\n|110836 |[?, HR, I]|3 |\n|1179601|[?, HR, I]|3 |\n|1459586|[?, HR, I]|3 |\n|1731081|[?, HR, I]|3 |\n+-------+----------+-------+\n\n+-------+------+\n|num_nat| count|\n+-------+------+\n| 2|116981|\n| 3| 736|\n+-------+------+\n\n+----------+-----+\n| nats|count|\n+----------+-----+\n| [?, I]|59888|\n| [?, RO]|26709|\n| [?, H]|13653|\n| [?, CH]| 5182|\n| [?, HR]| 4710|\n| [HR, I]| 4546|\n| [?, SLO]| 1884|\n|[?, HR, I]| 722|\n| [?, MP]| 229|\n| [CH, I]| 180|\n|[?, CH, I]| 14|\n+----------+-----+\n\nres134: Long = 112255\n"}]},"apps":[],"jobName":"paragraph_1512206780131_-1317910453","id":"20171202-102620_1941235943","dateCreated":"2017-12-02T10:26:20+0100","dateStarted":"2017-12-02T10:30:42+0100","dateFinished":"2017-12-02T10:30:44+0100","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:2725"},{"text":"%spark\n\nval fixed = readParquet(\"/home/asolimando/git/trap2017spark/data/fixed.parquet\").filter(month(col(\"timestamp\")) === 1)\n","user":"anonymous","dateUpdated":"2017-12-03T11:23:54+0100","config":{"colWidth":12,"enabled":true,"results":{},"editorSetting":{"language":"scala"},"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"fixed: org.apache.spark.sql.Dataset[org.apache.spark.sql.Row] = [plate: int, gate: int ... 3 more fields]\n"}]},"apps":[],"jobName":"paragraph_1512226824381_1308540573","id":"20171202-160024_1325491242","dateCreated":"2017-12-02T16:00:24+0100","dateStarted":"2017-12-03T11:23:54+0100","dateFinished":"2017-12-03T11:23:56+0100","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:2726"},{"text":"%spark\n\nfixed.filter(col(\"plate\") === 259).groupBy(col(\"plate\"), dayofmonth(col(\"timestamp\"))).count.filter(col(\"count\") <= 50).orderBy(desc(\"count\")).show(false)\n\n//fixed.filter(col(\"plate\") === 259).groupBy().agg(countDistinct(dayofmonth(col(\"timestamp\")))).show\n\n//fixed.show(false)","user":"anonymous","dateUpdated":"2017-12-03T11:36:24+0100","config":{"colWidth":12,"enabled":true,"results":{},"editorSetting":{"language":"scala"},"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"+-----+---------------------+-----+\n|plate|dayofmonth(timestamp)|count|\n+-----+---------------------+-----+\n|259 |7 |50 |\n|259 |25 |49 |\n|259 |28 |47 |\n|259 |18 |47 |\n|259 |29 |47 |\n|259 |27 |46 |\n|259 |23 |43 |\n|259 |21 |38 |\n|259 |16 |36 |\n|259 |26 |33 |\n|259 |12 |32 |\n|259 |30 |31 |\n|259 |6 |30 |\n|259 |19 |29 |\n|259 |8 |28 |\n|259 |3 |26 |\n+-----+---------------------+-----+\n\n"}]},"apps":[],"jobName":"paragraph_1512296577875_-206100494","id":"20171203-112257_1283972234","dateCreated":"2017-12-03T11:22:57+0100","dateStarted":"2017-12-03T11:36:24+0100","dateFinished":"2017-12-03T11:36:32+0100","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:2727"},{"text":"%spark\nfixed.filter(col(\"plate\") === 259).orderBy(\"timestamp\").count\n\nval twofivenine = fixed.filter(col(\"plate\") === 259).orderBy(\"timestamp\")\n\ntwofivenine.coalesce(1).write.option(\"header\", true).csv(\"/home/asolimando/259.csv\")","user":"anonymous","dateUpdated":"2017-12-03T11:58:21+0100","config":{"colWidth":12,"enabled":true,"results":{},"editorSetting":{"language":"scala"},"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"res80: Long = 1202\ntwofivenine: org.apache.spark.sql.Dataset[org.apache.spark.sql.Row] = [plate: int, gate: int ... 3 more fields]\n"}]},"apps":[],"jobName":"paragraph_1512296639607_1648697333","id":"20171203-112359_1576668257","dateCreated":"2017-12-03T11:23:59+0100","dateStarted":"2017-12-03T11:58:21+0100","dateFinished":"2017-12-03T11:58:57+0100","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:2728"},{"text":"%spark\n","user":"anonymous","dateUpdated":"2017-12-03T11:47:14+0100","config":{"colWidth":12,"enabled":true,"results":{},"editorSetting":{"language":"scala"},"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"apps":[],"jobName":"paragraph_1512298034219_1965301349","id":"20171203-114714_1238835984","dateCreated":"2017-12-03T11:47:14+0100","status":"READY","progressUpdateIntervalMs":500,"$$hashKey":"object:2729"}],"name":"trap2017","id":"2D1983EH6","angularObjects":{"2D2HQB33E:shared_process":[],"2CYTB3AY1:shared_process":[],"2D2FHGFN1:shared_process":[],"2CYQG5P5C:shared_process":[],"2D1CVX7WC:shared_process":[],"2CZURDV7H:shared_process":[],"2CYCAWD3Q:shared_process":[],"2CYMR4JAK:shared_process":[],"2CY6FU7XH:shared_process":[],"2CZDXW1ZB:shared_process":[],"2CY64TSMW:shared_process":[],"2CZKJUJDS:shared_process":[],"2CZVFAFHS:shared_process":[],"2D12SBMN8:shared_process":[],"2D1PSUNGA:shared_process":[],"2D15BJE8A:shared_process":[],"2CZMGNGTF:shared_process":[],"2CYMM61DQ:shared_process":[],"2CZ4E8KB9:shared_process":[]},"config":{"looknfeel":"default","personalizedMode":"false"},"info":{}}