tweet preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ab8bb466",
   "metadata": {},
   "source": [
    "sparksession 생성 먼저 해야 spark dataframe으로 json 데이터들을 불러올 수 있습니다"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f73aa17f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.conf import SparkConf\n",
    "from pyspark.sql import SparkSession\n",
    "\n",
    "conf = SparkConf().setAll([('spark.jars.packages', 'org.mongodb.spark:mongo-spark-connector_2.11:2.3.1')])\n",
    "spark = SparkSession.builder \\\n",
    "    .appName(\"spark test\") \\\n",
    "    .config(conf=conf) \\\n",
    "    .getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c8a7bbb9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "            <div>\n",
       "                <p><b>SparkSession - hive</b></p>\n",
       "                \n",
       "        <div>\n",
       "            <p><b>SparkContext</b></p>\n",
       "\n",
       "            <p><a href=\"http://192.168.0.32:4041\">Spark UI</a></p>\n",
       "\n",
       "            <dl>\n",
       "              <dt>Version</dt>\n",
       "                <dd><code>v3.1.2</code></dd>\n",
       "              <dt>Master</dt>\n",
       "                <dd><code>local[*]</code></dd>\n",
       "              <dt>AppName</dt>\n",
       "                <dd><code>PySparkShell</code></dd>\n",
       "            </dl>\n",
       "        </div>\n",
       "        \n",
       "            </div>\n",
       "        "
      ],
      "text/plain": [
       "<pyspark.sql.session.SparkSession at 0x7f808a948a60>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "0b3b4bf0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+------------+-----------+--------------------+------------------+--------------------+--------------------+--------------------+--------------+---------+------------+----+-------------------+-------------------+-----------------------+---------------------+-------------------------+-------------------+-----------------------+---------------+----+-----------------------------+------------------+-----------+--------------------+-------------------+--------------------+-----------------------+-----------+-------------+---------+--------------------+--------------------+-------------------------+-------------+---------+--------------------+\n",
      "|                 _id|contributors|coordinates|          created_at|display_text_range|            entities|   extended_entities|      extended_tweet|favorite_count|favorited|filter_level| geo|                 id|             id_str|in_reply_to_screen_name|in_reply_to_status_id|in_reply_to_status_id_str|in_reply_to_user_id|in_reply_to_user_id_str|is_quote_status|lang|                        place|possibly_sensitive|quote_count|       quoted_status|   quoted_status_id|quoted_status_id_str|quoted_status_permalink|reply_count|retweet_count|retweeted|    retweeted_status|              source|                     text| timestamp_ms|truncated|                user|\n",
      "+--------------------+------------+-----------+--------------------+------------------+--------------------+--------------------+--------------------+--------------+---------+------------+----+-------------------+-------------------+-----------------------+---------------------+-------------------------+-------------------+-----------------------+---------------+----+-----------------------------+------------------+-----------+--------------------+-------------------+--------------------+-----------------------+-----------+-------------+---------+--------------------+--------------------+-------------------------+-------------+---------+--------------------+\n",
      "|{60e51343ad59db3e...|        null|       null|Wed Jul 07 02:11:...|           [0, 85]|{[], [{null, pic....|{[{null, pic.twit...|                null|             0|    false|         low|null|1412595075171848192|1412595075171848192|                   null|                 null|                     null|               null|                   null|          false|  ko|{KR, 대한민국, 대한민국 창...|             false|          0|                null|               null|                null|                   null|          0|            0|    false|                null|<a href=\"http://t...|NCT DREAM(엔시티 드림)...|1625623885325|    false|{false, Wed Nov 1...|\n",
      "|{60e513cc36acd74d...|        null|       null|Wed Jul 07 02:39:...|              null|{[], null, [], [{...|                null|                null|             0|    false|         low|null|1412602031739604992|1412602031739604992|                   null|                 null|                     null|               null|                   null|          false|  in|                         null|              null|          0|                null|               null|                null|                   null|          0|            0|    false|{null, null, Tue ...|<a href=\"http://t...|     RT @Bubuyongi: NC...|1625625543900|    false|{false, Sun Dec 3...|\n",
      "|{60e513d136acd74d...|        null|       null|Wed Jul 07 02:39:...|          [13, 36]|{[], null, [{twit...|                null|                null|             0|    false|         low|null|1412602031647334405|1412602031647334405|            markleebase|  1412599759399886852|      1412599759399886852|1294253093580726272|    1294253093580726272|           true| und|                         null|             false|          0|{null, null, Wed ...|1412577812897165319| 1412577812897165319|   {https://twitter....|          0|            0|    false|                null|<a href=\"http://t...|     @markleebase http...|1625625543878|    false|{false, Sat May 0...|\n",
      "|{60e513d636acd74d...|        null|       null|Wed Jul 07 02:39:...|              null|{[], null, [{nave...|                null|                null|             0|    false|         low|null|1412602032138035205|1412602032138035205|                   null|                 null|                     null|               null|                   null|          false|  ko|                         null|             false|          0|                null|               null|                null|                   null|          0|            0|    false|{null, null, Wed ...|<a href=\"http://t...|  RT @mbcplusm: 주간아...|1625625543995|    false|{false, Tue Sep 3...|\n",
      "|{60e513db36acd74d...|        null|       null|Wed Jul 07 02:39:...|              null|{[], [{null, pic....|{[{null, pic.twit...|                null|             0|    false|         low|null|1412602031978647558|1412602031978647558|                   null|                 null|                     null|               null|                   null|          false|  en|                         null|             false|          0|                null|               null|                null|                   null|          0|            0|    false|{null, null, Wed ...|<a href=\"http://t...|     RT @captainuwu: N...|1625625543957|    false|{false, Sat May 2...|\n",
      "|{60e513e036acd74d...|        null|       null|Wed Jul 07 02:39:...|              null|{[], [{{, true, f...|{[{{, true, false...|                null|             0|    false|         low|null|1412602032213487616|1412602032213487616|                   null|                 null|                     null|               null|                   null|          false|  en|                         null|             false|          0|                null|               null|                null|                   null|          0|            0|    false|{null, null, Tue ...|<a href=\"http://t...|     RT @najaeminpics:...|1625625544013|    false|{false, Fri Sep 1...|\n",
      "|{60e513e536acd74d...|        null|       null|Wed Jul 07 02:39:...|              null|{[{[106, 114], JA...|                null|                null|             0|    false|         low|null|1412602032284831749|1412602032284831749|                   null|                 null|                     null|               null|                   null|          false|  en|                         null|              null|          0|                null|               null|                null|                   null|          0|            0|    false|{null, null, Wed ...|<a href=\"http://t...|     RT @styleofjaehyu...|1625625544030|    false|{false, Fri Oct 1...|\n",
      "|{60e513ea36acd74d...|        null|       null|Wed Jul 07 02:39:...|              null|{[{[46, 50], NCT}...|                null|                null|             0|    false|         low|null|1412602032456798209|1412602032456798209|                   null|                 null|                     null|               null|                   null|          false|  th|                         null|              null|          0|                null|               null|                null|                   null|          0|            0|    false|{null, null, Wed ...|<a href=\"http://t...|     RT @hellohaechan_...|1625625544071|    false|{false, Fri Oct 2...|\n",
      "|{60e513f036acd74d...|        null|       null|Wed Jul 07 02:39:...|              null|{[], null, [{twit...|                null|{[0, 273], {[], n...|             0|    false|         low|null|1412602032368668683|1412602032368668683|                   null|                 null|                     null|               null|                   null|           true|  in|                         null|              null|          0|{null, null, Mon ...|1411928603500056577| 1411928603500056577|   {https://twitter....|          0|            0|    false|                null|<a href=\"http://t...|     rt(s) are really ...|1625625544050|     true|{false, Wed Dec 0...|\n",
      "|{60e513f536acd74d...|        null|       null|Wed Jul 07 02:39:...|              null|{[], null, [{twit...|                null|{[0, 158], {[], n...|             0|    false|         low|null|1412602033052405763|1412602033052405763|           snghoontopia|  1412599614977417217|      1412599614977417217|1409862301176397826|    1409862301176397826|          false|  en|                         null|              null|          0|                null|               null|                null|                   null|          0|            0|    false|                null|<a href=\"https://...|     third : taeyong a...|1625625544213|     true|{false, Tue Jun 2...|\n",
      "|{60e513fa36acd74d...|        null|       null|Wed Jul 07 02:39:...|              null|{[{[38, 46], TAEY...|{[{null, pic.twit...|                null|             0|    false|         low|null|1412602033228587010|1412602033228587010|                   null|                 null|                     null|               null|                   null|          false|  en|                         null|             false|          0|                null|               null|                null|                   null|          0|            0|    false|{null, null, Wed ...|<a href=\"http://t...|     RT @sungjaeyong_:...|1625625544255|    false|{false, Thu Apr 3...|\n",
      "|{60e513ff36acd74d...|        null|       null|Wed Jul 07 02:39:...|              null|{[], null, [{twit...|                null|{[0, 231], {[{[21...|             0|    false|         low|null|1412602033195130885|1412602033195130885|                   null|                 null|                     null|               null|                   null|          false|  en|                         null|              null|          0|                null|               null|                null|                   null|          0|            0|    false|                null|<a href=\"http://t...|     He debuted with 1...|1625625544247|     true|{false, Wed Jul 2...|\n",
      "|{60e5140436acd74d...|        null|       null|Wed Jul 07 02:39:...|              null|{[], null, [], [{...|                null|                null|             0|    false|         low|null|1412602033966706694|1412602033966706694|                   null|                 null|                     null|               null|                   null|          false|  in|                         null|              null|          0|                null|               null|                null|                   null|          0|            0|    false|{null, null, Wed ...|<a href=\"http://t...|     RT @nct_menfess: ...|1625625544431|    false|{false, Sat Dec 0...|\n",
      "|{60e5140936acd74d...|        null|       null|Wed Jul 07 02:39:...|              null|{[], [{null, pic....|{[{null, pic.twit...|                null|             0|    false|         low|null|1412602033975164929|1412602033975164929|                   null|                 null|                     null|               null|                   null|          false|  in|                         null|             false|          0|                null|               null|                null|                   null|          0|            0|    false|{null, null, Tue ...|<a href=\"http://t...|     RT @amourhyuckie:...|1625625544433|    false|{false, Mon Oct 1...|\n",
      "|{60e5140e36acd74d...|        null|       null|Wed Jul 07 02:39:...|              null|  {[], null, [], []}|                null|                null|             0|    false|         low|null|1412602033832669184|1412602033832669184|                   null|                 null|                     null|               null|                   null|           true|  en|                         null|              null|          0|{null, null, Tue ...|1412475912063434756| 1412475912063434756|   {https://twitter....|          0|            0|    false|                null|<a href=\"https://...|     My NCT ult is Hae...|1625625544399|    false|{false, Tue Apr 2...|\n",
      "|{60e5141336acd74d...|        null|       null|Wed Jul 07 02:39:...|              null|{[{[52, 59], NCT1...|{[{null, pic.twit...|                null|             0|    false|         low|null|1412602034004496395|1412602034004496395|                   null|                 null|                     null|               null|                   null|          false|  en|                         null|             false|          0|                null|               null|                null|                   null|          0|            0|    false|{null, null, Tue ...|<a href=\"http://t...|     RT @NCTupdating: ...|1625625544440|    false|{false, Thu Oct 1...|\n",
      "|{60e5141836acd74d...|        null|       null|Wed Jul 07 02:39:...|              null|{[], null, [], [{...|                null|                null|             0|    false|         low|null|1412602034239414273|1412602034239414273|                   null|                 null|                     null|               null|                   null|          false|  th|                         null|              null|          0|                null|               null|                null|                   null|          0|            0|    false|{null, null, Wed ...|<a href=\"http://t...|     RT @TAEIL_TH: 🌹:...|1625625544496|    false|{false, Sun Nov 1...|\n",
      "|{60e5141d36acd74d...|        null|       null|Wed Jul 07 02:39:...|              null|{[], null, [], [{...|                null|                null|             0|    false|         low|null|1412602034306519040|1412602034306519040|                   null|                 null|                     null|               null|                   null|          false|  in|                         null|              null|          0|                null|               null|                null|                   null|          0|            0|    false|{null, null, Wed ...|<a href=\"http://t...|     RT @nct_menfess: ...|1625625544512|    false|{false, Thu Feb 1...|\n",
      "|{60e5142236acd74d...|        null|       null|Wed Jul 07 02:39:...|              null|{[{[119, 130], ตล...|                null|                null|             0|    false|         low|null|1412602034960752644|1412602034960752644|                   null|                 null|                     null|               null|                   null|          false|  th|                         null|              null|          0|                null|               null|                null|                   null|          0|            0|    false|{null, null, Wed ...|<a href=\"http://t...|     RT @jinnie7777_: ...|1625625544668|    false|{false, Thu May 1...|\n",
      "|{60e5142736acd74d...|        null|       null|Wed Jul 07 02:39:...|              null|{[{[38, 57], OURL...|{[{null, pic.twit...|                null|             0|    false|         low|null|1412602034839244804|1412602034839244804|                   null|                 null|                     null|               null|                   null|          false|  en|                         null|             false|          0|                null|               null|                null|                   null|          0|            0|    false|{null, null, Tue ...|<a href=\"http://t...|     RT @JJHLOOKS: 5 y...|1625625544639|    false|{false, Thu Jun 1...|\n",
      "+--------------------+------------+-----------+--------------------+------------------+--------------------+--------------------+--------------------+--------------+---------+------------+----+-------------------+-------------------+-----------------------+---------------------+-------------------------+-------------------+-----------------------+---------------+----+-----------------------------+------------------+-----------+--------------------+-------------------+--------------------+-----------------------+-----------+-------------+---------+--------------------+--------------------+-------------------------+-------------+---------+--------------------+\n",
      "only showing top 20 rows\n",
      "\n",
      "root\n",
      " |-- _id: struct (nullable = true)\n",
      " |    |-- oid: string (nullable = true)\n",
      " |-- contributors: null (nullable = true)\n",
      " |-- coordinates: null (nullable = true)\n",
      " |-- created_at: string (nullable = true)\n",
      " |-- display_text_range: array (nullable = true)\n",
      " |    |-- element: long (containsNull = true)\n",
      " |-- entities: struct (nullable = true)\n",
      " |    |-- hashtags: array (nullable = true)\n",
      " |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |-- text: string (nullable = true)\n",
      " |    |-- media: array (nullable = true)\n",
      " |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |-- additional_media_info: struct (nullable = true)\n",
      " |    |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |    |-- embeddable: boolean (nullable = true)\n",
      " |    |    |    |    |-- monetizable: boolean (nullable = true)\n",
      " |    |    |    |    |-- title: string (nullable = true)\n",
      " |    |    |    |-- display_url: string (nullable = true)\n",
      " |    |    |    |-- expanded_url: string (nullable = true)\n",
      " |    |    |    |-- id: long (nullable = true)\n",
      " |    |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |-- media_url: string (nullable = true)\n",
      " |    |    |    |-- media_url_https: string (nullable = true)\n",
      " |    |    |    |-- sizes: struct (nullable = true)\n",
      " |    |    |    |    |-- small: struct (nullable = true)\n",
      " |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |-- large: struct (nullable = true)\n",
      " |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |-- thumb: struct (nullable = true)\n",
      " |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |-- medium: struct (nullable = true)\n",
      " |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |-- source_status_id: long (nullable = true)\n",
      " |    |    |    |-- source_status_id_str: string (nullable = true)\n",
      " |    |    |    |-- source_user_id: long (nullable = true)\n",
      " |    |    |    |-- source_user_id_str: string (nullable = true)\n",
      " |    |    |    |-- type: string (nullable = true)\n",
      " |    |    |    |-- url: string (nullable = true)\n",
      " |    |-- urls: array (nullable = true)\n",
      " |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |-- display_url: string (nullable = true)\n",
      " |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |-- expanded_url: string (nullable = true)\n",
      " |    |    |    |-- url: string (nullable = true)\n",
      " |    |-- user_mentions: array (nullable = true)\n",
      " |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |-- screen_name: string (nullable = true)\n",
      " |    |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |    |-- name: string (nullable = true)\n",
      " |    |    |    |-- id: long (nullable = true)\n",
      " |-- extended_entities: struct (nullable = true)\n",
      " |    |-- media: array (nullable = true)\n",
      " |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |-- additional_media_info: struct (nullable = true)\n",
      " |    |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |    |-- embeddable: boolean (nullable = true)\n",
      " |    |    |    |    |-- monetizable: boolean (nullable = true)\n",
      " |    |    |    |    |-- title: string (nullable = true)\n",
      " |    |    |    |-- display_url: string (nullable = true)\n",
      " |    |    |    |-- expanded_url: string (nullable = true)\n",
      " |    |    |    |-- id: long (nullable = true)\n",
      " |    |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |-- media_url: string (nullable = true)\n",
      " |    |    |    |-- media_url_https: string (nullable = true)\n",
      " |    |    |    |-- sizes: struct (nullable = true)\n",
      " |    |    |    |    |-- small: struct (nullable = true)\n",
      " |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |-- large: struct (nullable = true)\n",
      " |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |-- thumb: struct (nullable = true)\n",
      " |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |-- medium: struct (nullable = true)\n",
      " |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |-- source_status_id: long (nullable = true)\n",
      " |    |    |    |-- source_status_id_str: string (nullable = true)\n",
      " |    |    |    |-- source_user_id: long (nullable = true)\n",
      " |    |    |    |-- source_user_id_str: string (nullable = true)\n",
      " |    |    |    |-- type: string (nullable = true)\n",
      " |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |    |-- video_info: struct (nullable = true)\n",
      " |    |    |    |    |-- aspect_ratio: array (nullable = true)\n",
      " |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |-- duration_millis: long (nullable = true)\n",
      " |    |    |    |    |-- variants: array (nullable = true)\n",
      " |    |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |    |-- bitrate: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- content_type: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- url: string (nullable = true)\n",
      " |-- extended_tweet: struct (nullable = true)\n",
      " |    |-- display_text_range: array (nullable = true)\n",
      " |    |    |-- element: long (containsNull = true)\n",
      " |    |-- entities: struct (nullable = true)\n",
      " |    |    |-- hashtags: array (nullable = true)\n",
      " |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |-- text: string (nullable = true)\n",
      " |    |    |-- media: array (nullable = true)\n",
      " |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |    |-- display_url: string (nullable = true)\n",
      " |    |    |    |    |-- expanded_url: string (nullable = true)\n",
      " |    |    |    |    |-- id: long (nullable = true)\n",
      " |    |    |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |-- media_url: string (nullable = true)\n",
      " |    |    |    |    |-- media_url_https: string (nullable = true)\n",
      " |    |    |    |    |-- sizes: struct (nullable = true)\n",
      " |    |    |    |    |    |-- small: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |-- large: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |-- thumb: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |-- medium: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |-- source_status_id: long (nullable = true)\n",
      " |    |    |    |    |-- source_status_id_str: string (nullable = true)\n",
      " |    |    |    |    |-- source_user_id: long (nullable = true)\n",
      " |    |    |    |    |-- source_user_id_str: string (nullable = true)\n",
      " |    |    |    |    |-- type: string (nullable = true)\n",
      " |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |    |    |-- video_info: struct (nullable = true)\n",
      " |    |    |    |    |    |-- aspect_ratio: array (nullable = true)\n",
      " |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |-- variants: array (nullable = true)\n",
      " |    |    |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |    |    |-- content_type: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- bitrate: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |-- urls: array (nullable = true)\n",
      " |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |-- display_url: string (nullable = true)\n",
      " |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |-- expanded_url: string (nullable = true)\n",
      " |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |-- user_mentions: array (nullable = true)\n",
      " |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |-- screen_name: string (nullable = true)\n",
      " |    |    |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |    |    |-- name: string (nullable = true)\n",
      " |    |    |    |    |-- id: long (nullable = true)\n",
      " |    |-- extended_entities: struct (nullable = true)\n",
      " |    |    |-- media: array (nullable = true)\n",
      " |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |    |-- display_url: string (nullable = true)\n",
      " |    |    |    |    |-- expanded_url: string (nullable = true)\n",
      " |    |    |    |    |-- id: long (nullable = true)\n",
      " |    |    |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |-- media_url: string (nullable = true)\n",
      " |    |    |    |    |-- media_url_https: string (nullable = true)\n",
      " |    |    |    |    |-- sizes: struct (nullable = true)\n",
      " |    |    |    |    |    |-- small: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |-- large: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |-- thumb: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |-- medium: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |-- source_status_id: long (nullable = true)\n",
      " |    |    |    |    |-- source_status_id_str: string (nullable = true)\n",
      " |    |    |    |    |-- source_user_id: long (nullable = true)\n",
      " |    |    |    |    |-- source_user_id_str: string (nullable = true)\n",
      " |    |    |    |    |-- type: string (nullable = true)\n",
      " |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |    |    |-- video_info: struct (nullable = true)\n",
      " |    |    |    |    |    |-- aspect_ratio: array (nullable = true)\n",
      " |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |-- variants: array (nullable = true)\n",
      " |    |    |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |    |    |-- content_type: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- bitrate: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |-- full_text: string (nullable = true)\n",
      " |-- favorite_count: long (nullable = true)\n",
      " |-- favorited: boolean (nullable = true)\n",
      " |-- filter_level: string (nullable = true)\n",
      " |-- geo: null (nullable = true)\n",
      " |-- id: long (nullable = true)\n",
      " |-- id_str: string (nullable = true)\n",
      " |-- in_reply_to_screen_name: string (nullable = true)\n",
      " |-- in_reply_to_status_id: long (nullable = true)\n",
      " |-- in_reply_to_status_id_str: string (nullable = true)\n",
      " |-- in_reply_to_user_id: long (nullable = true)\n",
      " |-- in_reply_to_user_id_str: string (nullable = true)\n",
      " |-- is_quote_status: boolean (nullable = true)\n",
      " |-- lang: string (nullable = true)\n",
      " |-- place: struct (nullable = true)\n",
      " |    |-- country_code: string (nullable = true)\n",
      " |    |-- country: string (nullable = true)\n",
      " |    |-- full_name: string (nullable = true)\n",
      " |    |-- bounding_box: struct (nullable = true)\n",
      " |    |    |-- coordinates: array (nullable = true)\n",
      " |    |    |    |-- element: array (containsNull = true)\n",
      " |    |    |    |    |-- element: array (containsNull = true)\n",
      " |    |    |    |    |    |-- element: double (containsNull = true)\n",
      " |    |    |-- type: string (nullable = true)\n",
      " |    |-- place_type: string (nullable = true)\n",
      " |    |-- name: string (nullable = true)\n",
      " |    |-- id: string (nullable = true)\n",
      " |    |-- url: string (nullable = true)\n",
      " |-- possibly_sensitive: boolean (nullable = true)\n",
      " |-- quote_count: long (nullable = true)\n",
      " |-- quoted_status: struct (nullable = true)\n",
      " |    |-- contributors: null (nullable = true)\n",
      " |    |-- coordinates: null (nullable = true)\n",
      " |    |-- created_at: string (nullable = true)\n",
      " |    |-- display_text_range: array (nullable = true)\n",
      " |    |    |-- element: long (containsNull = true)\n",
      " |    |-- entities: struct (nullable = true)\n",
      " |    |    |-- hashtags: array (nullable = true)\n",
      " |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |-- text: string (nullable = true)\n",
      " |    |    |-- media: array (nullable = true)\n",
      " |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |-- display_url: string (nullable = true)\n",
      " |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |-- sizes: struct (nullable = true)\n",
      " |    |    |    |    |    |-- small: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |-- large: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |-- thumb: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |-- medium: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |    |    |-- expanded_url: string (nullable = true)\n",
      " |    |    |    |    |-- media_url_https: string (nullable = true)\n",
      " |    |    |    |    |-- id: long (nullable = true)\n",
      " |    |    |    |    |-- type: string (nullable = true)\n",
      " |    |    |    |    |-- media_url: string (nullable = true)\n",
      " |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |-- urls: array (nullable = true)\n",
      " |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |-- display_url: string (nullable = true)\n",
      " |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |-- expanded_url: string (nullable = true)\n",
      " |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |-- user_mentions: array (nullable = true)\n",
      " |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |-- screen_name: string (nullable = true)\n",
      " |    |    |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |    |    |-- name: string (nullable = true)\n",
      " |    |    |    |    |-- id: long (nullable = true)\n",
      " |    |-- extended_entities: struct (nullable = true)\n",
      " |    |    |-- media: array (nullable = true)\n",
      " |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |-- display_url: string (nullable = true)\n",
      " |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |-- sizes: struct (nullable = true)\n",
      " |    |    |    |    |    |-- small: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |-- large: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |-- thumb: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |-- medium: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |    |    |-- expanded_url: string (nullable = true)\n",
      " |    |    |    |    |-- media_url_https: string (nullable = true)\n",
      " |    |    |    |    |-- id: long (nullable = true)\n",
      " |    |    |    |    |-- type: string (nullable = true)\n",
      " |    |    |    |    |-- media_url: string (nullable = true)\n",
      " |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |-- extended_tweet: struct (nullable = true)\n",
      " |    |    |-- display_text_range: array (nullable = true)\n",
      " |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |-- entities: struct (nullable = true)\n",
      " |    |    |    |-- hashtags: array (nullable = true)\n",
      " |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |-- text: string (nullable = true)\n",
      " |    |    |    |-- media: array (nullable = true)\n",
      " |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |-- additional_media_info: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- embeddable: boolean (nullable = true)\n",
      " |    |    |    |    |    |    |-- monetizable: boolean (nullable = true)\n",
      " |    |    |    |    |    |    |-- title: string (nullable = true)\n",
      " |    |    |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |    |    |-- display_url: string (nullable = true)\n",
      " |    |    |    |    |    |-- expanded_url: string (nullable = true)\n",
      " |    |    |    |    |    |-- id: long (nullable = true)\n",
      " |    |    |    |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |-- media_url: string (nullable = true)\n",
      " |    |    |    |    |    |-- media_url_https: string (nullable = true)\n",
      " |    |    |    |    |    |-- sizes: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- small: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- large: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- thumb: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- medium: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |-- source_status_id: long (nullable = true)\n",
      " |    |    |    |    |    |-- source_status_id_str: string (nullable = true)\n",
      " |    |    |    |    |    |-- source_user_id: long (nullable = true)\n",
      " |    |    |    |    |    |-- source_user_id_str: string (nullable = true)\n",
      " |    |    |    |    |    |-- type: string (nullable = true)\n",
      " |    |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |    |    |    |-- video_info: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- aspect_ratio: array (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |    |-- duration_millis: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- variants: array (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |    |    |    |-- bitrate: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- content_type: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |    |-- urls: array (nullable = true)\n",
      " |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |-- display_url: string (nullable = true)\n",
      " |    |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |-- expanded_url: string (nullable = true)\n",
      " |    |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |    |-- user_mentions: array (nullable = true)\n",
      " |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |-- screen_name: string (nullable = true)\n",
      " |    |    |    |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |    |    |    |-- name: string (nullable = true)\n",
      " |    |    |    |    |    |-- id: long (nullable = true)\n",
      " |    |    |-- extended_entities: struct (nullable = true)\n",
      " |    |    |    |-- media: array (nullable = true)\n",
      " |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |-- additional_media_info: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- embeddable: boolean (nullable = true)\n",
      " |    |    |    |    |    |    |-- monetizable: boolean (nullable = true)\n",
      " |    |    |    |    |    |    |-- title: string (nullable = true)\n",
      " |    |    |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |    |    |-- display_url: string (nullable = true)\n",
      " |    |    |    |    |    |-- expanded_url: string (nullable = true)\n",
      " |    |    |    |    |    |-- id: long (nullable = true)\n",
      " |    |    |    |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |-- media_url: string (nullable = true)\n",
      " |    |    |    |    |    |-- media_url_https: string (nullable = true)\n",
      " |    |    |    |    |    |-- sizes: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- small: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- large: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- thumb: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- medium: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |-- source_status_id: long (nullable = true)\n",
      " |    |    |    |    |    |-- source_status_id_str: string (nullable = true)\n",
      " |    |    |    |    |    |-- source_user_id: long (nullable = true)\n",
      " |    |    |    |    |    |-- source_user_id_str: string (nullable = true)\n",
      " |    |    |    |    |    |-- type: string (nullable = true)\n",
      " |    |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |    |    |    |-- video_info: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- aspect_ratio: array (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |    |-- duration_millis: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- variants: array (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |    |    |    |-- bitrate: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- content_type: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |-- full_text: string (nullable = true)\n",
      " |    |-- favorite_count: long (nullable = true)\n",
      " |    |-- favorited: boolean (nullable = true)\n",
      " |    |-- filter_level: string (nullable = true)\n",
      " |    |-- geo: null (nullable = true)\n",
      " |    |-- id: long (nullable = true)\n",
      " |    |-- id_str: string (nullable = true)\n",
      " |    |-- in_reply_to_screen_name: string (nullable = true)\n",
      " |    |-- in_reply_to_status_id: long (nullable = true)\n",
      " |    |-- in_reply_to_status_id_str: string (nullable = true)\n",
      " |    |-- in_reply_to_user_id: long (nullable = true)\n",
      " |    |-- in_reply_to_user_id_str: string (nullable = true)\n",
      " |    |-- is_quote_status: boolean (nullable = true)\n",
      " |    |-- lang: string (nullable = true)\n",
      " |    |-- place: null (nullable = true)\n",
      " |    |-- possibly_sensitive: boolean (nullable = true)\n",
      " |    |-- quote_count: long (nullable = true)\n",
      " |    |-- quoted_status_id: long (nullable = true)\n",
      " |    |-- quoted_status_id_str: string (nullable = true)\n",
      " |    |-- reply_count: long (nullable = true)\n",
      " |    |-- retweet_count: long (nullable = true)\n",
      " |    |-- retweeted: boolean (nullable = true)\n",
      " |    |-- source: string (nullable = true)\n",
      " |    |-- text: string (nullable = true)\n",
      " |    |-- truncated: boolean (nullable = true)\n",
      " |    |-- user: struct (nullable = true)\n",
      " |    |    |-- contributors_enabled: boolean (nullable = true)\n",
      " |    |    |-- created_at: string (nullable = true)\n",
      " |    |    |-- default_profile: boolean (nullable = true)\n",
      " |    |    |-- default_profile_image: boolean (nullable = true)\n",
      " |    |    |-- description: string (nullable = true)\n",
      " |    |    |-- favourites_count: long (nullable = true)\n",
      " |    |    |-- follow_request_sent: null (nullable = true)\n",
      " |    |    |-- followers_count: long (nullable = true)\n",
      " |    |    |-- following: null (nullable = true)\n",
      " |    |    |-- friends_count: long (nullable = true)\n",
      " |    |    |-- geo_enabled: boolean (nullable = true)\n",
      " |    |    |-- id: long (nullable = true)\n",
      " |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |-- is_translator: boolean (nullable = true)\n",
      " |    |    |-- lang: null (nullable = true)\n",
      " |    |    |-- listed_count: long (nullable = true)\n",
      " |    |    |-- location: string (nullable = true)\n",
      " |    |    |-- name: string (nullable = true)\n",
      " |    |    |-- notifications: null (nullable = true)\n",
      " |    |    |-- profile_background_color: string (nullable = true)\n",
      " |    |    |-- profile_background_image_url: string (nullable = true)\n",
      " |    |    |-- profile_background_image_url_https: string (nullable = true)\n",
      " |    |    |-- profile_background_tile: boolean (nullable = true)\n",
      " |    |    |-- profile_banner_url: string (nullable = true)\n",
      " |    |    |-- profile_image_url: string (nullable = true)\n",
      " |    |    |-- profile_image_url_https: string (nullable = true)\n",
      " |    |    |-- profile_link_color: string (nullable = true)\n",
      " |    |    |-- profile_sidebar_border_color: string (nullable = true)\n",
      " |    |    |-- profile_sidebar_fill_color: string (nullable = true)\n",
      " |    |    |-- profile_text_color: string (nullable = true)\n",
      " |    |    |-- profile_use_background_image: boolean (nullable = true)\n",
      " |    |    |-- protected: boolean (nullable = true)\n",
      " |    |    |-- screen_name: string (nullable = true)\n",
      " |    |    |-- statuses_count: long (nullable = true)\n",
      " |    |    |-- time_zone: null (nullable = true)\n",
      " |    |    |-- translator_type: string (nullable = true)\n",
      " |    |    |-- url: string (nullable = true)\n",
      " |    |    |-- utc_offset: null (nullable = true)\n",
      " |    |    |-- verified: boolean (nullable = true)\n",
      " |-- quoted_status_id: long (nullable = true)\n",
      " |-- quoted_status_id_str: string (nullable = true)\n",
      " |-- quoted_status_permalink: struct (nullable = true)\n",
      " |    |-- expanded: string (nullable = true)\n",
      " |    |-- display: string (nullable = true)\n",
      " |    |-- url: string (nullable = true)\n",
      " |-- reply_count: long (nullable = true)\n",
      " |-- retweet_count: long (nullable = true)\n",
      " |-- retweeted: boolean (nullable = true)\n",
      " |-- retweeted_status: struct (nullable = true)\n",
      " |    |-- contributors: null (nullable = true)\n",
      " |    |-- coordinates: null (nullable = true)\n",
      " |    |-- created_at: string (nullable = true)\n",
      " |    |-- display_text_range: array (nullable = true)\n",
      " |    |    |-- element: long (containsNull = true)\n",
      " |    |-- entities: struct (nullable = true)\n",
      " |    |    |-- hashtags: array (nullable = true)\n",
      " |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |-- text: string (nullable = true)\n",
      " |    |    |-- media: array (nullable = true)\n",
      " |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |-- additional_media_info: struct (nullable = true)\n",
      " |    |    |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |    |    |-- embeddable: boolean (nullable = true)\n",
      " |    |    |    |    |    |-- monetizable: boolean (nullable = true)\n",
      " |    |    |    |    |    |-- title: string (nullable = true)\n",
      " |    |    |    |    |-- display_url: string (nullable = true)\n",
      " |    |    |    |    |-- expanded_url: string (nullable = true)\n",
      " |    |    |    |    |-- id: long (nullable = true)\n",
      " |    |    |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |-- media_url: string (nullable = true)\n",
      " |    |    |    |    |-- media_url_https: string (nullable = true)\n",
      " |    |    |    |    |-- sizes: struct (nullable = true)\n",
      " |    |    |    |    |    |-- small: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |-- large: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |-- thumb: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |-- medium: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |-- source_status_id: long (nullable = true)\n",
      " |    |    |    |    |-- source_status_id_str: string (nullable = true)\n",
      " |    |    |    |    |-- source_user_id: long (nullable = true)\n",
      " |    |    |    |    |-- source_user_id_str: string (nullable = true)\n",
      " |    |    |    |    |-- type: string (nullable = true)\n",
      " |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |-- urls: array (nullable = true)\n",
      " |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |-- display_url: string (nullable = true)\n",
      " |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |-- expanded_url: string (nullable = true)\n",
      " |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |-- user_mentions: array (nullable = true)\n",
      " |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |-- screen_name: string (nullable = true)\n",
      " |    |    |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |    |    |-- name: string (nullable = true)\n",
      " |    |    |    |    |-- id: long (nullable = true)\n",
      " |    |-- extended_entities: struct (nullable = true)\n",
      " |    |    |-- media: array (nullable = true)\n",
      " |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |-- additional_media_info: struct (nullable = true)\n",
      " |    |    |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |    |    |-- embeddable: boolean (nullable = true)\n",
      " |    |    |    |    |    |-- monetizable: boolean (nullable = true)\n",
      " |    |    |    |    |    |-- title: string (nullable = true)\n",
      " |    |    |    |    |-- display_url: string (nullable = true)\n",
      " |    |    |    |    |-- expanded_url: string (nullable = true)\n",
      " |    |    |    |    |-- id: long (nullable = true)\n",
      " |    |    |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |-- media_url: string (nullable = true)\n",
      " |    |    |    |    |-- media_url_https: string (nullable = true)\n",
      " |    |    |    |    |-- sizes: struct (nullable = true)\n",
      " |    |    |    |    |    |-- small: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |-- large: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |-- thumb: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |-- medium: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |-- source_status_id: long (nullable = true)\n",
      " |    |    |    |    |-- source_status_id_str: string (nullable = true)\n",
      " |    |    |    |    |-- source_user_id: long (nullable = true)\n",
      " |    |    |    |    |-- source_user_id_str: string (nullable = true)\n",
      " |    |    |    |    |-- type: string (nullable = true)\n",
      " |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |    |    |-- video_info: struct (nullable = true)\n",
      " |    |    |    |    |    |-- aspect_ratio: array (nullable = true)\n",
      " |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |-- duration_millis: long (nullable = true)\n",
      " |    |    |    |    |    |-- variants: array (nullable = true)\n",
      " |    |    |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |    |    |-- bitrate: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- content_type: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |-- extended_tweet: struct (nullable = true)\n",
      " |    |    |-- display_text_range: array (nullable = true)\n",
      " |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |-- entities: struct (nullable = true)\n",
      " |    |    |    |-- hashtags: array (nullable = true)\n",
      " |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |-- text: string (nullable = true)\n",
      " |    |    |    |-- media: array (nullable = true)\n",
      " |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |-- additional_media_info: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- embeddable: boolean (nullable = true)\n",
      " |    |    |    |    |    |    |-- monetizable: boolean (nullable = true)\n",
      " |    |    |    |    |    |    |-- title: string (nullable = true)\n",
      " |    |    |    |    |    |-- display_url: string (nullable = true)\n",
      " |    |    |    |    |    |-- expanded_url: string (nullable = true)\n",
      " |    |    |    |    |    |-- id: long (nullable = true)\n",
      " |    |    |    |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |-- media_url: string (nullable = true)\n",
      " |    |    |    |    |    |-- media_url_https: string (nullable = true)\n",
      " |    |    |    |    |    |-- sizes: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- small: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- large: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- thumb: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- medium: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |-- source_status_id: long (nullable = true)\n",
      " |    |    |    |    |    |-- source_status_id_str: string (nullable = true)\n",
      " |    |    |    |    |    |-- source_user_id: long (nullable = true)\n",
      " |    |    |    |    |    |-- source_user_id_str: string (nullable = true)\n",
      " |    |    |    |    |    |-- type: string (nullable = true)\n",
      " |    |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |    |    |    |-- video_info: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- aspect_ratio: array (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |    |-- duration_millis: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- variants: array (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |    |    |    |-- bitrate: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- content_type: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |    |-- urls: array (nullable = true)\n",
      " |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |-- display_url: string (nullable = true)\n",
      " |    |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |-- expanded_url: string (nullable = true)\n",
      " |    |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |    |-- user_mentions: array (nullable = true)\n",
      " |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |-- screen_name: string (nullable = true)\n",
      " |    |    |    |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |    |    |    |-- name: string (nullable = true)\n",
      " |    |    |    |    |    |-- id: long (nullable = true)\n",
      " |    |    |-- extended_entities: struct (nullable = true)\n",
      " |    |    |    |-- media: array (nullable = true)\n",
      " |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |-- additional_media_info: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- embeddable: boolean (nullable = true)\n",
      " |    |    |    |    |    |    |-- monetizable: boolean (nullable = true)\n",
      " |    |    |    |    |    |    |-- title: string (nullable = true)\n",
      " |    |    |    |    |    |-- display_url: string (nullable = true)\n",
      " |    |    |    |    |    |-- expanded_url: string (nullable = true)\n",
      " |    |    |    |    |    |-- id: long (nullable = true)\n",
      " |    |    |    |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |-- media_url: string (nullable = true)\n",
      " |    |    |    |    |    |-- media_url_https: string (nullable = true)\n",
      " |    |    |    |    |    |-- sizes: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- small: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- large: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- thumb: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- medium: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |-- source_status_id: long (nullable = true)\n",
      " |    |    |    |    |    |-- source_status_id_str: string (nullable = true)\n",
      " |    |    |    |    |    |-- source_user_id: long (nullable = true)\n",
      " |    |    |    |    |    |-- source_user_id_str: string (nullable = true)\n",
      " |    |    |    |    |    |-- type: string (nullable = true)\n",
      " |    |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |    |    |    |-- video_info: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- aspect_ratio: array (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |    |-- duration_millis: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- variants: array (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |    |    |    |-- bitrate: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- content_type: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |-- full_text: string (nullable = true)\n",
      " |    |-- favorite_count: long (nullable = true)\n",
      " |    |-- favorited: boolean (nullable = true)\n",
      " |    |-- filter_level: string (nullable = true)\n",
      " |    |-- geo: null (nullable = true)\n",
      " |    |-- id: long (nullable = true)\n",
      " |    |-- id_str: string (nullable = true)\n",
      " |    |-- in_reply_to_screen_name: string (nullable = true)\n",
      " |    |-- in_reply_to_status_id: long (nullable = true)\n",
      " |    |-- in_reply_to_status_id_str: string (nullable = true)\n",
      " |    |-- in_reply_to_user_id: long (nullable = true)\n",
      " |    |-- in_reply_to_user_id_str: string (nullable = true)\n",
      " |    |-- is_quote_status: boolean (nullable = true)\n",
      " |    |-- lang: string (nullable = true)\n",
      " |    |-- place: null (nullable = true)\n",
      " |    |-- possibly_sensitive: boolean (nullable = true)\n",
      " |    |-- quote_count: long (nullable = true)\n",
      " |    |-- quoted_status: struct (nullable = true)\n",
      " |    |    |-- contributors: null (nullable = true)\n",
      " |    |    |-- coordinates: null (nullable = true)\n",
      " |    |    |-- created_at: string (nullable = true)\n",
      " |    |    |-- display_text_range: array (nullable = true)\n",
      " |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |-- entities: struct (nullable = true)\n",
      " |    |    |    |-- hashtags: array (nullable = true)\n",
      " |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |-- text: string (nullable = true)\n",
      " |    |    |    |-- media: array (nullable = true)\n",
      " |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |-- display_url: string (nullable = true)\n",
      " |    |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |-- sizes: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- small: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- large: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- thumb: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- medium: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |    |    |    |-- expanded_url: string (nullable = true)\n",
      " |    |    |    |    |    |-- media_url_https: string (nullable = true)\n",
      " |    |    |    |    |    |-- id: long (nullable = true)\n",
      " |    |    |    |    |    |-- type: string (nullable = true)\n",
      " |    |    |    |    |    |-- media_url: string (nullable = true)\n",
      " |    |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |    |-- urls: array (nullable = true)\n",
      " |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |-- display_url: string (nullable = true)\n",
      " |    |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |-- expanded_url: string (nullable = true)\n",
      " |    |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |-- extended_entities: struct (nullable = true)\n",
      " |    |    |    |-- media: array (nullable = true)\n",
      " |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |-- display_url: string (nullable = true)\n",
      " |    |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |-- sizes: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- small: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- large: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- thumb: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- medium: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |    |    |    |-- expanded_url: string (nullable = true)\n",
      " |    |    |    |    |    |-- media_url_https: string (nullable = true)\n",
      " |    |    |    |    |    |-- id: long (nullable = true)\n",
      " |    |    |    |    |    |-- type: string (nullable = true)\n",
      " |    |    |    |    |    |-- media_url: string (nullable = true)\n",
      " |    |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |-- extended_tweet: struct (nullable = true)\n",
      " |    |    |    |-- display_text_range: array (nullable = true)\n",
      " |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |-- entities: struct (nullable = true)\n",
      " |    |    |    |    |-- hashtags: array (nullable = true)\n",
      " |    |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |    |-- text: string (nullable = true)\n",
      " |    |    |    |    |-- media: array (nullable = true)\n",
      " |    |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |    |-- additional_media_info: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- embeddable: boolean (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- monetizable: boolean (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- title: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- display_url: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- expanded_url: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- id: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |    |-- media_url: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- media_url_https: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- sizes: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- small: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- large: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- thumb: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- medium: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- source_status_id: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- source_status_id_str: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- source_user_id: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- source_user_id_str: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- type: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- video_info: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- aspect_ratio: array (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |    |    |-- duration_millis: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- variants: array (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |    |    |    |    |-- bitrate: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |    |-- content_type: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |    |    |-- urls: array (nullable = true)\n",
      " |    |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |    |-- display_url: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |    |-- expanded_url: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |    |-- extended_entities: struct (nullable = true)\n",
      " |    |    |    |    |-- media: array (nullable = true)\n",
      " |    |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |    |-- additional_media_info: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- embeddable: boolean (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- monetizable: boolean (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- title: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- display_url: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- expanded_url: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- id: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |    |-- media_url: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- media_url_https: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- sizes: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- small: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- large: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- thumb: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- medium: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- source_status_id: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- source_status_id_str: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- source_user_id: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- source_user_id_str: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- type: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- video_info: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- aspect_ratio: array (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |    |    |-- duration_millis: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- variants: array (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |    |    |    |    |-- bitrate: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |    |-- content_type: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |    |-- full_text: string (nullable = true)\n",
      " |    |    |-- favorite_count: long (nullable = true)\n",
      " |    |    |-- favorited: boolean (nullable = true)\n",
      " |    |    |-- filter_level: string (nullable = true)\n",
      " |    |    |-- geo: null (nullable = true)\n",
      " |    |    |-- id: long (nullable = true)\n",
      " |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |-- in_reply_to_screen_name: null (nullable = true)\n",
      " |    |    |-- in_reply_to_status_id: null (nullable = true)\n",
      " |    |    |-- in_reply_to_status_id_str: null (nullable = true)\n",
      " |    |    |-- in_reply_to_user_id: null (nullable = true)\n",
      " |    |    |-- in_reply_to_user_id_str: null (nullable = true)\n",
      " |    |    |-- is_quote_status: boolean (nullable = true)\n",
      " |    |    |-- lang: string (nullable = true)\n",
      " |    |    |-- place: null (nullable = true)\n",
      " |    |    |-- possibly_sensitive: boolean (nullable = true)\n",
      " |    |    |-- quote_count: long (nullable = true)\n",
      " |    |    |-- quoted_status_id: long (nullable = true)\n",
      " |    |    |-- quoted_status_id_str: string (nullable = true)\n",
      " |    |    |-- reply_count: long (nullable = true)\n",
      " |    |    |-- retweet_count: long (nullable = true)\n",
      " |    |    |-- retweeted: boolean (nullable = true)\n",
      " |    |    |-- source: string (nullable = true)\n",
      " |    |    |-- text: string (nullable = true)\n",
      " |    |    |-- truncated: boolean (nullable = true)\n",
      " |    |    |-- user: struct (nullable = true)\n",
      " |    |    |    |-- contributors_enabled: boolean (nullable = true)\n",
      " |    |    |    |-- created_at: string (nullable = true)\n",
      " |    |    |    |-- default_profile: boolean (nullable = true)\n",
      " |    |    |    |-- default_profile_image: boolean (nullable = true)\n",
      " |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |-- favourites_count: long (nullable = true)\n",
      " |    |    |    |-- follow_request_sent: null (nullable = true)\n",
      " |    |    |    |-- followers_count: long (nullable = true)\n",
      " |    |    |    |-- following: null (nullable = true)\n",
      " |    |    |    |-- friends_count: long (nullable = true)\n",
      " |    |    |    |-- geo_enabled: boolean (nullable = true)\n",
      " |    |    |    |-- id: long (nullable = true)\n",
      " |    |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |    |-- is_translator: boolean (nullable = true)\n",
      " |    |    |    |-- lang: null (nullable = true)\n",
      " |    |    |    |-- listed_count: long (nullable = true)\n",
      " |    |    |    |-- location: string (nullable = true)\n",
      " |    |    |    |-- name: string (nullable = true)\n",
      " |    |    |    |-- notifications: null (nullable = true)\n",
      " |    |    |    |-- profile_background_color: string (nullable = true)\n",
      " |    |    |    |-- profile_background_image_url: string (nullable = true)\n",
      " |    |    |    |-- profile_background_image_url_https: string (nullable = true)\n",
      " |    |    |    |-- profile_background_tile: boolean (nullable = true)\n",
      " |    |    |    |-- profile_banner_url: string (nullable = true)\n",
      " |    |    |    |-- profile_image_url: string (nullable = true)\n",
      " |    |    |    |-- profile_image_url_https: string (nullable = true)\n",
      " |    |    |    |-- profile_link_color: string (nullable = true)\n",
      " |    |    |    |-- profile_sidebar_border_color: string (nullable = true)\n",
      " |    |    |    |-- profile_sidebar_fill_color: string (nullable = true)\n",
      " |    |    |    |-- profile_text_color: string (nullable = true)\n",
      " |    |    |    |-- profile_use_background_image: boolean (nullable = true)\n",
      " |    |    |    |-- protected: boolean (nullable = true)\n",
      " |    |    |    |-- screen_name: string (nullable = true)\n",
      " |    |    |    |-- statuses_count: long (nullable = true)\n",
      " |    |    |    |-- time_zone: null (nullable = true)\n",
      " |    |    |    |-- translator_type: string (nullable = true)\n",
      " |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |    |-- utc_offset: null (nullable = true)\n",
      " |    |    |    |-- verified: boolean (nullable = true)\n",
      " |    |-- quoted_status_id: long (nullable = true)\n",
      " |    |-- quoted_status_id_str: string (nullable = true)\n",
      " |    |-- quoted_status_permalink: struct (nullable = true)\n",
      " |    |    |-- expanded: string (nullable = true)\n",
      " |    |    |-- display: string (nullable = true)\n",
      " |    |    |-- url: string (nullable = true)\n",
      " |    |-- reply_count: long (nullable = true)\n",
      " |    |-- retweet_count: long (nullable = true)\n",
      " |    |-- retweeted: boolean (nullable = true)\n",
      " |    |-- source: string (nullable = true)\n",
      " |    |-- text: string (nullable = true)\n",
      " |    |-- truncated: boolean (nullable = true)\n",
      " |    |-- user: struct (nullable = true)\n",
      " |    |    |-- contributors_enabled: boolean (nullable = true)\n",
      " |    |    |-- created_at: string (nullable = true)\n",
      " |    |    |-- default_profile: boolean (nullable = true)\n",
      " |    |    |-- default_profile_image: boolean (nullable = true)\n",
      " |    |    |-- description: string (nullable = true)\n",
      " |    |    |-- favourites_count: long (nullable = true)\n",
      " |    |    |-- follow_request_sent: null (nullable = true)\n",
      " |    |    |-- followers_count: long (nullable = true)\n",
      " |    |    |-- following: null (nullable = true)\n",
      " |    |    |-- friends_count: long (nullable = true)\n",
      " |    |    |-- geo_enabled: boolean (nullable = true)\n",
      " |    |    |-- id: long (nullable = true)\n",
      " |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |-- is_translator: boolean (nullable = true)\n",
      " |    |    |-- lang: null (nullable = true)\n",
      " |    |    |-- listed_count: long (nullable = true)\n",
      " |    |    |-- location: string (nullable = true)\n",
      " |    |    |-- name: string (nullable = true)\n",
      " |    |    |-- notifications: null (nullable = true)\n",
      " |    |    |-- profile_background_color: string (nullable = true)\n",
      " |    |    |-- profile_background_image_url: string (nullable = true)\n",
      " |    |    |-- profile_background_image_url_https: string (nullable = true)\n",
      " |    |    |-- profile_background_tile: boolean (nullable = true)\n",
      " |    |    |-- profile_banner_url: string (nullable = true)\n",
      " |    |    |-- profile_image_url: string (nullable = true)\n",
      " |    |    |-- profile_image_url_https: string (nullable = true)\n",
      " |    |    |-- profile_link_color: string (nullable = true)\n",
      " |    |    |-- profile_sidebar_border_color: string (nullable = true)\n",
      " |    |    |-- profile_sidebar_fill_color: string (nullable = true)\n",
      " |    |    |-- profile_text_color: string (nullable = true)\n",
      " |    |    |-- profile_use_background_image: boolean (nullable = true)\n",
      " |    |    |-- protected: boolean (nullable = true)\n",
      " |    |    |-- screen_name: string (nullable = true)\n",
      " |    |    |-- statuses_count: long (nullable = true)\n",
      " |    |    |-- time_zone: null (nullable = true)\n",
      " |    |    |-- translator_type: string (nullable = true)\n",
      " |    |    |-- url: string (nullable = true)\n",
      " |    |    |-- utc_offset: null (nullable = true)\n",
      " |    |    |-- verified: boolean (nullable = true)\n",
      " |-- source: string (nullable = true)\n",
      " |-- text: string (nullable = true)\n",
      " |-- timestamp_ms: string (nullable = true)\n",
      " |-- truncated: boolean (nullable = true)\n",
      " |-- user: struct (nullable = true)\n",
      " |    |-- contributors_enabled: boolean (nullable = true)\n",
      " |    |-- created_at: string (nullable = true)\n",
      " |    |-- default_profile: boolean (nullable = true)\n",
      " |    |-- default_profile_image: boolean (nullable = true)\n",
      " |    |-- description: string (nullable = true)\n",
      " |    |-- favourites_count: long (nullable = true)\n",
      " |    |-- follow_request_sent: null (nullable = true)\n",
      " |    |-- followers_count: long (nullable = true)\n",
      " |    |-- following: null (nullable = true)\n",
      " |    |-- friends_count: long (nullable = true)\n",
      " |    |-- geo_enabled: boolean (nullable = true)\n",
      " |    |-- id: long (nullable = true)\n",
      " |    |-- id_str: string (nullable = true)\n",
      " |    |-- is_translator: boolean (nullable = true)\n",
      " |    |-- lang: null (nullable = true)\n",
      " |    |-- listed_count: long (nullable = true)\n",
      " |    |-- location: string (nullable = true)\n",
      " |    |-- name: string (nullable = true)\n",
      " |    |-- notifications: null (nullable = true)\n",
      " |    |-- profile_background_color: string (nullable = true)\n",
      " |    |-- profile_background_image_url: string (nullable = true)\n",
      " |    |-- profile_background_image_url_https: string (nullable = true)\n",
      " |    |-- profile_background_tile: boolean (nullable = true)\n",
      " |    |-- profile_banner_url: string (nullable = true)\n",
      " |    |-- profile_image_url: string (nullable = true)\n",
      " |    |-- profile_image_url_https: string (nullable = true)\n",
      " |    |-- profile_link_color: string (nullable = true)\n",
      " |    |-- profile_sidebar_border_color: string (nullable = true)\n",
      " |    |-- profile_sidebar_fill_color: string (nullable = true)\n",
      " |    |-- profile_text_color: string (nullable = true)\n",
      " |    |-- profile_use_background_image: boolean (nullable = true)\n",
      " |    |-- protected: boolean (nullable = true)\n",
      " |    |-- screen_name: string (nullable = true)\n",
      " |    |-- statuses_count: long (nullable = true)\n",
      " |    |-- time_zone: null (nullable = true)\n",
      " |    |-- translator_type: string (nullable = true)\n",
      " |    |-- url: string (nullable = true)\n",
      " |    |-- utc_offset: null (nullable = true)\n",
      " |    |-- verified: boolean (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df = spark.read.format(\"com.mongodb.spark.sql.DefaultSource\") \\\n",
    "            .option(\"spark.mongodb.input.uri\", 'mongodb://localhost:27017/nct.twitterdata') \\\n",
    "            .option(\"multiLine\",\"true\").load()\n",
    " \n",
    "df.show() # spark datafrmae 출력\n",
    "df.printSchema() # 스키마 출력\n",
    "# df.columns # 모든 컬럼명 리스트로 출력\n",
    "# df.count() # 행 개수 출력"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ad162b56",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+------------+-----------+--------------------+------------------+--------------------+--------------------+--------------+--------------+---------+------------+----+-------------------+-------------------+-----------------------+---------------------+-------------------------+-------------------+-----------------------+---------------+----+-----------------------------+------------------+-----------+--------------------+-------------------+--------------------+-----------------------+-----------+-------------+---------+--------------------+--------------------+-------------------------+-------------+---------+--------------------+\n",
      "|                 _id|contributors|coordinates|          created_at|display_text_range|            entities|   extended_entities|extended_tweet|favorite_count|favorited|filter_level| geo|                 id|             id_str|in_reply_to_screen_name|in_reply_to_status_id|in_reply_to_status_id_str|in_reply_to_user_id|in_reply_to_user_id_str|is_quote_status|lang|                        place|possibly_sensitive|quote_count|       quoted_status|   quoted_status_id|quoted_status_id_str|quoted_status_permalink|reply_count|retweet_count|retweeted|    retweeted_status|              source|                     text| timestamp_ms|truncated|                user|\n",
      "+--------------------+------------+-----------+--------------------+------------------+--------------------+--------------------+--------------+--------------+---------+------------+----+-------------------+-------------------+-----------------------+---------------------+-------------------------+-------------------+-----------------------+---------------+----+-----------------------------+------------------+-----------+--------------------+-------------------+--------------------+-----------------------+-----------+-------------+---------+--------------------+--------------------+-------------------------+-------------+---------+--------------------+\n",
      "|{60e51343ad59db3e...|        null|       null|Wed Jul 07 02:11:...|           [0, 85]|{[], [{null, pic....|{[{null, pic.twit...|          null|             0|    false|         low|null|1412595075171848192|1412595075171848192|                   null|                 null|                     null|               null|                   null|          false|  ko|{KR, 대한민국, 대한민국 창...|             false|          0|                null|               null|                null|                   null|          0|            0|    false|                null|<a href=\"http://t...|NCT DREAM(엔시티 드림)...|1625623885325|    false|{false, Wed Nov 1...|\n",
      "|{60e513cc36acd74d...|        null|       null|Wed Jul 07 02:39:...|              null|{[], null, [], [{...|                null|          null|             0|    false|         low|null|1412602031739604992|1412602031739604992|                   null|                 null|                     null|               null|                   null|          false|  in|                         null|              null|          0|                null|               null|                null|                   null|          0|            0|    false|{null, null, Tue ...|<a href=\"http://t...|     RT @Bubuyongi: NC...|1625625543900|    false|{false, Sun Dec 3...|\n",
      "|{60e513d136acd74d...|        null|       null|Wed Jul 07 02:39:...|          [13, 36]|{[], null, [{twit...|                null|          null|             0|    false|         low|null|1412602031647334405|1412602031647334405|            markleebase|  1412599759399886852|      1412599759399886852|1294253093580726272|    1294253093580726272|           true| und|                         null|             false|          0|{null, null, Wed ...|1412577812897165319| 1412577812897165319|   {https://twitter....|          0|            0|    false|                null|<a href=\"http://t...|     @markleebase http...|1625625543878|    false|{false, Sat May 0...|\n",
      "|{60e513d636acd74d...|        null|       null|Wed Jul 07 02:39:...|              null|{[], null, [{nave...|                null|          null|             0|    false|         low|null|1412602032138035205|1412602032138035205|                   null|                 null|                     null|               null|                   null|          false|  ko|                         null|             false|          0|                null|               null|                null|                   null|          0|            0|    false|{null, null, Wed ...|<a href=\"http://t...|  RT @mbcplusm: 주간아...|1625625543995|    false|{false, Tue Sep 3...|\n",
      "|{60e513db36acd74d...|        null|       null|Wed Jul 07 02:39:...|              null|{[], [{null, pic....|{[{null, pic.twit...|          null|             0|    false|         low|null|1412602031978647558|1412602031978647558|                   null|                 null|                     null|               null|                   null|          false|  en|                         null|             false|          0|                null|               null|                null|                   null|          0|            0|    false|{null, null, Wed ...|<a href=\"http://t...|     RT @captainuwu: N...|1625625543957|    false|{false, Sat May 2...|\n",
      "+--------------------+------------+-----------+--------------------+------------------+--------------------+--------------------+--------------+--------------+---------+------------+----+-------------------+-------------------+-----------------------+---------------------+-------------------------+-------------------+-----------------------+---------------+----+-----------------------------+------------------+-----------+--------------------+-------------------+--------------------+-----------------------+-----------+-------------+---------+--------------------+--------------------+-------------------------+-------------+---------+--------------------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# spark sql을 사용해 spark dataframe에 쿼리를 하기 위해선 임시 뷰를 생성해야합니다.\n",
    "df.createOrReplaceTempView(\"twitterdata\")\n",
    "sql_result = spark.sql(\"SELECT * FROM twitterdata\").show(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2f34057d",
   "metadata": {},
   "source": [
    "** 건의할 사항\n",
    "한국어랑 영어(ko, en) 트윗만 불러오는게 어떨지"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "56296905",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----+\n",
      "|lang|\n",
      "+----+\n",
      "|  ko|\n",
      "|  in|\n",
      "| und|\n",
      "|  ko|\n",
      "|  en|\n",
      "+----+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.select('lang').show(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b6eb48f7",
   "metadata": {},
   "source": [
    "그냥 df_name.select 으로 원하는 열만 뽑아올 수도 있습니다.\n",
    "\n",
    "ex. 가장 상위에 있는 json의 경우 단순 쿼리로 뽑아내기 가능\n",
    "\n",
    "id, created_at, text, retweet_count, favorite_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "bdf3fdfe",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------------+--------------------+-------------------------+-------------+--------------+\n",
      "|                 id|          created_at|                     text|retweet_count|favorite_count|\n",
      "+-------------------+--------------------+-------------------------+-------------+--------------+\n",
      "|1412595075171848192|Wed Jul 07 02:11:...|NCT DREAM(엔시티 드림)...|            0|             0|\n",
      "|1412602031739604992|Wed Jul 07 02:39:...|     RT @Bubuyongi: NC...|            0|             0|\n",
      "|1412602031647334405|Wed Jul 07 02:39:...|     @markleebase http...|            0|             0|\n",
      "|1412602032138035205|Wed Jul 07 02:39:...|  RT @mbcplusm: 주간아...|            0|             0|\n",
      "|1412602031978647558|Wed Jul 07 02:39:...|     RT @captainuwu: N...|            0|             0|\n",
      "+-------------------+--------------------+-------------------------+-------------+--------------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "depth0 = df.select(\"id\",\"created_at\", \"text\", \"retweet_count\", \"favorite_count\")\n",
    "depth0.show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "218deb69",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "576"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "depth0.count()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "402e5989",
   "metadata": {},
   "source": [
    "행에 nested 되어있을 경우 (user)\n",
    "\n",
    "user - name, screen_name, profile_image_url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "38b26721",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----------------+-------------+--------------------+\n",
      "|             name|  screen_name|   profile_image_url|\n",
      "+-----------------+-------------+--------------------+\n",
      "|교보핫트랙스 창원| hottracks_cw|http://pbs.twimg....|\n",
      "|              za.|blueitubiruu_|http://pbs.twimg....|\n",
      "|fariha | 31 dwm📌|     eymarkeu|http://pbs.twimg....|\n",
      "|               🐧|   parkjiajun|http://pbs.twimg....|\n",
      "|  จอง เนียร์young| ianew_skpm10|http://pbs.twimg....|\n",
      "+-----------------+-------------+--------------------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "user_df = df.select(\"user.*\")\n",
    "user_df.createOrReplaceTempView(\"user_sql\")\n",
    "user = spark.sql(\"select name, screen_name, profile_image_url from user_sql\")\n",
    "user.show(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d6c97f73",
   "metadata": {},
   "source": [
    "retweeted_status - id\n",
    "\n",
    ": 리트윗한 트윗일 경우 원본 트윗의 id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "159d5504",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------------+\n",
      "|              RT_id|\n",
      "+-------------------+\n",
      "|               null|\n",
      "|1412440500527796228|\n",
      "|               null|\n",
      "|1412588863088848900|\n",
      "|1412564786848534533|\n",
      "+-------------------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from pyspark.sql.functions import expr\n",
    "\n",
    "\n",
    "retweet_df = df.select(\"retweeted_status.*\")\n",
    "retweet_df.createOrReplaceTempView(\"ret_sql\")\n",
    "retweet_sql = spark.sql(\"select id from ret_sql\")\n",
    "# id 행 이미 있으니까 RT_id 라고 행 이름 변경\n",
    "retweet = retweet_sql.withColumn(\"RT_id\", expr(\"id\"))\n",
    "retweet = retweet.drop('id')\n",
    "retweet.show(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cf9bfc50",
   "metadata": {},
   "source": [
    "위에서 만든거 합침"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "bd34d345",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql.functions import monotonically_increasing_id\n",
    "\n",
    "DF1 = depth0.withColumn(\"row_id\", monotonically_increasing_id())\n",
    "DF2 = user.withColumn(\"row_id\", monotonically_increasing_id())\n",
    "dep_usr = DF1.join(DF2, (\"row_id\")).drop(\"row_id\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "60b5c385",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------------+--------------------+--------------------+-------------+--------------+-----------------+------------+--------------------+\n",
      "|                 id|          created_at|                text|retweet_count|favorite_count|             name| screen_name|   profile_image_url|\n",
      "+-------------------+--------------------+--------------------+-------------+--------------+-----------------+------------+--------------------+\n",
      "|1412602037892567041|Wed Jul 07 02:39:...|RT @iukkxrddfhan:...|            0|             0|           rpzlee| Rapunzelx31|http://pbs.twimg....|\n",
      "|1412602038236680193|Wed Jul 07 02:39:...|RT @nct_menfess: ...|            0|             0|   Kasihh.tryanaa|KasihhTryana|http://pbs.twimg....|\n",
      "|1412657043043852289|Wed Jul 07 06:17:...|RT @NCT127STRMth:...|            0|             0|      🌈🌤🌱✋🏼🙂|    knick_jj|http://pbs.twimg....|\n",
      "|1412602051884851207|Wed Jul 07 02:39:...|RT @markeu_yayy: ...|            0|             0|kyy ◡̎ | 31dwm 📌|  markyourss|http://pbs.twimg....|\n",
      "|1412616883702927364|Wed Jul 07 03:38:...|RT @HanteoNews: #...|            0|             0|             syaa| myleejeno00|http://pbs.twimg....|\n",
      "+-------------------+--------------------+--------------------+-------------+--------------+-----------------+------------+--------------------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "dep_usr.show(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2b4ce8e9",
   "metadata": {},
   "source": [
    "dep_usr 을 monotonically_increasing_id 로 임시 row를 생성해서 retweet이랑 join 시도\n",
    "\n",
    "monotonically_increasing_id 자체가 unique 와 increase 속성은 보장하지만 consecutive 한 속성은 보장하지 않아서 DF1 의 row_id가 0,1,2,8589934592,8589934593,... 이런식으로 인덱싱됨\n",
    "\n",
    "DF2의 row_id는 0,1,2,3,.. 으로 들어가서 두개 join하면 0,1,2행밖에 조인이 안됨\n",
    "\n",
    "연속적인 숫자를 생성하는 row_number() 함수를 사용"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "18302a87",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------------+--------------------+--------------------+-------------+--------------+-----------------+------------+--------------------+-------------------+\n",
      "|                 id|          created_at|                text|retweet_count|favorite_count|             name| screen_name|   profile_image_url|              RT_id|\n",
      "+-------------------+--------------------+--------------------+-------------+--------------+-----------------+------------+--------------------+-------------------+\n",
      "|1412602037892567041|Wed Jul 07 02:39:...|RT @iukkxrddfhan:...|            0|             0|           rpzlee| Rapunzelx31|http://pbs.twimg....|               null|\n",
      "|1412602038236680193|Wed Jul 07 02:39:...|RT @nct_menfess: ...|            0|             0|   Kasihh.tryanaa|KasihhTryana|http://pbs.twimg....|1412440500527796228|\n",
      "|1412657043043852289|Wed Jul 07 06:17:...|RT @NCT127STRMth:...|            0|             0|      🌈🌤🌱✋🏼🙂|    knick_jj|http://pbs.twimg....|               null|\n",
      "|1412602051884851207|Wed Jul 07 02:39:...|RT @markeu_yayy: ...|            0|             0|kyy ◡̎ | 31dwm 📌|  markyourss|http://pbs.twimg....|1412588863088848900|\n",
      "|1412616883702927364|Wed Jul 07 03:38:...|RT @HanteoNews: #...|            0|             0|             syaa| myleejeno00|http://pbs.twimg....|1412564786848534533|\n",
      "+-------------------+--------------------+--------------------+-------------+--------------+-----------------+------------+--------------------+-------------------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from pyspark.sql.functions import *\n",
    "from pyspark.sql.window import *\n",
    "\n",
    "DF1_tmp = dep_usr.withColumn(\"row_id\", monotonically_increasing_id())\n",
    "window = Window.orderBy(col('row_id'))\n",
    "DF1 = DF1_tmp.withColumn('row_id', row_number().over(window))\n",
    "\n",
    "DF2_tmp = retweet.withColumn(\"row_id\", monotonically_increasing_id())\n",
    "DF2 = DF2_tmp.withColumn('row_id', row_number().over(window))\n",
    "result = DF1.join(DF2, (\"row_id\")).drop(\"row_id\"); result.show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "3bf9d27e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "576"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "85cf6758",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------+\n",
      "|row_id|\n",
      "+------+\n",
      "|     1|\n",
      "|     2|\n",
      "|     3|\n",
      "|     4|\n",
      "|     5|\n",
      "|     6|\n",
      "|     7|\n",
      "|     8|\n",
      "|     9|\n",
      "|    10|\n",
      "|    11|\n",
      "|    12|\n",
      "|    13|\n",
      "|    14|\n",
      "|    15|\n",
      "|    16|\n",
      "|    17|\n",
      "|    18|\n",
      "|    19|\n",
      "|    20|\n",
      "+------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "DF1.select(\"row_id\").show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "5daa73cc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------+\n",
      "|row_id|\n",
      "+------+\n",
      "|     1|\n",
      "|     2|\n",
      "|     3|\n",
      "|     4|\n",
      "|     5|\n",
      "|     6|\n",
      "|     7|\n",
      "|     8|\n",
      "|     9|\n",
      "|    10|\n",
      "|    11|\n",
      "|    12|\n",
      "|    13|\n",
      "|    14|\n",
      "|    15|\n",
      "|    16|\n",
      "|    17|\n",
      "|    18|\n",
      "|    19|\n",
      "|    20|\n",
      "+------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "DF2.select(\"row_id\").show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "74fe6cf9",
   "metadata": {},
   "source": [
    "spark dataframe local에 저장"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d09bf0d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# json으로 저장\n",
    "# df_name.write.format(\"json\").mode(\"overwrite\").save(\"file.json\")\n",
    "# csv 로 저장\n",
    "result.write.option(\"header\", True).format(\"csv\").mode(\"overwrite\").save(\"result.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b9a2221a",
   "metadata": {},
   "source": [
    "pandas dataframe으로 변경"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "809c3331",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>created_at</th>\n",
       "      <th>text</th>\n",
       "      <th>retweet_count</th>\n",
       "      <th>favorite_count</th>\n",
       "      <th>name</th>\n",
       "      <th>screen_name</th>\n",
       "      <th>profile_image_url</th>\n",
       "      <th>RT_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1412602037892567041</td>\n",
       "      <td>Wed Jul 07 02:39:05 +0000 2021</td>\n",
       "      <td>RT @iukkxrddfhan: pls rt แจก 🌈🌤\\nบั้มเปล่าจีเว...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>rpzlee</td>\n",
       "      <td>Rapunzelx31</td>\n",
       "      <td>http://pbs.twimg.com/profile_images/1410818825...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1412602038236680193</td>\n",
       "      <td>Wed Jul 07 02:39:05 +0000 2021</td>\n",
       "      <td>RT @nct_menfess: Kata Mark Haechan itu mirip b...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Kasihh.tryanaa</td>\n",
       "      <td>KasihhTryana</td>\n",
       "      <td>http://pbs.twimg.com/profile_images/1387236819...</td>\n",
       "      <td>1.412441e+18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1412657043043852289</td>\n",
       "      <td>Wed Jul 07 06:17:39 +0000 2021</td>\n",
       "      <td>RT @NCT127STRMth: 210707 แทอิล (mo.on_air) กดฟ...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>🌈🌤🌱✋🏼🙂</td>\n",
       "      <td>knick_jj</td>\n",
       "      <td>http://pbs.twimg.com/profile_images/9577156416...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1412602051884851207</td>\n",
       "      <td>Wed Jul 07 02:39:08 +0000 2021</td>\n",
       "      <td>RT @markeu_yayy: ✧☞ day 6🐯🍉\\n\\n[Q : iconic wor...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>kyy ◡̎ | 31dwm 📌</td>\n",
       "      <td>markyourss</td>\n",
       "      <td>http://pbs.twimg.com/profile_images/1410500304...</td>\n",
       "      <td>1.412589e+18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1412616883702927364</td>\n",
       "      <td>Wed Jul 07 03:38:04 +0000 2021</td>\n",
       "      <td>RT @HanteoNews: #HANTEO GLOBAL K-POP REPORT : ...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>syaa</td>\n",
       "      <td>myleejeno00</td>\n",
       "      <td>http://pbs.twimg.com/profile_images/1412616011...</td>\n",
       "      <td>1.412565e+18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>571</th>\n",
       "      <td>1412657035833876491</td>\n",
       "      <td>Wed Jul 07 06:17:37 +0000 2021</td>\n",
       "      <td>RT @vlive_th: เย็นวันนี้❣️ ซีจือนี่เตรียมแท่งไ...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>&amp;</td>\n",
       "      <td>mrsjeong5</td>\n",
       "      <td>http://pbs.twimg.com/profile_images/1396818007...</td>\n",
       "      <td>1.413492e+18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>572</th>\n",
       "      <td>1412657036362407944</td>\n",
       "      <td>Wed Jul 07 06:17:38 +0000 2021</td>\n",
       "      <td>RT @SMerch_store: Sharing Postcard book NCT 12...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>#sociallydistant</td>\n",
       "      <td>richardhdent</td>\n",
       "      <td>http://pbs.twimg.com/profile_images/1318765142...</td>\n",
       "      <td>1.413510e+18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>573</th>\n",
       "      <td>1412616889583370242</td>\n",
       "      <td>Wed Jul 07 03:38:06 +0000 2021</td>\n",
       "      <td>กติกา\\n1.กดฟอลไอจี  kpopshop_everyty \\n2.กดฟอล...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>ตอบแชทไวมาก🔥 รับผ่อนอัลบั้ม</td>\n",
       "      <td>everytyshop</td>\n",
       "      <td>http://pbs.twimg.com/profile_images/1394989610...</td>\n",
       "      <td>1.413493e+18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>574</th>\n",
       "      <td>1412657040871276546</td>\n",
       "      <td>Wed Jul 07 06:17:39 +0000 2021</td>\n",
       "      <td>RT @Billboard_JAPAN: 【今週のダウンロード・アルバム・チャート“Down...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>김⁷ |🍷𝑃𝑒𝑟𝑚𝑖𝑠𝑠𝑖𝑜𝑛 𝑇𝑜 𝐷𝑎𝑛𝑐𝑒🤠⅓</td>\n",
       "      <td>parkjm_0613</td>\n",
       "      <td>http://pbs.twimg.com/profile_images/1412565215...</td>\n",
       "      <td>1.413493e+18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>575</th>\n",
       "      <td>1412657058650890242</td>\n",
       "      <td>Wed Jul 07 06:17:43 +0000 2021</td>\n",
       "      <td>RT @asmoa_: แจก30฿ ค่ะ เพราะรูปนี้เฮียหล่อมากก...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>แบมลีส✨🌷 คัมแบคๆ ฟีทชั้นรักแฮชาน ชั้นรักกุกๆ</td>\n",
       "      <td>amperpn</td>\n",
       "      <td>http://pbs.twimg.com/profile_images/1409919818...</td>\n",
       "      <td>1.413509e+18</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>576 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                      id                      created_at  \\\n",
       "0    1412602037892567041  Wed Jul 07 02:39:05 +0000 2021   \n",
       "1    1412602038236680193  Wed Jul 07 02:39:05 +0000 2021   \n",
       "2    1412657043043852289  Wed Jul 07 06:17:39 +0000 2021   \n",
       "3    1412602051884851207  Wed Jul 07 02:39:08 +0000 2021   \n",
       "4    1412616883702927364  Wed Jul 07 03:38:04 +0000 2021   \n",
       "..                   ...                             ...   \n",
       "571  1412657035833876491  Wed Jul 07 06:17:37 +0000 2021   \n",
       "572  1412657036362407944  Wed Jul 07 06:17:38 +0000 2021   \n",
       "573  1412616889583370242  Wed Jul 07 03:38:06 +0000 2021   \n",
       "574  1412657040871276546  Wed Jul 07 06:17:39 +0000 2021   \n",
       "575  1412657058650890242  Wed Jul 07 06:17:43 +0000 2021   \n",
       "\n",
       "                                                  text  retweet_count  \\\n",
       "0    RT @iukkxrddfhan: pls rt แจก 🌈🌤\\nบั้มเปล่าจีเว...              0   \n",
       "1    RT @nct_menfess: Kata Mark Haechan itu mirip b...              0   \n",
       "2    RT @NCT127STRMth: 210707 แทอิล (mo.on_air) กดฟ...              0   \n",
       "3    RT @markeu_yayy: ✧☞ day 6🐯🍉\\n\\n[Q : iconic wor...              0   \n",
       "4    RT @HanteoNews: #HANTEO GLOBAL K-POP REPORT : ...              0   \n",
       "..                                                 ...            ...   \n",
       "571  RT @vlive_th: เย็นวันนี้❣️ ซีจือนี่เตรียมแท่งไ...              0   \n",
       "572  RT @SMerch_store: Sharing Postcard book NCT 12...              0   \n",
       "573  กติกา\\n1.กดฟอลไอจี  kpopshop_everyty \\n2.กดฟอล...              0   \n",
       "574  RT @Billboard_JAPAN: 【今週のダウンロード・アルバム・チャート“Down...              0   \n",
       "575  RT @asmoa_: แจก30฿ ค่ะ เพราะรูปนี้เฮียหล่อมากก...              0   \n",
       "\n",
       "     favorite_count                                          name  \\\n",
       "0                 0                                        rpzlee   \n",
       "1                 0                                Kasihh.tryanaa   \n",
       "2                 0                                        🌈🌤🌱✋🏼🙂   \n",
       "3                 0                              kyy ◡̎ | 31dwm 📌   \n",
       "4                 0                                          syaa   \n",
       "..              ...                                           ...   \n",
       "571               0                                             &   \n",
       "572               0                              #sociallydistant   \n",
       "573               0                   ตอบแชทไวมาก🔥 รับผ่อนอัลบั้ม   \n",
       "574               0                    김⁷ |🍷𝑃𝑒𝑟𝑚𝑖𝑠𝑠𝑖𝑜𝑛 𝑇𝑜 𝐷𝑎𝑛𝑐𝑒🤠⅓   \n",
       "575               0  แบมลีส✨🌷 คัมแบคๆ ฟีทชั้นรักแฮชาน ชั้นรักกุกๆ   \n",
       "\n",
       "      screen_name                                  profile_image_url  \\\n",
       "0     Rapunzelx31  http://pbs.twimg.com/profile_images/1410818825...   \n",
       "1    KasihhTryana  http://pbs.twimg.com/profile_images/1387236819...   \n",
       "2        knick_jj  http://pbs.twimg.com/profile_images/9577156416...   \n",
       "3      markyourss  http://pbs.twimg.com/profile_images/1410500304...   \n",
       "4     myleejeno00  http://pbs.twimg.com/profile_images/1412616011...   \n",
       "..            ...                                                ...   \n",
       "571     mrsjeong5  http://pbs.twimg.com/profile_images/1396818007...   \n",
       "572  richardhdent  http://pbs.twimg.com/profile_images/1318765142...   \n",
       "573   everytyshop  http://pbs.twimg.com/profile_images/1394989610...   \n",
       "574   parkjm_0613  http://pbs.twimg.com/profile_images/1412565215...   \n",
       "575       amperpn  http://pbs.twimg.com/profile_images/1409919818...   \n",
       "\n",
       "            RT_id  \n",
       "0             NaN  \n",
       "1    1.412441e+18  \n",
       "2             NaN  \n",
       "3    1.412589e+18  \n",
       "4    1.412565e+18  \n",
       "..            ...  \n",
       "571  1.413492e+18  \n",
       "572  1.413510e+18  \n",
       "573  1.413493e+18  \n",
       "574  1.413493e+18  \n",
       "575  1.413509e+18  \n",
       "\n",
       "[576 rows x 9 columns]"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "pandas = result.select(\"*\").toPandas()\n",
    "pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "e90f2b99",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "576"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pandas['id'].count()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3c8bddeb",
   "metadata": {},
   "source": [
    "빅쿼리로 보내기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8ced95be",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Saving the data to BigQuery\n",
    "word_count.write.format('bigquery') \\\n",
    "  .option('table', 'wordcount_dataset.wordcount_output') \\\n",
    "  .save()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}