Download data for live games

bkowshik · Sep 14, 2024 · 243120d · 243120d
1 parent 8a30f42
commit 243120d
Show file tree

Hide file tree

Showing 10 changed files with 163 additions and 94 deletions.
diff --git a/isl_2024/_modidx.py b/isl_2024/_modidx.py
@@ -6,7 +6,9 @@
                 'git_url': 'https://github.com/bkowshik/isl-2024',
                 'lib_path': 'isl_2024'},
   'syms': { 'isl_2024.core': {'isl_2024.core.foo': ('core.html#foo', 'isl_2024/core.py')},
-            'isl_2024.scrape_live_stats': {},
+            'isl_2024.scrape_live_stats': { 'isl_2024.scrape_live_stats.fetch_live_stats': ( 'scrape_live_stats.html#fetch_live_stats',
+                                                                                             'isl_2024/scrape_live_stats.py')},
             'isl_2024.scrape_matches': {},
-            'isl_2024.scrape_wallstream': {},
+            'isl_2024.scrape_wallstream': { 'isl_2024.scrape_wallstream.fetch_wallstream': ( 'scrape_wallstream.html#fetch_wallstream',
+                                                                                             'isl_2024/scrape_wallstream.py')},
             'isl_2024.utils': {'isl_2024.utils.get_live_matches': ('utils.html#get_live_matches', 'isl_2024/utils.py')}}}
diff --git a/isl_2024/scrape_live_stats.py b/isl_2024/scrape_live_stats.py
@@ -3,7 +3,7 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/02_scrape_live_stats.ipynb.
 
 # %% auto 0
-__all__ = ['parent_dir', 'log_dir', 'data_dir', 'match_id', 'url', 'headers', 'response']
+__all__ = ['parent_dir', 'log_dir', 'data_dir', 'match_ids', 'fetch_live_stats']
 
 # %% ../nbs/02_scrape_live_stats.ipynb 2
 import warnings
@@ -14,6 +14,10 @@
 import os
 import requests
 
+# NOTE: Had to install the package with the following command for the import to work.
+# python3 -m pip install -e '.[dev]'
+from .utils import *
+
 # %% ../nbs/02_scrape_live_stats.ipynb 4
 try:
     # This will work when running as a script
@@ -35,19 +39,26 @@
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s', filename=os.path.join(log_dir, 'scrape_live_stats.log'), filemode='a')
 
 # %% ../nbs/02_scrape_live_stats.ipynb 5
-match_id = 66796
-url = f'https://www.indiansuperleague.com/football/live/india_sl_stats/json/{match_id}.json'
-headers = {
-    'accept': '*/*',
-    'referer': 'https://www.indiansuperleague.com/',
-    'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36'
-}
-response = requests.get(url, headers=headers)
+def fetch_live_stats(match_id):
+    url = f'https://www.indiansuperleague.com/football/live/india_sl_stats/json/{match_id}.json'
+    headers = {
+        'accept': '*/*',
+        'referer': 'https://www.indiansuperleague.com/',
+        'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36'
+    }
+    response = requests.get(url, headers=headers)
+
+    if response.status_code == 200:
+        logging.info('API request successful. Content length: {}'.format(len(response.content)))
+        with open(os.path.join(data_dir, f'{match_id}.txt'), 'a') as f:
+            f.write(response.text + "\n")
+    else:
+        logging.error('API request failed. Status code: {}'.format(response.status_code))
 
 # %% ../nbs/02_scrape_live_stats.ipynb 6
-if response.status_code == 200:
-    logging.info('API request successful. Content length: {}'.format(len(response.content)))
-    with open(os.path.join(data_dir, f'{match_id}.txt'), 'a') as f:
-        f.write(response.text + "\n")
-else:
-    logging.error('API request failed. Status code: {}'.format(response.status_code))
+# Fetch IDs of live matches using the matches summary.
+match_ids = get_live_matches()
+logging.info('Live matches: {} [{}]'.format(len(match_ids), ', '.join(match_ids)))
+
+for match_id in match_ids:
+    fetch_live_stats(match_id)
diff --git a/isl_2024/scrape_wallstream.py b/isl_2024/scrape_wallstream.py
@@ -3,7 +3,7 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/03_scrape_wallstream.ipynb.
 
 # %% auto 0
-__all__ = ['parent_dir', 'log_dir', 'data_dir', 'match_id', 'url', 'headers', 'response']
+__all__ = ['parent_dir', 'log_dir', 'data_dir', 'fetch_wallstream']
 
 # %% ../nbs/03_scrape_wallstream.ipynb 2
 import warnings
@@ -14,6 +14,10 @@
 import os
 import requests
 
+# NOTE: Had to install the package with the following command for the import to work.
+# python3 -m pip install -e '.[dev]'
+from .utils import *
+
 # %% ../nbs/03_scrape_wallstream.ipynb 4
 try:
     # This will work when running as a script
@@ -34,20 +38,19 @@
 
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s', filename=os.path.join(log_dir, 'scrape_wallstream.log'), filemode='a')
 
-# %% ../nbs/03_scrape_wallstream.ipynb 5
-match_id = 66796
-url = f"https://www.indiansuperleague.com/functions/wallstream/?sport_id=2&client_id=5KEUfrMT/+2lgecJyh42zA==&match_id={match_id}"
-headers = {
-    'accept': '*/*',
-    'referer': 'https://www.indiansuperleague.com/',
-    'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36'
-}
-response = requests.get(url, headers=headers)
-
 # %% ../nbs/03_scrape_wallstream.ipynb 6
-if response.status_code == 200:
-    logging.info('API request successful. Content length: {}'.format(len(response.content)))
-    with open(os.path.join(data_dir, f'{match_id}.txt'), 'a') as f:
-        f.write(response.text + "\n")
-else:
-    logging.error('API request failed. Status code: {}'.format(response.status_code))
+def fetch_wallstream(match_id):
+    url = f"https://www.indiansuperleague.com/functions/wallstream/?sport_id=2&client_id=5KEUfrMT/+2lgecJyh42zA==&match_id={match_id}"
+    headers = {
+        'accept': '*/*',
+        'referer': 'https://www.indiansuperleague.com/',
+        'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36'
+    }
+    response = requests.get(url, headers=headers)
+
+    if response.status_code == 200:
+        logging.info('API request successful. Content length: {}'.format(len(response.content)))
+        with open(os.path.join(data_dir, f'{match_id}.txt'), 'a') as f:
+            f.write(response.text + "\n")
+    else:
+        logging.error('API request failed. Status code: {}'.format(response.status_code))
diff --git a/isl_2024/utils.py b/isl_2024/utils.py
@@ -8,7 +8,6 @@
 warnings.filterwarnings('ignore')
 
 import json
-import logging
 import os
 import requests
 import datetime
@@ -34,8 +33,6 @@
 if not os.path.exists(data_dir):
     os.makedirs(data_dir)
 
-logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s', filename=os.path.join(log_dir, 'utils.log'), filemode='a')
-
 # %% ../nbs/00_utils.ipynb 6
 def get_live_matches(now = datetime.datetime.now(pytz.timezone('Asia/Kolkata'))):
     with open(os.path.join(data_dir, 'matches.txt'), 'r') as f:
@@ -58,8 +55,7 @@ def get_live_matches(now = datetime.datetime.now(pytz.timezone('Asia/Kolkata')))
 
     live_matches = matches_df[
         (now >= matches_df["start_at"] - datetime.timedelta(minutes = 15)) &
-        (now <= matches_df["end_at"] + datetime.timedelta(minutes = 60))
+        (now <= matches_df["end_at"] + datetime.timedelta(minutes = 30))
     ]
     game_ids = list(live_matches['game_id'].values)
-    logging.info('Live matches: {} [{}]'.format(len(game_ids), ', '.join(game_ids)))
     return(game_ids)
diff --git a/nbs/00_utils.ipynb b/nbs/00_utils.ipynb
@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 11,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -21,7 +21,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 12,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -31,7 +31,6 @@
     "warnings.filterwarnings('ignore')\n",
     "\n",
     "import json\n",
-    "import logging\n",
     "import os\n",
     "import requests\n",
     "import datetime\n",
@@ -42,7 +41,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 13,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -52,7 +51,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 14,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -72,9 +71,7 @@
     "    os.makedirs(log_dir)\n",
     "\n",
     "if not os.path.exists(data_dir):\n",
-    "    os.makedirs(data_dir)\n",
-    "\n",
-    "logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s', filename=os.path.join(log_dir, 'utils.log'), filemode='a')"
+    "    os.makedirs(data_dir)"
    ]
   },
   {
@@ -86,7 +83,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 15,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -112,25 +109,24 @@
     "\n",
     "    live_matches = matches_df[\n",
     "        (now >= matches_df[\"start_at\"] - datetime.timedelta(minutes = 15)) &\n",
-    "        (now <= matches_df[\"end_at\"] + datetime.timedelta(minutes = 60))\n",
+    "        (now <= matches_df[\"end_at\"] + datetime.timedelta(minutes = 30))\n",
     "    ]\n",
     "    game_ids = list(live_matches['game_id'].values)\n",
-    "    logging.info('Live matches: {} [{}]'.format(len(game_ids), ', '.join(game_ids)))\n",
     "    return(game_ids)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 16,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "['66795', '66796']"
+       "[]"
       ]
      },
-     "execution_count": null,
+     "execution_count": 16,
      "metadata": {},
      "output_type": "execute_result"
     }

diff --git a/nbs/01_scrape_matches.ipynb b/nbs/01_scrape_matches.ipynb
@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 15,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -21,7 +21,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 16,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -38,7 +38,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 17,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -48,7 +48,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 18,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -75,7 +75,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 19,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -91,7 +91,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 20,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -106,7 +106,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 21,
    "metadata": {},
    "outputs": [],
    "source": [

diff --git a/nbs/02_scrape_live_stats.ipynb b/nbs/02_scrape_live_stats.ipynb
@@ -33,7 +33,11 @@
     "import json\n",
     "import logging\n",
     "import os\n",
-    "import requests"
+    "import requests\n",
+    "\n",
+    "# NOTE: Had to install the package with the following command for the import to work.\n",
+    "# python3 -m pip install -e '.[dev]'\n",
+    "from isl_2024.utils import *"
    ]
   },
   {
@@ -80,14 +84,21 @@
    "outputs": [],
    "source": [
     "#| export\n",
-    "match_id = 66796\n",
-    "url = f'https://www.indiansuperleague.com/football/live/india_sl_stats/json/{match_id}.json'\n",
-    "headers = {\n",
-    "    'accept': '*/*',\n",
-    "    'referer': 'https://www.indiansuperleague.com/',\n",
-    "    'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36'\n",
-    "}\n",
-    "response = requests.get(url, headers=headers)"
+    "def fetch_live_stats(match_id):\n",
+    "    url = f'https://www.indiansuperleague.com/football/live/india_sl_stats/json/{match_id}.json'\n",
+    "    headers = {\n",
+    "        'accept': '*/*',\n",
+    "        'referer': 'https://www.indiansuperleague.com/',\n",
+    "        'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36'\n",
+    "    }\n",
+    "    response = requests.get(url, headers=headers)\n",
+    "\n",
+    "    if response.status_code == 200:\n",
+    "        logging.info('API request successful. Content length: {}'.format(len(response.content)))\n",
+    "        with open(os.path.join(data_dir, f'{match_id}.txt'), 'a') as f:\n",
+    "            f.write(response.text + \"\\n\")\n",
+    "    else:\n",
+    "        logging.error('API request failed. Status code: {}'.format(response.status_code))"
    ]
   },
   {
@@ -97,14 +108,22 @@
    "outputs": [],
    "source": [
     "#| export\n",
-    "if response.status_code == 200:\n",
-    "    logging.info('API request successful. Content length: {}'.format(len(response.content)))\n",
-    "    with open(os.path.join(data_dir, f'{match_id}.txt'), 'a') as f:\n",
-    "        f.write(response.text + \"\\n\")\n",
-    "else:\n",
-    "    logging.error('API request failed. Status code: {}'.format(response.status_code))"
+    "\n",
+    "# Fetch IDs of live matches using the matches summary.\n",
+    "match_ids = get_live_matches()\n",
+    "logging.info('Live matches: {} [{}]'.format(len(match_ids), ', '.join(match_ids)))\n",
+    "\n",
+    "for match_id in match_ids:\n",
+    "    fetch_live_stats(match_id)"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
   {
    "cell_type": "code",
    "execution_count": null,