Live stats are stored as one file per match

bkowshik · Sep 14, 2024 · 2db98b6 · 2db98b6
1 parent f8701ec
commit 2db98b6
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 7 deletions.
diff --git a/isl_2024/scrape_live_stats.py b/isl_2024/scrape_live_stats.py
@@ -3,7 +3,7 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/01_scrape_live_stats.ipynb.
 
 # %% auto 0
-__all__ = ['parent_dir', 'log_dir', 'data_dir', 'url', 'headers', 'response']
+__all__ = ['parent_dir', 'log_dir', 'data_dir', 'match_id', 'url', 'headers', 'response']
 
 # %% ../nbs/01_scrape_live_stats.ipynb 2
 import warnings
@@ -24,7 +24,7 @@
 
 parent_dir = os.path.abspath(os.path.join(script_dir, os.pardir))
 log_dir = os.path.join(parent_dir, 'logs')
-data_dir = os.path.join(parent_dir, 'data')
+data_dir = os.path.join(parent_dir, 'data/live_stats')
 
 if not os.path.exists(log_dir):
     os.makedirs(log_dir)
@@ -35,7 +35,8 @@
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s', filename=os.path.join(log_dir, 'scrape_live_stats.log'), filemode='a')
 
 # %% ../nbs/01_scrape_live_stats.ipynb 5
-url = 'https://www.indiansuperleague.com/football/live/india_sl_stats/json/66794.json'
+match_id = 66794
+url = f'https://www.indiansuperleague.com/football/live/india_sl_stats/json/{match_id}.json'
 headers = {
     'accept': '*/*',
     'referer': 'https://www.indiansuperleague.com/',
@@ -46,7 +47,7 @@
 # %% ../nbs/01_scrape_live_stats.ipynb 6
 if response.status_code == 200:
     logging.info('API request successful. Content length: {}'.format(len(response.content)))
-    with open(os.path.join(data_dir, 'scrape_live_stats.txt'), 'a') as f:
+    with open(os.path.join(data_dir, f'{match_id}.txt'), 'a') as f:
         f.write(response.text + "\n")
 else:
     logging.error('API request failed. Status code: {}'.format(response.status_code))
diff --git a/nbs/01_scrape_live_stats.ipynb b/nbs/01_scrape_live_stats.ipynb
@@ -62,7 +62,7 @@
     "\n",
     "parent_dir = os.path.abspath(os.path.join(script_dir, os.pardir))\n",
     "log_dir = os.path.join(parent_dir, 'logs')\n",
-    "data_dir = os.path.join(parent_dir, 'data')\n",
+    "data_dir = os.path.join(parent_dir, 'data/live_stats')\n",
     "    \n",
     "if not os.path.exists(log_dir):\n",
     "    os.makedirs(log_dir)\n",
@@ -80,7 +80,8 @@
    "outputs": [],
    "source": [
     "#| export\n",
-    "url = 'https://www.indiansuperleague.com/football/live/india_sl_stats/json/66794.json'\n",
+    "match_id = 66794\n",
+    "url = f'https://www.indiansuperleague.com/football/live/india_sl_stats/json/{match_id}.json'\n",
     "headers = {\n",
     "    'accept': '*/*',\n",
     "    'referer': 'https://www.indiansuperleague.com/',\n",
@@ -98,7 +99,7 @@
     "#| export\n",
     "if response.status_code == 200:\n",
     "    logging.info('API request successful. Content length: {}'.format(len(response.content)))\n",
-    "    with open(os.path.join(data_dir, 'scrape_live_stats.txt'), 'a') as f:\n",
+    "    with open(os.path.join(data_dir, f'{match_id}.txt'), 'a') as f:\n",
     "        f.write(response.text + \"\\n\")\n",
     "else:\n",
     "    logging.error('API request failed. Status code: {}'.format(response.status_code))"