From 89a54b4ed1197bb8607690c1855d94ba5e54a3f3 Mon Sep 17 00:00:00 2001 From: Bhargav Kowshik Date: Sat, 14 Sep 2024 19:22:25 +0530 Subject: [PATCH] Function to get live matches --- isl_2024/_modidx.py | 3 +- isl_2024/scrape_live_stats.py | 2 +- isl_2024/scrape_wallstream.py | 2 +- isl_2024/utils.py | 65 ++++++++++++++ nbs/00_utils.ipynb | 160 +++++++++++++++++++++++++++++++++ nbs/02_scrape_live_stats.ipynb | 2 +- nbs/03_scrape_wallstream.ipynb | 2 +- 7 files changed, 231 insertions(+), 5 deletions(-) create mode 100644 isl_2024/utils.py create mode 100644 nbs/00_utils.ipynb diff --git a/isl_2024/_modidx.py b/isl_2024/_modidx.py index afd3e9b..3908dcd 100644 --- a/isl_2024/_modidx.py +++ b/isl_2024/_modidx.py @@ -8,4 +8,5 @@ 'syms': { 'isl_2024.core': {'isl_2024.core.foo': ('core.html#foo', 'isl_2024/core.py')}, 'isl_2024.scrape_live_stats': {}, 'isl_2024.scrape_matches': {}, - 'isl_2024.scrape_wallstream': {}}} + 'isl_2024.scrape_wallstream': {}, + 'isl_2024.utils': {'isl_2024.utils.get_live_matches': ('utils.html#get_live_matches', 'isl_2024/utils.py')}}} diff --git a/isl_2024/scrape_live_stats.py b/isl_2024/scrape_live_stats.py index 5c75d89..065c0d1 100644 --- a/isl_2024/scrape_live_stats.py +++ b/isl_2024/scrape_live_stats.py @@ -35,7 +35,7 @@ logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s', filename=os.path.join(log_dir, 'scrape_live_stats.log'), filemode='a') # %% ../nbs/02_scrape_live_stats.ipynb 5 -match_id = 66795 +match_id = 66796 url = f'https://www.indiansuperleague.com/football/live/india_sl_stats/json/{match_id}.json' headers = { 'accept': '*/*', diff --git a/isl_2024/scrape_wallstream.py b/isl_2024/scrape_wallstream.py index 6be426d..9e99530 100644 --- a/isl_2024/scrape_wallstream.py +++ b/isl_2024/scrape_wallstream.py @@ -35,7 +35,7 @@ logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s', filename=os.path.join(log_dir, 'scrape_wallstream.log'), filemode='a') # %% ../nbs/03_scrape_wallstream.ipynb 5 -match_id = 66795 +match_id = 66796 url = f"https://www.indiansuperleague.com/functions/wallstream/?sport_id=2&client_id=5KEUfrMT/+2lgecJyh42zA==&match_id={match_id}" headers = { 'accept': '*/*', diff --git a/isl_2024/utils.py b/isl_2024/utils.py new file mode 100644 index 0000000..da554d3 --- /dev/null +++ b/isl_2024/utils.py @@ -0,0 +1,65 @@ +# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/00_utils.ipynb. + +# %% auto 0 +__all__ = ['parent_dir', 'log_dir', 'data_dir', 'get_live_matches'] + +# %% ../nbs/00_utils.ipynb 2 +import warnings +warnings.filterwarnings('ignore') + +import json +import logging +import os +import requests +import datetime +import pytz + +import pandas as pd + +# %% ../nbs/00_utils.ipynb 4 +try: + # This will work when running as a script + script_dir = os.path.dirname(os.path.abspath(__file__)) +except NameError: + # This will work when running in a Jupyter notebook + script_dir = os.getcwd() + +parent_dir = os.path.abspath(os.path.join(script_dir, os.pardir)) +log_dir = os.path.join(parent_dir, 'logs') +data_dir = os.path.join(parent_dir, 'data') + +if not os.path.exists(log_dir): + os.makedirs(log_dir) + +if not os.path.exists(data_dir): + os.makedirs(data_dir) + +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s', filename=os.path.join(log_dir, 'utils.log'), filemode='a') + +# %% ../nbs/00_utils.ipynb 6 +def get_live_matches(now = datetime.datetime.now(pytz.timezone('Asia/Kolkata'))): + with open(os.path.join(data_dir, 'matches.txt'), 'r') as f: + logs = f.readlines() + + latest = logs[-1] + matches = json.loads(latest)['matches'] + + matches_df = [] + for match in matches: + matches_df.append({ + 'start_at': match['start_date'], + 'end_at': match['end_date'], + 'game_id': match['game_id'], + }) + + matches_df = pd.DataFrame(matches_df) + matches_df['start_at'] = pd.to_datetime(matches_df['start_at']) + matches_df['end_at'] = pd.to_datetime(matches_df['end_at']) + + live_matches = matches_df[ + (now >= matches_df["start_at"] - datetime.timedelta(minutes = 15)) & + (now <= matches_df["end_at"] + datetime.timedelta(minutes = 60)) + ] + game_ids = list(live_matches['game_id'].values) + logging.info('Live matches: {} [{}]'.format(len(game_ids), ', '.join(game_ids))) + return(game_ids) diff --git a/nbs/00_utils.ipynb b/nbs/00_utils.ipynb new file mode 100644 index 0000000..950ca51 --- /dev/null +++ b/nbs/00_utils.ipynb @@ -0,0 +1,160 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#| hide\n", + "#| default_exp utils" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Utils\n", + "\n", + "Utility functions to be used in other scripts." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "\n", + "import warnings\n", + "warnings.filterwarnings('ignore')\n", + "\n", + "import json\n", + "import logging\n", + "import os\n", + "import requests\n", + "import datetime\n", + "import pytz\n", + "\n", + "import pandas as pd" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#| hide\n", + "from nbdev.showdoc import *" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "try:\n", + " # This will work when running as a script\n", + " script_dir = os.path.dirname(os.path.abspath(__file__))\n", + "except NameError:\n", + " # This will work when running in a Jupyter notebook\n", + " script_dir = os.getcwd()\n", + "\n", + "parent_dir = os.path.abspath(os.path.join(script_dir, os.pardir))\n", + "log_dir = os.path.join(parent_dir, 'logs')\n", + "data_dir = os.path.join(parent_dir, 'data')\n", + " \n", + "if not os.path.exists(log_dir):\n", + " os.makedirs(log_dir)\n", + "\n", + "if not os.path.exists(data_dir):\n", + " os.makedirs(data_dir)\n", + "\n", + "logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s', filename=os.path.join(log_dir, 'utils.log'), filemode='a')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def get_live_matches(now = datetime.datetime.now(pytz.timezone('Asia/Kolkata'))):\n", + " with open(os.path.join(data_dir, 'matches.txt'), 'r') as f:\n", + " logs = f.readlines()\n", + "\n", + " latest = logs[-1]\n", + " matches = json.loads(latest)['matches']\n", + "\n", + " matches_df = []\n", + " for match in matches:\n", + " matches_df.append({\n", + " 'start_at': match['start_date'],\n", + " 'end_at': match['end_date'],\n", + " 'game_id': match['game_id'],\n", + " })\n", + "\n", + " matches_df = pd.DataFrame(matches_df)\n", + " matches_df['start_at'] = pd.to_datetime(matches_df['start_at'])\n", + " matches_df['end_at'] = pd.to_datetime(matches_df['end_at'])\n", + "\n", + " live_matches = matches_df[\n", + " (now >= matches_df[\"start_at\"] - datetime.timedelta(minutes = 15)) &\n", + " (now <= matches_df[\"end_at\"] + datetime.timedelta(minutes = 60))\n", + " ]\n", + " game_ids = list(live_matches['game_id'].values)\n", + " logging.info('Live matches: {} [{}]'.format(len(game_ids), ', '.join(game_ids)))\n", + " return(game_ids)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['66795', '66796']" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "#| hide\n", + "get_live_matches()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "python3", + "language": "python", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/nbs/02_scrape_live_stats.ipynb b/nbs/02_scrape_live_stats.ipynb index 852c170..674a0fd 100644 --- a/nbs/02_scrape_live_stats.ipynb +++ b/nbs/02_scrape_live_stats.ipynb @@ -80,7 +80,7 @@ "outputs": [], "source": [ "#| export\n", - "match_id = 66795\n", + "match_id = 66796\n", "url = f'https://www.indiansuperleague.com/football/live/india_sl_stats/json/{match_id}.json'\n", "headers = {\n", " 'accept': '*/*',\n", diff --git a/nbs/03_scrape_wallstream.ipynb b/nbs/03_scrape_wallstream.ipynb index 7d654e0..3e7761d 100644 --- a/nbs/03_scrape_wallstream.ipynb +++ b/nbs/03_scrape_wallstream.ipynb @@ -80,7 +80,7 @@ "outputs": [], "source": [ "#| export\n", - "match_id = 66795\n", + "match_id = 66796\n", "url = f\"https://www.indiansuperleague.com/functions/wallstream/?sport_id=2&client_id=5KEUfrMT/+2lgecJyh42zA==&match_id={match_id}\"\n", "headers = {\n", " 'accept': '*/*',\n",