refactor generatejob

dev/findProMatches.mts

@@ -1,6 +1,6 @@
 const { insertMatch } = await import('../util/insert.js');
 const { db } = await import('../store/db.js');
-const { generateJob, getSteamAPIData } = await import('../util/utility.js');
+const { getSteamAPIData, SteamAPIUrls } = await import('../util/utility.js');
 
 async function getPage(url: string, leagueid: number) {
   const data: any = await getSteamAPIData({ url });
@@ -12,10 +12,9 @@ async function getPage(url: string, leagueid: number) {
   for (let i = 0; i < data.result.matches.length; i++) {
     const match = data.results.matches[i];
     console.log(match.match_id);
-    const job = generateJob('api_details', {
+    const url = SteamAPIUrls.api_details({
       match_id: match.match_id,
     });
-    const { url } = job;
     const body: any = await getSteamAPIData({
       url,
     });
@@ -25,11 +24,11 @@ async function getPage(url: string, leagueid: number) {
     }
   }
   if (data.result.results_remaining) {
-    const url2 = generateJob('api_history', {
+    const url2 = SteamAPIUrls.api_history({
       leagueid,
       start_at_match_id:
         data.result.matches[data.result.matches.length - 1].match_id - 1,
-    }).url;
+    });
     return getPage(url2, leagueid);
   }
 }
@@ -43,7 +42,7 @@ const data: any = await db
 const leagueIds = data.map((l: any) => l.leagueid);
 // NOTE: there could be a lot of leagueids
 leagueIds.forEach(async (leagueid: number) => {
-  const { url } = generateJob('api_history', {
+  const url = SteamAPIUrls.api_history({
     leagueid,
   });
   return getPage(url, leagueid);
@@ -56,10 +55,9 @@ const data = await getDataPromise(leagueUrl);
   const leagueIds = data.result.leagues.map(l => l.leagueid);
     // iterate through leagueids and use getmatchhistory to retrieve matches for each
  leagueIds.forEach(leagueid) => {
-    const url = generateJob('api_history',
-      {
+    const url = SteamAPIUrls.api_history({
         leagueid,
-      }).url;
+      });
     return getPage(url, leagueid, cb);
   }, (err) => {
     process.exit(Number(err));

dev/fixProMatches.mts

@@ -1,6 +1,6 @@
 const { insertMatch } = await import('../util/insert.js');
 const { db } = await import('../store/db.js');
-const { generateJob, getSteamAPIData, isProMatch } = await import(
+const { SteamAPIUrls, getSteamAPIData, isProMatch } = await import(
   '../util/utility.js'
 );
 
@@ -11,10 +11,9 @@ const { rows } = await db.raw(
 for (let i = 0; i < rows.length; i++) {
   const match = rows[i];
   console.log(match.match_id);
-  const job = generateJob('api_details', {
+  const url = SteamAPIUrls.api_details({
     match_id: match.match_id,
   });
-  const { url } = job;
   const body: any = await getSteamAPIData({
     url,
   });

fetcher/getApiData.ts

@@ -1,4 +1,4 @@
-import { generateJob, getSteamAPIData, redisCount } from '../util/utility';
+import { SteamAPIUrls, getSteamAPIData, redisCount } from '../util/utility';
 import { Archive } from '../store/archive';
 import cassandra from '../store/cassandra';
 import { type ApiMatch } from '../util/pgroup';
@@ -56,10 +56,9 @@ async function getOrFetchApiData(matchId: number): Promise<{
   if (saved) {
     return { data: saved, error: null };
   }
-  const job = generateJob('api_details', {
+  const url = SteamAPIUrls.api_details({
     match_id: matchId,
   });
-  const { url } = job;
   let match;
   try {
     // We currently can't fetch because the Steam GetMatchDetails API is broken

svc/backfill.ts

@@ -2,7 +2,7 @@
 import config from '../config';
 import { Archive } from '../store/archive';
 import type { ApiMatch } from '../util/pgroup';
-import { generateJob, getSteamAPIData, transformMatch } from '../util/utility';
+import { SteamAPIUrls, getSteamAPIData, transformMatch } from '../util/utility';
 import fs from 'fs';
 
 // following need to be set
@@ -25,13 +25,13 @@ async function scanApi() {
       process.exit(0);
     }
     const begin = Date.now();
-    const container = generateJob('api_sequence', {
+    const url = SteamAPIUrls.api_sequence({
       start_at_match_seq_num: seqNum,
     });
     let data = null;
     try {
       data = await getSteamAPIData({
-        url: container.url,
+        url,
         // We could rotate through proxies here to ensure consistent load
         // proxy: apiHosts,
       });

svc/backupscanner.ts

@@ -17,10 +17,9 @@ async function processMatch(matchId: number) {
   // Check if exists
   const res = await redis.get(`scanner_insert:${matchId}`);
   if (!res) {
-    const job = generateJob('api_details', {
+    const url = SteamAPIUrls.api_details({
       match_id: matchId,
     });
-    const { url } = job;
     const body = await getSteamAPIData({
       url,
       delay,
@@ -39,11 +38,11 @@ async function processMatch(matchId: number) {
   }
 }
 async function processPlayer(accountId: string) {
-  const ajob = generateJob('api_history', {
+  const url = SteamAPIUrls.api_history({
     account_id: accountId,
   });
   const body = await getSteamAPIData({
-    url: ajob.url,
+    url,
     delay,
   });
   if (!body || !body.result || !body.result.matches) {

svc/fullhistory.ts

@@ -4,7 +4,7 @@ import config from '../config';
 import {
   redisCount,
   getSteamAPIData,
-  generateJob,
+  SteamAPIUrls,
   eachLimitPromise,
 } from '../util/utility';
 import db from '../store/db';
@@ -42,14 +42,13 @@ async function processFullHistory(job: FullHistoryJob) {
   // by default, fetch only 100 matches, unless long_history is specified then fetch 500
   // As of December 2021 filtering by hero ID doesn't work
   // const heroArray = config.NODE_ENV === 'test' ? ['0'] : Object.keys(heroes);
-  const heroId = '0';
+  // const heroId = '0';
   // use steamapi via specific player history and specific hero id (up to 500 games per hero)
   const matchesToProcess: Record<string, ApiMatch> = {};
   let isMatchDataDisabled = null;
   // make a request for every possible hero
-  const container = generateJob('api_history', {
+  const url = SteamAPIUrls.api_history({
     account_id: player.account_id,
-    hero_id: heroId,
     matches_requested: 100,
   });
   const getApiMatchPage = async (
@@ -96,7 +95,7 @@ async function processFullHistory(job: FullHistoryJob) {
     return getApiMatchPage(player, url);
   };
   // Fetches 1-5 pages of matches for the players and updates the match_ids object
-  await getApiMatchPage(player, container.url);
+  await getApiMatchPage(player, url);
   if (Object.keys(matchesToProcess).length > 0) {
     isMatchDataDisabled = false;
     // check what matches the player is already associated with
@@ -125,10 +124,10 @@ async function processFullHistory(job: FullHistoryJob) {
         // Disabled due to Steam GetMatchDetails being broken
         // This would update the match blob with the visibility and update player caches to make them show up under a player
         // Could possibly queue these matches for GC data fetch and then trigger a reconciliation from our own DB (similar to proposed change after parsing a match)
-        // const container = generateJob('api_details', {
+        // const url = SteamAPIUrls.api_details({
         //   match_id: Number(matchId),
         // });
-        // const body = await getSteamAPIData({ url: container.url });
+        // const body = await getSteamAPIData({ url });
         // const match = body.result;
         // Don't insert match blob to avoid overwriting with less data from API
         // Only update player_caches to associate the match with players

svc/heroes.ts

@@ -3,16 +3,16 @@ import axios from 'axios';
 import db from '../store/db';
 import { upsert } from '../util/insert';
 import {
-  generateJob,
+  SteamAPIUrls,
   getSteamAPIData,
   invokeIntervalAsync,
 } from '../util/utility';
 
 async function doHeroes() {
-  const container = generateJob('api_heroes', {
+  const url = SteamAPIUrls.api_heroes({
     language: 'english',
   });
-  const body = await getSteamAPIData({ url: container.url });
+  const body = await getSteamAPIData({ url });
   if (!body || !body.result || !body.result.heroes) {
     return;
   }

svc/items.ts

@@ -3,16 +3,16 @@ import { items } from 'dotaconstants';
 import db from '../store/db';
 import { upsert } from '../util/insert';
 import {
-  generateJob,
+  SteamAPIUrls,
   getSteamAPIData,
   invokeIntervalAsync,
 } from '../util/utility';
 
 async function doItems() {
-  const container = generateJob('api_items', {
+  const url = SteamAPIUrls.api_items({
     language: 'english',
   });
-  const body = await getSteamAPIData({ url: container.url });
+  const body = await getSteamAPIData({ url });
   if (!body || !body.result || !body.result.data) {
     throw new Error('invalid body');
   }

svc/livegames.ts

@@ -3,7 +3,7 @@ import JSONbig from 'json-bigint';
 import redis from '../store/redis';
 import db from '../store/db';
 import {
-  generateJob,
+  SteamAPIUrls,
   getSteamAPIData,
   invokeIntervalAsync,
 } from '../util/utility';
@@ -12,8 +12,8 @@ async function doLiveGames() {
   // Get the list of pro players
   const proPlayers: ProPlayer[] = await db.select().from('notable_players');
   // Get the list of live games
-  const container = generateJob('api_top_live_game', {});
-  const body = await getSteamAPIData({ url: container.url, raw: true });
+  const url = SteamAPIUrls.api_top_live_game();
+  const body = await getSteamAPIData({ url, raw: true });
   const json = JSONbig.parse(body);
   // If a match contains a pro player
   // add their name to the match object, save it to redis zset, keyed by server_steam_id
@@ -43,9 +43,9 @@ async function doLiveGames() {
       await redis.zremrangebyrank('liveGames', '0', '-101');
     }
     // Get detailed stats for each live game
-    // const { url } = utility.generateJob('api_realtime_stats', {
+    // const url = SteamAPIUrls.api_realtime_stats({
     //   server_steam_id: match.server_steam_id
-    // }).url;
+    // });
   }
 }
 invokeIntervalAsync(doLiveGames, 60 * 1000);

svc/profiler.ts

@@ -3,7 +3,7 @@ import { upsertPlayer, bulkIndexPlayer } from '../util/insert';
 import db from '../store/db';
 import {
   getSteamAPIData,
-  generateJob,
+  SteamAPIUrls,
   convert64to32,
   invokeIntervalAsync,
 } from '../util/utility';
@@ -15,11 +15,11 @@ async function doProfiler() {
   const result = await db.raw(
     'SELECT account_id from players TABLESAMPLE SYSTEM_ROWS(100)',
   );
-  const container = generateJob('api_summaries', {
+  const url = SteamAPIUrls.api_summaries({
     players: result.rows,
   });
   // We can also queue a rank tier/MMR request for these players
-  const body = await getSteamAPIData({ url: container.url });
+  const body = await getSteamAPIData({ url });
   const results = body.response.players.filter(
     (player: User) => player.steamid,
   );

svc/scanner.ts

@@ -4,7 +4,7 @@ import redis from '../store/redis';
 import { insertMatch } from '../util/insert';
 import type { ApiMatch } from '../util/pgroup';
 import {
-  generateJob,
+  SteamAPIUrls,
   getApiHosts,
   getSteamAPIData,
   redisCount,
@@ -38,13 +38,13 @@ async function scanApi() {
     const apiHosts = await getApiHosts();
     const parallelism = Math.min(apiHosts.length, API_KEYS.length);
     const scannerWaitCatchup = SCANNER_WAIT / parallelism;
-    const container = generateJob('api_sequence', {
+    const url = SteamAPIUrls.api_sequence({
       start_at_match_seq_num: seqNum,
     });
     let data = null;
     try {
       data = await getSteamAPIData({
-        url: container.url,
+        url,
         proxy: apiHosts,
       });
     } catch (err: any) {
@@ -133,9 +133,9 @@ async function start() {
     let numResult = await getCurrentSeqNum();
     if (!numResult) {
       // Never do this in production to avoid skipping sequence number if we didn't pull .env properly
-      const container = generateJob('api_history', {});
+      const url = SteamAPIUrls.api_history({});
       // Just get the approximate current seq num
-      const data = await getSteamAPIData({ url: container.url });
+      const data = await getSteamAPIData({ url });
       numResult = data.result.matches[0].match_seq_num;
       await db.raw(
         'INSERT INTO last_seq_num(match_seq_num) VALUES (?) ON CONFLICT DO NOTHING',

svc/teams.ts

@@ -3,7 +3,7 @@ import axios from 'axios';
 import db from '../store/db';
 import { upsert } from '../util/insert';
 import {
-  generateJob,
+  SteamAPIUrls,
   getSteamAPIData,
   invokeIntervalAsync,
 } from '../util/utility';
@@ -20,18 +20,18 @@ async function doTeams() {
     await new Promise((resolve) => setTimeout(resolve, 1000));
     // GetTeamInfo disabled as of october 2017
     /*
-    const container = utility.generateJob('api_teams', {
+    const url = SteamAPIUrls.api_teams({
       // 2 is the smallest team id, use as default
       team_id: m.team_id || 2,
     });
     */
-    const container = generateJob('api_team_info_by_team_id', {
+    const url = SteamAPIUrls.api_team_info_by_team_id({
       start_at_team_id: m.team_id,
     });
     let body;
     try {
       body = await getSteamAPIData({
-        url: container.url,
+        url,
         raw: true,
       });
     } catch (err) {
@@ -51,7 +51,7 @@ async function doTeams() {
     const logoRegex = /^"logo":(.*),$/m;
     const match = logoRegex.exec(raw);
     const logoUgc = match?.[1];
-    const ugcJob = generateJob('api_get_ugc_file_details', {
+    const ugcUrl = SteamAPIUrls.api_get_ugc_file_details({
       ugcid: logoUgc,
     });
     // Steam's CDN sometimes has better versions of team logos available
@@ -71,7 +71,7 @@ async function doTeams() {
       // Try getting image from ugc
       try {
         const ugcBody = await getSteamAPIData({
-          url: ugcJob.url,
+          url: ugcUrl,
         });
         t.team_id = m.team_id;
         if (ugcBody && ugcBody.data) {