Check DB before processing repo

If we find a repo entry in the DB, we can just use that data

build-filtered-data.mjs

@@ -96,6 +96,10 @@ export async function analyseRepo (repo) {
     await repoData.fetchAndValidateRepoInfo()
     repoData.log('repo metadata and latest commit details fetched and validated.')
     repoData.log(`GraphQL rate limit remaining: ${repoData.graphQLRateLimit.remaining}`)
+    if (db.isRepoUpToDate(repoOwner, repoName, repoData.lastUpdated)) {
+      repoData.log('repo has had no updates since we last checked. Skipping.')
+      return db.getRepoData(repoOwner, repoName)
+    }
     await repoData.fetchAndValidateRepoTree()
     repoData.log('tree fetched and validated.')
 

helpers/database.mjs

@@ -24,6 +24,34 @@ export class RepoDB {
       )
     `)
     setup.run()
+
+    // Create an index on repoOwner and repoName for faster lookups
+    const indexSetup = this.db.prepare(`
+      CREATE INDEX IF NOT EXISTS idx_repo_owner_name ON repos (repoOwner, repoName)
+    `)
+    indexSetup.run()
+  }
+
+  getRepoData (repoOwner, repoName) {
+    const query = this.db.prepare(`
+      SELECT * FROM repos
+      WHERE repoOwner = ? AND repoName = ?
+    `)
+
+    const result = query.get(repoOwner, repoName)
+    if (!result) {
+      return null
+    }
+    return this.denormaliseData(result)
+  }
+
+  isRepoUpToDate (repoOwner, repoName, lastUpdated) {
+    const query = this.db.prepare(`
+      SELECT lastUpdated FROM repos
+      WHERE repoOwner = ? AND repoName = ?
+    `)
+    const result = query.get(repoOwner, repoName)
+    return result && result.lastUpdated === lastUpdated
   }
 
   insertRepoData (repo) {
@@ -112,6 +140,24 @@ export class RepoDB {
     return normalisedData
   }
 
+  denormaliseData (dataset) {
+    const denormalisedData = {}
+    for (const [key, value] of Object.entries(dataset)) {
+      if (value === 0 || value === 1) {
+        denormalisedData[key] = value === 1
+      } else if (value === '') {
+        denormalisedData[key] = null
+      } else {
+        try {
+          denormalisedData[key] = JSON.parse(value)
+        } catch (error) {
+          denormalisedData[key] = value
+        }
+      }
+    }
+    return denormalisedData
+  }
+
   close () {
     this.db.close()
   }