From 885bb1048a74acd82c9ed9c41de8e010db407414 Mon Sep 17 00:00:00 2001
From: Brett Kyle <brett.kyle@digital.cabinet-office.gov.uk>
Date: Fri, 3 Jan 2025 20:59:16 +0000
Subject: [PATCH] Check DB before processing repo

If we find a repo entry in the DB, we can just use that data
---
 build-filtered-data.mjs |  4 ++++
 helpers/database.mjs    | 46 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 50 insertions(+)

diff --git a/build-filtered-data.mjs b/build-filtered-data.mjs
index 0235f3d2..833e476f 100644
--- a/build-filtered-data.mjs
+++ b/build-filtered-data.mjs
@@ -96,6 +96,10 @@ export async function analyseRepo (repo) {
     await repoData.fetchAndValidateRepoInfo()
     repoData.log('repo metadata and latest commit details fetched and validated.')
     repoData.log(`GraphQL rate limit remaining: ${repoData.graphQLRateLimit.remaining}`)
+    if (db.isRepoUpToDate(repoOwner, repoName, repoData.lastUpdated)) {
+      repoData.log('repo has had no updates since we last checked. Skipping.')
+      return db.getRepoData(repoOwner, repoName)
+    }
     await repoData.fetchAndValidateRepoTree()
     repoData.log('tree fetched and validated.')
 
diff --git a/helpers/database.mjs b/helpers/database.mjs
index 504913d4..b455130c 100644
--- a/helpers/database.mjs
+++ b/helpers/database.mjs
@@ -24,6 +24,34 @@ export class RepoDB {
       )
     `)
     setup.run()
+
+    // Create an index on repoOwner and repoName for faster lookups
+    const indexSetup = this.db.prepare(`
+      CREATE INDEX IF NOT EXISTS idx_repo_owner_name ON repos (repoOwner, repoName)
+    `)
+    indexSetup.run()
+  }
+
+  getRepoData (repoOwner, repoName) {
+    const query = this.db.prepare(`
+      SELECT * FROM repos
+      WHERE repoOwner = ? AND repoName = ?
+    `)
+
+    const result = query.get(repoOwner, repoName)
+    if (!result) {
+      return null
+    }
+    return this.denormaliseData(result)
+  }
+
+  isRepoUpToDate (repoOwner, repoName, lastUpdated) {
+    const query = this.db.prepare(`
+      SELECT lastUpdated FROM repos
+      WHERE repoOwner = ? AND repoName = ?
+    `)
+    const result = query.get(repoOwner, repoName)
+    return result && result.lastUpdated === lastUpdated
   }
 
   insertRepoData (repo) {
@@ -112,6 +140,24 @@ export class RepoDB {
     return normalisedData
   }
 
+  denormaliseData (dataset) {
+    const denormalisedData = {}
+    for (const [key, value] of Object.entries(dataset)) {
+      if (value === 0 || value === 1) {
+        denormalisedData[key] = value === 1
+      } else if (value === '') {
+        denormalisedData[key] = null
+      } else {
+        try {
+          denormalisedData[key] = JSON.parse(value)
+        } catch (error) {
+          denormalisedData[key] = value
+        }
+      }
+    }
+    return denormalisedData
+  }
+
   close () {
     this.db.close()
   }