diff --git a/build-filtered-data.mjs b/build-filtered-data.mjs index 0235f3d2..833e476f 100644 --- a/build-filtered-data.mjs +++ b/build-filtered-data.mjs @@ -96,6 +96,10 @@ export async function analyseRepo (repo) { await repoData.fetchAndValidateRepoInfo() repoData.log('repo metadata and latest commit details fetched and validated.') repoData.log(`GraphQL rate limit remaining: ${repoData.graphQLRateLimit.remaining}`) + if (db.isRepoUpToDate(repoOwner, repoName, repoData.lastUpdated)) { + repoData.log('repo has had no updates since we last checked. Skipping.') + return db.getRepoData(repoOwner, repoName) + } await repoData.fetchAndValidateRepoTree() repoData.log('tree fetched and validated.') diff --git a/helpers/database.mjs b/helpers/database.mjs index 504913d4..b455130c 100644 --- a/helpers/database.mjs +++ b/helpers/database.mjs @@ -24,6 +24,34 @@ export class RepoDB { ) `) setup.run() + + // Create an index on repoOwner and repoName for faster lookups + const indexSetup = this.db.prepare(` + CREATE INDEX IF NOT EXISTS idx_repo_owner_name ON repos (repoOwner, repoName) + `) + indexSetup.run() + } + + getRepoData (repoOwner, repoName) { + const query = this.db.prepare(` + SELECT * FROM repos + WHERE repoOwner = ? AND repoName = ? + `) + + const result = query.get(repoOwner, repoName) + if (!result) { + return null + } + return this.denormaliseData(result) + } + + isRepoUpToDate (repoOwner, repoName, lastUpdated) { + const query = this.db.prepare(` + SELECT lastUpdated FROM repos + WHERE repoOwner = ? AND repoName = ? + `) + const result = query.get(repoOwner, repoName) + return result && result.lastUpdated === lastUpdated } insertRepoData (repo) { @@ -112,6 +140,24 @@ export class RepoDB { return normalisedData } + denormaliseData (dataset) { + const denormalisedData = {} + for (const [key, value] of Object.entries(dataset)) { + if (value === 0 || value === 1) { + denormalisedData[key] = value === 1 + } else if (value === '') { + denormalisedData[key] = null + } else { + try { + denormalisedData[key] = JSON.parse(value) + } catch (error) { + denormalisedData[key] = value + } + } + } + return denormalisedData + } + close () { this.db.close() }