From be7225d90396c03ef572ea6461630cde54327d44 Mon Sep 17 00:00:00 2001 From: Brett Kyle Date: Wed, 1 Jan 2025 20:15:12 +0000 Subject: [PATCH] Get some repo info using the GraphQL API This has a separate rate limit, and getting this information basically doesn't impact it at all. Should save a few thousand REST API calls. --- build-filtered-data.mjs | 3 +- helpers/octokit.mjs | 62 +++++++++++++++++-------------- helpers/repo-data.mjs | 45 +++++++++------------- helpers/repo-data.test.mjs | 76 +++++++++++++++++++------------------- package-lock.json | 36 ++++++++++++------ package.json | 1 + todo.md | 4 ++ 7 files changed, 120 insertions(+), 107 deletions(-) diff --git a/build-filtered-data.mjs b/build-filtered-data.mjs index f5884499..99c3d35a 100644 --- a/build-filtered-data.mjs +++ b/build-filtered-data.mjs @@ -71,7 +71,8 @@ export async function analyseRepo (repo) { } repoData.log('analyzing...') - await repoData.fetchAndValidateMetaData() + await repoData.fetchAndValidateRepoInfo() + repoData.log('repo metadata and latest commit details fetched and validated.') await repoData.fetchAndValidateRepoTree() repoData.log('tree fetched and validated.') diff --git a/helpers/octokit.mjs b/helpers/octokit.mjs index b27605dc..7e07b54e 100644 --- a/helpers/octokit.mjs +++ b/helpers/octokit.mjs @@ -1,5 +1,6 @@ import { Octokit } from 'octokit' import { throttling } from '@octokit/plugin-throttling' +import { graphql } from '@octokit/graphql' const MyOctokit = Octokit.plugin(throttling) const octokit = new MyOctokit({ @@ -31,35 +32,40 @@ const octokit = new MyOctokit({ }, }) -/** - * Gets repo metadata - * - * @param {string} repoOwner - The owner of the repo - * @param {string} repoName - The name of the repo - * @returns {Promise>} - * @throws {RequestError} - If the request fails - */ -export async function getRepoMetaData (repoOwner, repoName) { - return await octokit.rest.repos.get({ - owner: repoOwner, - repo: repoName, - }) -} +const graphQLAuth = graphql.defaults({ + headers: { + authorization: `token ${process.env.GITHUB_AUTH_TOKEN}`, + } +}) -/** - * Gets the latest commit for a repo - * @param {string} repoOwner - The owner of the repo - * @param {string} repoName - The name of the repo - * @returns {Promise>} - * @throws {RequestError} - If the request fails - */ -export async function getLatestCommit (repoOwner, repoName) { - const commits = await octokit.rest.repos.listCommits({ - owner: repoOwner, - repo: repoName, - per_page: 1, - }) - return commits.data[0] +export async function getRepoInfo (owner, name) { + const query = ` + query($owner: String!, $name: String!) { + repository(owner: $owner, name: $name) { + createdAt + pushedAt + defaultBranchRef { + target { + ... on Commit { + oid + } + } + } + } + rateLimit { + cost + remaining + resetAt + } + } + ` + + const variables = { + owner, + name, + } + + return await graphQLAuth(query, variables) } /** diff --git a/helpers/repo-data.mjs b/helpers/repo-data.mjs index a3a70f5c..7e91d1b4 100644 --- a/helpers/repo-data.mjs +++ b/helpers/repo-data.mjs @@ -1,7 +1,6 @@ import { + getRepoInfo, getFileContent, - getLatestCommit, - getRepoMetaData, getRepoTree, } from './octokit.mjs' import * as yarnLock from '@yarnpkg/lockfile' @@ -43,6 +42,8 @@ export class RepoData { this.errorThrown = null this.repoTree = null this.frontendVersions = [] + this.latestCommitSHA = null + this.graphQLRateLimit = null } /** @@ -58,24 +59,28 @@ export class RepoData { } /** - * Fetches and validates repo metadata + * Fetches metadata and repo tree using GraphQL * * @throws {NoMetaDataError} - If metadata could not be fetched + * @throws {NoRepoTreeError} - If the tree could not be fetched * @throws {RequestError} - If the request fails - * */ - async fetchAndValidateMetaData () { - const repoMetaData = await getRepoMetaData(this.repoOwner, this.repoName) - if (repoMetaData) { - this.lastUpdated = repoMetaData.data.pushed_at - this.repoCreated = repoMetaData.data.created_at - } + async fetchAndValidateRepoInfo () { + const response = await getRepoInfo(this.repoOwner, this.repoName) + + this.repoCreated = response.repository?.createdAt + this.lastUpdated = response.repository?.pushedAt + this.latestCommitSHA = response.repository?.defaultBranchRef?.target?.oid + this.graphQLRateLimit = response.rateLimit // Some repos won't have a pushed_at if (!this.repoCreated) { throw new NoMetaDataError() } - this.log('metadata fetched and validated.') + + if (!this.latestCommitSHA) { + throw new NoCommitsError() + } } /** @@ -85,32 +90,16 @@ export class RepoData { * @throws {RequestError} - If the request fails */ async fetchAndValidateRepoTree () { - const latestCommitSha = await this.getLatestCommitSha() this.repoTree = await getRepoTree( this.repoOwner, this.repoName, - latestCommitSha + this.latestCommitSHA ) if (!this.repoTree || !this.repoTree.data || !this.repoTree.data.tree) { throw new NoRepoTreeError() } } - /** - * Gets the SHA of the latest commit - * - * @returns {string} - The SHA of the latest commit - * @throws {NoCommitsError} - If the repo has no commits - * @throws {RequestError} - If the request fails - */ - async getLatestCommitSha () { - const latestCommit = await getLatestCommit(this.repoOwner, this.repoName) - if (latestCommit === undefined) { - throw new NoCommitsError() - } - return latestCommit.sha - } - /** * Checks if repo is a prototype * diff --git a/helpers/repo-data.test.mjs b/helpers/repo-data.test.mjs index a4382f55..726fffd2 100644 --- a/helpers/repo-data.test.mjs +++ b/helpers/repo-data.test.mjs @@ -2,16 +2,14 @@ import { describe, it, expect, vi } from 'vitest' import { RepoData, NoMetaDataError, NoRepoTreeError, NoCommitsError, UnsupportedLockFileError } from './repo-data.mjs' import { getFileContent, - getLatestCommit, - getRepoMetaData, getRepoTree, + getRepoInfo } from './octokit.mjs' // Mock the octokit functions vi.mock('./octokit.mjs', () => ({ getFileContent: vi.fn(), - getLatestCommit: vi.fn(), - getRepoMetaData: vi.fn(), + getRepoInfo: vi.fn(), getRepoTree: vi.fn(), })) @@ -57,40 +55,62 @@ describe('RepoData', () => { ) }) - describe('fetchAndValidateMetaData', () => { + describe('fetchAndValidateRepoInfo', () => { it('should throw a NoMetaDataError if metadata is missing', async () => { const repoData = new RepoData(repoOwner, repoName, serviceOwners) - getRepoMetaData.mockResolvedValue({ + getRepoInfo.mockResolvedValue({ data: { - pushed_at: null, - created_at: null, - }, + repository: { + createdAt: '2022-01-01T00:00:00Z', + pushedAt: '2023-01-01T00:00:00Z' + } + } }) - await expect(repoData.fetchAndValidateMetaData()).rejects.toThrow( + await expect(repoData.fetchAndValidateRepoInfo()).rejects.toThrow( NoMetaDataError ) }) - it('should fetch and validate metadata', async () => { + it('should throw a NoCommitsError if no latest commit', async () => { const repoData = new RepoData(repoOwner, repoName, serviceOwners) - getRepoMetaData.mockResolvedValue({ - data: { - pushed_at: '2023-01-01T00:00:00Z', - created_at: '2022-01-01T00:00:00Z', - }, + getRepoInfo.mockResolvedValue({ + repository: { + createdAt: '2022-01-01T00:00:00Z', + pushedAt: '2023-01-01T00:00:00Z' + } + }) + + await expect(repoData.fetchAndValidateRepoInfo()).rejects.toThrow( + NoCommitsError + ) + }) + + it('should fetch and validate repo info', async () => { + const repoData = new RepoData(repoOwner, repoName, serviceOwners) + getRepoInfo.mockResolvedValue({ + repository: { + createdAt: '2022-01-01T00:00:00Z', + pushedAt: '2023-01-01T00:00:00Z', + defaultBranchRef: { + target: { + oid: 'test-sha' + } + } + } }) - await repoData.fetchAndValidateMetaData() + await repoData.fetchAndValidateRepoInfo() expect(repoData.lastUpdated).toBe('2023-01-01T00:00:00Z') expect(repoData.repoCreated).toBe('2022-01-01T00:00:00Z') + expect(repoData.latestCommitSHA).toBe('test-sha') }) }) describe('fetchAndValidateRepoTree', () => { it('should throw a NoRepoTreeError if repo tree is missing', async () => { const repoData = new RepoData(repoOwner, repoName, serviceOwners) - vi.spyOn(repoData, 'getLatestCommitSha').mockResolvedValue('test-sha') + repoData.latestCommitSHA = 'test-sha' getRepoTree.mockResolvedValue({ data: { tree: null, @@ -104,7 +124,6 @@ describe('RepoData', () => { it('should fetch and validate repo tree', async () => { const repoData = new RepoData(repoOwner, repoName, serviceOwners) - getLatestCommit.mockResolvedValue({ sha: 'test-sha' }) getRepoTree.mockResolvedValue({ data: { tree: [] } }) await repoData.fetchAndValidateRepoTree() @@ -112,25 +131,6 @@ describe('RepoData', () => { }) }) - describe('getLatestCommitSha', () => { - it('should throw a NoCommitsError if the repo has no commits', async () => { - const repoData = new RepoData(repoOwner, repoName, serviceOwners) - getLatestCommit.mockResolvedValue(undefined) - - await expect(repoData.getLatestCommitSha()).rejects.toThrow( - NoCommitsError - ) - }) - - it('should get the SHA of the latest commit', async () => { - const repoData = new RepoData(repoOwner, repoName, serviceOwners) - getLatestCommit.mockResolvedValue({ sha: 'test-sha' }) - - const sha = await repoData.getLatestCommitSha() - expect(sha).toBe('test-sha') - }) - }) - describe('checkPrototype', () => { it('should assume prototype if usage_data.js present', async () => { const repoData = new RepoData(repoOwner, repoName, serviceOwners) diff --git a/package-lock.json b/package-lock.json index ff007f84..5b11cc4a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,6 +10,7 @@ "@babel/eslint-parser": "^7.25.9", "@babel/plugin-syntax-import-assertions": "^7.26.0", "@eslint/js": "^9.17.0", + "@octokit/graphql": "^8.1.2", "@octokit/plugin-throttling": "^9.3.0", "@yarnpkg/lockfile": "^1.1.0", "eslint": "^9.17.0", @@ -1241,13 +1242,14 @@ } }, "node_modules/@octokit/graphql": { - "version": "8.1.1", - "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-8.1.1.tgz", - "integrity": "sha512-ukiRmuHTi6ebQx/HFRCXKbDlOh/7xEV6QUXaE7MJEKGNAncGI/STSbOkl12qVXZrfZdpXctx5O9X1AIaebiDBg==", + "version": "8.1.2", + "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-8.1.2.tgz", + "integrity": "sha512-bdlj/CJVjpaz06NBpfHhp4kGJaRZfz7AzC+6EwUImRtrwIw8dIgJ63Xg0OzV9pRn3rIzrt5c2sa++BL0JJ8GLw==", "dev": true, + "license": "MIT", "dependencies": { - "@octokit/request": "^9.0.0", - "@octokit/types": "^13.0.0", + "@octokit/request": "^9.1.4", + "@octokit/types": "^13.6.2", "universal-user-agent": "^7.0.0" }, "engines": { @@ -1385,14 +1387,16 @@ } }, "node_modules/@octokit/request": { - "version": "9.1.1", - "resolved": "https://registry.npmjs.org/@octokit/request/-/request-9.1.1.tgz", - "integrity": "sha512-pyAguc0p+f+GbQho0uNetNQMmLG1e80WjkIaqqgUkihqUp0boRU6nKItXO4VWnr+nbZiLGEyy4TeKRwqaLvYgw==", + "version": "9.1.4", + "resolved": "https://registry.npmjs.org/@octokit/request/-/request-9.1.4.tgz", + "integrity": "sha512-tMbOwGm6wDII6vygP3wUVqFTw3Aoo0FnVQyhihh8vVq12uO3P+vQZeo2CKMpWtPSogpACD0yyZAlVlQnjW71DA==", "dev": true, + "license": "MIT", "dependencies": { "@octokit/endpoint": "^10.0.0", "@octokit/request-error": "^6.0.1", - "@octokit/types": "^13.1.0", + "@octokit/types": "^13.6.2", + "fast-content-type-parse": "^2.0.0", "universal-user-agent": "^7.0.2" }, "engines": { @@ -1412,10 +1416,11 @@ } }, "node_modules/@octokit/types": { - "version": "13.5.0", - "resolved": "https://registry.npmjs.org/@octokit/types/-/types-13.5.0.tgz", - "integrity": "sha512-HdqWTf5Z3qwDVlzCrP8UJquMwunpDiMPt5er+QjGzL4hqr/vBVY/MauQgS1xWxCDT1oMx1EULyqxncdCY/NVSQ==", + "version": "13.6.2", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-13.6.2.tgz", + "integrity": "sha512-WpbZfZUcZU77DrSW4wbsSgTPfKcp286q3ItaIgvSbBpZJlu6mnYXAkjZz6LVZPXkEvLIM8McanyZejKTYUHipA==", "dev": true, + "license": "MIT", "dependencies": { "@octokit/openapi-types": "^22.2.0" } @@ -3557,6 +3562,13 @@ "node": ">=12.0.0" } }, + "node_modules/fast-content-type-parse": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fast-content-type-parse/-/fast-content-type-parse-2.0.0.tgz", + "integrity": "sha512-fCqg/6Sps8tqk8p+kqyKqYfOF0VjPNYrqpLiqNl0RBKmD80B080AJWVV6EkSkscjToNExcXg1+Mfzftrx6+iSA==", + "dev": true, + "license": "MIT" + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", diff --git a/package.json b/package.json index a03af6f0..3476084d 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "@babel/eslint-parser": "^7.25.9", "@babel/plugin-syntax-import-assertions": "^7.26.0", "@eslint/js": "^9.17.0", + "@octokit/graphql": "^8.1.2", "@octokit/plugin-throttling": "^9.3.0", "@yarnpkg/lockfile": "^1.1.0", "eslint": "^9.17.0", diff --git a/todo.md b/todo.md index 716ca697..0cf7d0d2 100644 --- a/todo.md +++ b/todo.md @@ -1,7 +1,11 @@ # TODOs +## Optimisation and behaviour + - [*] Handle multiple packagefiles - [ ] Handle nested (multiple?) lock files +- [ ] Fetch metadata, repotree and latest commit SHA in one API call (3*4600 API calls is a big saving!) + - Make sure to check cost of GraphQL query - the rate limit is 10,000 - but mixing that and REST might give us more? ## Manual ports (STRETCH)