Feat: allow for exemption of repos from dependency graph integrator

packages/common/prisma/schema.prisma

@@ -389,8 +389,6 @@ model github_repository_custom_properties {
   property_name  String
   repository_id  BigInt
   value          String?
-
-  @@ignore
 }
 
 model github_team_repositories {

packages/repocop/src/index.ts

@@ -1,5 +1,6 @@
 import { CloudWatchClient } from '@aws-sdk/client-cloudwatch';
 import type {
+	github_repository_custom_properties,
 	guardian_github_actions_usage,
 	PrismaClient,
 	repocop_github_repository_rules,
@@ -22,6 +23,7 @@ import {
 	getRepoOwnership,
 	getRepositories,
 	getRepositoryBranches,
+	getRepositoryCustomProperties,
 	getRepositoryLanguages,
 	getSnykIssues,
 	getSnykProjects,
@@ -160,13 +162,22 @@ export async function main() {
 		nonPlaygroundStacks,
 	);
 
+	const customProperties = await getRepositoryCustomProperties(prisma);
+	const customPropertiesExemptedFromDepGraphIntegration: github_repository_custom_properties[] =
+		customProperties.filter((property) => {
+			return (
+				property.property_name === 'gu_dependency_graph_integrator_ignore' &&
+				property.value
+			);
+		});
 	const dependencyGraphIntegratorRepoCount = 5;
 
 	await sendReposToDependencyGraphIntegrator(
 		config,
 		repoLanguages,
 		productionRepos,
 		productionWorkflowUsages,
+		customPropertiesExemptedFromDepGraphIntegration,
 		repoOwners,
 		dependencyGraphIntegratorRepoCount,
 		octokit,

packages/repocop/src/query.ts

@@ -1,6 +1,7 @@
 import type {
 	github_languages,
 	github_repository_branches,
+	github_repository_custom_properties,
 	guardian_github_actions_usage,
 	PrismaClient,
 	view_repo_ownership,
@@ -200,3 +201,11 @@ export async function getProductionWorkflowUsages(
 	});
 	return toNonEmptyArray(actions_usage);
 }
+
+export async function getRepositoryCustomProperties(
+	client: PrismaClient,
+): Promise<NonEmptyArray<github_repository_custom_properties>> {
+	const custom_properties =
+		await client.github_repository_custom_properties.findMany({});
+	return toNonEmptyArray(custom_properties);
+}

packages/repocop/src/remediation/dependency_graph-integrator/send-to-sns.test.ts

@@ -1,5 +1,6 @@
 import type {
 	github_languages,
+	github_repository_custom_properties,
 	guardian_github_actions_usage,
 	view_repo_ownership,
 } from '@prisma/client';
@@ -15,12 +16,13 @@ import {
 	createSnsEventsForDependencyGraphIntegration,
 	doesRepoHaveDepSubmissionWorkflowForLanguage,
 	getExistingPullRequest,
-	getReposWithoutWorkflows,
+	getSuitableReposWithoutWorkflows,
 } from './send-to-sns';
 
 const fullName = 'guardian/repo-name';
 const fullName2 = 'guardian/repo2';
-const scalaLang = 'Scala';
+const scala = 'Scala';
+const kotlin = 'Kotlin';
 
 function createActionsUsage(
 	fullName: string,
@@ -74,8 +76,11 @@ function repositoryWithDepGraphLanguage(
 	};
 }
 
-function repoWithTargetLanguage(fullName: string): github_languages {
-	return repoWithLanguages(fullName, ['Scala', 'TypeScript']);
+function repoWithTargetLanguage(
+	fullName: string,
+	language: DepGraphLanguage,
+): github_languages {
+	return repoWithLanguages(fullName, [language, 'TypeScript']);
 }
 
 function repoWithoutTargetLanguage(fullName: string): github_languages {
@@ -103,8 +108,8 @@ describe('When trying to find repos using Scala', () => {
 	test('return true if Scala is found in the repo', () => {
 		const result = checkRepoForLanguage(
 			repository(fullName),
-			[repoWithTargetLanguage(fullName)],
-			scalaLang,
+			[repoWithTargetLanguage(fullName, scala)],
+			scala,
 		);
 
 		expect(result).toBe(true);
@@ -113,7 +118,7 @@ describe('When trying to find repos using Scala', () => {
 		const result = checkRepoForLanguage(
 			repository(fullName),
 			[repoWithoutTargetLanguage(fullName)],
-			scalaLang,
+			scala,
 		);
 		expect(result).toBe(false);
 	});
@@ -124,60 +129,132 @@ describe('When checking a repo for an existing dependency submission workflow',
 		const result = doesRepoHaveDepSubmissionWorkflowForLanguage(
 			repository(fullName),
 			[repoWithDepSubmissionWorkflow(fullName)],
-			'Scala',
+			scala,
 		);
 		expect(result).toBe(true);
 	});
 	test('return false if workflow is not present', () => {
 		const result = doesRepoHaveDepSubmissionWorkflowForLanguage(
 			repository(fullName),
 			[repoWithoutWorkflow(fullName)],
-			'Scala',
+			scala,
 		);
 		expect(result).toBe(false);
 	});
 });
 
-describe('When getting suitable events to send to SNS', () => {
+describe('When getting suitable repos to send to SNS', () => {
 	test('return the repo when a Scala repo is found without an existing workflow', () => {
-		const result = getReposWithoutWorkflows(
-			[repoWithTargetLanguage(fullName)],
+		const result = getSuitableReposWithoutWorkflows(
+			[repoWithTargetLanguage(fullName, scala)],
 			[repository(fullName)],
 			[repoWithoutWorkflow(fullName)],
+			[],
 		);
-		const expected = [repositoryWithDepGraphLanguage(fullName, 'Scala')];
+		const expected = [repositoryWithDepGraphLanguage(fullName, scala)];
 
 		expect(result).toEqual(expected);
 	});
 	test('return empty repo array when a Scala repo is found with an existing workflow', () => {
-		const result = getReposWithoutWorkflows(
-			[repoWithTargetLanguage(fullName)],
+		const result = getSuitableReposWithoutWorkflows(
+			[repoWithTargetLanguage(fullName, scala)],
 			[repository(fullName)],
 			[repoWithDepSubmissionWorkflow(fullName)],
+			[],
 		);
 		expect(result).toEqual([]);
 	});
-	test('return empty array when non-Scala repo is found with without an existing workflow', () => {
-		const result = getReposWithoutWorkflows(
+	test('return empty array when non-Scala/Kotlin repo is found with without an existing workflow', () => {
+		const result = getSuitableReposWithoutWorkflows(
 			[repoWithoutTargetLanguage(fullName)],
 			[repository(fullName)],
 			[repoWithoutWorkflow(fullName)],
+			[],
 		);
 		expect(result).toEqual([]);
 	});
-	test('return 2 events when 2 Scala repos are found without an existing workflow', () => {
-		const result = getReposWithoutWorkflows(
-			[repoWithTargetLanguage(fullName), repoWithTargetLanguage(fullName2)],
+	test('return both repos when 2 Scala repos are found without an existing workflow', () => {
+		const result = getSuitableReposWithoutWorkflows(
+			[
+				repoWithTargetLanguage(fullName, scala),
+				repoWithTargetLanguage(fullName2, scala),
+			],
 			[repository(fullName), repository(fullName2)],
 			[repoWithoutWorkflow(fullName), repoWithoutWorkflow(fullName2)],
+			[],
 		);
 		const expected = [
-			repositoryWithDepGraphLanguage(fullName, 'Scala'),
-			repositoryWithDepGraphLanguage(fullName2, 'Scala'),
+			repositoryWithDepGraphLanguage(fullName, scala),
+			repositoryWithDepGraphLanguage(fullName2, scala),
 		];
 
 		expect(result).toEqual(expected);
 	});
+	function exemptedCustomProperty(): github_repository_custom_properties {
+		return {
+			cq_sync_time: null,
+			cq_source_name: null,
+			cq_id: 'id1',
+			cq_parent_id: null,
+			org: 'guardian',
+			property_name: 'gu_dependency_graph_integrator_ignore',
+			repository_id: BigInt(1),
+			value: scala,
+		};
+	}
+
+	function nonExemptedCustomProperty(): github_repository_custom_properties {
+		return {
+			cq_sync_time: null,
+			cq_source_name: null,
+			cq_id: 'id1',
+			cq_parent_id: null,
+			org: 'guardian',
+			property_name: 'gu_dependency_graph_integrator_ignore',
+			repository_id: BigInt(12345),
+			value: null,
+		};
+	}
+	test('return the repo when a Scala repo is found without an existing workflow and repo is not exempt', () => {
+		const result = getSuitableReposWithoutWorkflows(
+			[repoWithTargetLanguage(fullName, scala)],
+			[repository(fullName)],
+			[repoWithoutWorkflow(fullName)],
+			[nonExemptedCustomProperty()],
+		);
+		const expected = [repositoryWithDepGraphLanguage(fullName, scala)];
+
+		expect(result).toEqual(expected);
+	});
+	test('return the repo when a Kotlin repo is found without an existing workflow and repo is not exempt', () => {
+		const result = getSuitableReposWithoutWorkflows(
+			[repoWithTargetLanguage(fullName, kotlin)],
+			[repository(fullName)],
+			[repoWithoutWorkflow(fullName)],
+			[nonExemptedCustomProperty()],
+		);
+		const expected = [repositoryWithDepGraphLanguage(fullName, kotlin)];
+
+		expect(result).toEqual(expected);
+	});
+	test('return empty repo array when a Scala repo is found without an existing workflow but is exempt', () => {
+		const result = getSuitableReposWithoutWorkflows(
+			[repoWithTargetLanguage(fullName, scala)],
+			[repository(fullName)],
+			[repoWithoutWorkflow(fullName)],
+			[exemptedCustomProperty()],
+		);
+		expect(result).toEqual([]);
+	});
+	test('return empty repo array when a Kotlin repo is found without an existing workflow but is exempt', () => {
+		const result = getSuitableReposWithoutWorkflows(
+			[repoWithTargetLanguage(fullName, kotlin)],
+			[repository(fullName)],
+			[repoWithoutWorkflow(fullName)],
+			[{ ...exemptedCustomProperty(), value: kotlin }],
+		);
+		expect(result).toEqual([]);
+	});
 
 	const ownershipRecord1: view_repo_ownership = {
 		full_repo_name: fullName,
@@ -200,13 +277,13 @@ describe('When getting suitable events to send to SNS', () => {
 		};
 
 		const result = createSnsEventsForDependencyGraphIntegration(
-			[repositoryWithDepGraphLanguage(fullName, 'Scala')],
+			[repositoryWithDepGraphLanguage(fullName, scala)],
 			[ownershipRecord1, ownershipRecord2],
 		);
 		expect(result).toEqual([
 			{
 				name: removeRepoOwner(fullName),
-				language: 'Scala',
+				language: scala,
 				admins: ['team-slug', 'team-slug2'],
 			},
 		]);
@@ -219,13 +296,13 @@ describe('When getting suitable events to send to SNS', () => {
 		};
 
 		const result = createSnsEventsForDependencyGraphIntegration(
-			[repositoryWithDepGraphLanguage(fullName, 'Scala')],
+			[repositoryWithDepGraphLanguage(fullName, scala)],
 			[ownershipRecord],
 		);
 		expect(result).toEqual([
 			{
 				name: removeRepoOwner(fullName),
-				language: 'Scala',
+				language: scala,
 				admins: [],
 			},
 		]);

packages/repocop/src/remediation/dependency_graph-integrator/send-to-sns.ts

@@ -2,11 +2,13 @@ import { PublishCommand, SNSClient } from '@aws-sdk/client-sns';
 import type { Endpoints } from '@octokit/types';
 import type {
 	github_languages,
+	github_repository_custom_properties,
 	guardian_github_actions_usage,
 	view_repo_ownership,
 } from '@prisma/client';
 import { awsClientConfig } from 'common/src/aws';
 import { shuffle } from 'common/src/functions';
+import { logger } from 'common/src/logs';
 import type {
 	DependencyGraphIntegratorEvent,
 	DepGraphLanguage,
@@ -119,10 +121,30 @@ async function sendOneRepoToDepGraphIntegrator(
 	}
 }
 
-export function getReposWithoutWorkflows(
+export function repoIsExempted(
+	repo: Repository,
+	exemptedCustomProperties: github_repository_custom_properties[],
+	language: DepGraphLanguage,
+): boolean {
+	const exemptedRepo: github_repository_custom_properties | undefined =
+		exemptedCustomProperties.find(
+			(property) =>
+				repo.id === property.repository_id && language === property.value,
+		);
+	if (exemptedRepo) {
+		logger.log({
+			message: `${repo.name} is exempted from dependency graph integration for ${language}`,
+			numexemptedCustomProperties: exemptedCustomProperties.length,
+		});
+	}
+	return exemptedRepo !== undefined;
+}
+
+export function getSuitableReposWithoutWorkflows(
 	languages: github_languages[],
 	productionRepos: Repository[],
 	productionWorkflowUsages: guardian_github_actions_usage[],
+	exemptedCustomProperties: github_repository_custom_properties[],
 ): RepositoryWithDepGraphLanguage[] {
 	const depGraphLanguages: DepGraphLanguage[] = ['Scala', 'Kotlin'];
 
@@ -136,6 +158,9 @@ export function getReposWithoutWorkflows(
 			);
 
 			return reposWithDepGraphLanguages
+				.filter(
+					(repo) => !repoIsExempted(repo, exemptedCustomProperties, language),
+				)
 				.filter((repo) => {
 					const workflowUsagesForRepo = productionWorkflowUsages.filter(
 						(workflow) => workflow.full_name === repo.full_name,
@@ -160,15 +185,17 @@ export async function sendReposToDependencyGraphIntegrator(
 	repoLanguages: github_languages[],
 	productionRepos: Repository[],
 	productionWorkflowUsages: guardian_github_actions_usage[],
+	repoCustomProperties: github_repository_custom_properties[],
 	repoOwners: view_repo_ownership[],
 	repoCount: number,
 	octokit: Octokit,
 ): Promise<void> {
 	const reposRequiringDepGraphIntegration: RepositoryWithDepGraphLanguage[] =
-		getReposWithoutWorkflows(
+		getSuitableReposWithoutWorkflows(
 			repoLanguages,
 			productionRepos,
 			productionWorkflowUsages,
+			repoCustomProperties,
 		);
 
 	if (reposRequiringDepGraphIntegration.length !== 0) {