[WIP] feat(fern-bot): update FDR repo and PR DB

.github/workflows/deploy-fern-bot-dev.yml

@@ -24,6 +24,7 @@ env:
   FERNIE_SLACK_APP_TOKEN: ${{ secrets.FERNIE_SLACK_APP_TOKEN }}
   CUSTOMER_ALERTS_SLACK_CHANNEL: "customer-upgrades-dev"
   CO_API_KEY: ${{ secrets.DEV_CO_API_KEY }}
+  FERN_TOKEN: ${{ secrets.FERN_TOKEN }}
 
 jobs:
   deploy_dev:

.github/workflows/deploy-fern-bot-prod.yml

@@ -20,6 +20,7 @@ env:
   DEFAULT_VENUS_ORIGIN: "https://venus.buildwithfern.com"
   DEFAULT_FDR_ORIGIN: "https://registry.buildwithfern.com"
   CO_API_KEY: ${{ secrets.PROD_CO_API_KEY }}
+  FERN_TOKEN: ${{ secrets.FERN_TOKEN }}
 
 jobs:
   deploy_prod:

servers/fern-bot/package.json

@@ -22,7 +22,7 @@
     "@fern-api/core-utils": "0.15.0-rc63",
     "@fern-api/github": "workspace:*",
     "@fern-api/venus-api-sdk": "0.8.1-1-gd6d1a5b",
-    "@fern-fern/generators-sdk": "0.109.0-21be2e5be",
+    "@fern-fern/generators-sdk": "0.110.0-ba7050020",
     "@octokit/openapi-types": "^22.1.0",
     "@slack/web-api": "^6.9.0",
     "cohere-ai": "^7.9.5",

servers/fern-bot/serverless.yml

@@ -28,6 +28,8 @@ provider:
     FERNIE_SLACK_APP_TOKEN: ${env:FERNIE_SLACK_APP_TOKEN, 'placeholder'}
     CUSTOMER_ALERTS_SLACK_CHANNEL: ${env:CUSTOMER_ALERTS_SLACK_CHANNEL, 'placeholder'}
     REPO_TO_RUN_ON: ${env:REPO_TO_RUN_ON, 'OMIT'}
+    FERN_TOKEN: ${env:FERN_TOKEN, 'OMIT'}
+
   # Roles for the lambda functions
   iam:
     role:
@@ -81,6 +83,14 @@ functions:
     layers:
       - arn:aws:lambda:us-east-1:553035198032:layer:git-lambda2:8
 
+  updateFDRRepoData:
+    timeout: 900
+    memorySize: 5120
+    ephemeralStorageSize: 10240
+    handler: "src/functions/update-fdr-repo-data/updateFDRRepoData.handler"
+    layers:
+      - arn:aws:lambda:us-east-1:553035198032:layer:git-lambda2:8
+
   # =============================================
 
   # For on-demand invoking update, especially useful for triggering one-off updates
@@ -185,6 +195,39 @@ stepFunctions:
                           # the output is useless to us.
                           ResultPath: null
                           OutputPath: null
+              - StartAt: UpdateFDRRepoDataBranch
+                States:
+                  UpdateGeneratorsBranch:
+                    Type: Map
+                    End: true
+                    MaxConcurrency: 50
+                    ItemReader:
+                      ReaderConfig:
+                        InputType: CSV
+                        CSVHeaderLocation: FIRST_ROW
+                      Resource: "arn:aws:states:::s3:getObject"
+                      Parameters:
+                        Bucket: "fern-bot-${opt:stage}"
+                        Key: "lambdas/repos.csv"
+                    ItemProcessor:
+                      ProcessorConfig:
+                        Mode: DISTRIBUTED
+                        ExecutionType: STANDARD
+                      StartAt: HandleRepoForGenerator
+                      States:
+                        HandleRepoForGenerator:
+                          Type: Task
+                          Resource: "arn:aws:states:::lambda:invoke"
+                          Parameters:
+                            "Payload.$": "$"
+                            FunctionName:
+                              Fn::GetAtt: [updateFDRRepoData, Arn]
+                          End: true
+                          # Try to discard whatever result AWS has for us
+                          # since we'll hit a size limit otherwise, and
+                          # the output is useless to us.
+                          ResultPath: null
+                          OutputPath: null
             End: true
             # Same as above, try to discard the result
             ResultPath: null

servers/fern-bot/src/functions/generator-updates/shared/updateGeneratorInternal.ts

@@ -2,10 +2,9 @@
 import { FernVenusApi, FernVenusApiClient } from "@fern-api/venus-api-sdk";
 import { FernRegistryClient } from "@fern-fern/generators-sdk";
 import { ChangelogResponse } from "@fern-fern/generators-sdk/api/resources/generators";
-import { execFernCli } from "@libs/fern";
+import { execFernCli, getGenerators, NO_API_FALLBACK_KEY } from "@libs/fern";
 import { DEFAULT_REMOTE_NAME, cloneRepo, configureGit, type Repository } from "@libs/github/utilities";
 import { GeneratorMessageMetadata, SlackService } from "@libs/slack/SlackService";
-import yaml from "js-yaml";
 import { Octokit } from "octokit";
 import SemVer from "semver";
 import { CleanOptions, SimpleGit } from "simple-git";
@@ -86,7 +85,7 @@
     // > N additional updates, see more
 
     if (changelogs.length === 0) {
         throw new Error("Version difference was found, but no changelog entries were found. This is unexpected.");
     }
 
     // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
@@ -130,17 +129,6 @@
     return undefined;
 }
 
-// This type is meant to mirror the data model for the `generator list` command
-// defined in the OSS repo.
-type GeneratorList = Record<string, Record<string, string[]>>;
-const NO_API_FALLBACK_KEY = "NO_API_FALLBACK";
-async function getGenerators(fullRepoPath: string): Promise<GeneratorList> {
-    // Note since this is multi-line, we do not call `cleanStdout` on it, but it should be parsed ok.
-    const response = await execFernCli(`generator list --api-fallback ${NO_API_FALLBACK_KEY}`, fullRepoPath);
-
-    return yaml.load(response.stdout) as GeneratorList;
-}
-
 // We pollute stdout with a version upgrade log, this tries to ignore that by only consuming the first line
 // Exported to leverage in tests
 export function cleanStdout(stdout: string): string {

servers/fern-bot/src/functions/update-fdr-repo-data/actions/updateFDRRepoData.ts

@@ -0,0 +1,211 @@
+import { FernRegistryClient } from "@fern-fern/generators-sdk";
+import { Env } from "@libs/env";
+import { setupGithubApp } from "@libs/github";
+import { App, Octokit } from "octokit";
+import { readFile } from "fs/promises";
+import { Repository, PullRequest } from "@libs/github";
+import tmp from "tmp-promise";
+import { cleanFernStdout, execFernCli, getGenerators, NO_API_FALLBACK_KEY } from "@libs/fern";
+import { cloneRepo, configureGit } from "@libs/github/utilities";
+import { RepoData } from "@libs/schemas";
+import { PullRequestReviewer, PullRequestState } from "@fern-fern/generators-sdk/api";
+
+// Note given we're making requests to FDR, this could take time, so we're parallelizing this function with a Map step in
+// the step function, as we do for all the other actions.
+export async function updateFDRRepoDataInternal(env: Env, repoData: RepoData | undefined): Promise<void> {
+    const app: App = setupGithubApp(env);
+
+    // Get repo data for the given repo
+    await app.eachRepository(async (installation) => {
+        if (
+            (repoData && installation.repository.full_name !== repoData.full_name) ||
+            installation.repository.full_name === "fern-api/fern-platform"
+        ) {
+            return;
+        }
+        await updateRepoDb(
+            app,
+            installation.repository,
+            installation.octokit,
+            env.GITHUB_APP_LOGIN_NAME,
+            env.GITHUB_APP_LOGIN_ID,
+            env.DEFAULT_FDR_ORIGIN,
+            env.FERN_TOKEN,
+        );
+    });
+}
+
+async function updateRepoDb(
+    app: App,
+    repository: Repository,
+    octokit: Octokit,
+    fernBotLoginName: string,
+    fernBotLoginId: string,
+    fdrUrl: string,
+    fernToken: string,
+): Promise<void> {
+    console.log(`Updating repo data at  ${fdrUrl} with token ${fernToken}`);
+    const client = new FernRegistryClient({ environment: fdrUrl, token: fernToken });
+
+    const [git, fullRepoPath] = await configureGit(repository);
+    console.log(`Cloning repo: ${repository.clone_url} to ${fullRepoPath}`);
+    await cloneRepo(git, repository, octokit, fernBotLoginName, fernBotLoginId);
+
+    try {
+        // Try this to see if it's a fern config repo, there are probably better ways to do this
+        const generatorsList = await getGenerators(fullRepoPath);
+        const organizationId = cleanFernStdout((await execFernCli("organization", fullRepoPath)).stdout);
+
+        // Update config repo in FDR
+        console.log("trying upsertRepository");
+        const upsertResponse = await client.git.upsertRepository({
+            type: "config",
+            id: {
+                type: "github",
+                id: repository.id.toString(),
+            },
+            name: repository.name,
+            owner: repository.owner.login,
+            fullName: repository.full_name,
+            url: repository.html_url,
+            repositoryOwnerOrganizationId: organizationId,
+            // TODO(FER-2517): actually track and action checks
+            defaultBranchChecks: [],
+        });
+
+        if (!upsertResponse.ok) {
+            console.log(`Failed to upsert configuration repo, bailing out: ${JSON.stringify(upsertResponse.error)}`);
+            return;
+        }
+        await getAndUpsertPulls(client, octokit, repository);
+
+        for (const [apiName, api] of Object.entries(generatorsList)) {
+            for (const [groupName, group] of Object.entries(api)) {
+                for (const generator of group) {
+                    const tmpDir = await tmp.dir();
+                    const repoJsonFileName = `${tmpDir.path}/${repository.id.toString()}.json`;
+                    let command = `generator get --repository --language --generator ${generator} --group ${groupName} -o ${repoJsonFileName}`;
+                    if (apiName !== NO_API_FALLBACK_KEY) {
+                        command += ` --api ${apiName}`;
+                    }
+
+                    await execFernCli(command, fullRepoPath);
+                    const maybeRepo = await readFile(repoJsonFileName, "utf8");
+                    if (maybeRepo?.length > 0) {
+                        // Of the form { repository: string, language: string }
+                        const generatorsYmlRepo = JSON.parse(maybeRepo);
+                        if ("repository" in generatorsYmlRepo && generatorsYmlRepo.repository.length > 0) {
+                            const octokitRepo = await getRepository(app, generatorsYmlRepo.repository);
+                            if (octokitRepo) {
+                                await client.git.upsertRepository({
+                                    type: "sdk",
+                                    sdkLanguage: generatorsYmlRepo.language,
+                                    id: {
+                                        type: "github",
+                                        id: octokitRepo.id.toString(),
+                                    },
+                                    name: octokitRepo.name,
+                                    owner: octokitRepo.owner.login,
+                                    fullName: octokitRepo.full_name,
+                                    url: octokitRepo.url,
+                                    repositoryOwnerOrganizationId: organizationId,
+                                    // TODO(FER-2517): actually track and action checks
+                                    defaultBranchChecks: [],
+                                });
+
+                                await getAndUpsertPulls(client, octokit, octokitRepo);
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    } catch (e) {
+        console.log(
+            `Found a repo that was not a Fern config repo, or not a high enough version, skipping...: ${(e as Error).message}`,
+        );
+    }
+}
+
+async function getAndUpsertPulls(client: FernRegistryClient, octokit: Octokit, repository: Repository) {
+    // Get all PRs on repo, update PRs in FDR
+    const pulls = await octokit.rest.pulls.list({
+        state: "all",
+        owner: repository.owner.login,
+        repo: repository.name,
+    });
+
+    for (const pull of pulls.data) {
+        try {
+            await client.git.upsertPullRequest({
+                pullRequestNumber: pull.number,
+                repositoryName: repository.name,
+                repositoryOwner: repository.owner.login,
+                author: pull.user
+                    ? {
+                          username: pull.user.login,
+                          // These can be null or undefined, but we're just taking them as the same and making them undefined
+                          email: pull.user.email ?? undefined,
+                          name: pull.user.name ?? undefined,
+                      }
+                    : undefined,
+                reviewers: getReviewers(pull as PullRequest),
+                title: pull.title,
+                url: pull.html_url,
+                // TODO(FER-2517): actually track and action checks
+                checks: [],
+                // This should be a safe cast
+                state: pull.state as PullRequestState,
+                createdAt: new Date(pull.created_at),
+                updatedAt: new Date(pull.updated_at),
+                mergedAt: pull.merged_at != null ? new Date(pull.merged_at) : undefined,
+                closedAt: pull.closed_at != null ? new Date(pull.closed_at) : undefined,
+            });
+        } catch (e) {
+            console.error(
+                `Error updating PR ${pull.number} on repo ${repository.full_name}: ${e}, likely we have not registered this repo, quitting.`,
+            );
+            return;
+        }
+    }
+}
+
+function getReviewers(pull: PullRequest): PullRequestReviewer[] {
+    const reviewers: PullRequestReviewer[] = [];
+    if (pull.requested_reviewers != null) {
+        for (const reviewer of pull.requested_reviewers) {
+            reviewers.push({
+                type: "user",
+                username: reviewer.login,
+                // These can be null or undefined, but we're just taking them as the same and making them undefined
+                email: reviewer.email ?? undefined,
+                name: reviewer.name ?? undefined,
+            });
+        }
+    }
+
+    if (pull.requested_teams != null) {
+        for (const team_reviewer of pull.requested_teams) {
+            reviewers.push({
+                type: "team",
+                name: team_reviewer.name,
+                teamId: team_reviewer.id.toString(),
+            });
+        }
+    }
+
+    return reviewers;
+}
+
+// Does octokit not expose a better way to do this???
+async function getRepository(app: App, repositoryFullName: string): Promise<Repository | undefined> {
+    let maybeRepo: Repository | undefined = undefined;
+    await app.eachRepository((installation) => {
+        // repo and organization names are case insensitive, so the full name is as well
+        if (installation.repository.full_name.toLowerCase() === repositoryFullName.toLowerCase()) {
+            maybeRepo = installation.repository;
+        }
+    });
+
+    return maybeRepo;
+}

servers/fern-bot/src/functions/update-fdr-repo-data/updateFDRRepoData.ts

@@ -0,0 +1,13 @@
+import { evaluateEnv } from "@libs/env";
+import { handlerWrapper } from "@libs/handler-wrapper";
+import { updateFDRRepoDataInternal } from "./actions/updateFDRRepoData";
+import { RepoData } from "@libs/schemas";
+
+const updateFDRRepoData = async (event: unknown) => {
+    console.debug("Beginning scheduled run of `updateRepoData`, received event:", event);
+    const env = evaluateEnv();
+    console.debug("Environment evaluated, continuing to actual action execution.");
+    return await updateFDRRepoDataInternal(env, event as RepoData | undefined);
+};
+
+export const handler = handlerWrapper(updateFDRRepoData);

servers/fern-bot/src/libs/env.ts

@@ -17,6 +17,7 @@ export interface Env {
     DEFAULT_FDR_ORIGIN: string;
     CUSTOMER_ALERTS_SLACK_CHANNEL: string;
     FERNIE_SLACK_APP_TOKEN: string;
+    FERN_TOKEN: string;
 }
 
 export function evaluateEnv(): Env {
@@ -40,5 +41,6 @@ export function evaluateEnv(): Env {
         DEFAULT_FDR_ORIGIN: process?.env.DEFAULT_FDR_ORIGIN!,
         FERNIE_SLACK_APP_TOKEN: process?.env.FERNIE_SLACK_APP_TOKEN!,
         CUSTOMER_ALERTS_SLACK_CHANNEL: process?.env.CUSTOMER_ALERTS_SLACK_CHANNEL!,
+        FERN_TOKEN: process?.env.FERN_TOKEN!,
     };
 }

servers/fern-bot/src/libs/fern.ts

@@ -2,6 +2,9 @@ import execa from "execa";
 import tmp from "tmp-promise";
 import { doesPathExist } from "./fs";
 
+import { readFile } from "fs/promises";
+import yaml from "js-yaml";
+
 export async function execFernCli(
     command: string,
     cwd?: string,
@@ -43,3 +46,24 @@ export async function execFernCli(
         throw error;
     }
 }
+
+// We pollute stdout with a version upgrade log, this tries to ignore that by only consuming the first line
+// Exported to leverage in tests
+export function cleanFernStdout(stdout: string): string {
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+    return stdout.split("╭─")[0]!.split("\n")[0]!.trim();
+}
+
+// This type is meant to mirror the data model for the `generator list` command
+// defined in the OSS repo.
+type GeneratorList = Record<string, Record<string, string[]>>;
+export const NO_API_FALLBACK_KEY = "NO_API_FALLBACK";
+export async function getGenerators(fullRepoPath: string): Promise<GeneratorList> {
+    const tmpDir = await tmp.dir();
+    const outputPath = `${tmpDir.path}/gen_list.yml`;
+    await execFernCli(`generator list --api-fallback ${NO_API_FALLBACK_KEY} -o ${outputPath}`, fullRepoPath);
+
+    const data = await readFile(outputPath, "utf-8");
+
+    return yaml.load(data) as GeneratorList;
+}