(feature, generator-cli): Add ability to clone repository (#1028)

.gitattributes

@@ -1,4 +1,4 @@
-packages/generator-cli/configuration/generated/** linguist-generated=true
+clis/generator-cli/configuration/generated/** linguist-generated=true
 packages/template-resolver/src/generated/** linguist-generated=true
 packages/fdr-sdk/src/client/generated/** linguist-generated=true
 servers/fdr/src/api/generated/** linguist-generated=true

clis/generator-cli/package.json

@@ -24,6 +24,7 @@
     "depcheck": "depcheck"
   },
   "devDependencies": {
+    "@fern-api/github": "workspace:*",
     "@fern-api/fs-utils": "0.15.0-rc63",
     "@types/jest": "^29.5.11",
     "@types/lodash-es": "^4.17.12",

clis/generator-cli/src/configuration/generated/api/resources/readme/types/GithubRemote.ts

@@ -0,0 +1,10 @@
+/**
+ * This file was auto-generated by Fern from our API Definition.
+ */
+
+export interface GithubRemote {
+    /** A full repo url (i.e. https://github.com/fern-api/fern) */
+    repoUrl: string;
+    /** The token used to clone the GitHub repository. */
+    installationToken: string;
+}

clis/generator-cli/src/configuration/generated/api/resources/readme/types/ReadmeConfig.ts

@@ -12,6 +12,8 @@ import * as FernGeneratorCli from "../../../index";
  * that comes from each generator (i.e. features, requirements, and more).
  */
 export interface ReadmeConfig {
+    /** If specified, the original README.md will be fetched from this remote (if it exists). */
+    remote?: FernGeneratorCli.Remote;
     language: FernGeneratorCli.LanguageInfo;
     organization: string;
     bannerLink?: string;

clis/generator-cli/src/configuration/generated/api/resources/readme/types/Remote.ts

@@ -0,0 +1,49 @@
+/**
+ * This file was auto-generated by Fern from our API Definition.
+ */
+
+import * as FernGeneratorCli from "../../../index";
+
+/**
+ * The remote where the README.md is hosted.
+ */
+export type Remote = FernGeneratorCli.Remote.Github;
+
+export declare namespace Remote {
+    interface Github extends FernGeneratorCli.GithubRemote, _Utils {
+        type: "github";
+    }
+
+    interface _Utils {
+        _visit: <_Result>(visitor: FernGeneratorCli.Remote._Visitor<_Result>) => _Result;
+    }
+
+    interface _Visitor<_Result> {
+        github: (value: FernGeneratorCli.GithubRemote) => _Result;
+        _other: (value: { type: string }) => _Result;
+    }
+}
+
+export const Remote = {
+    github: (value: FernGeneratorCli.GithubRemote): FernGeneratorCli.Remote.Github => {
+        return {
+            ...value,
+            type: "github",
+            _visit: function <_Result>(
+                this: FernGeneratorCli.Remote.Github,
+                visitor: FernGeneratorCli.Remote._Visitor<_Result>
+            ) {
+                return FernGeneratorCli.Remote._visit(this, visitor);
+            },
+        };
+    },
+
+    _visit: <_Result>(value: FernGeneratorCli.Remote, visitor: FernGeneratorCli.Remote._Visitor<_Result>): _Result => {
+        switch (value.type) {
+            case "github":
+                return visitor.github(value);
+            default:
+                return visitor._other(value as any);
+        }
+    },
+} as const;

clis/generator-cli/src/configuration/generated/api/resources/readme/types/index.ts

@@ -1,4 +1,6 @@
 export * from "./ReadmeConfig";
+export * from "./Remote";
+export * from "./GithubRemote";
 export * from "./ReadmeFeature";
 export * from "./LanguageInfo";
 export * from "./TypescriptInfo";

clis/generator-cli/src/configuration/generated/serialization/resources/readme/types/GithubRemote.ts

@@ -0,0 +1,22 @@
+/**
+ * This file was auto-generated by Fern from our API Definition.
+ */
+
+import * as serializers from "../../../index";
+import * as FernGeneratorCli from "../../../../api/index";
+import * as core from "../../../../core";
+
+export const GithubRemote: core.serialization.ObjectSchema<
+    serializers.GithubRemote.Raw,
+    FernGeneratorCli.GithubRemote
+> = core.serialization.object({
+    repoUrl: core.serialization.string(),
+    installationToken: core.serialization.string(),
+});
+
+export declare namespace GithubRemote {
+    interface Raw {
+        repoUrl: string;
+        installationToken: string;
+    }
+}

clis/generator-cli/src/configuration/generated/serialization/resources/readme/types/ReadmeConfig.ts

@@ -5,13 +5,15 @@
 import * as serializers from "../../../index";
 import * as FernGeneratorCli from "../../../../api/index";
 import * as core from "../../../../core";
+import { Remote } from "./Remote";
 import { LanguageInfo } from "./LanguageInfo";
 import { ReadmeFeature } from "./ReadmeFeature";
 
 export const ReadmeConfig: core.serialization.ObjectSchema<
     serializers.ReadmeConfig.Raw,
     FernGeneratorCli.ReadmeConfig
 > = core.serialization.object({
+    remote: Remote.optional(),
     language: LanguageInfo,
     organization: core.serialization.string(),
     bannerLink: core.serialization.string().optional(),
@@ -22,6 +24,7 @@ export const ReadmeConfig: core.serialization.ObjectSchema<
 
 export declare namespace ReadmeConfig {
     interface Raw {
+        remote?: Remote.Raw | null;
         language: LanguageInfo.Raw;
         organization: string;
         bannerLink?: string | null;

clis/generator-cli/src/configuration/generated/serialization/resources/readme/types/Remote.ts

@@ -0,0 +1,32 @@
+/**
+ * This file was auto-generated by Fern from our API Definition.
+ */
+
+import * as serializers from "../../../index";
+import * as FernGeneratorCli from "../../../../api/index";
+import * as core from "../../../../core";
+import { GithubRemote } from "./GithubRemote";
+
+export const Remote: core.serialization.Schema<serializers.Remote.Raw, FernGeneratorCli.Remote> = core.serialization
+    .union("type", {
+        github: GithubRemote,
+    })
+    .transform<FernGeneratorCli.Remote>({
+        transform: (value) => {
+            switch (value.type) {
+                case "github":
+                    return FernGeneratorCli.Remote.github(value);
+                default:
+                    return value as FernGeneratorCli.Remote;
+            }
+        },
+        untransform: ({ _visit, ...value }) => value as any,
+    });
+
+export declare namespace Remote {
+    type Raw = Remote.Github;
+
+    interface Github extends GithubRemote.Raw {
+        type: "github";
+    }
+}

clis/generator-cli/src/configuration/generated/serialization/resources/readme/types/index.ts

@@ -1,4 +1,6 @@
 export * from "./ReadmeConfig";
+export * from "./Remote";
+export * from "./GithubRemote";
 export * from "./ReadmeFeature";
 export * from "./LanguageInfo";
 export * from "./TypescriptInfo";

clis/generator-cli/src/readme/ReadmeGenerator.ts

@@ -1,3 +1,4 @@
+import { cloneRepository } from "@fern-api/github";
 import fs from "fs";
 import { camelCase, upperFirst } from "lodash-es";
 import { FernGeneratorCli } from "../configuration/generated";
@@ -36,7 +37,7 @@ export class ReadmeGenerator {
         this.writeHeader({ writer });
         this.writeBlocks({
             writer,
-            blocks: this.mergeBlocks({ blocks }),
+            blocks: await this.mergeBlocks({ blocks }),
         });
         writer.end();
 
@@ -93,18 +94,33 @@ export class ReadmeGenerator {
         });
     }
 
-    private mergeBlocks({ blocks }: { blocks: Block[] }): Block[] {
-        if (this.originalReadme == null) {
+    private async mergeBlocks({ blocks }: { blocks: Block[] }): Promise<Block[]> {
+        const originalReadmeContent = await this.getOriginalReadmeContent();
+        if (originalReadmeContent == null) {
             return blocks;
         }
-        const parsed = this.readmeParser.parse({ content: this.originalReadme });
+        const parsed = this.readmeParser.parse({ content: originalReadmeContent });
         const merger = new BlockMerger({
             original: parsed.blocks,
             updated: blocks,
         });
         return merger.merge();
     }
 
+    private async getOriginalReadmeContent(): Promise<string | undefined> {
+        if (this.originalReadme != null) {
+            return this.originalReadme;
+        }
+        if (this.readmeConfig.remote != null) {
+            const clonedRepository = await cloneRepository({
+                githubRepository: this.readmeConfig.remote.repoUrl,
+                installationToken: this.readmeConfig.remote.installationToken,
+            });
+            return await clonedRepository.getReadme();
+        }
+        return undefined;
+    }
+
     private writeBlocks({ writer, blocks }: { writer: Writer; blocks: Block[] }): void {
         for (const block of blocks) {
             block.write(writer);

clis/generator-cli/tsconfig.json

@@ -1,5 +1,6 @@
 {
   "extends": "../../shared/tsconfig.shared.json",
   "compilerOptions": { "composite": true, "outDir": "dist", "rootDir": "." },
-  "include": ["./src/**/*"]
+  "include": ["./src/**/*"],
+  "references": [{ "path": "../../packages/commons/github" }]
 }

fern/apis/generator-cli/definition/readme.yml

@@ -9,6 +9,10 @@ types:
       (i.e. specified in the generators.yml), and dynamically generated information
       that comes from each generator (i.e. features, requirements, and more).
     properties:
+      remote:
+        docs: |
+          If specified, the original README.md will be fetched from this remote (if it exists).
+        type: optional<Remote>
       language: LanguageInfo
       organization: string
       bannerLink: optional<string>
@@ -20,6 +24,21 @@ types:
           The features are rendered in the order they're specified.
         type: optional<list<ReadmeFeature>>
 
+  Remote:
+    docs: |
+      The remote where the README.md is hosted.
+    union:
+      github: GithubRemote
+
+  GithubRemote:
+    properties:
+      repoUrl:
+        docs: A full repo url (i.e. https://github.com/fern-api/fern)
+        type: string
+      installationToken:
+        docs: The token used to clone the GitHub repository.
+        type: string
+
   ReadmeFeature:
     docs: |
       A single feature supported by a generator (e.g. PAGINATION).

fern/apis/generator-cli/generators.yml

@@ -6,7 +6,7 @@ groups:
         version: 0.20.9
         output:
           location: local-file-system
-          path: ../../../packages/generator-cli/src/configuration/generated
+          path: ../../../clis/generator-cli/src/configuration/generated
         config:
           includeUtilsOnUnionMembers: true
           useBrandedStringAliases: true

packages/commons/github/package.json

@@ -35,13 +35,15 @@
     "depcheck": "depcheck"
   },
   "dependencies": {
-    "octokit": "^3.2.0"
+    "octokit": "^3.2.0",
+    "tmp-promise": "^3.0.3"
   },
   "devDependencies": {
     "@fern-platform/configs": "workspace:*",
     "@types/node": "^18.7.18",
     "depcheck": "^1.4.3",
     "eslint": "^8.56.0",
+    "simple-git": "^3.24.0",
     "vitest": "^1.5.0",
     "organize-imports-cli": "^0.10.0",
     "prettier": "^3.2.4",

packages/commons/github/src/ClonedRepository.ts

@@ -0,0 +1,37 @@
+import { lstat, readFile } from "fs/promises";
+import path from "path";
+import { SimpleGit } from "simple-git";
+import { README_FILEPATH } from "./constants";
+
+// ClonedRepository is a repository that has been successfully cloned to the local file system
+// and is ready to be used.
+export class ClonedRepository {
+    private git: SimpleGit;
+    private clonePath: string;
+
+    constructor({ git, clonePath }: { git: SimpleGit; clonePath: string }) {
+        this.git = git;
+        this.clonePath = clonePath;
+    }
+
+    public async getReadme(): Promise<string | undefined> {
+        return await this.readFile({ relativeFilePath: README_FILEPATH });
+    }
+
+    private async readFile({ relativeFilePath }: { relativeFilePath: string }): Promise<string | undefined> {
+        const absoluteFilePath = path.join(this.clonePath, relativeFilePath);
+        if (!doesPathExist(absoluteFilePath)) {
+            return undefined;
+        }
+        return await readFile(absoluteFilePath, "utf-8");
+    }
+}
+
+async function doesPathExist(filepath: string): Promise<boolean> {
+    try {
+        await lstat(filepath);
+        return true;
+    } catch {
+        return false;
+    }
+}

packages/commons/github/src/RepositoryReference.ts

@@ -0,0 +1,11 @@
+// RepositoryReference is a parsed GitHub repository reference, which
+// contains a variety of formats.
+export interface RepositoryReference {
+    remote: string; // e.g. github.com
+    owner: string; // e.g. fern-api
+    repo: string; // e.g. fern
+    repoUrl: string; // e.g. https://github.com/fern-api/fern
+    cloneUrl: string; // e.g. https://github.ccom/fern-api/fern.git
+
+    getAuthedCloneUrl: (installationToken: string) => string;
+}

packages/commons/github/src/__test__/cloneRepository.test.ts

@@ -0,0 +1,28 @@
+import { cloneRepository } from "../cloneRepository";
+
+describe("cloneRepository", () => {
+    it("fern-api/docs-starter-openapi", async () => {
+        const repository = await cloneRepository({
+            githubRepository: "github.com/fern-api/docs-starter-openapi",
+            installationToken: undefined,
+        });
+        const readme = await repository.getReadme();
+        expect(readme).contains("Fern");
+    });
+    it("invalid installation token", async () => {
+        await expect(async () => {
+            await cloneRepository({
+                githubRepository: "https://github.com/fern-api/fern-platform",
+                installationToken: "ghp_xyz",
+            });
+        }).rejects.toThrow();
+    });
+    it("repository does not exist", async () => {
+        await expect(async () => {
+            await cloneRepository({
+                githubRepository: "https://github.com/fern-api/does-not-exist",
+                installationToken: undefined,
+            });
+        }).rejects.toThrow();
+    });
+});