diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b910971e..f282de76 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,6 +18,12 @@ jobs: os: [macOS-latest, windows-latest, ubuntu-22.04-xl] steps: + # Some test cases are sensitive to line endings. Disable autocrlf on + # Windows to ensure consistent behavior. + - name: Disable autocrlf + if: runner.os == 'Windows' + run: git config --global core.autocrlf false + - name: Setup repo uses: actions/checkout@v3 diff --git a/action/deps.js b/action/deps.js index 6fd1219c..c4ac3421 100644 --- a/action/deps.js +++ b/action/deps.js @@ -4354,7 +4354,15 @@ function include(path, include, exclude) { } return true; } -async function walk(cwd, dir, files, options) { +async function walk(cwd, dir, options) { + const hashPathMap = new Map(); + const manifestEntries = await walkInner(cwd, dir, hashPathMap, options); + return { + manifestEntries, + hashPathMap + }; +} +async function walkInner(cwd, dir, hashPathMap, options) { const entries = {}; for await (const file of Deno.readDir(dir)){ const path = join2(dir, file.name); @@ -4371,12 +4379,12 @@ async function walk(cwd, dir, files, options) { gitSha1, size: data.byteLength }; - files.set(gitSha1, path); + hashPathMap.set(gitSha1, path); } else if (file.isDirectory) { if (relative === "/.git") continue; entry = { kind: "directory", - entries: await walk(cwd, path, files, options) + entries: await walkInner(cwd, path, hashPathMap, options) }; } else if (file.isSymlink) { const target = await Deno.readLink(path); diff --git a/action/index.js b/action/index.js index 252c50d5..77a613b6 100644 --- a/action/index.js +++ b/action/index.js @@ -79,11 +79,14 @@ async function main() { if (!includes.some((i) => i.includes("node_modules"))) { excludes.push("**/node_modules"); } - const assets = new Map(); - const entries = await walk(cwd, cwd, assets, { - include: includes.map(convertPatternToRegExp), - exclude: excludes.map(convertPatternToRegExp), - }); + const { manifestEntries: entries, hashPathMap: assets } = await walk( + cwd, + cwd, + { + include: includes.map(convertPatternToRegExp), + exclude: excludes.map(convertPatternToRegExp), + }, + ); core.debug(`Discovered ${assets.size} assets`); const api = new API(`GitHubOIDC ${token}`, ORIGIN); diff --git a/src/subcommands/deploy.ts b/src/subcommands/deploy.ts index 7bb6659b..24cd32e1 100644 --- a/src/subcommands/deploy.ts +++ b/src/subcommands/deploy.ts @@ -1,6 +1,6 @@ // Copyright 2021 Deno Land Inc. All rights reserved. MIT license. -import { fromFileUrl, type Spinner } from "../../deps.ts"; +import { fromFileUrl, relative, type Spinner, yellow } from "../../deps.ts"; import { envVarsFromArgs } from "../utils/env_vars.ts"; import { wait } from "../utils/spinner.ts"; import configFile from "../config_file.ts"; @@ -8,7 +8,11 @@ import { error } from "../error.ts"; import { API, APIError, endpoint } from "../utils/api.ts"; import type { ManifestEntry } from "../utils/api_types.ts"; import { parseEntrypoint } from "../utils/entrypoint.ts"; -import { convertPatternToRegExp, walk } from "../utils/walk.ts"; +import { + containsEntryInManifest, + convertPatternToRegExp, + walk, +} from "../utils/manifest.ts"; import TokenProvisioner from "../utils/access_token.ts"; import type { Args as RawArgs } from "../args.ts"; import organization from "../utils/organization.ts"; @@ -273,14 +277,46 @@ async function deploy(opts: DeployOpts): Promise { if (opts.static) { wait("").start().info(`Uploading all files from the current dir (${cwd})`); const assetSpinner = wait("Finding static assets...").start(); - const assets = new Map(); const include = opts.include.map(convertPatternToRegExp); const exclude = opts.exclude.map(convertPatternToRegExp); - const entries = await walk(cwd, cwd, assets, { include, exclude }); + const { manifestEntries: entries, hashPathMap: assets } = await walk( + cwd, + cwd, + { include, exclude }, + ); assetSpinner.succeed( `Found ${assets.size} asset${assets.size === 1 ? "" : "s"}.`, ); + // If the import map is specified but not in the manifest, error out. + if ( + opts.importMapUrl !== null && + !containsEntryInManifest( + entries, + relative(cwd, fromFileUrl(opts.importMapUrl)), + ) + ) { + error( + `Import map ${opts.importMapUrl} not found in the assets to be uploaded. Please check --include and --exclude options to make sure the import map is included.`, + ); + } + + // If the config file is present but not in the manifest, show a warning + // that any import map settings in the config file will not be used. + if ( + opts.importMapUrl === null && opts.config !== null && + !containsEntryInManifest( + entries, + relative(cwd, opts.config), + ) + ) { + wait("").start().warn( + yellow( + `Config file ${opts.config} not found in the assets to be uploaded; any import map settings in the config file will not be applied during deployment. If this is not your intention, please check --include and --exclude options to make sure the config file is included.`, + ), + ); + } + uploadSpinner = wait("Determining assets to upload...").start(); const neededHashes = await api.projectNegotiateAssets(project.id, { entries, diff --git a/src/utils/walk.ts b/src/utils/manifest.ts similarity index 63% rename from src/utils/walk.ts rename to src/utils/manifest.ts index 5bfc458a..5ce45f3d 100644 --- a/src/utils/walk.ts +++ b/src/utils/manifest.ts @@ -1,4 +1,5 @@ import { globToRegExp, isGlob, join, normalize } from "../../deps.ts"; +import { unreachable } from "../../tests/deps.ts"; import type { ManifestEntry } from "./api_types.ts"; /** Calculate git object hash, like `git hash-object` does. */ @@ -38,7 +39,25 @@ function include( export async function walk( cwd: string, dir: string, - files: Map, + options: { include: RegExp[]; exclude: RegExp[] }, +): Promise< + { + manifestEntries: Record; + hashPathMap: Map; + } +> { + const hashPathMap = new Map(); + const manifestEntries = await walkInner(cwd, dir, hashPathMap, options); + return { + manifestEntries, + hashPathMap, + }; +} + +async function walkInner( + cwd: string, + dir: string, + hashPathMap: Map, options: { include: RegExp[]; exclude: RegExp[] }, ): Promise> { const entries: Record = {}; @@ -65,12 +84,12 @@ export async function walk( gitSha1, size: data.byteLength, }; - files.set(gitSha1, path); + hashPathMap.set(gitSha1, path); } else if (file.isDirectory) { if (relative === "/.git") continue; entry = { kind: "directory", - entries: await walk(cwd, path, files, options), + entries: await walkInner(cwd, path, hashPathMap, options), }; } else if (file.isSymlink) { const target = await Deno.readLink(path); @@ -98,3 +117,41 @@ export function convertPatternToRegExp(pattern: string): RegExp { ? new RegExp(globToRegExp(normalize(pattern)).toString().slice(1, -2)) : new RegExp(`^${normalize(pattern)}`); } + +/** + * Determines if the manifest contains the entry at the given relative path. + * + * @param manifestEntries manifest entries to search + * @param entryRelativePathToLookup a relative path to look up in the manifest + * @returns `true` if the manifest contains the entry at the given relative path + */ +export function containsEntryInManifest( + manifestEntries: Record, + entryRelativePathToLookup: string, +): boolean { + for (const [entryName, entry] of Object.entries(manifestEntries)) { + switch (entry.kind) { + case "file": + case "symlink": { + if (entryName === entryRelativePathToLookup) { + return true; + } + break; + } + case "directory": { + if (!entryRelativePathToLookup.startsWith(entryName)) { + break; + } + + const relativePath = entryRelativePathToLookup.slice( + entryName.length + 1, + ); + return containsEntryInManifest(entry.entries, relativePath); + } + default: + unreachable(); + } + } + + return false; +} diff --git a/src/utils/manifest_test.ts b/src/utils/manifest_test.ts new file mode 100644 index 00000000..5afd8690 --- /dev/null +++ b/src/utils/manifest_test.ts @@ -0,0 +1,295 @@ +import { dirname, fromFileUrl, join } from "../../deps.ts"; +import { assert, assertEquals, assertFalse } from "../../tests/deps.ts"; +import type { ManifestEntry } from "./api_types.ts"; +import { + containsEntryInManifest, + convertPatternToRegExp, + walk, +} from "./manifest.ts"; + +Deno.test({ + name: "convertPatternToRegExp", + ignore: Deno.build.os === "windows", + fn: () => { + assertEquals(convertPatternToRegExp("foo"), new RegExp("^foo")); + assertEquals(convertPatternToRegExp(".././foo"), new RegExp("^../foo")); + assertEquals(convertPatternToRegExp("*.ts"), new RegExp("^[^/]*\\.ts/*")); + }, +}); + +Deno.test({ + name: "walk and containsEntryInManifest", + fn: async (t) => { + type Test = { + name: string; + input: { + testdir: string; + include: readonly string[]; + exclude: readonly string[]; + }; + expected: { + entries: Record; + containedEntries: readonly string[]; + notContainedEntries: readonly string[]; + }; + }; + + const tests: Test[] = [ + { + name: "single_file", + input: { + testdir: "single_file", + include: [], + exclude: [], + }, + expected: { + entries: { + "a.txt": { + kind: "file", + gitSha1: "78981922613b2afb6025042ff6bd878ac1994e85", + size: 2, + }, + }, + containedEntries: ["a.txt"], + notContainedEntries: ["b.txt", ".git", "deno.json"], + }, + }, + { + name: "single_file with include", + input: { + testdir: "single_file", + include: ["a.txt"], + exclude: [], + }, + expected: { + entries: { + "a.txt": { + kind: "file", + gitSha1: "78981922613b2afb6025042ff6bd878ac1994e85", + size: 2, + }, + }, + containedEntries: ["a.txt"], + notContainedEntries: ["b.txt", ".git", "deno.json"], + }, + }, + { + name: "single_file with include 2", + input: { + testdir: "single_file", + include: ["*.txt"], + exclude: [], + }, + expected: { + entries: { + "a.txt": { + kind: "file", + gitSha1: "78981922613b2afb6025042ff6bd878ac1994e85", + size: 2, + }, + }, + containedEntries: ["a.txt"], + notContainedEntries: ["b.txt", ".git", "deno.json"], + }, + }, + { + name: "single_file with exclude", + input: { + testdir: "single_file", + include: [], + exclude: ["a.txt"], + }, + expected: { + entries: {}, + containedEntries: [], + notContainedEntries: ["a.txt", "b.txt", ".git", "deno.json"], + }, + }, + { + name: "two_levels", + input: { + testdir: "two_levels", + include: [], + exclude: [], + }, + expected: { + entries: { + "a.txt": { + kind: "file", + gitSha1: "78981922613b2afb6025042ff6bd878ac1994e85", + size: 2, + }, + "inner": { + kind: "directory", + entries: { + "b.txt": { + kind: "file", + gitSha1: "61780798228d17af2d34fce4cfbdf35556832472", + size: 2, + }, + }, + }, + }, + containedEntries: ["a.txt", "inner/b.txt"], + notContainedEntries: [ + "b.txt", + "inner/a.txt", + ".git", + "deno.json", + "inner", + ], + }, + }, + { + name: "two_levels with include", + input: { + testdir: "two_levels", + include: ["**/b.txt"], + exclude: [], + }, + expected: { + entries: { + "inner": { + kind: "directory", + entries: { + "b.txt": { + kind: "file", + gitSha1: "61780798228d17af2d34fce4cfbdf35556832472", + size: 2, + }, + }, + }, + }, + containedEntries: ["inner/b.txt"], + notContainedEntries: [ + "a.txt", + "b.txt", + "inner/a.txt", + ".git", + "deno.json", + "inner", + ], + }, + }, + { + name: "two_levels with exclude", + input: { + testdir: "two_levels", + include: [], + exclude: ["*.txt"], + }, + expected: { + entries: { + "inner": { + kind: "directory", + entries: { + "b.txt": { + kind: "file", + gitSha1: "61780798228d17af2d34fce4cfbdf35556832472", + size: 2, + }, + }, + }, + }, + containedEntries: ["inner/b.txt"], + notContainedEntries: [ + "a.txt", + "b.txt", + "inner/a.txt", + ".git", + "deno.json", + "inner", + ], + }, + }, + { + name: "complex", + input: { + testdir: "complex", + include: [], + exclude: [], + }, + expected: { + entries: { + "a.txt": { + kind: "file", + gitSha1: "78981922613b2afb6025042ff6bd878ac1994e85", + size: 2, + }, + "inner1": { + kind: "directory", + entries: { + "b.txt": { + kind: "file", + gitSha1: "61780798228d17af2d34fce4cfbdf35556832472", + size: 2, + }, + }, + }, + "inner2": { + kind: "directory", + entries: { + "b.txt": { + kind: "file", + gitSha1: "61780798228d17af2d34fce4cfbdf35556832472", + size: 2, + }, + }, + }, + }, + containedEntries: ["a.txt", "inner1/b.txt", "inner2/b.txt"], + notContainedEntries: [ + "b.txt", + "inner1/a.txt", + "inner2/a.txt", + ".git", + "deno.json", + "inner1", + "inner2", + ], + }, + }, + ]; + + for (const test of tests) { + await t.step({ + name: test.name, + fn: async () => { + const { manifestEntries } = await walk( + join( + fromFileUrl(dirname(import.meta.url)), + "manifest_testdata", + test.input.testdir, + ), + join( + fromFileUrl(dirname(import.meta.url)), + "manifest_testdata", + test.input.testdir, + ), + { + include: test.input.include.map(convertPatternToRegExp), + exclude: test.input.exclude.map(convertPatternToRegExp), + }, + ); + assertEquals(manifestEntries, test.expected.entries); + + for (const entry of test.expected.containedEntries) { + const contained = containsEntryInManifest(manifestEntries, entry); + assert( + contained, + `Expected ${entry} to be contained in the manifest`, + ); + } + + for (const entry of test.expected.notContainedEntries) { + const contained = containsEntryInManifest(manifestEntries, entry); + assertFalse( + contained, + `Expected ${entry} to *not* be contained in the manifest`, + ); + } + }, + }); + } + }, +}); diff --git a/src/utils/manifest_testdata/complex/a.txt b/src/utils/manifest_testdata/complex/a.txt new file mode 100644 index 00000000..78981922 --- /dev/null +++ b/src/utils/manifest_testdata/complex/a.txt @@ -0,0 +1 @@ +a diff --git a/src/utils/manifest_testdata/complex/inner1/b.txt b/src/utils/manifest_testdata/complex/inner1/b.txt new file mode 100644 index 00000000..61780798 --- /dev/null +++ b/src/utils/manifest_testdata/complex/inner1/b.txt @@ -0,0 +1 @@ +b diff --git a/src/utils/manifest_testdata/complex/inner2/b.txt b/src/utils/manifest_testdata/complex/inner2/b.txt new file mode 100644 index 00000000..61780798 --- /dev/null +++ b/src/utils/manifest_testdata/complex/inner2/b.txt @@ -0,0 +1 @@ +b diff --git a/src/utils/manifest_testdata/single_file/a.txt b/src/utils/manifest_testdata/single_file/a.txt new file mode 100644 index 00000000..78981922 --- /dev/null +++ b/src/utils/manifest_testdata/single_file/a.txt @@ -0,0 +1 @@ +a diff --git a/src/utils/manifest_testdata/two_levels/a.txt b/src/utils/manifest_testdata/two_levels/a.txt new file mode 100644 index 00000000..78981922 --- /dev/null +++ b/src/utils/manifest_testdata/two_levels/a.txt @@ -0,0 +1 @@ +a diff --git a/src/utils/manifest_testdata/two_levels/inner/b.txt b/src/utils/manifest_testdata/two_levels/inner/b.txt new file mode 100644 index 00000000..61780798 --- /dev/null +++ b/src/utils/manifest_testdata/two_levels/inner/b.txt @@ -0,0 +1 @@ +b diff --git a/src/utils/mod.ts b/src/utils/mod.ts index 65075ed1..ceaeb6fe 100644 --- a/src/utils/mod.ts +++ b/src/utils/mod.ts @@ -2,7 +2,7 @@ import { semverGreaterThanOrEquals, semverParse } from "../../deps.ts"; export { parseEntrypoint } from "./entrypoint.ts"; export { API, APIError } from "./api.ts"; -export { convertPatternToRegExp, walk } from "./walk.ts"; +export { convertPatternToRegExp, walk } from "./manifest.ts"; export { fromFileUrl, resolve } from "../../deps.ts"; // deno-lint-ignore no-explicit-any diff --git a/src/utils/walk_test.ts b/src/utils/walk_test.ts deleted file mode 100644 index 827af2df..00000000 --- a/src/utils/walk_test.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { assertEquals } from "../../tests/deps.ts"; -import { convertPatternToRegExp } from "./walk.ts"; - -Deno.test({ - name: "convertPatternToRegExp", - ignore: Deno.build.os === "windows", - fn: () => { - assertEquals(convertPatternToRegExp("foo"), new RegExp("^foo")); - assertEquals(convertPatternToRegExp(".././foo"), new RegExp("^../foo")); - assertEquals(convertPatternToRegExp("*.ts"), new RegExp("^[^/]*\\.ts/*")); - }, -});