Skip to content

Commit

Permalink
Add --omit-paths flag to templates apply (#868)
Browse files Browse the repository at this point in the history 
* updating omitting extraction of paths from getBlob() tarball

* add --omit-paths flag to templates apply command

* update tests

* Add example

* refactor to match spec

* missing linter

* dont change the expected output stored in "files"
  • Loading branch information
@joshspicer
joshspicer authored Aug 12, 2024
1 parent ed43a5b commit fcefe52
Show file tree
Hide file tree
Showing 4 changed files with 161 additions and 25 deletions.
41 changes: 27 additions & 14 deletions src/spec-configuration/containerCollectionsOCI.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -504,7 +504,7 @@ export async function getPublishedTags(params: CommonParams, ref: OCIRef): Promi
}
}

export async function getBlob(params: CommonParams, url: string, ociCacheDir: string, destCachePath: string, ociRef: OCIRef, expectedDigest: string, ignoredFilesDuringExtraction: string[] = [], metadataFile?: string): Promise<{ files: string[]; metadata: {} | undefined } | undefined> {
export async function getBlob(params: CommonParams, url: string, ociCacheDir: string, destCachePath: string, ociRef: OCIRef, expectedDigest: string, omitDuringExtraction: string[] = [], metadataFile?: string): Promise<{ files: string[]; metadata: {} | undefined } | undefined> {
// TODO: Parallelize if multiple layers (not likely).
// TODO: Seeking might be needed if the size is too large.

Expand Down Expand Up @@ -543,24 +543,37 @@ export async function getBlob(params: CommonParams, url: string, ociCacheDir: st
await mkdirpLocal(destCachePath);
await writeLocalFile(tempTarballPath, resBody);

// https://github.com/devcontainers/spec/blob/main/docs/specs/devcontainer-templates.md#the-optionalpaths-property
const directoriesToOmit = omitDuringExtraction.filter(f => f.endsWith('/*')).map(f => f.slice(0, -1));
const filesToOmit = omitDuringExtraction.filter(f => !f.endsWith('/*'));

output.write(`omitDuringExtraction: '${omitDuringExtraction.join(', ')}`, LogLevel.Trace);
output.write(`Files to omit: '${filesToOmit.join(', ')}'`, LogLevel.Info);
if (directoriesToOmit.length) {
output.write(`Dirs to omit : '${directoriesToOmit.join(', ')}'`, LogLevel.Info);
}

const files: string[] = [];
await tar.x(
{
file: tempTarballPath,
cwd: destCachePath,
filter: (path: string, stat: tar.FileStat) => {
// Skip files that are in the ignore list
if (ignoredFilesDuringExtraction.some(f => path.indexOf(f) !== -1)) {
// Skip.
output.write(`Skipping file '${path}' during blob extraction`, LogLevel.Trace);
return false;
filter: (tPath: string, stat: tar.FileStat) => {
output.write(`Testing '${tPath}'(${stat.type})`, LogLevel.Trace);
const cleanedPath = tPath
.replace(/\\/g, '/')
.replace(/^\.\//, '');

if (filesToOmit.includes(cleanedPath) || directoriesToOmit.some(d => cleanedPath.startsWith(d))) {
output.write(` Omitting '${tPath}'`, LogLevel.Trace);
return false; // Skip
}
// Keep track of all files extracted, in case the caller is interested.
output.write(`${path} : ${stat.type}`, LogLevel.Trace);
if ((stat.type.toString() === 'File')) {
files.push(path);

if (stat.type.toString() === 'File') {
files.push(tPath);
}
return true;

return true; // Keep
}
}
);
Expand All @@ -576,8 +589,8 @@ export async function getBlob(params: CommonParams, url: string, ociCacheDir: st
{
file: tempTarballPath,
cwd: ociCacheDir,
filter: (path: string, _: tar.FileStat) => {
return path === `./${metadataFile}`;
filter: (tPath: string, _: tar.FileStat) => {
return tPath === `./${metadataFile}`;
}
});
const pathToMetadataFile = path.join(ociCacheDir, metadataFile);
Expand Down
5 changes: 3 additions & 2 deletions src/spec-configuration/containerTemplatesOCI.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,13 @@ export interface SelectedTemplate {
id: string;
options: TemplateOptions;
features: TemplateFeatureOption[];
omitPaths: string[];
}

export async function fetchTemplate(params: CommonParams, selectedTemplate: SelectedTemplate, templateDestPath: string, userProvidedTmpDir?: string): Promise<string[] | undefined> {
const { output } = params;

let { id: userSelectedId, options: userSelectedOptions } = selectedTemplate;
let { id: userSelectedId, options: userSelectedOptions, omitPaths } = selectedTemplate;
const templateRef = getRef(output, userSelectedId);
if (!templateRef) {
output.write(`Failed to parse template ref for ${userSelectedId}`, LogLevel.Error);
Expand All @@ -46,7 +47,7 @@ export async function fetchTemplate(params: CommonParams, selectedTemplate: Sele
output.write(`blob url: ${blobUrl}`, LogLevel.Trace);

const tmpDir = userProvidedTmpDir || path.join(os.tmpdir(), 'vsch-template-temp', `${Date.now()}`);
const blobResult = await getBlob(params, blobUrl, tmpDir, templateDestPath, templateRef, blobDigest, ['devcontainer-template.json', 'README.md', 'NOTES.md'], 'devcontainer-template.json');
const blobResult = await getBlob(params, blobUrl, tmpDir, templateDestPath, templateRef, blobDigest, [...omitPaths, 'devcontainer-template.json', 'README.md', 'NOTES.md'], 'devcontainer-template.json');

if (!blobResult) {
throw new Error(`Failed to download package for ${templateRef.resource}`);
Expand Down
16 changes: 14 additions & 2 deletions src/spec-node/templatesCLI/apply.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ export function templateApplyOptions(y: Argv) {
'features': { type: 'string', alias: 'f', default: '[]', description: 'Features to add to the provided Template, provided as JSON.' },
'log-level': { choices: ['info' as 'info', 'debug' as 'debug', 'trace' as 'trace'], default: 'info' as 'info', description: 'Log level.' },
'tmp-dir': { type: 'string', description: 'Directory to use for temporary files. If not provided, the system default will be inferred.' },
'omit-paths': { type: 'string', default: '[]', description: 'List of paths within the Template to omit applying, provided as JSON. To ignore a directory append \'/*\'. Eg: \'[".github/*", "dir/a/*", "file.ts"]\'' },
})
.check(_argv => {
return true;
Expand All @@ -34,6 +35,7 @@ async function templateApply({
'features': featuresArgs,
'log-level': inputLogLevel,
'tmp-dir': userProvidedTmpDir,
'omit-paths': omitPathsArg,
}: TemplateApplyArgs) {
const disposables: (() => Promise<unknown> | undefined)[] = [];
const dispose = async () => {
Expand Down Expand Up @@ -65,13 +67,23 @@ async function templateApply({
process.exit(1);
}

let omitPaths: string[] = [];
if (omitPathsArg) {
let omitPathsErrors: jsonc.ParseError[] = [];
omitPaths = jsonc.parse(omitPathsArg, omitPathsErrors);
if (!Array.isArray(omitPaths)) {
output.write('Invalid \'--omitPaths\' argument provided. Provide as a JSON array, eg: \'[".github/*", "dir/a/*", "file.ts"]\'', LogLevel.Error);
process.exit(1);
}
}

const selectedTemplate: SelectedTemplate = {
id: templateId,
options,
features
features,
omitPaths,
};


const files = await fetchTemplate({ output, env: process.env }, selectedTemplate, workspaceFolder, userProvidedTmpDir);
if (!files) {
output.write(`Failed to fetch template '${id}'.`, LogLevel.Error);
Expand Down
124 changes: 117 additions & 7 deletions src/test/container-templates/containerTemplatesOCI.test.ts
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
import { createPlainLog, LogLevel, makeLog } from '../../spec-utils/log';
import * as assert from 'assert';
import * as os from 'os';
import * as path from 'path';
import { createPlainLog, LogLevel, makeLog } from '../../spec-utils/log';
export const output = makeLog(createPlainLog(text => process.stdout.write(text), () => LogLevel.Trace));
import { fetchTemplate, SelectedTemplate } from '../../spec-configuration/containerTemplatesOCI';
import * as path from 'path';
import { readLocalFile } from '../../spec-utils/pfs';

describe('fetchTemplate', async function () {
Expand All @@ -14,7 +15,8 @@ describe('fetchTemplate', async function () {
const selectedTemplate: SelectedTemplate = {
id: 'ghcr.io/devcontainers/templates/docker-from-docker:latest',
options: { 'installZsh': 'false', 'upgradePackages': 'true', 'dockerVersion': '20.10', 'moby': 'true' },
features: []
features: [],
omitPaths: [],
};

const dest = path.relative(process.cwd(), path.join(__dirname, 'tmp1'));
Expand Down Expand Up @@ -43,7 +45,8 @@ describe('fetchTemplate', async function () {
const selectedTemplate: SelectedTemplate = {
id: 'ghcr.io/devcontainers/templates/docker-from-docker:latest',
options: {},
features: []
features: [],
omitPaths: [],
};

const dest = path.relative(process.cwd(), path.join(__dirname, 'tmp2'));
Expand Down Expand Up @@ -72,7 +75,8 @@ describe('fetchTemplate', async function () {
const selectedTemplate: SelectedTemplate = {
id: 'ghcr.io/devcontainers/templates/docker-from-docker:latest',
options: { 'installZsh': 'false', 'upgradePackages': 'true', 'dockerVersion': '20.10', 'moby': 'true', 'enableNonRootDocker': 'true' },
features: [{ id: 'ghcr.io/devcontainers/features/azure-cli:1', options: {} }]
features: [{ id: 'ghcr.io/devcontainers/features/azure-cli:1', options: {} }],
omitPaths: [],
};

const dest = path.relative(process.cwd(), path.join(__dirname, 'tmp3'));
Expand Down Expand Up @@ -104,7 +108,8 @@ describe('fetchTemplate', async function () {
const selectedTemplate: SelectedTemplate = {
id: 'ghcr.io/devcontainers/templates/anaconda-postgres:latest',
options: { 'nodeVersion': 'lts/*' },
features: [{ id: 'ghcr.io/devcontainers/features/azure-cli:1', options: {} }, { id: 'ghcr.io/devcontainers/features/git:1', options: { 'version': 'latest', ppa: true } }]
features: [{ id: 'ghcr.io/devcontainers/features/azure-cli:1', options: {} }, { id: 'ghcr.io/devcontainers/features/git:1', options: { 'version': 'latest', ppa: true } }],
omitPaths: [],
};

const dest = path.relative(process.cwd(), path.join(__dirname, 'tmp4'));
Expand All @@ -123,4 +128,109 @@ describe('fetchTemplate', async function () {
assert.match(devcontainer, /"ghcr.io\/devcontainers\/features\/azure-cli:1": {}/);
assert.match(devcontainer, /"ghcr.io\/devcontainers\/features\/git:1": {\n\t\t\t"version": "latest",\n\t\t\t"ppa": true/);
});
});

describe('omit-path', async function () {
this.timeout('120s');

// https://github.com/codspace/templates/pkgs/container/templates%2Fmytemplate/255979159?tag=1.0.4
const id = 'ghcr.io/codspace/templates/mytemplate@sha256:57cbf968907c74c106b7b2446063d114743ab3f63345f7c108c577915c535185';
const templateFiles = [
'./c1.ts',
'./c2.ts',
'./c3.ts',
'./.devcontainer/devcontainer.json',
'./.github/dependabot.yml',
'./assets/hello.md',
'./assets/hi.md',
'./example-projects/exampleA/a1.ts',
'./example-projects/exampleA/.github/dependabot.yml',
'./example-projects/exampleA/subFolderA/a2.ts',
'./example-projects/exampleB/b1.ts',
'./example-projects/exampleB/.github/dependabot.yml',
'./example-projects/exampleB/subFolderB/b2.ts',
];

// NOTE: Certain files, like the 'devcontainer-template.json', are always filtered
// out as they are not part of the Template.
it('Omit nothing', async () => {
const selectedTemplate: SelectedTemplate = {
id,
options: {},
features: [],
omitPaths: [],
};

const files = await fetchTemplate(
{ output, env: process.env },
selectedTemplate,
path.join(os.tmpdir(), 'vsch-test-template-temp', `${Date.now()}`)
);

assert.ok(files);
assert.strictEqual(files.length, templateFiles.length);
for (const file of templateFiles) {
assert.ok(files.includes(file));
}
});

it('Omit nested folder', async () => {
const selectedTemplate: SelectedTemplate = {
id,
options: {},
features: [],
omitPaths: ['example-projects/exampleB/*'],
};

const files = await fetchTemplate(
{ output, env: process.env },
selectedTemplate,
path.join(os.tmpdir(), 'vsch-test-template-temp', `${Date.now()}`)
);

const expectedRemovedFiles = [
'./example-projects/exampleB/b1.ts',
'./example-projects/example/.github/dependabot.yml',
'./example-projects/exampleB/subFolderB/b2.ts',
];

assert.ok(files);
assert.strictEqual(files.length, templateFiles.length - 3);
for (const file of expectedRemovedFiles) {
assert.ok(!files.includes(file));
}
});

it('Omit single file, root folder, and nested folder', async () => {
const selectedTemplate: SelectedTemplate = {
id,
options: {},
features: [],
omitPaths: ['.github/*', 'example-projects/exampleA/*', 'c1.ts'],
};

const files = await fetchTemplate(
{ output, env: process.env },
selectedTemplate,
path.join(os.tmpdir(), 'vsch-test-template-temp', `${Date.now()}`)
);

const expectedRemovedFiles = [
'./c1.ts',
'./.github/dependabot.yml',
'./example-projects/exampleA/a1.ts',
'./example-projects/exampleA/.github/dependabot.yml',
'./example-projects/exampleA/subFolderA/a2.ts',
];

assert.ok(files);
assert.strictEqual(files.length, templateFiles.length - 5);
for (const file of expectedRemovedFiles) {
assert.ok(!files.includes(file));
}
});
});


});


0 comments on commit fcefe52

Please sign in to comment.