Skip to content

Commit fcefe52

Browse files
joshspicerjoshspicer
authored
Add --omit-paths flag to templates apply (#868)
* updating omitting extraction of paths from getBlob() tarball * add --omit-paths flag to templates apply command * update tests * Add example * refactor to match spec * missing linter * dont change the expected output stored in "files"
1 parent ed43a5b commit fcefe52

File tree

4 files changed

+161
-25
lines changed

4 files changed

+161
-25
lines changed

‎src/spec-configuration/containerCollectionsOCI.ts‎

Lines changed: 27 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -504,7 +504,7 @@ export async function getPublishedTags(params: CommonParams, ref: OCIRef): Promi
504504
}
505505
}
506506

507-
export async function getBlob(params: CommonParams, url: string, ociCacheDir: string, destCachePath: string, ociRef: OCIRef, expectedDigest: string, ignoredFilesDuringExtraction: string[] = [], metadataFile?: string): Promise<{ files: string[]; metadata: {} | undefined } | undefined> {
507+
export async function getBlob(params: CommonParams, url: string, ociCacheDir: string, destCachePath: string, ociRef: OCIRef, expectedDigest: string, omitDuringExtraction: string[] = [], metadataFile?: string): Promise<{ files: string[]; metadata: {} | undefined } | undefined> {
508508
// TODO: Parallelize if multiple layers (not likely).
509509
// TODO: Seeking might be needed if the size is too large.
510510

@@ -543,24 +543,37 @@ export async function getBlob(params: CommonParams, url: string, ociCacheDir: st
543543
await mkdirpLocal(destCachePath);
544544
await writeLocalFile(tempTarballPath, resBody);
545545

546+
// https://github.com/devcontainers/spec/blob/main/docs/specs/devcontainer-templates.md#the-optionalpaths-property
547+
const directoriesToOmit = omitDuringExtraction.filter(f => f.endsWith('/*')).map(f => f.slice(0, -1));
548+
const filesToOmit = omitDuringExtraction.filter(f => !f.endsWith('/*'));
549+
550+
output.write(`omitDuringExtraction: '${omitDuringExtraction.join(', ')}`, LogLevel.Trace);
551+
output.write(`Files to omit: '${filesToOmit.join(', ')}'`, LogLevel.Info);
552+
if (directoriesToOmit.length) {
553+
output.write(`Dirs to omit : '${directoriesToOmit.join(', ')}'`, LogLevel.Info);
554+
}
555+
546556
const files: string[] = [];
547557
await tar.x(
548558
{
549559
file: tempTarballPath,
550560
cwd: destCachePath,
551-
filter: (path: string, stat: tar.FileStat) => {
552-
// Skip files that are in the ignore list
553-
if (ignoredFilesDuringExtraction.some(f => path.indexOf(f) !== -1)) {
554-
// Skip.
555-
output.write(`Skipping file '${path}' during blob extraction`, LogLevel.Trace);
556-
return false;
561+
filter: (tPath: string, stat: tar.FileStat) => {
562+
output.write(`Testing '${tPath}'(${stat.type})`, LogLevel.Trace);
563+
const cleanedPath = tPath
564+
.replace(/\\/g, '/')
565+
.replace(/^\.\//, '');
566+
567+
if (filesToOmit.includes(cleanedPath) || directoriesToOmit.some(d => cleanedPath.startsWith(d))) {
568+
output.write(` Omitting '${tPath}'`, LogLevel.Trace);
569+
return false; // Skip
557570
}
558-
// Keep track of all files extracted, in case the caller is interested.
559-
output.write(`${path} : ${stat.type}`, LogLevel.Trace);
560-
if ((stat.type.toString() === 'File')) {
561-
files.push(path);
571+
572+
if (stat.type.toString() === 'File') {
573+
files.push(tPath);
562574
}
563-
return true;
575+
576+
return true; // Keep
564577
}
565578
}
566579
);
@@ -576,8 +589,8 @@ export async function getBlob(params: CommonParams, url: string, ociCacheDir: st
576589
{
577590
file: tempTarballPath,
578591
cwd: ociCacheDir,
579-
filter: (path: string, _: tar.FileStat) => {
580-
return path === `./${metadataFile}`;
592+
filter: (tPath: string, _: tar.FileStat) => {
593+
return tPath === `./${metadataFile}`;
581594
}
582595
});
583596
const pathToMetadataFile = path.join(ociCacheDir, metadataFile);

‎src/spec-configuration/containerTemplatesOCI.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,13 @@ export interface SelectedTemplate {
1919
id: string;
2020
options: TemplateOptions;
2121
features: TemplateFeatureOption[];
22+
omitPaths: string[];
2223
}
2324

2425
export async function fetchTemplate(params: CommonParams, selectedTemplate: SelectedTemplate, templateDestPath: string, userProvidedTmpDir?: string): Promise<string[] | undefined> {
2526
const { output } = params;
2627

27-
let { id: userSelectedId, options: userSelectedOptions } = selectedTemplate;
28+
let { id: userSelectedId, options: userSelectedOptions, omitPaths } = selectedTemplate;
2829
const templateRef = getRef(output, userSelectedId);
2930
if (!templateRef) {
3031
output.write(`Failed to parse template ref for ${userSelectedId}`, LogLevel.Error);
@@ -46,7 +47,7 @@ export async function fetchTemplate(params: CommonParams, selectedTemplate: Sele
4647
output.write(`blob url: ${blobUrl}`, LogLevel.Trace);
4748

4849
const tmpDir = userProvidedTmpDir || path.join(os.tmpdir(), 'vsch-template-temp', `${Date.now()}`);
49-
const blobResult = await getBlob(params, blobUrl, tmpDir, templateDestPath, templateRef, blobDigest, ['devcontainer-template.json', 'README.md', 'NOTES.md'], 'devcontainer-template.json');
50+
const blobResult = await getBlob(params, blobUrl, tmpDir, templateDestPath, templateRef, blobDigest, [...omitPaths, 'devcontainer-template.json', 'README.md', 'NOTES.md'], 'devcontainer-template.json');
5051

5152
if (!blobResult) {
5253
throw new Error(`Failed to download package for ${templateRef.resource}`);

‎src/spec-node/templatesCLI/apply.ts‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ export function templateApplyOptions(y: Argv) {
1515
'features': { type: 'string', alias: 'f', default: '[]', description: 'Features to add to the provided Template, provided as JSON.' },
1616
'log-level': { choices: ['info' as 'info', 'debug' as 'debug', 'trace' as 'trace'], default: 'info' as 'info', description: 'Log level.' },
1717
'tmp-dir': { type: 'string', description: 'Directory to use for temporary files. If not provided, the system default will be inferred.' },
18+
'omit-paths': { type: 'string', default: '[]', description: 'List of paths within the Template to omit applying, provided as JSON. To ignore a directory append \'/*\'. Eg: \'[".github/*", "dir/a/*", "file.ts"]\'' },
1819
})
1920
.check(_argv => {
2021
return true;
@@ -34,6 +35,7 @@ async function templateApply({
3435
'features': featuresArgs,
3536
'log-level': inputLogLevel,
3637
'tmp-dir': userProvidedTmpDir,
38+
'omit-paths': omitPathsArg,
3739
}: TemplateApplyArgs) {
3840
const disposables: (() => Promise<unknown> | undefined)[] = [];
3941
const dispose = async () => {
@@ -65,13 +67,23 @@ async function templateApply({
6567
process.exit(1);
6668
}
6769

70+
let omitPaths: string[] = [];
71+
if (omitPathsArg) {
72+
let omitPathsErrors: jsonc.ParseError[] = [];
73+
omitPaths = jsonc.parse(omitPathsArg, omitPathsErrors);
74+
if (!Array.isArray(omitPaths)) {
75+
output.write('Invalid \'--omitPaths\' argument provided. Provide as a JSON array, eg: \'[".github/*", "dir/a/*", "file.ts"]\'', LogLevel.Error);
76+
process.exit(1);
77+
}
78+
}
79+
6880
const selectedTemplate: SelectedTemplate = {
6981
id: templateId,
7082
options,
71-
features
83+
features,
84+
omitPaths,
7285
};
7386

74-
7587
const files = await fetchTemplate({ output, env: process.env }, selectedTemplate, workspaceFolder, userProvidedTmpDir);
7688
if (!files) {
7789
output.write(`Failed to fetch template '${id}'.`, LogLevel.Error);

‎src/test/container-templates/containerTemplatesOCI.test.ts‎

Lines changed: 117 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
1-
import { createPlainLog, LogLevel, makeLog } from '../../spec-utils/log';
21
import * as assert from 'assert';
2+
import * as os from 'os';
3+
import * as path from 'path';
4+
import { createPlainLog, LogLevel, makeLog } from '../../spec-utils/log';
35
export const output = makeLog(createPlainLog(text => process.stdout.write(text), () => LogLevel.Trace));
46
import { fetchTemplate, SelectedTemplate } from '../../spec-configuration/containerTemplatesOCI';
5-
import * as path from 'path';
67
import { readLocalFile } from '../../spec-utils/pfs';
78

89
describe('fetchTemplate', async function () {
@@ -14,7 +15,8 @@ describe('fetchTemplate', async function () {
1415
const selectedTemplate: SelectedTemplate = {
1516
id: 'ghcr.io/devcontainers/templates/docker-from-docker:latest',
1617
options: { 'installZsh': 'false', 'upgradePackages': 'true', 'dockerVersion': '20.10', 'moby': 'true' },
17-
features: []
18+
features: [],
19+
omitPaths: [],
1820
};
1921

2022
const dest = path.relative(process.cwd(), path.join(__dirname, 'tmp1'));
@@ -43,7 +45,8 @@ describe('fetchTemplate', async function () {
4345
const selectedTemplate: SelectedTemplate = {
4446
id: 'ghcr.io/devcontainers/templates/docker-from-docker:latest',
4547
options: {},
46-
features: []
48+
features: [],
49+
omitPaths: [],
4750
};
4851

4952
const dest = path.relative(process.cwd(), path.join(__dirname, 'tmp2'));
@@ -72,7 +75,8 @@ describe('fetchTemplate', async function () {
7275
const selectedTemplate: SelectedTemplate = {
7376
id: 'ghcr.io/devcontainers/templates/docker-from-docker:latest',
7477
options: { 'installZsh': 'false', 'upgradePackages': 'true', 'dockerVersion': '20.10', 'moby': 'true', 'enableNonRootDocker': 'true' },
75-
features: [{ id: 'ghcr.io/devcontainers/features/azure-cli:1', options: {} }]
78+
features: [{ id: 'ghcr.io/devcontainers/features/azure-cli:1', options: {} }],
79+
omitPaths: [],
7680
};
7781

7882
const dest = path.relative(process.cwd(), path.join(__dirname, 'tmp3'));
@@ -104,7 +108,8 @@ describe('fetchTemplate', async function () {
104108
const selectedTemplate: SelectedTemplate = {
105109
id: 'ghcr.io/devcontainers/templates/anaconda-postgres:latest',
106110
options: { 'nodeVersion': 'lts/*' },
107-
features: [{ id: 'ghcr.io/devcontainers/features/azure-cli:1', options: {} }, { id: 'ghcr.io/devcontainers/features/git:1', options: { 'version': 'latest', ppa: true } }]
111+
features: [{ id: 'ghcr.io/devcontainers/features/azure-cli:1', options: {} }, { id: 'ghcr.io/devcontainers/features/git:1', options: { 'version': 'latest', ppa: true } }],
112+
omitPaths: [],
108113
};
109114

110115
const dest = path.relative(process.cwd(), path.join(__dirname, 'tmp4'));
@@ -123,4 +128,109 @@ describe('fetchTemplate', async function () {
123128
assert.match(devcontainer, /"ghcr.io\/devcontainers\/features\/azure-cli:1": {}/);
124129
assert.match(devcontainer, /"ghcr.io\/devcontainers\/features\/git:1": {\n\t\t\t"version": "latest",\n\t\t\t"ppa": true/);
125130
});
126-
});
131+
132+
describe('omit-path', async function () {
133+
this.timeout('120s');
134+
135+
// https://github.com/codspace/templates/pkgs/container/templates%2Fmytemplate/255979159?tag=1.0.4
136+
const id = 'ghcr.io/codspace/templates/mytemplate@sha256:57cbf968907c74c106b7b2446063d114743ab3f63345f7c108c577915c535185';
137+
const templateFiles = [
138+
'./c1.ts',
139+
'./c2.ts',
140+
'./c3.ts',
141+
'./.devcontainer/devcontainer.json',
142+
'./.github/dependabot.yml',
143+
'./assets/hello.md',
144+
'./assets/hi.md',
145+
'./example-projects/exampleA/a1.ts',
146+
'./example-projects/exampleA/.github/dependabot.yml',
147+
'./example-projects/exampleA/subFolderA/a2.ts',
148+
'./example-projects/exampleB/b1.ts',
149+
'./example-projects/exampleB/.github/dependabot.yml',
150+
'./example-projects/exampleB/subFolderB/b2.ts',
151+
];
152+
153+
// NOTE: Certain files, like the 'devcontainer-template.json', are always filtered
154+
// out as they are not part of the Template.
155+
it('Omit nothing', async () => {
156+
const selectedTemplate: SelectedTemplate = {
157+
id,
158+
options: {},
159+
features: [],
160+
omitPaths: [],
161+
};
162+
163+
const files = await fetchTemplate(
164+
{ output, env: process.env },
165+
selectedTemplate,
166+
path.join(os.tmpdir(), 'vsch-test-template-temp', `${Date.now()}`)
167+
);
168+
169+
assert.ok(files);
170+
assert.strictEqual(files.length, templateFiles.length);
171+
for (const file of templateFiles) {
172+
assert.ok(files.includes(file));
173+
}
174+
});
175+
176+
it('Omit nested folder', async () => {
177+
const selectedTemplate: SelectedTemplate = {
178+
id,
179+
options: {},
180+
features: [],
181+
omitPaths: ['example-projects/exampleB/*'],
182+
};
183+
184+
const files = await fetchTemplate(
185+
{ output, env: process.env },
186+
selectedTemplate,
187+
path.join(os.tmpdir(), 'vsch-test-template-temp', `${Date.now()}`)
188+
);
189+
190+
const expectedRemovedFiles = [
191+
'./example-projects/exampleB/b1.ts',
192+
'./example-projects/example/.github/dependabot.yml',
193+
'./example-projects/exampleB/subFolderB/b2.ts',
194+
];
195+
196+
assert.ok(files);
197+
assert.strictEqual(files.length, templateFiles.length - 3);
198+
for (const file of expectedRemovedFiles) {
199+
assert.ok(!files.includes(file));
200+
}
201+
});
202+
203+
it('Omit single file, root folder, and nested folder', async () => {
204+
const selectedTemplate: SelectedTemplate = {
205+
id,
206+
options: {},
207+
features: [],
208+
omitPaths: ['.github/*', 'example-projects/exampleA/*', 'c1.ts'],
209+
};
210+
211+
const files = await fetchTemplate(
212+
{ output, env: process.env },
213+
selectedTemplate,
214+
path.join(os.tmpdir(), 'vsch-test-template-temp', `${Date.now()}`)
215+
);
216+
217+
const expectedRemovedFiles = [
218+
'./c1.ts',
219+
'./.github/dependabot.yml',
220+
'./example-projects/exampleA/a1.ts',
221+
'./example-projects/exampleA/.github/dependabot.yml',
222+
'./example-projects/exampleA/subFolderA/a2.ts',
223+
];
224+
225+
assert.ok(files);
226+
assert.strictEqual(files.length, templateFiles.length - 5);
227+
for (const file of expectedRemovedFiles) {
228+
assert.ok(!files.includes(file));
229+
}
230+
});
231+
});
232+
233+
234+
});
235+
236+

0 commit comments

Comments
 (0)