Skip to content

Commit 340a8cc

Browse files
scottdallamurascottdallamura
authored
Merge pull request #4566 from Microsoft/users/scdallam/downloadbuildartifact119
DownloadBuildArtifact: FileContainer
2 parents 7509f9b + fa4148f commit 340a8cc

File tree

15 files changed

+456
-4
lines changed

15 files changed

+456
-4
lines changed

‎Tasks/DownloadBuildArtifact/ArtifactProvider.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
import {BuildArtifact} from 'vso-node-api/interfaces/BuildInterfaces';
2+
3+
export interface ArtifactProvider {
4+
supportsArtifactType(artifactType: string): boolean;
5+
downloadArtifact(artifact: BuildArtifact, targetPath: string): Promise<void>;
6+
}

‎Tasks/DownloadBuildArtifact/Downloader.ts‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
import * as path from 'path';
2+
import * as tl from 'vsts-task-lib/task';
3+
import * as fs from 'fs';
4+
5+
/**
6+
* Represents an item to be downloaded
7+
*/
8+
export interface DownloadItem<T> {
9+
/**
10+
* The path to the item, relative to the target path
11+
*/
12+
relativePath: string;
13+
14+
/**
15+
* Artifact-specific data
16+
*/
17+
data: T;
18+
}
19+
20+
/**
21+
* Downloads items
22+
* @param items the items to download
23+
* @param targetPath the folder that will hold the downloaded items
24+
* @param maxConcurrency the maximum number of items to download simultaneously
25+
* @param downloader a function that, given a DownloadItem, will return a content stream
26+
*/
27+
export async function download<T>(items: DownloadItem<T>[], targetPath: string, maxConcurrency: number, downloader: (item: DownloadItem<T>) => Promise<fs.ReadStream>): Promise<void> {
28+
// keep track of folders we've touched so we don't call mkdirP for every single file
29+
let createdFolders: { [key: string]: boolean } = {};
30+
let downloaders: Promise<{}>[] = [];
31+
32+
let fileCount: number = items.length;
33+
let logProgress = fileCount < 100 ? logProgressFilename : logProgressPercentage;
34+
35+
maxConcurrency = Math.min(maxConcurrency, items.length);
36+
for (let i = 0; i < maxConcurrency; ++i) {
37+
downloaders.push(new Promise(async (resolve, reject) => {
38+
try {
39+
while (items.length > 0) {
40+
let item = items.pop();
41+
let fileIndex = fileCount - items.length;
42+
43+
// the full path of the downloaded file
44+
let outputFilename = path.join(targetPath, item.relativePath);
45+
46+
// create the folder if necessary
47+
let folder = path.dirname(outputFilename);
48+
if (!createdFolders.hasOwnProperty(folder)) {
49+
if (!tl.exist(folder)) {
50+
tl.mkdirP(folder);
51+
}
52+
createdFolders[folder] = true;
53+
}
54+
55+
logProgress(item.relativePath, outputFilename, fileIndex, fileCount);
56+
await new Promise(async (downloadResolve, downloadReject) => {
57+
try {
58+
// get the content stream from the provider
59+
let contentStream = await downloader(item);
60+
61+
// create the target stream
62+
let outputStream = fs.createWriteStream(outputFilename);
63+
64+
// pipe the content to the target
65+
contentStream.pipe(outputStream);
66+
contentStream.on('end', () => {
67+
tl.debug(`Downloaded '${item.relativePath}' to '${outputFilename}'`);
68+
downloadResolve();
69+
});
70+
}
71+
catch (err) {
72+
console.log(tl.loc("DownloadError", item.relativePath));
73+
downloadReject(err);
74+
}
75+
});
76+
}
77+
resolve();
78+
}
79+
catch (err) {
80+
reject(err);
81+
}
82+
}));
83+
}
84+
85+
await Promise.all(downloaders);
86+
}
87+
88+
function logProgressFilename(relativePath: string, outputFilename: string, fileIndex: number, fileCount: number): void {
89+
console.log(tl.loc("DownloadingFile", relativePath, outputFilename, fileIndex, fileCount));
90+
}
91+
92+
function logProgressPercentage(relativePath: string, outputFilename: string, fileIndex: number, fileCount: number): void {
93+
let percentage = (fileIndex / fileCount) * 100;
94+
if (Math.floor(percentage) > Math.floor(((fileIndex - 1) / fileCount) * 100)) {
95+
console.log(tl.loc("DownloadingPercentage", percentage.toFixed(2), fileIndex, fileCount));
96+
}
97+
}

‎Tasks/DownloadBuildArtifact/FileContainer.ts‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
import * as tl from 'vsts-task-lib/task';
2+
import * as fs from 'fs';
3+
import * as path from 'path';
4+
5+
import {BuildArtifact} from 'vso-node-api/interfaces/BuildInterfaces';
6+
import {FileContainerItem, ContainerItemType} from 'vso-node-api/interfaces/FileContainerInterfaces';
7+
import {IFileContainerApi} from 'vso-node-api/FileContainerApi';
8+
import {WebApi, getHandlerFromToken} from 'vso-node-api/WebApi';
9+
10+
import {DownloadItem, download} from './Downloader';
11+
import {ArtifactProvider} from './ArtifactProvider';
12+
13+
export class FileContainerProvider implements ArtifactProvider {
14+
public supportsArtifactType(artifactType: string): boolean {
15+
return !!artifactType && artifactType.toLowerCase() === "container";
16+
}
17+
18+
public async downloadArtifact(artifact: BuildArtifact, targetPath: string): Promise<void> {
19+
if (!artifact || !artifact.resource || !artifact.resource.data) {
20+
throw new Error(tl.loc("FileContainerInvalidArtifact"));
21+
}
22+
23+
let containerParts: string[] = artifact.resource.data.split('/', 3);
24+
if (containerParts.length !== 3) {
25+
throw new Error(tl.loc("FileContainerInvalidArtifactData"));
26+
}
27+
28+
let containerId: number = parseInt(containerParts[1]);
29+
let containerPath: string = containerParts[2];
30+
31+
let accessToken = tl.getEndpointAuthorizationParameter('SYSTEMVSSCONNECTION', 'AccessToken', false);
32+
let credentialHandler = getHandlerFromToken(accessToken);
33+
let collectionUrl = tl.getEndpointUrl('SYSTEMVSSCONNECTION', false);
34+
let vssConnection = new WebApi(collectionUrl, credentialHandler);
35+
36+
let fileContainerApi = vssConnection.getFileContainerApi();
37+
38+
// get all items
39+
let items: FileContainerItem[] = await fileContainerApi.getItems(containerId, null, containerPath, false, null, null, false, false);
40+
41+
// ignore folders
42+
items = items.filter(item => item.itemType === ContainerItemType.File);
43+
tl.debug(`Found ${items.length} File items in #/${containerId}/${containerPath}`);
44+
45+
let downloadItems: DownloadItem<FileContainerItem>[] = items.map((item) => {
46+
return {
47+
relativePath: item.path,
48+
data: item
49+
};
50+
})
51+
52+
// download the items
53+
await download(downloadItems, targetPath, 8, (item: DownloadItem<FileContainerItem>) => new Promise(async (resolve, reject) => {
54+
try {
55+
let downloadFilename = item.data.path.substring(item.data.path.lastIndexOf("/") + 1);
56+
let itemResponse = await fileContainerApi.getItem(containerId, null, item.data.path, downloadFilename);
57+
if (itemResponse.statusCode === 200) {
58+
resolve(itemResponse.result);
59+
}
60+
else {
61+
// TODO: decide whether to retry or bail
62+
reject(itemResponse);
63+
}
64+
}
65+
catch (err) {
66+
reject(err);
67+
}
68+
}));
69+
}
70+
}
71+
72+
function getAuthToken() {
73+
let auth = tl.getEndpointAuthorization('SYSTEMVSSCONNECTION', false);
74+
if (auth.scheme.toLowerCase() === 'oauth') {
75+
return auth.parameters['AccessToken'];
76+
}
77+
else {
78+
throw new Error(tl.loc("CredentialsNotFound"))
79+
}
80+
}

‎Tasks/DownloadBuildArtifact/FilePath.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
import * as tl from 'vsts-task-lib/task';
2+
3+
import {BuildArtifact} from 'vso-node-api/interfaces/BuildInterfaces';
4+
5+
import {ArtifactProvider} from './ArtifactProvider';
6+
7+
export class FilePathProvider implements ArtifactProvider {
8+
public supportsArtifactType(artifactType: string): boolean {
9+
return !!artifactType && artifactType.toLowerCase() === "filepath";
10+
}
11+
12+
public async downloadArtifact(artifact: BuildArtifact, targetPath: string): Promise<void> {
13+
throw new Error(tl.loc("FilePathNotSupported"));
14+
}
15+
}

‎Tasks/DownloadBuildArtifact/Strings/resources.resjson/en-US/resources.resjson‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
{
2+
"loc.friendlyName": "Download Build Artifact",
3+
"loc.helpMarkDown": "",
4+
"loc.description": "Download a build artifact.",
5+
"loc.instanceNameFormat": "Download build artifact $(artifactName)",
6+
"loc.input.label.buildId": "Build",
7+
"loc.input.help.buildId": "The build from which to download the artifact",
8+
"loc.input.label.artifactName": "Artifact name",
9+
"loc.input.help.artifactName": "The name of the artifact to download",
10+
"loc.input.label.downloadPath": "Destination directory",
11+
"loc.input.help.downloadPath": "Path on the agent machine where the artifact will be downloaded",
12+
"loc.messages.FileContainerCredentialsNotFound": "Could not determine credentials to connect to file container service.",
13+
"loc.messages.FileContainerInvalidArtifact": "Invalid file container artifact",
14+
"loc.messages.FileContainerInvalidArtifactData": "Invalid file container artifact. Resource data must be in the format #/{container id}/path",
15+
"loc.messages.FilePathNotSupported": "File share artifacts are not yet supported in the early preview. Coming soon.",
16+
"loc.messages.ArtifactProviderNotFound": "Could not determine a provider to download artifact of type %s",
17+
"loc.messages.DownloadError": "Error downloading file %s",
18+
"loc.messages.DownloadingFile": "Downloading '%s' to '%s' (file %d of %d)",
19+
"loc.messages.DownloadingPercentage": "Downloading... %d%% (%d of %d)",
20+
"loc.messages.InvalidBuildId": "Invalid build id specified (%s)"
21+
}

‎Tasks/DownloadBuildArtifact/icon.png‎

1.52 KB
Loading

‎Tasks/DownloadBuildArtifact/icon.svg‎

Lines changed: 20 additions & 0 deletions
Loading

‎Tasks/DownloadBuildArtifact/main.ts‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
import path = require('path');
2+
3+
import {BuildArtifact, ArtifactResource} from 'vso-node-api/interfaces/BuildInterfaces';
4+
import {WebApi, getHandlerFromToken} from 'vso-node-api/WebApi';
5+
import * as tl from 'vsts-task-lib/task';
6+
7+
import {ArtifactProvider} from './ArtifactProvider';
8+
import {FileContainerProvider} from './FileContainer';
9+
import {FilePathProvider} from './FilePath';
10+
11+
async function main(): Promise<void> {
12+
try {
13+
tl.setResourcePath(path.join(__dirname, 'task.json'));
14+
15+
let projectId = tl.getVariable('System.TeamProjectId');
16+
let artifactName = tl.getInput("artifactName");
17+
let downloadPath = tl.getPathInput("downloadPath");
18+
let buildId = parseInt(tl.getInput("buildId"));
19+
20+
if (isNaN(buildId)) {
21+
throw new Error(tl.loc("InvalidBuildId", tl.getInput("buildId")));
22+
}
23+
24+
let accessToken = tl.getEndpointAuthorizationParameter('SYSTEMVSSCONNECTION', 'AccessToken', false);
25+
let credentialHandler = getHandlerFromToken(accessToken);
26+
let collectionUrl = tl.getEndpointUrl('SYSTEMVSSCONNECTION', false);
27+
let vssConnection = new WebApi(collectionUrl, credentialHandler);
28+
29+
// get the artifact metadata
30+
let buildApi = vssConnection.getBuildApi();
31+
let artifact = await buildApi.getArtifact(buildId, artifactName, projectId);
32+
33+
let providers: ArtifactProvider[] = [
34+
new FileContainerProvider(),
35+
new FilePathProvider()
36+
];
37+
38+
let provider = providers.filter((provider) => provider.supportsArtifactType(artifact.resource.type))[0];
39+
if (provider) {
40+
await provider.downloadArtifact(artifact, downloadPath);
41+
}
42+
else {
43+
throw new Error(tl.loc("ArtifactProviderNotFound", artifact.resource.type));
44+
}
45+
46+
tl.setResult(tl.TaskResult.Succeeded, "");
47+
}
48+
catch (err) {
49+
tl.setResult(tl.TaskResult.Failed, err);
50+
}
51+
}
52+
53+
main();

‎Tasks/DownloadBuildArtifact/package.json‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
{
2+
"name": "downloadbuildartifact",
3+
"version": "0.1.0",
4+
"description": "Download Build Artifact Task",
5+
"main": "download.js",
6+
"scripts": {
7+
"test": "echo \"Error: no test specified\" && exit 1"
8+
},
9+
"repository": {
10+
"type": "git",
11+
"url": "git+https://github.com/Microsoft/vsts-tasks.git"
12+
},
13+
"author": "Microsoft Corporation",
14+
"license": "MIT",
15+
"bugs": {
16+
"url": "https://github.com/Microsoft/vsts-tasks/issues"
17+
},
18+
"homepage": "https://github.com/Microsoft/vsts-tasks#readme",
19+
"dependencies": {
20+
"@types/node": "^6.0.78",
21+
"vso-node-api": "^6.2.5-preview",
22+
"vsts-task-lib": "^2.0.3-preview"
23+
}
24+
}

‎Tasks/DownloadBuildArtifact/task.json‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
{
2+
"id": "a433f589-fce1-4460-9ee6-44a624aeb1fb",
3+
"name": "DownloadBuildArtifact",
4+
"friendlyName": "Download Build Artifact",
5+
"description": "Download a build artifact.",
6+
"helpMarkDown": "",
7+
"category": "Utility",
8+
"author": "Microsoft Corporation",
9+
"preview": true,
10+
"version": {
11+
"Major": 0,
12+
"Minor": 1,
13+
"Patch": 57
14+
},
15+
"demands": [],
16+
"inputs": [
17+
{
18+
"name": "buildId",
19+
"type": "pickList",
20+
"label": "Build",
21+
"required": false,
22+
"helpMarkDown": "The build from which to download the artifact",
23+
"defaultValue": "$(Build.BuildId)",
24+
"options": {
25+
"$(Build.BuildId)": "The current build"
26+
}
27+
},
28+
{
29+
"name": "artifactName",
30+
"type": "string",
31+
"label": "Artifact name",
32+
"defaultValue": "drop",
33+
"required": true,
34+
"helpMarkDown": "The name of the artifact to download"
35+
},
36+
{
37+
"name": "downloadPath",
38+
"type": "string",
39+
"label": "Destination directory",
40+
"defaultValue": "$(System.ArtifactsDirectory)",
41+
"required": true,
42+
"helpMarkDown": "Path on the agent machine where the artifact will be downloaded"
43+
}
44+
],
45+
"dataSourceBindings": [
46+
],
47+
"instanceNameFormat": "Download build artifact $(artifactName)",
48+
"execution": {
49+
"Node": {
50+
"target": "main.js",
51+
"argumentFormat": ""
52+
}
53+
},
54+
"messages": {
55+
"FileContainerCredentialsNotFound": "Could not determine credentials to connect to file container service.",
56+
"FileContainerInvalidArtifact": "Invalid file container artifact",
57+
"FileContainerInvalidArtifactData": "Invalid file container artifact. Resource data must be in the format #/{container id}/path",
58+
"FilePathNotSupported": "File share artifacts are not yet supported in the early preview. Coming soon.",
59+
"ArtifactProviderNotFound": "Could not determine a provider to download artifact of type %s",
60+
"DownloadError": "Error downloading file %s",
61+
"DownloadingFile": "Downloading '%s' to '%s' (file %d of %d)",
62+
"DownloadingPercentage": "Downloading... %d%% (%d of %d)",
63+
"InvalidBuildId": "Invalid build id specified (%s)"
64+
}
65+
}

0 commit comments

Comments
 (0)