Skip to content

Commit c888a59

Browse files
thesattirajuthesattiraju
authored
[JenkinsDownloadArtifacts] Supporting download to a relative path (#6… (#6035)
* [JenkinsDownloadArtifacts] Supporting download to a relative path (#6020) * Bumping version * Supporting download to a relative path * Supporting download of more than 5000 files * throwing error for getArtifactItems * resolving cr comments * string fix * Removing unused reference * strings
1 parent 704d9ff commit c888a59

File tree

8 files changed

+67
-30
lines changed

8 files changed

+67
-30
lines changed

‎Tasks/Common/azure-blobstorage-artifactProvider/Strings/resources.resjson/en-US/resources.resjson‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
{
2-
"loc.messages.ArtifactItemsTruncationWarning": "Downloaded items may be trucated at 5000 items, all artifact items might not have been downloaded.",
32
"loc.messages.CreatedBlobForItem": "Created blob for item {0}. Blob uri: {1}",
43
"loc.messages.CreatedContainer": "Created container {0}.",
54
"loc.messages.ErrorInReadStream": "Error in Read stream: {0}",
@@ -9,5 +8,7 @@
98
"loc.messages.FailedToListItemInsideContainer": "Failed to list items inside container: {0}. Error: {1}",
109
"loc.messages.SuccessFullyFetchedItemList": "Successfully fetcted list of items",
1110
"loc.messages.UnableToFetchItem": "Unable to fetch item: {0}. Error: {1}",
12-
"loc.messages.UploadingItem": "Uploading {0}."
11+
"loc.messages.UploadingItem": "Uploading {0}.",
12+
"loc.messages.ContinuationTokenExistsFetchingRemainingFiles": "Continuation token exists, trying to fetch the list of remaining files.",
13+
"loc.messages.GetArtifactItemsNotSupported": "Get artifact items not supported, invalid code path"
1314
}

‎Tasks/Common/azure-blobstorage-artifactProvider/azureBlobStorageProvider.ts‎

Lines changed: 45 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,18 @@ import models = require('artifact-engine/Models');
55
import tl = require('vsts-task-lib/task');
66

77
export class AzureBlobProvider implements models.IArtifactProvider {
8-
constructor(storageAccount: string, container: string, accessKey: string, prefixFolderPath?: string, host?: string) {
8+
9+
constructor(storageAccount: string, container: string, accessKey: string, prefixFolderPath?: string, host?: string, addPrefixToDownloadedItems?: boolean) {
910
this._storageAccount = storageAccount;
1011
this._accessKey = accessKey;
1112
this._container = container;
12-
this._prefixFolderPath = prefixFolderPath;
13+
if (!!prefixFolderPath) {
14+
this._prefixFolderPath = prefixFolderPath.endsWith("/") ? prefixFolderPath : prefixFolderPath + "/";
15+
} else {
16+
this._prefixFolderPath = "";
17+
}
1318
this._blobSvc = azureStorage.createBlobService(this._storageAccount, this._accessKey, host);
19+
this._addPrefixToDownloadedItems = !!addPrefixToDownloadedItems;
1420
}
1521

1622
public putArtifactItem(item: models.ArtifactItem, readStream: NodeJS.ReadableStream): Promise<models.ArtifactItem> {
@@ -52,12 +58,18 @@ export class AzureBlobProvider implements models.IArtifactProvider {
5258
}
5359

5460
public getArtifactItems(artifactItem: models.ArtifactItem): Promise<models.ArtifactItem[]> {
55-
return this._getItems(this._container, artifactItem.path);
61+
throw new Error(tl.loc("GetArtifactItemsNotSupported"));
5662
}
5763

5864
public getArtifactItem(artifactItem: models.ArtifactItem): Promise<NodeJS.ReadableStream> {
5965
return new Promise((resolve, reject) => {
60-
var readStream: NodeJS.ReadableStream = this._blobSvc.createReadStream(this._container, artifactItem.path, null);
66+
var readStream: NodeJS.ReadableStream;
67+
if (!this._addPrefixToDownloadedItems && !!this._prefixFolderPath) {
68+
// Adding prefix path to get the absolute path
69+
readStream = this._blobSvc.createReadStream(this._container, this._prefixFolderPath + artifactItem.path, null);
70+
} else {
71+
readStream = this._blobSvc.createReadStream(this._container, artifactItem.path, null);
72+
}
6173
resolve(readStream);
6274
});
6375
}
@@ -83,20 +95,34 @@ export class AzureBlobProvider implements models.IArtifactProvider {
8395
}
8496

8597
private _getItems(container: string, parentRelativePath?: string): Promise<models.ArtifactItem[]> {
86-
var promise = new Promise<models.ArtifactItem[]>((resolve, reject) => {
98+
var promise = new Promise<models.ArtifactItem[]>(async (resolve, reject) => {
8799
var items: models.ArtifactItem[] = [];
100+
var continuationToken = null;
101+
var result;
102+
do {
103+
result = await this._getListOfItemsInsideContainer(container, parentRelativePath, continuationToken);
104+
items = items.concat(this._convertBlobResultToArtifactItem(result.entries));
105+
continuationToken = result.continuationToken;
106+
if (!!continuationToken) {
107+
console.log(tl.loc("ContinuationTokenExistsFetchingRemainingFiles"));
108+
}
109+
} while (continuationToken);
110+
111+
console.log(tl.loc("SuccessFullyFetchedItemList"));
112+
resolve(items);
113+
});
88114

89-
this._blobSvc.listBlobsSegmentedWithPrefix(container, parentRelativePath, null, (error, result) => {
115+
return promise;
116+
}
117+
118+
private async _getListOfItemsInsideContainer(container, parentRelativePath, continuationToken): Promise<azureStorage.BlobService.ListBlobsResult> {
119+
var promise = new Promise<azureStorage.BlobService.ListBlobsResult>((resolve, reject) => {
120+
this._blobSvc.listBlobsSegmentedWithPrefix(container, parentRelativePath, continuationToken, async (error, result) => {
90121
if (!!error) {
91122
console.log(tl.loc("FailedToListItemInsideContainer", container, error.message));
92123
reject(error);
93124
} else {
94-
console.log(tl.loc("SuccessFullyFetchedItemList"));
95-
if (result.continuationToken) {
96-
tl.warning(tl.loc("ArtifactItemsTruncationWarning"));
97-
}
98-
items = this._convertBlobResultToArtifactItem(result.entries);
99-
resolve(items);
125+
resolve(result);
100126
}
101127
});
102128
});
@@ -111,7 +137,12 @@ export class AzureBlobProvider implements models.IArtifactProvider {
111137
artifactitem.itemType = models.ItemType.File;
112138
artifactitem.fileLength = parseInt(element.contentLength);
113139
artifactitem.lastModified = new Date(element.lastModified + 'Z');
114-
artifactitem.path = element.name;
140+
if (!this._addPrefixToDownloadedItems && !!this._prefixFolderPath) {
141+
// Supplying relative path without prefix; removing the first occurence
142+
artifactitem.path = element.name.replace(this._prefixFolderPath, "").trim();
143+
} else {
144+
artifactitem.path = element.name;
145+
}
115146
artifactItems.push(artifactitem);
116147
});
117148

@@ -124,4 +155,5 @@ export class AzureBlobProvider implements models.IArtifactProvider {
124155
private _prefixFolderPath: string;
125156
private _isContainerExists: boolean = false;
126157
private _blobSvc: azureStorage.BlobService;
158+
private _addPrefixToDownloadedItems: boolean = false;
127159
}

‎Tasks/Common/azure-blobstorage-artifactProvider/blobservice.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ export class BlobService {
2020
var fileProvider = new artifactProviders.FilesystemProvider(source);
2121
var azureProvider = new azureBlobProvider.AzureBlobProvider(this._storageAccountName, container, this._storageAccessKey, prefixFolderPath, this._host);
2222
var processor = new artifactProcessor.ArtifactEngine();
23-
var processorOptions = new artifactProcessor.ArtifactEngineOptions();
23+
var processorOptions = new artifactProcessor.ArtifactEngineOptions();
2424
if (itemPattern) {
2525
processorOptions.itemPattern = itemPattern;
2626
}
@@ -37,11 +37,11 @@ export class BlobService {
3737
return uploadedUrls;
3838
}
3939

40-
public async downloadBlobs(destination: string, container: string, prefixFolderPath?: string, itemPattern?: string): Promise<void> {
40+
public async downloadBlobs(destination: string, container: string, prefixFolderPath?: string, itemPattern?: string, addPrefixToDownloadedItems?: boolean): Promise<void> {
4141
var fileProvider = new artifactProviders.FilesystemProvider(destination);
42-
var azureProvider = new azureBlobProvider.AzureBlobProvider(this._storageAccountName, container, this._storageAccessKey, prefixFolderPath, this._host);
42+
var azureProvider = new azureBlobProvider.AzureBlobProvider(this._storageAccountName, container, this._storageAccessKey, prefixFolderPath, this._host, !!addPrefixToDownloadedItems);
4343
var processor = new artifactProcessor.ArtifactEngine();
44-
var processorOptions = new artifactProcessor.ArtifactEngineOptions();
44+
var processorOptions = new artifactProcessor.ArtifactEngineOptions();
4545
if (itemPattern) {
4646
processorOptions.itemPattern = itemPattern;
4747
}

‎Tasks/Common/azure-blobstorage-artifactProvider/module.json‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
{
22
"messages": {
3-
"ArtifactItemsTruncationWarning":"Downloaded items may be trucated at 5000 items, all artifact items might not have been downloaded.",
43
"CreatedBlobForItem": "Created blob for item {0}. Blob uri: {1}",
54
"CreatedContainer": "Created container {0}.",
65
"ErrorInReadStream": "Error in Read stream: {0}",
@@ -10,6 +9,8 @@
109
"FailedToListItemInsideContainer": "Failed to list items inside container: {0}. Error: {1}",
1110
"SuccessFullyFetchedItemList": "Successfully fetcted list of items",
1211
"UnableToFetchItem": "Unable to fetch item: {0}. Error: {1}",
13-
"UploadingItem": "Uploading {0}."
12+
"UploadingItem": "Uploading {0}.",
13+
"ContinuationTokenExistsFetchingRemainingFiles": "Continuation token exists, trying to fetch the list of remaining files.",
14+
"GetArtifactItemsNotSupported": "Get artifact items not supported, invalid code path"
1415
}
1516
}

‎Tasks/JenkinsDownloadArtifacts/AzureStorageArtifacts/AzureStorageArtifactDownloader.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ export class AzureStorageArtifactDownloader {
2626

2727
const blobService = new BlobService.BlobService(storageAccount.name, storageAccount.primaryAccessKey);
2828

29-
await blobService.downloadBlobs(downloadToPath, this.containerName, this.commonVirtualPath, fileType || "**");
29+
await blobService.downloadBlobs(downloadToPath, this.containerName, this.commonVirtualPath, fileType || "**", false);
3030
}
3131

3232
private async _getStorageAccountDetails(): Promise<StorageAccountInfo> {

‎Tasks/JenkinsDownloadArtifacts/Strings/resources.resjson/en-US/resources.resjson‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,6 @@
109109
"loc.messages.ResourceNameCannotBeNull": "Resource name is required.",
110110
"loc.messages.activeDirectoryResourceIdUrlCannotBeEmpty": "Active directory resource url is required.",
111111
"loc.messages.StorageAccountCannotBeNull": "storage accountName is required.",
112-
"loc.messages.ArtifactItemsTruncationWarning": "Downloaded items may be trucated at 5000 items, all artifact items might not have been downloaded.",
113112
"loc.messages.CreatedBlobForItem": "Created blob for item {0}. Blob uri: {1}",
114113
"loc.messages.CreatedContainer": "Created container {0}.",
115114
"loc.messages.ErrorInReadStream": "Error in Read stream: {0}",
@@ -122,5 +121,7 @@
122121
"loc.messages.UploadingItem": "Uploading {0}.",
123122
"loc.messages.FoundJobType": "Found job type %s",
124123
"loc.messages.CannotFindJobType": "Could not detect job type",
125-
"loc.messages.InvalidJobName": "Invalid job name %s"
124+
"loc.messages.InvalidJobName": "Invalid job name %s",
125+
"loc.messages.ContinuationTokenExistsFetchingRemainingFiles": "Continuation token exists, trying to fetch the list of remaining files.",
126+
"loc.messages.GetArtifactItemsNotSupported": "Get artifact items not supported, invalid code path"
126127
}

‎Tasks/JenkinsDownloadArtifacts/task.json‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
"version": {
1919
"Major": 1,
2020
"Minor": 125,
21-
"Patch": 1
21+
"Patch": 2
2222
},
2323
"groups": [
2424
{
@@ -328,7 +328,6 @@
328328
"ResourceNameCannotBeNull": "Resource name is required.",
329329
"activeDirectoryResourceIdUrlCannotBeEmpty": "Active directory resource url is required.",
330330
"StorageAccountCannotBeNull": "storage accountName is required.",
331-
"ArtifactItemsTruncationWarning": "Downloaded items may be trucated at 5000 items, all artifact items might not have been downloaded.",
332331
"CreatedBlobForItem": "Created blob for item {0}. Blob uri: {1}",
333332
"CreatedContainer": "Created container {0}.",
334333
"ErrorInReadStream": "Error in Read stream: {0}",
@@ -341,6 +340,8 @@
341340
"UploadingItem": "Uploading {0}.",
342341
"FoundJobType": "Found job type %s",
343342
"CannotFindJobType": "Could not detect job type",
344-
"InvalidJobName": "Invalid job name %s"
343+
"InvalidJobName": "Invalid job name %s",
344+
"ContinuationTokenExistsFetchingRemainingFiles": "Continuation token exists, trying to fetch the list of remaining files.",
345+
"GetArtifactItemsNotSupported": "Get artifact items not supported, invalid code path"
345346
}
346347
}

‎Tasks/JenkinsDownloadArtifacts/task.loc.json‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
"version": {
1919
"Major": 1,
2020
"Minor": 125,
21-
"Patch": 1
21+
"Patch": 2
2222
},
2323
"groups": [
2424
{
@@ -328,7 +328,6 @@
328328
"ResourceNameCannotBeNull": "ms-resource:loc.messages.ResourceNameCannotBeNull",
329329
"activeDirectoryResourceIdUrlCannotBeEmpty": "ms-resource:loc.messages.activeDirectoryResourceIdUrlCannotBeEmpty",
330330
"StorageAccountCannotBeNull": "ms-resource:loc.messages.StorageAccountCannotBeNull",
331-
"ArtifactItemsTruncationWarning": "ms-resource:loc.messages.ArtifactItemsTruncationWarning",
332331
"CreatedBlobForItem": "ms-resource:loc.messages.CreatedBlobForItem",
333332
"CreatedContainer": "ms-resource:loc.messages.CreatedContainer",
334333
"ErrorInReadStream": "ms-resource:loc.messages.ErrorInReadStream",
@@ -341,6 +340,8 @@
341340
"UploadingItem": "ms-resource:loc.messages.UploadingItem",
342341
"FoundJobType": "ms-resource:loc.messages.FoundJobType",
343342
"CannotFindJobType": "ms-resource:loc.messages.CannotFindJobType",
344-
"InvalidJobName": "ms-resource:loc.messages.InvalidJobName"
343+
"InvalidJobName": "ms-resource:loc.messages.InvalidJobName",
344+
"ContinuationTokenExistsFetchingRemainingFiles": "ms-resource:loc.messages.ContinuationTokenExistsFetchingRemainingFiles",
345+
"GetArtifactItemsNotSupported": "ms-resource:loc.messages.GetArtifactItemsNotSupported"
345346
}
346347
}

0 commit comments

Comments
 (0)