Merge branch 'master' of https://github.com/Azure/BatchLabs

timotheeguerin · timotheeguerin · commit de00a0a26c7f · 2017-12-01T09:16:41.000-08:00
diff --git a/app/components/data/shared/cloud-file-picker/cloud-file-picker.component.ts b/app/components/data/shared/cloud-file-picker/cloud-file-picker.component.ts
@@ -40,7 +40,6 @@ export class CloudFilePickerComponent implements ControlValueAccessor, OnInit, O
     private _subscriptions: Subscription[] = [];
 
     constructor(private storageService: StorageService, private dialog: DialogService) {
-
         this.fileGroupsData = this.storageService.containerListView(storageService.ncjFileGroupPrefix);
         this.fileGroupsData.items.subscribe((fileGroups) => {
             this.fileGroups = fileGroups;
diff --git a/app/components/data/shared/cloud-file-picker/cloud-file-picker.html b/app/components/data/shared/cloud-file-picker/cloud-file-picker.html
@@ -4,4 +4,4 @@
         {{hint}}
     </mat-hint>
 </mat-form-field>
-<bl-button type="round" [action]="openFilePickerDialog"><i class="fa fa-file"></i></bl-button>
+<bl-button type="round" [action]="openFilePickerDialog" [disabled]="!containerId" title="Select file"><i class="fa fa-file"></i></bl-button>
diff --git a/app/components/data/shared/cloud-file-picker/cloud-file-picker.scss b/app/components/data/shared/cloud-file-picker/cloud-file-picker.scss
@@ -1,3 +1,6 @@
+@import "app/styles/variables";
+
+// todo: should probably share this CSS with file group sas
 bl-cloud-file-picker {
     display: flex;
     align-items: center;
@@ -8,5 +11,16 @@ bl-cloud-file-picker {
 
     bl-button {
         margin-left: 5px;
+
+        &.disabled[color] {
+            cursor: not-allowed;
+            background: map-get($primary, 50);
+            color: $whitesmoke;
+
+            &:hover, &:focus, &:active {
+                background: map-get($primary, 50);
+                color: $whitesmoke;
+            }
+        }
     }
 }
diff --git a/app/components/data/shared/data.shared.module.ts b/app/components/data/shared/data.shared.module.ts
@@ -5,10 +5,11 @@ import { FileBrowseModule } from "app/components/file/browse";
 import { CloudFilePickerComponent, CloudFilePickerDialogComponent } from "./cloud-file-picker";
 import { StorageErrorDisplayComponent } from "./errors";
 import { FileGroupPickerComponent } from "./file-group-picker";
+import { FileGroupSasComponent } from "./file-group-sas";
 import { FileGroupsPickerComponent } from "./file-groups-picker";
 
 const components = [
-    FileGroupPickerComponent, FileGroupsPickerComponent,
+    FileGroupPickerComponent, FileGroupSasComponent, FileGroupsPickerComponent,
     CloudFilePickerComponent, CloudFilePickerDialogComponent,
     StorageErrorDisplayComponent,
 ];
diff --git a/app/components/data/shared/file-group-sas/file-group-sas.component.ts b/app/components/data/shared/file-group-sas/file-group-sas.component.ts
@@ -0,0 +1,115 @@
+import { Component, Input, OnChanges, OnDestroy, OnInit, forwardRef } from "@angular/core";
+import {
+    ControlValueAccessor, FormControl, NG_VALIDATORS, NG_VALUE_ACCESSOR,
+} from "@angular/forms";
+import { autobind } from "core-decorators";
+import { List } from "immutable";
+import * as moment from "moment";
+import { Subscription } from "rxjs";
+
+import { BlobContainer } from "app/models";
+import { ListContainerParams, StorageService } from "app/services";
+import { ListView } from "app/services/core";
+
+import "./file-group-sas.scss";
+
+// tslint:disable:no-forward-ref
+@Component({
+    selector: "bl-file-group-sas",
+    templateUrl: "file-group-sas.html",
+    providers: [
+        { provide: NG_VALUE_ACCESSOR, useExisting: forwardRef(() => FileGroupSasComponent), multi: true },
+        { provide: NG_VALIDATORS, useExisting: forwardRef(() => FileGroupSasComponent), multi: true },
+    ],
+})
+export class FileGroupSasComponent implements ControlValueAccessor, OnChanges, OnInit, OnDestroy {
+    @Input() public label: string;
+    @Input() public hint: string;
+
+    /**
+     * Name of the file group from which to pick a file.
+     */
+    @Input() public containerId: string;
+
+    public fileGroups: List<BlobContainer>;
+    public value = new FormControl();
+    public fileGroupsData: ListView<BlobContainer, ListContainerParams>;
+    public warning = false;
+
+    private _propagateChange: (value: any[]) => void = null;
+    private _subscriptions: Subscription[] = [];
+
+    constructor(private storageService: StorageService) {
+        this.fileGroupsData = this.storageService.containerListView(storageService.ncjFileGroupPrefix);
+        this.fileGroupsData.items.subscribe((fileGroups) => {
+            this.fileGroups = fileGroups;
+        });
+
+        this._subscriptions.push(this.value.valueChanges.debounceTime(400).distinctUntilChanged().subscribe((value) => {
+            this._checkValid(value);
+            if (this._propagateChange) {
+                this._propagateChange(value);
+            }
+        }));
+    }
+
+    public ngOnChanges(inputs) {
+        if (this.containerId && inputs.containerId &&
+            inputs.containerId.currentValue !== inputs.containerId.previousValue) {
+            this.generateSasToken();
+        } else {
+            this.value.setValue(null);
+        }
+    }
+
+    public ngOnInit() {
+        this.fileGroupsData.fetchNext();
+    }
+
+    public ngOnDestroy() {
+        this._subscriptions.forEach(x => x.unsubscribe());
+        this.fileGroupsData.dispose();
+    }
+
+    public writeValue(value: string) {
+        this.value.setValue(value);
+    }
+
+    public registerOnChange(fn) {
+        this._propagateChange = fn;
+    }
+
+    public registerOnTouched() {
+        this.value.markAsTouched();
+    }
+
+    public validate(c: FormControl) {
+        return null;
+    }
+
+    @autobind()
+    public generateSasToken() {
+        /**
+         * Blob Container read access policy that is valid for 7 days, The maximum
+         * lifetime of a task in Batch.
+         */
+        const accessPolicy = {
+            AccessPolicy: {
+                Permissions: "rl",
+                ResourceTypes: "CONTAINER",
+                Services: "BLOB",
+                Start: moment.utc().add(-15, "minutes").toDate(),
+                Expiry: moment.utc().add(7, "day").toDate(),
+            },
+        };
+
+        this.storageService.generateSharedAccessContainerUrl(this.containerId, accessPolicy).subscribe((value) => {
+            this.value.setValue(value);
+        });
+    }
+
+    private _checkValid(value: string) {
+        const valid = !value || this.fileGroups.map(x => x.name).includes(value);
+        this.warning = !valid;
+    }
+}
diff --git a/app/components/data/shared/file-group-sas/file-group-sas.html b/app/components/data/shared/file-group-sas/file-group-sas.html
@@ -0,0 +1,7 @@
+<mat-form-field>
+    <input type="text" matInput [formControl]="value" [placeholder]="label">
+    <mat-hint *ngIf="hint" align="end">
+        {{hint}}
+    </mat-hint>
+</mat-form-field>
+<bl-button type="round" [action]="generateSasToken" [disabled]="!containerId" title="Generate SAS"><i class="fa fa-refresh"></i></bl-button>
diff --git a/app/components/data/shared/file-group-sas/file-group-sas.scss b/app/components/data/shared/file-group-sas/file-group-sas.scss
@@ -0,0 +1,30 @@
+@import "app/styles/variables";
+
+// todo: should probably share this CSS with file group picker
+bl-file-group-sas {
+    display: flex;
+    align-items: center;
+
+    mat-form-field {
+        flex: 1;
+    }
+
+    bl-button {
+        margin-left: 5px;
+
+        &.disabled[color] {
+            cursor: not-allowed;
+            background: map-get($primary, 50);
+            color: $whitesmoke;
+
+            &:hover, &:focus, &:active {
+                background: map-get($primary, 50);
+                color: $whitesmoke;
+            }
+        }
+
+        .fa-refresh {
+            color: $whitesmoke;
+        }
+    }
+}
diff --git a/app/components/data/shared/file-group-sas/index.ts b/app/components/data/shared/file-group-sas/index.ts
@@ -0,0 +1 @@
+export * from "./file-group-sas.component";
diff --git a/app/components/file/browse/node-file-browse.html b/app/components/file/browse/node-file-browse.html
@@ -2,5 +2,5 @@
 </bl-file-explorer>
 <div *ngIf="!isNodeAvailable" class="info-overlay node-unavailable">
     Node is currently
-    <span class="highlight">{{node.state}}&nbsp;</span>there is no files to view now.
+    <span class="highlight">'{{node?.state}}'&nbsp;</span>there is no files to view now.
 </div>
diff --git a/app/components/market/submit/market-application.model.ts b/app/components/market/submit/market-application.model.ts
@@ -7,6 +7,7 @@ export enum NcjParameterExtendedType {
     int = "int",
     fileGroup = "file-group",
     fileInFileGroup = "file-in-file-group",
+    fileGroupSas = "file-group-sas",
     dropDown = "drop-down",
 }
 
diff --git a/app/components/market/submit/parameter-input/parameter-input.html b/app/components/market/submit/parameter-input/parameter-input.html
@@ -9,6 +9,11 @@
         <bl-file-group-picker [formControl]="parameterValue" [label]="parameter.name" class="form-element" [hint]="parameter.description">
         </bl-file-group-picker>
     </div>
+    <div *ngSwitchCase="NcjParameterExtendedType.fileGroupSas">
+        <bl-file-group-sas [containerId]="getContainerFromFileGroup(parameterValues[parameter.dependsOn])" [formControl]="parameterValue"
+            [label]="parameter.name" class="form-element" [hint]="parameter.description">
+        </bl-file-group-sas>
+    </div>
     <div *ngSwitchCase="NcjParameterExtendedType.fileInFileGroup">
         <bl-cloud-file-picker [containerId]="getContainerFromFileGroup(parameterValues[parameter.dependsOn])" [formControl]="parameterValue"
             [label]="parameter.name" class="form-element" [hint]="parameter.description">
diff --git a/app/services/storage.service.ts b/app/services/storage.service.ts
@@ -5,6 +5,7 @@ import { AsyncSubject, Observable, Subject } from "rxjs";
 import { BackgroundTaskService } from "app/components/base/background-task";
 import { BlobContainer, File, ServerError } from "app/models";
 import { FileSystemService } from "app/services";
+import { SharedAccessPolicy } from "app/services/storage/models";
 import { Constants, log } from "app/utils";
 import {
     DataCache,
@@ -330,6 +331,17 @@ export class StorageService {
         });
     }
 
+    public generateSharedAccessContainerUrl(container: string, sharedAccessPolicy: SharedAccessPolicy)
+        : Observable<string> {
+        return this._callStorageClient((client) => {
+            const sasToken = client.generateSharedAccessSignature(container, sharedAccessPolicy);
+            return Promise.resolve(client.getUrl(container, null, sasToken));
+        }, (error) => {
+            // TODO-Andrew: test that errors are caught
+            log.error(`Error generating container SAS: ${container}`, { ...error });
+        });
+    }
+
     public uploadToSasUrl(sasUrl: string, filePath: string): Observable<any> {
         const subject = new AsyncSubject<storage.BlobService.BlobResult>();
 
diff --git a/app/services/storage/blob-storage-client-proxy.ts b/app/services/storage/blob-storage-client-proxy.ts
@@ -1,6 +1,6 @@
 import * as storage from "azure-storage";
 
-import { BlobStorageResult, StorageRequestOptions } from "./models";
+import { BlobStorageResult, SharedAccessPolicy, StorageRequestOptions } from "./models";
 
 export interface ListBlobOptions {
     /**
@@ -306,6 +306,29 @@ export class BlobStorageClientProxy {
         });
     }
 
+    /**
+     * Retrieves a shared access signature token.
+     * http://azure.github.io/azure-storage-node/BlobService.html#generateSharedAccessSignature__anchor
+     *
+     * @param {string} container - Name of the storage container
+     * @param {string} sharedAccessPolicy - The shared access policy
+     */
+    public generateSharedAccessSignature(container: string, sharedAccessPolicy: SharedAccessPolicy): string {
+        return this.client.generateSharedAccessSignature(container, null, sharedAccessPolicy, null);
+    }
+
+    /**
+     * Retrieves a blob or container URL.
+     * http://azure.github.io/azure-storage-node/BlobService.html#getUrl__anchor
+     *
+     * @param {string} container - Name of the storage container
+     * @param {string} blob - Optional blob name.
+     * @param {string} sasToken - The Shared Access Signature token.
+     */
+    public getUrl(container: string, blob?: string, sasToken?: string): string {
+        return this.client.getUrl(container, blob, sasToken);
+    }
+
     /**
      * Return the list of folder names return by listing blobs with a delimiter
      * @param response
diff --git a/app/services/storage/models/index.ts b/app/services/storage/models/index.ts
@@ -1,2 +1,3 @@
 export * from "./blobStorageResult";
+export * from "./sharedAccessPolicy";
 export * from "./storageRequestOptions";
diff --git a/app/services/storage/models/sharedAccessPolicy.ts b/app/services/storage/models/sharedAccessPolicy.ts
@@ -0,0 +1,55 @@
+/**
+ * Shared access policy for the request to generate a SAS token.
+ * Note that the definition is in PascalCase.
+ */
+
+export interface SharedAccessPolicy {
+    /**
+     * Optional signed identifier.
+     */
+    Id?: string;
+
+    /**
+     * Access policy with any permissions and start and expiry time.
+     */
+    AccessPolicy: AccessPolicy;
+}
+
+export interface AccessPolicy {
+    /**
+     * The permission type
+     */
+    Permissions: string;
+
+    /**
+     * The time at which the Shared Access Signature becomes valid (The UTC value will be used).
+     */
+    Start?: Date;
+
+    /**
+     * The time at which the Shared Access Signature becomes expired (The UTC value will be used).
+     */
+    Expiry?: Date;
+
+    /**
+     * An IP address or a range of IP addresses from which to accept requests. When specifying a range, note
+     * that the range is inclusive.
+     */
+    IPAddressOrRange?: string;
+
+    /**
+     * The protocol permitted for a request made with the SAS.
+     */
+    Protocols?: string;
+
+    /**
+     * The services (blob, file, queue, table) for a shared access signature associated with this shared
+     * access policy.
+     */
+    Services?: string;
+
+    /**
+     * The resource type for a shared access signature associated with this shared access policy.
+     */
+    ResourceTypes?: string;
+}
diff --git a/electron-builder.yml b/electron-builder.yml
@@ -10,7 +10,7 @@ files:
   - "package.json"
 
 extraResources:
-  - from: "python/build/"
+  - from: "python/dist/main"
     to: "python-rpc"
     filter:
       - "**/*"
diff --git a/package.json b/package.json
@@ -14,7 +14,7 @@
   "scripts": {
     "ts": "ts-node --project tsconfig.node.json",
     "start": "npm run -s electron:prod",
-    "clean": "rimraf build/* dll/* python/build/*",
+    "clean": "rimraf build/* dll/* python/dist/* python/build/*",
     "test": "npm run -s test-client && npm run -s test-app",
     "test-app": "karma start",
     "test-app:mem": "cross-env DEBUG_MEM=1 npm run -s test-app",
@@ -26,7 +26,7 @@
     "test-client-watch": "npm run -s test-client -- --watch",
     "build-client": "tsc -p tsconfig.node.json",
     "build-app": "npm run webpack -- --profile --colors --display-error-details --display-cached",
-    "build-python": "npm run -s ts config/build-python.ts",
+    "build-python": "pyinstaller python/main.spec -y --distpath=python/dist --workpath=python/build",
     "build": "npm run -s clean && npm run -s build-client && npm run -s build-app",
     "build:prod": "cross-env NODE_ENV=production npm run -s build",
     "watch": "npm run webpack -- --watch --progress --profile --colors --display-error-details --display-cached",
diff --git a/python/.gitignore b/python/.gitignore
@@ -29,7 +29,6 @@ wheels/
 #  Usually these files are written by a python script from a template
 #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 *.manifest
-*.spec
 
 # Installer logs
 pip-log.txt
diff --git a/python/main.spec b/python/main.spec
diff --git a/python/requirements.txt b/python/requirements.txt
diff --git a/python/server/app.py b/python/server/app.py
diff --git a/python/setup.py b/python/setup.py
diff --git a/test/app/components/market/submit/market-application.model.spec.ts b/test/app/components/market/submit/market-application.model.spec.ts
diff --git a/test/app/components/market/submit/parameter-input.component.spec.ts b/test/app/components/market/submit/parameter-input.component.spec.ts
diff --git a/yarn.lock b/yarn.lock

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+export * from "./file-group-sas.component";`
Original file line number	Diff line number	Diff line change
`@@ -7,6 +7,7 @@ export enum NcjParameterExtendedType {`
`7`	`7`	`int = "int",`
`8`	`8`	`fileGroup = "file-group",`
`9`	`9`	`fileInFileGroup = "file-in-file-group",`
	`10`	`+ fileGroupSas = "file-group-sas",`
`10`	`11`	`dropDown = "drop-down",`
`11`	`12`	`}`
`12`	`13`