[Test-Fragments] Add NOT_VALIDATED in file validation status terminal status (#105)

lib/constants.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.defaultYaml = void 0;
+exports.APIRoute = exports.testmanagerApiVersion = exports.defaultYaml = void 0;
 exports.defaultYaml = {
     version: 'v0.1',
     testId: 'SampleTest',
@@ -48,3 +48,13 @@ exports.defaultYaml = {
         }
     ]
 };
+exports.testmanagerApiVersion = "2024-07-01-preview";
+var BaseAPIRoute;
+(function (BaseAPIRoute) {
+    BaseAPIRoute.featureFlag = "featureFlags";
+})(BaseAPIRoute || (BaseAPIRoute = {}));
+var APIRoute;
+(function (APIRoute) {
+    const latestVersion = "api-version=" + exports.testmanagerApiVersion;
+    APIRoute.FeatureFlags = (flag) => `${BaseAPIRoute.featureFlag}/${flag}?${latestVersion}`;
+})(APIRoute || (exports.APIRoute = APIRoute = {}));

lib/main.js

@@ -39,6 +39,8 @@ const util = __importStar(require("./util"));
 const TestKind_1 = require("./engine/TestKind");
 const fs = __importStar(require("fs"));
 const util_1 = require("util");
+const FeatureFlagService_1 = require("./services/FeatureFlagService");
+const FeatureFlags_1 = require("./services/FeatureFlags");
 const resultFolder = 'loadTest';
 const reportZipFileName = 'report.zip';
 const resultZipFileName = 'results.zip';
@@ -74,8 +76,8 @@ function run() {
         }
     });
 }
-function getTestAPI(validate) {
-    return __awaiter(this, void 0, void 0, function* () {
+function getTestAPI(validate_1) {
+    return __awaiter(this, arguments, void 0, function* (validate, returnTestObj = false) {
         var _a, _b;
         var urlSuffix = "tests/" + testId + "?api-version=" + util.apiConstants.latestVersion;
         urlSuffix = baseURL + urlSuffix;
@@ -113,6 +115,9 @@ function getTestAPI(validate) {
             }
             var inputScriptFileInfo = testObj.kind == TestKind_1.TestKind.URL ? testObj.inputArtifacts.urlTestConfigFileInfo : testObj.inputArtifacts.testScriptFileInfo;
             if (validate) {
+                if (returnTestObj) {
+                    return [inputScriptFileInfo.validationStatus, testObj];
+                }
                 return inputScriptFileInfo.validationStatus;
             }
             else {
@@ -223,9 +228,10 @@ function uploadTestPlan() {
             var startTime = new Date();
             var maxAllowedTime = new Date(startTime.getTime() + minutesToAdd * 60000);
             var validationStatus = "VALIDATION_INITIATED";
+            var testObj;
             while (maxAllowedTime > (new Date()) && (validationStatus == "VALIDATION_INITIATED" || validationStatus == "NOT_VALIDATED" || validationStatus == null)) {
                 try {
-                    validationStatus = yield getTestAPI(true);
+                    [validationStatus, testObj] = yield getTestAPI(true, true);
                 }
                 catch (e) {
                     retry--;
@@ -235,8 +241,18 @@ function uploadTestPlan() {
                 }
                 yield util.sleep(5000);
             }
+            console.log("Validation status of the test plan: " + validationStatus);
             if (validationStatus == null || validationStatus == "VALIDATION_SUCCESS") {
                 console.log(`Validated test plan for the test successfully.`);
+                // Get errors from all files
+                var fileErrors = util.getAllFileErrors(testObj);
+                if (Object.keys(fileErrors).length > 0) {
+                    console.log("Validation failed for the following files:");
+                    for (const [file, error] of Object.entries(fileErrors)) {
+                        console.log(`File: ${file}, Error: ${error}`);
+                    }
+                    throw new Error("Validation of one or more files failed. Please correct the errors and try again.");
+                }
                 yield createTestRun();
             }
             else if (validationStatus == "VALIDATION_INITIATED" || validationStatus == "NOT_VALIDATED")
@@ -252,7 +268,7 @@ function uploadConfigFile() {
         if (configFiles != undefined && configFiles.length > 0) {
             for (let filepath of configFiles) {
                 let filename = map.getFileName(filepath);
-                var urlSuffix = "tests/" + testId + "/files/" + filename + "?api-version=" + util.apiConstants.latestVersion;
+                var urlSuffix = "tests/" + testId + "/files/" + filename + "?api-version=" + util.apiConstants.latestVersion + ("&fileType=" + FileType.ADDITIONAL_ARTIFACTS);
                 urlSuffix = baseURL + urlSuffix;
                 let headers = yield map.UploadAndValidateHeader();
                 let uploadresult = yield util.httpClientRetries(urlSuffix, headers, 'put', 3, filepath, true);
@@ -291,6 +307,13 @@ function uploadZipArtifacts() {
             let flagValidationPending = true;
             let zipInvalid = false;
             let zipFailureReason = "";
+            // TODO(harshanb): Remove this check once the feature flag is enabled by default.
+            let isTestScriptFragmentEnabled = yield FeatureFlagService_1.FeatureFlagService.isFeatureEnabledAsync(FeatureFlags_1.FeatureFlags.enableTestScriptFragments, baseURL);
+            let zipValidationTerminateStates = ["VALIDATION_SUCCESS", "VALIDATION_FAILURE"];
+            if (isTestScriptFragmentEnabled) {
+                // NOT_VALIDATED is a terminal state for the file validation and actual validation will be performed during test script upload
+                zipValidationTerminateStates.push("NOT_VALIDATED");
+            }
             while (maxAllowedTime > (new Date()) && flagValidationPending) {
                 var urlSuffix = "tests/" + testId + "?api-version=" + util.apiConstants.latestVersion;
                 urlSuffix = baseURL + urlSuffix;
@@ -304,7 +327,7 @@ function uploadZipArtifacts() {
                 flagValidationPending = false;
                 if (testObj && testObj.inputArtifacts && testObj.inputArtifacts.additionalFileInfo) {
                     for (const file of testObj.inputArtifacts.additionalFileInfo) {
-                        if (file.fileType == FileType.ZIPPED_ARTIFACTS && (file.validationStatus != "VALIDATION_SUCCESS" && file.validationStatus != "VALIDATION_FAILURE")) {
+                        if (file.fileType == FileType.ZIPPED_ARTIFACTS && (file.validationStatus in zipValidationTerminateStates)) {
                             flagValidationPending = true;
                             break;
                         }
@@ -329,7 +352,12 @@ function uploadZipArtifacts() {
             else if (flagValidationPending) {
                 throw new Error("Validation of one or more zip artifacts timed out. Please retry.");
             }
-            console.log(`Uploaded and validated ${zipFiles.length} zip artifact(s) for the test successfully.`);
+            if (isTestScriptFragmentEnabled) {
+                console.log(`Uploaded ${zipFiles.length} zip artifact(s) for the test successfully.`);
+            }
+            else {
+                console.log(`Uploaded and validated ${zipFiles.length} zip artifact(s) for the test successfully.`);
+            }
         }
         var statuscode = yield uploadPropertyFile();
         if (statuscode == 201) {

lib/mappers.js

@@ -32,7 +32,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getARMEndpoint = exports.getTenantId = exports.getFileName = exports.getTestId = exports.getZipFiles = exports.getConfigFiles = exports.getPropertyFile = exports.getTestFile = exports.getYamlPath = exports.getTestKind = exports.getDefaultTestRunName = exports.getDefaultTestName = exports.getSubName = exports.getInputParams = exports.getResourceId = exports.getTestHeader = exports.getTestRunHeader = exports.startTestData = exports.armTokenHeader = exports.UploadAndValidateHeader = exports.createTestHeader = exports.createTestData = void 0;
+exports.getARMEndpoint = exports.getTenantId = exports.getFileName = exports.getTestId = exports.getZipFiles = exports.getConfigFiles = exports.getPropertyFile = exports.getTestFile = exports.getYamlPath = exports.getTestKind = exports.getDefaultTestRunName = exports.getDefaultTestName = exports.getSubName = exports.getInputParams = exports.getResourceId = exports.getTestHeader = exports.getTestRunHeader = exports.startTestData = exports.armTokenHeader = exports.UploadAndValidateHeader = exports.createTestHeader = exports.createTestData = exports.getToken = void 0;
 const core = __importStar(require("@actions/core"));
 const yaml = require("js-yaml");
 const jwt_decode = __importStar(require("jwt-decode"));
@@ -100,6 +100,10 @@ function getExistingData() {
             envYaml[key] = null;
     }
 }
+function getToken() {
+    return token;
+}
+exports.getToken = getToken;
 function createTestData() {
     getExistingData();
     var data = {

lib/models/APIResponseModel.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

lib/services/FeatureFlagService.js

@@ -0,0 +1,73 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FeatureFlagService = void 0;
+const constants_1 = require("../constants");
+const map = __importStar(require("../mappers"));
+const util = __importStar(require("../util"));
+class FeatureFlagService {
+    static getFeatureFlagAsync(flag_1, baseUrl_1) {
+        return __awaiter(this, arguments, void 0, function* (flag, baseUrl, useCache = true) {
+            if (useCache && flag in this.featureFlagCache) {
+                return { featureFlag: flag, enabled: this.featureFlagCache[flag.toString()] };
+            }
+            let uri = baseUrl + constants_1.APIRoute.FeatureFlags(flag.toString());
+            let headers = {
+                'content-type': 'application/json',
+                'Authorization': 'Bearer ' + map.getToken()
+            };
+            let flagResponse = yield util.httpClientRetries(uri, headers, 'get', 3, "", false, false);
+            try {
+                let flagObj = (yield util.getResultObj(flagResponse));
+                this.featureFlagCache[flag.toString()] = flagObj.enabled;
+                return flagObj;
+            }
+            catch (error) {
+                // remove item from dict
+                // handle in case getFlag was called with cache true once and then with cache false, and failed during second call
+                // remove the item from cache so that it can be fetched again rather than using old value
+                delete this.featureFlagCache[flag.toString()];
+                return null;
+            }
+        });
+    }
+    static isFeatureEnabledAsync(flag_1, baseUrl_1) {
+        return __awaiter(this, arguments, void 0, function* (flag, baseUrl, useCache = true) {
+            let flagObj = yield this.getFeatureFlagAsync(flag, baseUrl, useCache);
+            return flagObj ? flagObj.enabled : false;
+        });
+    }
+}
+exports.FeatureFlagService = FeatureFlagService;
+FeatureFlagService.featureFlagCache = {};

lib/services/FeatureFlags.js

@@ -0,0 +1,7 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FeatureFlags = void 0;
+var FeatureFlags;
+(function (FeatureFlags) {
+    FeatureFlags["enableTestScriptFragments"] = "enableTestScriptFragments";
+})(FeatureFlags || (exports.FeatureFlags = FeatureFlags = {}));

lib/util.js

@@ -32,7 +32,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ErrorCorrection = exports.getResultObj = exports.validCriteria = exports.isTerminalTestStatus = exports.removeUnits = exports.indexOfFirstDigit = exports.deleteFile = exports.getReportFolder = exports.getResultFolder = exports.checkValidityYaml = exports.invalidDescription = exports.invalidDisplayName = exports.invalidName = exports.getUniqueId = exports.sleep = exports.printClientMetrics = exports.uploadFileToResultsFolder = exports.printCriteria = exports.printTestDuration = exports.checkFileTypes = exports.checkFileType = exports.httpClientRetries = exports.uploadFileData = exports.ManagedIdentityType = exports.apiConstants = void 0;
+exports.getAllFileErrors = exports.ErrorCorrection = exports.getResultObj = exports.validCriteria = exports.isTerminalTestStatus = exports.removeUnits = exports.indexOfFirstDigit = exports.deleteFile = exports.getReportFolder = exports.getResultFolder = exports.checkValidityYaml = exports.invalidDescription = exports.invalidDisplayName = exports.invalidName = exports.getUniqueId = exports.sleep = exports.printClientMetrics = exports.uploadFileToResultsFolder = exports.printCriteria = exports.printTestDuration = exports.checkFileTypes = exports.checkFileType = exports.httpClientRetries = exports.uploadFileData = exports.ManagedIdentityType = exports.apiConstants = void 0;
 const fs = __importStar(require("fs"));
 var path = require('path');
 var AdmZip = require("adm-zip");
@@ -87,7 +87,7 @@ function uploadFileData(filepath) {
 exports.uploadFileData = uploadFileData;
 const correlationHeader = 'x-ms-correlation-request-id';
 function httpClientRetries(urlSuffix_1, header_1, method_1) {
-    return __awaiter(this, arguments, void 0, function* (urlSuffix, header, method, retries = 1, data, isUploadCall = true) {
+    return __awaiter(this, arguments, void 0, function* (urlSuffix, header, method, retries = 1, data, isUploadCall = true, log = true) {
         let httpResponse;
         try {
             let correlationId = `gh-actions-${getUniqueId()}`;
@@ -118,7 +118,9 @@ function httpClientRetries(urlSuffix_1, header_1, method_1) {
             if (retries) {
                 let sleeptime = (5 - retries) * 1000 + Math.floor(Math.random() * 5001);
                 yield sleep(sleeptime);
-                console.log(`Failed to connect to ${urlSuffix} due to ${err.message}, retrying in ${sleeptime / 1000} seconds`);
+                if (log) {
+                    console.log(`Failed to connect to ${urlSuffix} due to ${err.message}, retrying in ${sleeptime / 1000} seconds`);
+                }
                 return httpClientRetries(urlSuffix, header, method, retries - 1, data);
             }
             else
@@ -542,3 +544,23 @@ function ErrorCorrection(result) {
     return "Unable to fetch the response. Please re-run or contact support if the issue persists. " + "Status code: " + result.message.statusCode;
 }
 exports.ErrorCorrection = ErrorCorrection;
+function getAllFileErrors(testObj) {
+    var allArtifacts = [];
+    for (var key in testObj.inputArtifacts) {
+        var artifacts = testObj.inputArtifacts[key];
+        if (artifacts instanceof Array) {
+            allArtifacts = allArtifacts.concat(artifacts.filter((artifact) => artifact !== null && artifact !== undefined));
+        }
+        else if (artifacts !== null && artifacts !== undefined) {
+            allArtifacts.push(artifacts);
+        }
+    }
+    var fileErrors = {};
+    for (const file of allArtifacts) {
+        if (file.validationStatus === "VALIDATION_FAILURE") {
+            fileErrors[file.fileName] = file.validationFailureDetails;
+        }
+    }
+    return fileErrors;
+}
+exports.getAllFileErrors = getAllFileErrors;

src/constants.ts

@@ -46,3 +46,14 @@ export const defaultYaml: any =
         }
     ]
 }
+
+export const testmanagerApiVersion = "2024-07-01-preview";
+
+namespace BaseAPIRoute {
+    export const featureFlag = "featureFlags";
+}
+
+export namespace APIRoute {
+    const latestVersion = "api-version="+testmanagerApiVersion;
+    export const FeatureFlags = (flag: string) => `${BaseAPIRoute.featureFlag}/${flag}?${latestVersion}`;
+}