Fix validation issues (#56)

* Adding test and validation stage before publishing

* Adding utils python package

* Adding ignored packages folder

* Fix test project

* Use correct publish machine

* Fix path for unity launcher in the validation phase

* Create Editor folder and add registry to manifest

* Add missing package to manifest

* Provide Test folder to package

* Fix project Version

* Disable tests and fix terminal tests path

* Reintroducing editmode tests

* Use correct version

* Fix solution file

Author: fabriziounity

.gitignore

@@ -143,7 +143,7 @@ publish/
 # NuGet Packages
 *.nupkg
 # The packages folder can be ignored because of Package Restore
-**/packages/*
+#**/packages/*
 # except build/, which is used as an MSBuild target.
 !**/packages/build/
 # Uncomment if necessary however generally it will be regenerated when needed

.gitlab-ci.yml

@@ -2,54 +2,78 @@ variables:
   GIT_SUBMODULE_STRATEGY: recursive
 
 stages:
-  - run_tests
+  - run_tests_on_mono
   - build_package
+  - run_package_tests
   - validate_package
   - publish_package
 
-run_tests:mac:
-  stage: run_tests
+run_tests_on_mono:mac:
+  stage: run_tests_on_mono
   tags:
     - mac
   script:
-    - cd src
+    - cd src/
     - nuget restore
     - msbuild Unity.Mathematics.sln
-    - mono packages/NUnit.ConsoleRunner.3.9.0/tools/nunit3-console.exe Unity.Mathematics.Tests/bin/Debug/Unity.Mathematics.Tests.dll --process=single
+    - mono packages/NUnit.ConsoleRunner.3.9.0/tools/nunit3-console.exe Tests/bin/Debug/Unity.Mathematics.Tests.dll --process=single
 
 build_package:mac:
   stage: build_package
   tags:
     - mac
   script:
-    - cd src/
+    - cd src
     - npm version 0.0.1
     - npm pack .
   artifacts:
     paths:
       - src/*.tgz
 
+run_package_tests:mac:
+  stage: run_package_tests
+  tags:
+    - mac
+  before_script:
+    - python Tools/CI/get_unity_launcher.py
+  script:
+    - tar xzvf src/com.unity.mathematics-0.0.1.tgz
+    - ./.UnityLauncher.Editor/osx.10.12-x64/publish/UnityLauncher.Editor -registryoverride https://bfartifactory.bf.unity3d.com/artifactory/api/npm/unity-upm-staging -timeout 1200 -scriptingBackend mono -unityexecutable ~/Editor/Unity.app/Contents/MacOS/Unity -projectpath Unity.Mathematics.TestProject -batchmode -silentcrashes -automated -logfile output-macOS-editmode-test.log -cleanedLogFile output-macOS-editmode-test-cleaned.log -runtests -testresults results.xml -testPlatform editmode --ignoreErrorsOnArtifactCreation
+
 validate_package:mac:
   stage: validate_package
   tags:
     - mac
+  only:
+    - tags
+  before_script:
+    - python Tools/CI/get_unity_launcher.py
   script:
+    - cp Tools/CI/ValidatePackage.cs Unity.Mathematics.TestProject/Assets/Editor/ValidatePackage.cs
     - tar xzvf src/com.unity.mathematics-0.0.1.tgz
-    - python Tools/CI/validation.py --package-path package
+    - cd package
+    - npm version $CI_COMMIT_TAG
+    - jq --arg rev $CI_COMMIT_SHA '.repository.revision=$rev' package.json > temp_package && mv temp_package package.json
+    - cd ..
+    - ./.UnityLauncher.Editor/osx.10.12-x64/publish/UnityLauncher.Editor -registryoverride https://bfartifactory.bf.unity3d.com/artifactory/api/npm/unity-upm-staging -timeout 1200 -unityexecutable ~/Editor/Unity.app/Contents/MacOS/Unity -executeMethod ValidatePackage.Validate ../package -projectpath Unity.Mathematics.TestProject -batchmode -silentcrashes -automated -logfile output-linux-validation.log -cleanedLogFile output-linux-validation-cleaned.log -quit -displayResolutionDialog disabled -addPackage [email protected]
   dependencies:
     - build_package:mac
+  artifacts:
+    paths:
+      - package/
+
 
 publish_package:mac:
   stage: publish_package
   tags:
-    - mac
+    - gamecode
+    - macOS
   only:
     - tags
   script:
-    - tar xzvf src/com.unity.mathematics-0.0.1.tgz
+    - python Tools/CI/validation.py --package-path package --commit-tag $CI_COMMIT_TAG
     - cd package
-    - npm version $CI_COMMIT_TAG
-    - npm pack .
-    - npm publish com.unity.mathematics-$CI_COMMIT_TAG.tgz
+      #- npm publish --registry https://staging-packages.unity.com
+    - echo "Would have published com.unity.mathematics $CI_COMMIT_TAG"
   dependencies:
-    - build_package:mac
+    - validate_package:mac

Tools/CI/ValidatePackage.cs

@@ -0,0 +1,92 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text.RegularExpressions;
+using Unity.Properties.Serialization;
+using UnityEditor;
+using UnityEditor.PackageManager.ValidationSuite;
+using UnityEngine;
+
+public class ValidatePackage
+{
+    static List<string> failuresWhitelist = new List<string>{};// "Restricted File Type Validation", "Tests Validation" };
+
+    [MenuItem("Window/Validate Mathematics")]
+    public static void ValidateMathematics()
+    {
+        ValidateInternal("../package");
+    }
+
+    static void FilterWhitelistedFailures(string report)
+    {
+        var reportLines = report.Split(new string[] {"\r\n\r\n"}, StringSplitOptions.RemoveEmptyEntries);
+        Array.Copy(reportLines, 2, reportLines, 0, reportLines.Length - 2);
+        Regex regex = new Regex($"(\\w+)[ -]+\"([\\w\\s]+)\"\\s+(.*)");
+
+        var issues = string.Empty;
+
+        foreach (var line in reportLines)
+        {
+            var match = regex.Match(line);
+            if (match.Success)
+            {
+                var status = match.Groups[1].Captures[0].Value;
+                if (status != "Failed")
+                    continue;
+                var check = match.Groups[2].Captures[0].Value;
+                var data = match.Groups[3].Captures[0].Value;
+
+                if (status == "Failed" && failuresWhitelist.Contains(check))
+                    continue;
+
+                issues += line + "\r\n";
+            }
+        }
+
+        if (issues != string.Empty)
+        {
+            throw new Exception(issues);
+        }
+    }
+
+    public static void ValidateInternal(string packagePath)
+    {
+        var packageManifest = Path.Combine(packagePath, "package.json");
+        if (!File.Exists(packageManifest))
+            throw new Exception($"File package.json not found in {packagePath}");
+
+        object dict;
+        if (!Json.TryDeserializeObject(File.ReadAllText(packageManifest), out dict))
+            throw new Exception($"Invalid package.json file in {packageManifest}");
+
+        Json.TryGetValue((IDictionary<string, object>)dict, "name", out string name);
+        Json.TryGetValue((IDictionary<string, object>)dict, "version", out string version);
+
+        var result = ValidationSuite.ValidatePackage($"{name}@{version}", ValidationType.Publishing);
+        if (!result)
+            FilterWhitelistedFailures(ValidationSuite.GetValidationSuiteReport(name, version));
+    }
+
+    public static void Validate()
+    {
+        var arguments = Environment.GetCommandLineArgs();
+        var executeMethodIndex = 0;
+        while (executeMethodIndex < arguments.Length)
+        {
+            if (arguments[executeMethodIndex] == "-executeMethod")
+                break;
+            executeMethodIndex++;
+        }
+
+        // Need to advance by 2 elements because the format is -executeMethod Builders.Build <Target>
+        // n.b.: executeMethodIndex is the index of -executeMethod
+        var targetIndex = executeMethodIndex + 2;
+        if (targetIndex >= arguments.Length || arguments[targetIndex].StartsWith("-"))
+            throw new Exception("Wrong argument. To execute this method run unity with the parameter -executeMethod ValidatePackage.Validate <path_to_package>");
+
+	    var packagePath = arguments[targetIndex];
+        ValidateInternal(packagePath);
+    }
+}

Tools/CI/get_unity_launcher.py

@@ -0,0 +1,43 @@
+import os
+import sys
+
+import utils
+
+artifactory_api_url = "https://artifactory.eu-cph-1.unityops.net"
+artifactory_api_repository = "core-automation"
+artifactory_download_url = "https://bfartifactory.bf.unity3d.com/artifactory/ie-generic-core-automation"
+bokken_artifactory_cache_url = "http://172.28.214.140/"
+build_version = '2018.2.*&latest=true'
+
+def download_unity_launcher(type):
+    if not os.path.exists(".tmp"):
+        os.mkdir(".tmp")
+    download_url = artifactory_download_url
+    if 'USE_UBERBUCKET' in os.environ:
+        download_url = bokken_artifactory_cache_url
+    utils.download_url("%s/tools/unity-launcher/UnityLauncher.%s.zip" % (download_url, type),
+                       ".tmp/UnityLauncher.%s.zip" % type)
+
+
+def extract_unity_launcher(type):
+    utils.extract_zip(".tmp/UnityLauncher.%s.zip" % type,
+                      ".UnityLauncher.%s" % type)
+    current_os = utils.get_current_os()
+    if current_os == "macOS":
+        os.chmod('.UnityLauncher.{0}/osx.10.12-x64/publish/UnityLauncher.{0}'.format(type), 0755)
+    elif current_os == "linux":
+        os.chmod('.UnityLauncher.{0}/linux-x64/publish/UnityLauncher.{0}'.format(type), 0755)
+
+
+def prepare_unity_launcher(run_type):
+
+    download_unity_launcher(run_type)
+    extract_unity_launcher(run_type)
+
+def main():
+    prepare_unity_launcher("Editor")
+    return
+
+
+if __name__ == "__main__":
+    main()

Tools/CI/utils.py

@@ -0,0 +1,108 @@
+import json
+import os
+import shutil
+import tarfile
+import zipfile
+import subprocess
+
+import requests
+
+import sys
+
+target_os = sys.platform
+
+def get_current_os():
+    p = sys.platform
+    if p == "darwin":
+        return "macOS"
+    if p == "win32":
+        return "windows"
+    return "linux"
+
+
+def get_target_os():
+    if target_os != sys.platform:
+        return target_os
+
+    return get_current_os()
+
+
+def set_target_os(target):
+    global target_os
+    target_os = target
+
+def get_url_json(url):
+    print "  Getting json from {0}".format(url)
+    import urllib2
+    response = urllib2.urlopen(url)
+    return json.loads(response.read())
+
+
+def extract_tarball(download_path, extract_path):
+    print "  Extracting %s into %s" % (download_path, extract_path)
+    tar = tarfile.open(download_path, "r:gz")
+    tar.extractall(extract_path)
+    tar.close()
+
+
+def download_url(url, filename):
+    print "  Downloading %s to %s" % (url, filename)
+
+    r = requests.get(url, stream=True)
+    with open(filename, 'wb') as f:
+        shutil.copyfileobj(r.raw, f)
+
+
+def extract_zip(archive, destination):
+    print "  Extracting %s into %s" % (archive, destination)
+    import zipfile
+    if get_current_os() == "windows":
+        zip_ref = ZipfileLongWindowsPaths(archive, 'r')
+    else:
+        zip_ref = zipfile.ZipFile(archive, 'r')
+    zip_ref.extractall(destination)
+    zip_ref.close()
+
+
+def winapi_path(dos_path, encoding=None):
+    path = os.path.abspath(dos_path)
+
+    if path.startswith("\\\\"):
+        path = "\\\\?\\UNC\\" + path[2:]
+    else:
+        path = "\\\\?\\" + path
+
+    return path
+
+
+def npm_cmd(cmd, registry):
+    registry_cmd = ''
+    if registry:
+        registry_cmd = "--registry {0}".format(registry)
+
+    formatted_cmd = 'npm {0} {1}'.format(cmd, registry_cmd)
+
+    print "  Running: {0}".format(formatted_cmd)
+    return subprocess.check_output(formatted_cmd, shell=True, stderr=subprocess.STDOUT)
+
+
+def git_cmd(cmd, print_command=True):
+    formatted_cmd = "git {0}".format(cmd)
+    if print_command:
+        print "  Running: {0}".format(formatted_cmd)
+    return subprocess.check_output(formatted_cmd, shell=True, stderr=subprocess.STDOUT)
+
+
+def git_cmd_code_only(cmd, print_command=True):
+    formatted_cmd = "git {0}".format(cmd)
+    if print_command:
+        print "  Running: {0}".format(formatted_cmd)
+    with open(os.devnull, 'w') as devnull:
+        return subprocess.call(formatted_cmd, shell=True, stderr=devnull, stdout=devnull)
+
+
+class ZipfileLongWindowsPaths(zipfile.ZipFile):
+
+    def _extract_member(self, member, targetpath, pwd):
+        targetpath = winapi_path(targetpath)
+        return zipfile.ZipFile._extract_member(self, member, targetpath, pwd)