Develop pro package refactor (#21)

* Removed com.unity.ml-agents package.

* Added com.unity.ml-agents submodule.

* Updated yamato pack to recurse submodules.

* Updated all yamato to recurse submodules.

* Fixed bug in precommit gh workflow.

* Updated validate meta files.

* Fix for protobuf gen test.

Author: miguelalonsojr

.github/workflows/pre-commit.yml

@@ -12,6 +12,9 @@ jobs:
     runs-on: [self-hosted, Linux, X64]
     steps:
     - uses: actions/checkout@v2
+      with:
+        token: ${{ secrets.GHE_ACCESS_TOKEN }}
+        submodules: recursive
     - uses: actions/setup-python@v2
       with:
         python-version: 3.7.x
@@ -37,6 +40,9 @@ jobs:
     runs-on: [self-hosted, Linux, X64]
     steps:
     - uses: actions/checkout@v2
+      with:
+          token: ${{ secrets.GHE_ACCESS_TOKEN }}
+          submodules: recursive
     - uses: actions/setup-python@v2
     - uses: actions/setup-node@v2
       with:
@@ -55,5 +61,8 @@ jobs:
     runs-on: [self-hosted, Linux, X64]
     steps:
     - uses: actions/checkout@v2
+      with:
+          token: ${{ secrets.GHE_ACCESS_TOKEN }}
+          submodules: recursive
     - uses: actions/setup-python@v2
     - run: python utils/validate_meta_files.py

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "com.unity.ml-agents"]
+	path = com.unity.ml-agents
+	url = ../com.unity.ml-agents.git

.yamato/com.unity.ml-agents-coverage.yml

@@ -12,6 +12,7 @@ test_coverage_{{ package.name }}_{{ platform.name }}_{{ editor.version }}:
         image: {{ platform.image }}
         flavor: {{ platform.flavor}}
     commands:
+        - git submodule update --init --recursive
         - npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm
         - upm-ci project test -u {{ editor.version }} --type project-tests --project-path {{ editor.testProject }} --package-filter {{ package.name }} {{ coverageOptions }} --extra-utr-arg "reruncount=2"
         - |

.yamato/com.unity.ml-agents-optional-dep-tests.yml

@@ -18,6 +18,7 @@ OptionalDependencyTests_{{ optional_dep.name }}:
         image: ml-agents/ml-agents-ubuntu-18.04:latest
         flavor: b1.medium
     commands:
+        - git submodule update --init --recursive
         - |
             curl -L https://artifactory.prd.it.unity3d.com/artifactory/api/gpg/key/public | sudo apt-key add -
             sudo sh -c "echo 'deb https://artifactory.prd.it.unity3d.com/artifactory/unity-apt-local bionic main' > /etc/apt/sources.list.d/unity.list"

.yamato/com.unity.ml-agents-pack.yml

@@ -5,6 +5,7 @@ pack:
     image: ml-agents/ml-agents-ubuntu-18.04:latest
     flavor: b1.small
   commands:
+    - git submodule update --init --recursive
     - |
       eval "$($HOME/anaconda/bin/conda shell.bash hook)"
       conda activate python3.7

.yamato/com.unity.ml-agents-performance.yml

@@ -12,6 +12,7 @@ Run_Mac_Perfomance_Tests{{ editor.version }}:
   variables:
     UNITY_VERSION: {{ editor.version }}
   commands:
+    - git submodule update --init --recursive
     - python3 -m pip install unity-downloader-cli --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple --upgrade
     - unity-downloader-cli -u {{ editor.version }} -c editor --wait --fast
     - curl -s https://artifactory.prd.it.unity3d.com/artifactory/unity-tools-local/utr-standalone/utr --output utr

.yamato/com.unity.ml-agents-test.yml

@@ -79,6 +79,7 @@ test_{{ package.name }}_{{ platform.name }}_{{ editor.version }}:
         image: {{ platform.image }}
         flavor: {{ platform.flavor}}
     commands:
+        - git submodule update --init --recursive
         - npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm
         - upm-ci package test -u {{ editor.version }} --package-path {{ package.name }} {{ noDefaultPackagesOptions }} --warnings-as-errors --extra-utr-arg "reruncount=2"
     artifacts:
@@ -123,6 +124,7 @@ test_{{ package.name }}_{{ platform.name }}_trunk:
         image: {{ platform.image }}
         flavor: {{ platform.flavor}}
     commands:
+        - git submodule update --init --recursive
         - |
 {% if platform.name == "linux" %}
           conda activate python3.7

.yamato/compressed-sensor-test.yml

@@ -10,6 +10,7 @@ test_compressed_obs_{{ editor.version }}:
   variables:
     UNITY_VERSION: {{ editor.version }}
   commands:
+    - git submodule update --init --recursive
     - |
       eval "$($HOME/anaconda/bin/conda shell.bash hook)"
       conda activate python3.7

.yamato/gym-interface-test.yml

@@ -10,6 +10,7 @@ test_gym_interface_{{ editor.version }}:
   variables:
     UNITY_VERSION: {{ editor.version }}
   commands:
+    - git submodule update --init --recursive
     - |
       eval "$($HOME/anaconda/bin/conda shell.bash hook)"
       conda activate python3.7

.yamato/protobuf-generation-test.yml

@@ -6,8 +6,9 @@ test_linux_protobuf_generation:
     flavor: b1.large
   variables:
     GRPC_VERSION: "1.14.1"
-    CS_PROTO_PATH: "com.unity.ml-agents/Runtime/Grpc/CommunicatorObjects"
+    CS_PROTO_PATH: "Runtime/Grpc/CommunicatorObjects"
   commands:
+    - git submodule update --init --recursive
     - |
       sudo apt-get update && sudo apt-get install -y nuget
       eval "$($HOME/anaconda/bin/conda shell.bash hook)"
@@ -22,10 +23,11 @@ test_linux_protobuf_generation:
       popd
       mkdir -p artifacts
       touch artifacts/proto.patch
+      cd com.unity.ml-agents
       git diff --exit-code -- :/ ":(exclude,top)$CS_PROTO_PATH/*.meta"                          \
       || { GIT_ERR=$?; echo "protobufs need to be regenerated, apply the patch uploaded to artifacts."; \
       echo "Apply the patch with the command 'git apply proto.patch'";                                  \
-      git diff -- :/ ":(exclude,top)$CS_PROTO_PATH/*.meta" > artifacts/proto.patch; exit $GIT_ERR; }
+      git diff -- :/ ":(exclude,top)$CS_PROTO_PATH/*.meta" > ../artifacts/proto.patch; exit $GIT_ERR; }
   triggers:
     cancel_old_ci: true
     expression: |