Merge pull request #2833 from Unity-Technologies/release-0.11.0

Release 0.11.0

Author: Jonathan Harper

.circleci/config.yml

@@ -33,8 +33,6 @@ jobs:
           keys:
           # Parameterize the cache so that different python versions can get different versions of the packages
           - v1-dependencies-py<< parameters.pyversion >>-{{ checksum "python_deps.txt" }}
-          # fallback to using the latest cache if no exact match is found
-          - v1-dependencies-py<< parameters.pyversion >>-
 
       - run:
           name: Install Dependencies
@@ -121,7 +119,97 @@ jobs:
             . venv/bin/activate
             pre-commit run --hook-stage manual markdown-link-check --all-files
 
+  protobuf_generation_check:
+    docker:
+      - image: circleci/python:3.7.3
+    working_directory: ~/repo/
+
+    steps:
+      - checkout
+      - run:
+          name: Combine proto files for caching
+          command: cat protobuf-definitions/proto/mlagents/envs/communicator_objects/*.proto > /tmp/proto_deps.txt
+
+      - restore_cache:
+          keys:
+          - v1-protobuf-gen-dependencies-{{ checksum "/tmp/proto_deps.txt" }}
+          - v1-protobuf-gen-dependencies-
+
+      - run:
+          name: Install Dependencies
+          command: |
+            sudo apt-get install nuget
+            nuget install Grpc.Tools -Version 1.14.1 -OutputDirectory protobuf-definitions/
+            python3 -m venv venv
+            . venv/bin/activate
+            pip install --upgrade pip
+            pip install grpcio-tools==1.13.0  --progress-bar=off
+            pip install mypy-protobuf==1.16.0 --progress-bar=off
+      - save_cache:
+          paths:
+            - ./venv
+          key: v1-protobuf-gen-dependencies-{{ checksum "/tmp/proto_deps.txt" }}
+
+      - run:
+          name: Generate protobufs
+          command: |
+            . venv/bin/activate
+            cd protobuf-definitions
+            chmod +x Grpc.Tools.1.14.1/tools/linux_x64/protoc
+            chmod +x Grpc.Tools.1.14.1/tools/linux_x64/grpc_csharp_plugin
+            COMPILER=Grpc.Tools.1.14.1/tools/linux_x64 ./make.sh
+            CS_PROTO_PATH=UnitySDK/Assets/ML-Agents/Scripts/Grpc/CommunicatorObjects
+            git diff --exit-code --quiet -- :/ ":(exclude,top)$CS_PROTO_PATH/*.meta" \
+              || { GIT_ERR=$?; echo "protobufs need to be regenerated, apply the patch uploaded to artifacts."; \
+              echo "Apply the patch with the command: git apply proto.patch"; \
+              git diff -- :/ ":(exclude,top)$CS_PROTO_PATH/*.meta" > /tmp/proto.patch; \
+              exit $GIT_ERR; }
+      - store_artifacts:
+            path: /tmp/proto.patch
+            destination: proto.patch
+
+  deploy:
+    parameters:
+      directory:
+        type: string
+        description: Local directory to use for publishing (e.g. ml-agents)
+    docker:
+      - image: circleci/python:3.6
+    steps:
+      - checkout
+      - run:
+          name: install python dependencies
+          command: |
+            python3 -m venv venv
+            . venv/bin/activate
+            pip install  --upgrade pip
+            pip install setuptools wheel twine
+      - run:
+          name: verify git tag vs. version
+          command: |
+            python3 -m venv venv
+            . venv/bin/activate
+            cd << parameters.directory >>
+            python setup.py verify
+      - run:
+          name: create packages
+          command: |
+            . venv/bin/activate
+            cd << parameters.directory >>
+            python setup.py sdist
+            python setup.py bdist_wheel
+      - run:
+          name: upload to pypi
+          # To upload to test, just add the following flag to twine upload:
+          # --repository-url https://test.pypi.org/legacy/
+          # and change the username to "mlagents-test"
+          command: |
+            . venv/bin/activate
+            cd << parameters.directory >>
+            twine upload -u mlagents -p $PYPI_PASSWORD dist/*
+
 workflows:
+  version: 2
   workflow:
     jobs:
       - build_python:
@@ -137,3 +225,28 @@ workflows:
           # Test python 3.7 with the newest supported versions
           pip_constraints: test_constraints_max_version.txt
       - markdown_link_check
+      - protobuf_generation_check
+      - deploy:
+          name: deploy ml-agents-envs
+          directory: ml-agents-envs
+          filters:
+            tags:
+              only: /[0-9]+(\.[0-9]+)*(\.dev[0-9]+)*/
+            branches:
+              ignore: /.*/
+      - deploy:
+          name: deploy ml-agents
+          directory: ml-agents
+          filters:
+            tags:
+              only: /[0-9]+(\.[0-9]+)*(\.dev[0-9]+)*/
+            branches:
+              ignore: /.*/
+      - deploy:
+          name: deploy gym-unity
+          directory: gym-unity
+          filters:
+            tags:
+              only: /[0-9]+(\.[0-9]+)*(\.dev[0-9]+)*/
+            branches:
+              ignore: /.*/

.pre-commit-config.yaml

@@ -3,6 +3,12 @@ repos:
     rev: 19.3b0
     hooks:
     -   id: black
+        exclude: >
+            (?x)^(
+                .*_pb2.py|
+                .*_pb2_grpc.py
+            )$
+
 -   repo: https://github.com/pre-commit/mirrors-mypy
     rev: v0.720
     hooks:
@@ -20,14 +26,16 @@ repos:
         name: mypy-gym-unity
         files: "gym-unity/.*"
         args: [--ignore-missing-imports, --disallow-incomplete-defs]
+
 -   repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v2.2.3
     hooks:
     -   id: mixed-line-ending
         exclude: >
             (?x)^(
                 .*cs.meta|
-                .*.css
+                .*.css|
+                .*.meta
             )$
         args: [--fix=lf]
     -   id: flake8
@@ -36,6 +44,16 @@ repos:
                 .*_pb2.py|
                 .*_pb2_grpc.py
             )$
+        additional_dependencies: [flake8-comprehensions]
+    -   id: trailing-whitespace
+        name: trailing-whitespace-markdown
+        types: [markdown]
+
+-   repo: https://github.com/pre-commit/pygrep-hooks
+    rev: v1.4.1  # Use the ref you want to point at
+    hooks:
+    -   id: python-check-mock-methods
+
 # "Local" hooks, see https://pre-commit.com/#repository-local-hooks
 -   repo: local
     hooks:
@@ -51,3 +69,8 @@ repos:
         exclude: ".*localized.*"
         # Only run manually, e.g. pre-commit run --hook-stage manual markdown-link-check
         stages: [manual]
+    -   id: validate-versions
+        name: validate library versions
+        language: script
+        entry: utils/validate_versions.py
+        files: ".*/setup.py"

.yamato/csharp-tests.yml

@@ -0,0 +1,30 @@
+test_editors:
+  - version: 2017.4
+  - version: 2018.4
+  - version: 2019.3
+---
+{% for editor in test_editors %}
+test_mac_editmode_{{ editor.version }}:
+  name: Test Mac EditMode {{ editor.version }}
+  agent:
+    type: Unity::VM::osx
+    image: ml-agents/ml-agents-bokken-mac:release
+    flavor: i1.small
+  variables:
+    UNITY_VERSION: {{ editor.version }}
+  commands:
+    - ./run-tests-editmode-osx-editor.sh
+  triggers:
+    branches:
+      only:
+        - "/develop-.*/"
+    targets:
+      only:
+        - "develop"
+    pull_requests:
+      - targets:
+          only:
+            - "master"
+            - "/release-.*/"
+            - "/hotfix-.*/"
+{% endfor %}

.yamato/standalone-build-test.yml

@@ -0,0 +1,30 @@
+test_editors:
+  - version: 2017.4
+  - version: 2018.4
+  - version: 2019.3
+---
+{% for editor in test_editors %}
+test_mac_standalone_{{ editor.version }}:
+  name: Test Mac Standalone {{ editor.version }}
+  agent:
+    type: Unity::VM::osx
+    image: ml-agents/ml-agents-bokken-mac:release
+    flavor: i1.small
+  variables:
+    UNITY_VERSION: {{ editor.version }}
+  commands:
+    - ./run-standalone-build-osx.sh
+  triggers:
+    branches:
+      only:
+        - "/develop-.*/"
+    targets:
+      only:
+        - "develop"
+    pull_requests:
+      - targets:
+          only:
+            - "master"
+            - "/release-.*/"
+            - "/hotfix-.*/"
+{% endfor %}

README.md

@@ -29,7 +29,6 @@ developer communities.
 * Support for multiple environment configurations and training scenarios
 * Train memory-enhanced agents using deep reinforcement learning
 * Easily definable Curriculum Learning and Generalization scenarios
-* Broadcasting of agent behavior for supervised learning
 * Built-in support for Imitation Learning
 * Flexible agent control with On Demand Decision Making
 * Visualizing network outputs within the environment

SURVEY.md

@@ -1,5 +1,5 @@
 # Unity ML-Agents Toolkit Survey
 
-Your opinion matters a great deal to us. Only by hearing your thoughts on the Unity ML-Agents Toolkit can we continue to improve and grow. Please take a few minutes to let us know about it. 
+Your opinion matters a great deal to us. Only by hearing your thoughts on the Unity ML-Agents Toolkit can we continue to improve and grow. Please take a few minutes to let us know about it.
 
-[Fill out the survey](https://goo.gl/forms/qFMYSYr5TlINvG6f1) 
+[Fill out the survey](https://goo.gl/forms/qFMYSYr5TlINvG6f1)