Pass artifactory url for all pip calls in yamato (#4282)

* pass artifactory url for all pip calls in yamato

* pip proto generation libs

* undo pyupgrade on generated code

Author: Chris Elion

.pre-commit-config.yaml

@@ -44,7 +44,12 @@ repos:
     hooks:
     -   id: pyupgrade
         args: [--py3-plus, --py36-plus]
-        exclude: .*barracuda.py
+        exclude: >
+            (?x)^(
+                .*barracuda.py|
+                .*_pb2.py|
+                .*_pb2_grpc.py
+            )$
 
 -   repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v2.5.0

.yamato/gym-interface-test.yml

@@ -11,7 +11,7 @@ test_gym_interface_{{ editor.version }}:
   variables:
     UNITY_VERSION: {{ editor.version }}
   commands:
-    - pip install pyyaml
+    - pip install pyyaml --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple
     - python -u -m ml-agents.tests.yamato.setup_venv
     - ./venv/bin/python ml-agents/tests/yamato/scripts/run_gym.py --env=artifacts/testPlayer-Basic
   dependencies:
@@ -21,12 +21,12 @@ test_gym_interface_{{ editor.version }}:
     expression: |
       (pull_request.target eq "master" OR
       pull_request.target match "release.+") AND
-      NOT pull_request.draft AND 
-      (pull_request.changes.any match "com.unity.ml-agents/**" OR 
-      pull_request.changes.any match "Project/**" OR 
-      pull_request.changes.any match "ml-agents/**" OR 
-      pull_request.changes.any match "ml-agents-envs/**" OR 
-      pull_request.changes.any match "gym-unity/**" OR 
+      NOT pull_request.draft AND
+      (pull_request.changes.any match "com.unity.ml-agents/**" OR
+      pull_request.changes.any match "Project/**" OR
+      pull_request.changes.any match "ml-agents/**" OR
+      pull_request.changes.any match "ml-agents-envs/**" OR
+      pull_request.changes.any match "gym-unity/**" OR
       pull_request.changes.any match ".yamato/gym-interface-test.yml") AND
       NOT pull_request.changes.all match "**/*.md"
 {% endfor %}

.yamato/protobuf-generation-test.yml

@@ -13,9 +13,8 @@ test_mac_protobuf_generation:
       nuget install Grpc.Tools -Version $GRPC_VERSION -OutputDirectory protobuf-definitions/
       python3 -m venv venv
       . venv/bin/activate
-      pip install --upgrade pip
-      pip install grpcio-tools==1.13.0  --progress-bar=off
-      pip install mypy-protobuf==1.16.0 --progress-bar=off
+      pip install --upgrade pip --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple
+      pip install grpcio==1.28.1 grpcio-tools==1.13.0 protobuf==3.11.3 six==1.14.0 mypy-protobuf==1.16.0  --progress-bar=off --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple
       cd protobuf-definitions
       chmod +x Grpc.Tools.$GRPC_VERSION/tools/macosx_x64/protoc
       chmod +x Grpc.Tools.$GRPC_VERSION/tools/macosx_x64/grpc_csharp_plugin

.yamato/python-ll-api-test.yml

@@ -11,9 +11,9 @@ test_mac_ll_api_{{ editor.version }}:
   variables:
     UNITY_VERSION: {{ editor.version }}
   commands:
-    - pip install pyyaml
+    - pip install pyyaml --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple
     - python -u -m ml-agents.tests.yamato.setup_venv
-    - ./venv/bin/python ml-agents/tests/yamato/scripts/run_llapi.py 
+    - ./venv/bin/python ml-agents/tests/yamato/scripts/run_llapi.py
     - ./venv/bin/python ml-agents/tests/yamato/scripts/run_llapi.py --env=artifacts/testPlayer-Basic
     - ./venv/bin/python ml-agents/tests/yamato/scripts/run_llapi.py --env=artifacts/testPlayer-WallJump
     - ./venv/bin/python ml-agents/tests/yamato/scripts/run_llapi.py --env=artifacts/testPlayer-Bouncer
@@ -24,11 +24,11 @@ test_mac_ll_api_{{ editor.version }}:
     expression: |
       (pull_request.target eq "master" OR
       pull_request.target match "release.+") AND
-      NOT pull_request.draft AND 
-      (pull_request.changes.any match "com.unity.ml-agents/**" OR 
-      pull_request.changes.any match "Project/**" OR 
-      pull_request.changes.any match "ml-agents/**" OR 
-      pull_request.changes.any match "ml-agents-envs/**" OR 
+      NOT pull_request.draft AND
+      (pull_request.changes.any match "com.unity.ml-agents/**" OR
+      pull_request.changes.any match "Project/**" OR
+      pull_request.changes.any match "ml-agents/**" OR
+      pull_request.changes.any match "ml-agents-envs/**" OR
       pull_request.changes.any match ".yamato/python-ll-api-test.yml") AND
       NOT pull_request.changes.all match "**/*.md"
 {% endfor %}

.yamato/standalone-build-test.yml

@@ -12,7 +12,7 @@ test_mac_standalone_{{ editor.version }}:
   variables:
     UNITY_VERSION: {{ editor.version }}
   commands:
-    - pip install pyyaml
+    - pip install pyyaml --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple
     - python -u -m ml-agents.tests.yamato.standalone_build_tests
     - python -u -m ml-agents.tests.yamato.standalone_build_tests --scene=Assets/ML-Agents/Examples/Basic/Scenes/Basic.unity
     - python -u -m ml-agents.tests.yamato.standalone_build_tests --scene=Assets/ML-Agents/Examples/Bouncer/Scenes/Bouncer.unity

.yamato/training-int-tests.yml

@@ -12,7 +12,7 @@ test_mac_training_int_{{ editor.version }}:
   variables:
     UNITY_VERSION: {{ editor.version }}
   commands:
-    - pip install pyyaml
+    - pip install pyyaml --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple
     - python -u -m ml-agents.tests.yamato.training_int_tests
     # Backwards-compatibility tests.
     # If we make a breaking change to the communication protocol, these will need

ml-agents-envs/mlagents_envs/communicator_objects/unity_to_external_pb2_grpc.py

@@ -4,7 +4,7 @@
 from mlagents_envs.communicator_objects import unity_message_pb2 as mlagents__envs_dot_communicator__objects_dot_unity__message__pb2
 
 
-class UnityToExternalProtoStub:
+class UnityToExternalProtoStub(object):
   # missing associated documentation comment in .proto file
   pass
 
@@ -21,7 +21,7 @@ def __init__(self, channel):
         )
 
 
-class UnityToExternalProtoServicer:
+class UnityToExternalProtoServicer(object):
   # missing associated documentation comment in .proto file
   pass
 

ml-agents/tests/yamato/yamato_utils.py

@@ -136,8 +136,9 @@ def init_venv(
     if extra_packages:
         pip_commands += extra_packages
     for cmd in pip_commands:
+        pip_index_url = "--index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple"
         subprocess.check_call(
-            f"source {venv_path}/bin/activate; python -m pip install -q {cmd}",
+            f"source {venv_path}/bin/activate; python -m pip install -q {cmd} {pip_index_url}",
             shell=True,
         )
     return venv_path