From e9b9af637ab19b82e0bed5074f64619af7039736 Mon Sep 17 00:00:00 2001 From: Ruo-Ping Dong Date: Mon, 30 Nov 2020 11:07:02 -0800 Subject: [PATCH 01/36] make actionSpec not read-only --- .../Runtime/Actuators/ActionSpec.cs | 30 ++++++++++++------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/com.unity.ml-agents/Runtime/Actuators/ActionSpec.cs b/com.unity.ml-agents/Runtime/Actuators/ActionSpec.cs index 309dfce795..eb19a3a280 100644 --- a/com.unity.ml-agents/Runtime/Actuators/ActionSpec.cs +++ b/com.unity.ml-agents/Runtime/Actuators/ActionSpec.cs @@ -1,15 +1,25 @@ using System; -using System.Collections.Generic; using System.Linq; using Unity.MLAgents.Policies; +using UnityEngine; namespace Unity.MLAgents.Actuators { /// /// Defines the structure of an Action Space to be used by the Actuator system. /// - public readonly struct ActionSpec + [Serializable] + public struct ActionSpec { + [SerializeField] + int[] m_BranchSizes; + [SerializeField] + int m_NumContinuousActions; + [SerializeField] + int m_NumDiscreteActions; + [SerializeField] + int m_SumOfDiscreteBranchSizes; + /// /// An array of branch sizes for our action space. /// @@ -20,23 +30,23 @@ public readonly struct ActionSpec /// /// For an IActuator with a Continuous it will be null. /// - public readonly int[] BranchSizes; + public int[] BranchSizes { get { return m_BranchSizes; } set { m_BranchSizes = value; } } /// /// The number of actions for a Continuous . /// - public int NumContinuousActions { get; } + public int NumContinuousActions { get { return m_NumContinuousActions; } set { m_NumContinuousActions = value; } } /// /// The number of branches for a Discrete . /// - public int NumDiscreteActions { get; } + public int NumDiscreteActions { get { return m_NumDiscreteActions; } set { m_NumDiscreteActions = value; } } /// /// Get the total number of Discrete Actions that can be taken by calculating the Sum /// of all of the Discrete Action branch sizes. /// - public int SumOfDiscreteBranchSizes { get; } + public int SumOfDiscreteBranchSizes { get { return m_SumOfDiscreteBranchSizes; } set { m_SumOfDiscreteBranchSizes = value; } } /// /// Creates a Continuous with the number of actions available. @@ -65,10 +75,10 @@ public static ActionSpec MakeDiscrete(params int[] branchSizes) internal ActionSpec(int numContinuousActions, int numDiscreteActions, int[] branchSizes = null) { - NumContinuousActions = numContinuousActions; - NumDiscreteActions = numDiscreteActions; - BranchSizes = branchSizes; - SumOfDiscreteBranchSizes = branchSizes?.Sum() ?? 0; + m_NumContinuousActions = numContinuousActions; + m_NumDiscreteActions = numDiscreteActions; + m_BranchSizes = branchSizes; + m_SumOfDiscreteBranchSizes = branchSizes?.Sum() ?? 0; } /// From 4840d09134d4366fc710b1c75ed55244fb2c5722 Mon Sep 17 00:00:00 2001 From: Ruo-Ping Dong Date: Mon, 30 Nov 2020 11:08:48 -0800 Subject: [PATCH 02/36] add actionSpec in BrainParameters and update BrainParameters Drawer --- .../Editor/BrainParametersDrawer.cs | 55 ++++++------------- .../Runtime/Policies/BrainParameters.cs | 6 +- 2 files changed, 22 insertions(+), 39 deletions(-) diff --git a/com.unity.ml-agents/Editor/BrainParametersDrawer.cs b/com.unity.ml-agents/Editor/BrainParametersDrawer.cs index a6e5db5a45..0fd1b5e98a 100644 --- a/com.unity.ml-agents/Editor/BrainParametersDrawer.cs +++ b/com.unity.ml-agents/Editor/BrainParametersDrawer.cs @@ -14,6 +14,10 @@ internal class BrainParametersDrawer : PropertyDrawer // The height of a line in the Unity Inspectors const float k_LineHeight = 17f; const int k_VecObsNumLine = 3; + const string k_ActionSpecName = "VectorActionSpec"; + const string k_ContinuousActionSizeName = "m_NumContinuousActions"; + const string k_DiscreteActionSizeName = "m_NumDiscreteActions"; + const string k_DiscreteBranchSizeName = "m_BranchSizes"; const string k_ActionSizePropName = "VectorActionSize"; const string k_ActionTypePropName = "VectorActionSpaceType"; const string k_ActionDescriptionPropName = "VectorActionDescriptions"; @@ -97,22 +101,10 @@ static void DrawVectorAction(Rect position, SerializedProperty property) EditorGUI.LabelField(position, "Vector Action"); position.y += k_LineHeight; EditorGUI.indentLevel++; - var bpVectorActionType = property.FindPropertyRelative(k_ActionTypePropName); - EditorGUI.PropertyField( - position, - bpVectorActionType, - new GUIContent("Space Type", - "Corresponds to whether state vector contains a single integer (Discrete) " + - "or a series of real-valued floats (Continuous).")); + var actionSpecProperty = property.FindPropertyRelative(k_ActionSpecName); + DrawContinuousVectorAction(position, actionSpecProperty); position.y += k_LineHeight; - if (bpVectorActionType.enumValueIndex == 1) - { - DrawContinuousVectorAction(position, property); - } - else - { - DrawDiscreteVectorAction(position, property); - } + DrawDiscreteVectorAction(position, actionSpecProperty); } /// @@ -123,21 +115,11 @@ static void DrawVectorAction(Rect position, SerializedProperty property) /// to make the custom GUI for. static void DrawContinuousVectorAction(Rect position, SerializedProperty property) { - var vecActionSize = property.FindPropertyRelative(k_ActionSizePropName); - - // This check is here due to: - // https://fogbugz.unity3d.com/f/cases/1246524/ - // If this case has been resolved, please remove this if condition. - if (vecActionSize.arraySize != 1) - { - vecActionSize.arraySize = 1; - } - var continuousActionSize = - vecActionSize.GetArrayElementAtIndex(0); + var continuousActionSize = property.FindPropertyRelative(k_ContinuousActionSizeName); EditorGUI.PropertyField( position, continuousActionSize, - new GUIContent("Space Size", "Length of continuous action vector.")); + new GUIContent("Continuous Action Size", "Length of continuous action vector.")); } /// @@ -148,27 +130,27 @@ static void DrawContinuousVectorAction(Rect position, SerializedProperty propert /// to make the custom GUI for. static void DrawDiscreteVectorAction(Rect position, SerializedProperty property) { - var vecActionSize = property.FindPropertyRelative(k_ActionSizePropName); + var branchSizes = property.FindPropertyRelative(k_DiscreteBranchSizeName); var newSize = EditorGUI.IntField( - position, "Branches Size", vecActionSize.arraySize); + position, "Discrete Branch Sizes", branchSizes.arraySize); // This check is here due to: // https://fogbugz.unity3d.com/f/cases/1246524/ // If this case has been resolved, please remove this if condition. - if (newSize != vecActionSize.arraySize) + if (newSize != branchSizes.arraySize) { - vecActionSize.arraySize = newSize; + branchSizes.arraySize = newSize; } position.y += k_LineHeight; position.x += 20; position.width -= 20; for (var branchIndex = 0; - branchIndex < vecActionSize.arraySize; + branchIndex < branchSizes.arraySize; branchIndex++) { var branchActionSize = - vecActionSize.GetArrayElementAtIndex(branchIndex); + branchSizes.GetArrayElementAtIndex(branchIndex); EditorGUI.PropertyField( position, @@ -185,11 +167,8 @@ static void DrawDiscreteVectorAction(Rect position, SerializedProperty property) /// The height of the drawer of the Vector Action. static float GetHeightDrawVectorAction(SerializedProperty property) { - var actionSize = 2 + property.FindPropertyRelative(k_ActionSizePropName).arraySize; - if (property.FindPropertyRelative(k_ActionTypePropName).enumValueIndex == 0) - { - actionSize += 1; - } + var actionSpecProperty = property.FindPropertyRelative(k_ActionSpecName); + var actionSize = 3 + actionSpecProperty.FindPropertyRelative("m_BranchSizes").arraySize; return actionSize * k_LineHeight; } } diff --git a/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs b/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs index dade3b4773..5d673fc762 100644 --- a/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs +++ b/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs @@ -1,6 +1,7 @@ using System; using UnityEngine; using UnityEngine.Serialization; +using Unity.MLAgents.Actuators; namespace Unity.MLAgents.Policies { @@ -50,6 +51,8 @@ public class BrainParameters [FormerlySerializedAs("numStackedVectorObservations")] [Range(1, 50)] public int NumStackedVectorObservations = 1; + public ActionSpec VectorActionSpec = new ActionSpec(0, 0, new int[] { }); + /// /// The size of the action space. /// @@ -106,7 +109,8 @@ public BrainParameters Clone() NumStackedVectorObservations = NumStackedVectorObservations, VectorActionSize = (int[])VectorActionSize.Clone(), VectorActionDescriptions = (string[])VectorActionDescriptions.Clone(), - VectorActionSpaceType = VectorActionSpaceType + VectorActionSpaceType = VectorActionSpaceType, + VectorActionSpec = new ActionSpec(VectorActionSpec.NumContinuousActions, VectorActionSpec.NumDiscreteActions, VectorActionSpec.BranchSizes) }; } } From d7080bea1fead865e12cd6cab06269a55b10aecd Mon Sep 17 00:00:00 2001 From: Ruo-Ping Dong Date: Mon, 30 Nov 2020 15:52:47 -0800 Subject: [PATCH 03/36] Integrate BrainParameters with ActionSpec, deprecate some fields --- .../Editor/BrainParametersDrawer.cs | 2 +- .../Editor/DemonstrationDrawer.cs | 1 + .../Runtime/Actuators/VectorActuator.cs | 26 ++------ com.unity.ml-agents/Runtime/Agent.cs | 4 +- .../Runtime/Communicator/GrpcExtensions.cs | 59 ++++++++++++++----- .../Inference/BarracudaModelParamLoader.cs | 16 ++--- .../Runtime/Policies/BrainParameters.cs | 23 +++++--- 7 files changed, 70 insertions(+), 61 deletions(-) diff --git a/com.unity.ml-agents/Editor/BrainParametersDrawer.cs b/com.unity.ml-agents/Editor/BrainParametersDrawer.cs index 0fd1b5e98a..49bab1bcce 100644 --- a/com.unity.ml-agents/Editor/BrainParametersDrawer.cs +++ b/com.unity.ml-agents/Editor/BrainParametersDrawer.cs @@ -132,7 +132,7 @@ static void DrawDiscreteVectorAction(Rect position, SerializedProperty property) { var branchSizes = property.FindPropertyRelative(k_DiscreteBranchSizeName); var newSize = EditorGUI.IntField( - position, "Discrete Branch Sizes", branchSizes.arraySize); + position, "Discrete Branch Size", branchSizes.arraySize); // This check is here due to: // https://fogbugz.unity3d.com/f/cases/1246524/ diff --git a/com.unity.ml-agents/Editor/DemonstrationDrawer.cs b/com.unity.ml-agents/Editor/DemonstrationDrawer.cs index 006a36f211..7869c5d25a 100644 --- a/com.unity.ml-agents/Editor/DemonstrationDrawer.cs +++ b/com.unity.ml-agents/Editor/DemonstrationDrawer.cs @@ -72,6 +72,7 @@ static string BuildIntArrayLabel(SerializedProperty actionSizeProperty) /// void MakeActionsProperty(SerializedProperty property) { + // TODO: update deprecated fields to ActionSpec var actSizeProperty = property.FindPropertyRelative("VectorActionSize"); var actSpaceTypeProp = property.FindPropertyRelative("VectorActionSpaceType"); diff --git a/com.unity.ml-agents/Runtime/Actuators/VectorActuator.cs b/com.unity.ml-agents/Runtime/Actuators/VectorActuator.cs index 918cff2ea7..aa4d60a05e 100644 --- a/com.unity.ml-agents/Runtime/Actuators/VectorActuator.cs +++ b/com.unity.ml-agents/Runtime/Actuators/VectorActuator.cs @@ -22,33 +22,15 @@ internal ActionBuffers ActionBuffers /// Create a VectorActuator that forwards to the provided IActionReceiver. /// /// The used for OnActionReceived and WriteDiscreteActionMask. - /// For discrete action spaces, the branch sizes for each action. - /// For continuous action spaces, the number of actions is the 0th element. - /// + /// /// - /// Thrown for invalid public VectorActuator(IActionReceiver actionReceiver, - int[] vectorActionSize, - SpaceType spaceType, + ActionSpec actionSpec, string name = "VectorActuator") { m_ActionReceiver = actionReceiver; - string suffix; - switch (spaceType) - { - case SpaceType.Continuous: - ActionSpec = ActionSpec.MakeContinuous(vectorActionSize[0]); - suffix = "-Continuous"; - break; - case SpaceType.Discrete: - ActionSpec = ActionSpec.MakeDiscrete(vectorActionSize); - suffix = "-Discrete"; - break; - default: - throw new ArgumentOutOfRangeException(nameof(spaceType), - spaceType, - "Unknown enum value."); - } + ActionSpec = actionSpec; + string suffix = $"-Continuous-{actionSpec.NumContinuousActions}-Discrete-{actionSpec.NumDiscreteActions}"; Name = name + suffix; } diff --git a/com.unity.ml-agents/Runtime/Agent.cs b/com.unity.ml-agents/Runtime/Agent.cs index c2aee50cfe..7622a76e86 100644 --- a/com.unity.ml-agents/Runtime/Agent.cs +++ b/com.unity.ml-agents/Runtime/Agent.cs @@ -888,7 +888,7 @@ public virtual void Initialize() { } public virtual void Heuristic(in ActionBuffers actionsOut) { // For backward compatibility - switch (m_PolicyFactory.BrainParameters.VectorActionSpaceType) + switch (m_PolicyFactory.BrainParameters.VectorActionSpaceTypeDeprecated) { case SpaceType.Continuous: Heuristic(actionsOut.ContinuousActions.Array); @@ -988,7 +988,7 @@ void InitializeActuators() // Support legacy OnActionReceived // TODO don't set this up if the sizes are 0? var param = m_PolicyFactory.BrainParameters; - m_VectorActuator = new VectorActuator(this, param.VectorActionSize, param.VectorActionSpaceType); + m_VectorActuator = new VectorActuator(this, param.VectorActionSpec); m_ActuatorManager = new ActuatorManager(attachedActuators.Length + 1); m_LegacyActionCache = new float[m_VectorActuator.TotalNumberOfActions()]; diff --git a/com.unity.ml-agents/Runtime/Communicator/GrpcExtensions.cs b/com.unity.ml-agents/Runtime/Communicator/GrpcExtensions.cs index 46edaa5aab..4e2958d46a 100644 --- a/com.unity.ml-agents/Runtime/Communicator/GrpcExtensions.cs +++ b/com.unity.ml-agents/Runtime/Communicator/GrpcExtensions.cs @@ -103,10 +103,11 @@ public static BrainParametersProto ToProto(this BrainParameters bp, string name, { var brainParametersProto = new BrainParametersProto { - VectorActionSizeDeprecated = { bp.VectorActionSize }, - VectorActionSpaceTypeDeprecated = (SpaceTypeProto)bp.VectorActionSpaceType, + VectorActionSizeDeprecated = { bp.VectorActionSizeDeprecated }, + VectorActionSpaceTypeDeprecated = (SpaceTypeProto)bp.VectorActionSpaceTypeDeprecated, BrainName = name, - IsTraining = isTraining + IsTraining = isTraining, + ActionSpec = ToActionSpecProto(bp.VectorActionSpec), }; if (bp.VectorActionDescriptions != null) { @@ -127,18 +128,9 @@ public static BrainParametersProto ToBrainParametersProto(this ActionSpec action var brainParametersProto = new BrainParametersProto { BrainName = name, - IsTraining = isTraining + IsTraining = isTraining, + ActionSpec = ToActionSpecProto(actionSpec), }; - var actionSpecProto = new ActionSpecProto - { - NumContinuousActions = actionSpec.NumContinuousActions, - NumDiscreteActions = actionSpec.NumDiscreteActions, - }; - if (actionSpec.BranchSizes != null) - { - actionSpecProto.DiscreteBranchSizes.AddRange(actionSpec.BranchSizes); - } - brainParametersProto.ActionSpec = actionSpecProto; var supportHybrid = Academy.Instance.TrainerCapabilities == null || Academy.Instance.TrainerCapabilities.HybridActions; if (!supportHybrid) @@ -169,13 +161,48 @@ public static BrainParameters ToBrainParameters(this BrainParametersProto bpp) { var bp = new BrainParameters { - VectorActionSize = bpp.VectorActionSizeDeprecated.ToArray(), + VectorActionSizeDeprecated = bpp.VectorActionSizeDeprecated.ToArray(), VectorActionDescriptions = bpp.VectorActionDescriptionsDeprecated.ToArray(), - VectorActionSpaceType = (SpaceType)bpp.VectorActionSpaceTypeDeprecated + VectorActionSpaceTypeDeprecated = (SpaceType)bpp.VectorActionSpaceTypeDeprecated, + VectorActionSpec = ToActionSpec(bpp.ActionSpec), }; return bp; } + /// + /// Convert a ActionSpecProto to a ActionSpec struct. + /// + /// An instance of an action spec protobuf object. + /// An ActionSpec struct. + public static ActionSpec ToActionSpec(this ActionSpecProto actionSpecProto) + { + var actionSpec = new ActionSpec + ( + actionSpecProto.NumContinuousActions, + actionSpecProto.NumDiscreteActions, + actionSpecProto.DiscreteBranchSizes.ToArray() + ); + return actionSpec; + } + + /// + /// Convert a ActionSpec struct to a ActionSpecProto. + /// + /// An instance of an action spec struct. + /// An ActionSpecProto. + public static ActionSpecProto ToActionSpecProto(this ActionSpec actionSpec) + { + var actionSpecProto = new ActionSpecProto + { + NumContinuousActions = actionSpec.NumContinuousActions, + NumDiscreteActions = actionSpec.NumDiscreteActions, + }; + if (actionSpec.BranchSizes != null) + { + actionSpecProto.DiscreteBranchSizes.AddRange(actionSpec.BranchSizes); + } + return actionSpecProto; + } #endregion #region DemonstrationMetaData diff --git a/com.unity.ml-agents/Runtime/Inference/BarracudaModelParamLoader.cs b/com.unity.ml-agents/Runtime/Inference/BarracudaModelParamLoader.cs index 46083ec925..d71edc2fef 100644 --- a/com.unity.ml-agents/Runtime/Inference/BarracudaModelParamLoader.cs +++ b/com.unity.ml-agents/Runtime/Inference/BarracudaModelParamLoader.cs @@ -397,8 +397,8 @@ static string CheckPreviousActionShape( BrainParameters brainParameters, TensorProxy tensorProxy, SensorComponent[] sensorComponents, int observableAttributeTotalSize) { - // TODO: Update this check after intergrating ActionSpec into BrainParameters - var numberActionsBp = brainParameters.VectorActionSize.Length; + var numberActionsBp = brainParameters.VectorActionSpec.NumContinuousActions + + brainParameters.VectorActionSpec.NumDiscreteActions; var numberActionsT = tensorProxy.shape[tensorProxy.shape.Length - 1]; if (numberActionsBp != numberActionsT) { @@ -489,11 +489,7 @@ static IEnumerable CheckOutputTensorShape( static string CheckDiscreteActionOutputShape( BrainParameters brainParameters, ActuatorComponent[] actuatorComponents, TensorShape? shape, int modelContinuousActionSize, int modelSumDiscreteBranchSizes) { - var sumOfDiscreteBranchSizes = 0; - if (brainParameters.VectorActionSpaceType == SpaceType.Discrete) - { - sumOfDiscreteBranchSizes += brainParameters.VectorActionSize.Sum(); - } + var sumOfDiscreteBranchSizes = brainParameters.VectorActionSpec.SumOfDiscreteBranchSizes; foreach (var actuatorComponent in actuatorComponents) { @@ -529,11 +525,7 @@ static string CheckDiscreteActionOutputShape( static string CheckContinuousActionOutputShape( BrainParameters brainParameters, ActuatorComponent[] actuatorComponents, TensorShape? shape, int modelContinuousActionSize, int modelSumDiscreteBranchSizes) { - var numContinuousActions = 0; - if (brainParameters.VectorActionSpaceType == SpaceType.Continuous) - { - numContinuousActions += brainParameters.NumActions; - } + var numContinuousActions = brainParameters.VectorActionSpec.NumContinuousActions; foreach (var actuatorComponent in actuatorComponents) { diff --git a/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs b/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs index 5d673fc762..b3df636bf9 100644 --- a/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs +++ b/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs @@ -1,4 +1,5 @@ using System; +using System.Linq; using UnityEngine; using UnityEngine.Serialization; using Unity.MLAgents.Actuators; @@ -64,7 +65,8 @@ public class BrainParameters /// For the discrete action space: the number of branches in the action space. /// [FormerlySerializedAs("vectorActionSize")] - public int[] VectorActionSize = new[] { 1 }; + [FormerlySerializedAs("VectorActionSize")] + public int[] VectorActionSizeDeprecated = new[] { 1 }; /// /// The list of strings describing what the actions correspond to. @@ -76,21 +78,26 @@ public class BrainParameters /// Defines if the action is discrete or continuous. /// [FormerlySerializedAs("vectorActionSpaceType")] - public SpaceType VectorActionSpaceType = SpaceType.Discrete; + [FormerlySerializedAs("VectorActionSpaceType")] + public SpaceType VectorActionSpaceTypeDeprecated = SpaceType.Discrete; + + [SerializeField] + [HideInInspector] + internal bool hasUpgradedBrainParametersWithActionSpec; /// /// The number of actions specified by this Brain. /// - public int NumActions + public int NumActionsDeprecated { get { - switch (VectorActionSpaceType) + switch (VectorActionSpaceTypeDeprecated) { case SpaceType.Discrete: - return VectorActionSize.Length; + return VectorActionSizeDeprecated.Length; case SpaceType.Continuous: - return VectorActionSize[0]; + return VectorActionSizeDeprecated[0]; default: return 0; } @@ -107,9 +114,9 @@ public BrainParameters Clone() { VectorObservationSize = VectorObservationSize, NumStackedVectorObservations = NumStackedVectorObservations, - VectorActionSize = (int[])VectorActionSize.Clone(), + VectorActionSizeDeprecated = (int[])VectorActionSizeDeprecated.Clone(), VectorActionDescriptions = (string[])VectorActionDescriptions.Clone(), - VectorActionSpaceType = VectorActionSpaceType, + VectorActionSpaceTypeDeprecated = VectorActionSpaceTypeDeprecated, VectorActionSpec = new ActionSpec(VectorActionSpec.NumContinuousActions, VectorActionSpec.NumDiscreteActions, VectorActionSpec.BranchSizes) }; } From a1c81e909eed18b8ca1416769e90e264b1439189 Mon Sep 17 00:00:00 2001 From: Ruo-Ping Dong Date: Mon, 30 Nov 2020 15:53:24 -0800 Subject: [PATCH 04/36] add serialization callbacks --- .../Runtime/Policies/BrainParameters.cs | 38 ++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs b/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs index b3df636bf9..bcbbed3801 100644 --- a/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs +++ b/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs @@ -33,7 +33,7 @@ public enum SpaceType /// [GameObject]: https://docs.unity3d.com/Manual/GameObjects.html /// [Serializable] - public class BrainParameters + public class BrainParameters : ISerializationCallbackReceiver { /// /// The number of the observations that are added in @@ -120,5 +120,41 @@ public BrainParameters Clone() VectorActionSpec = new ActionSpec(VectorActionSpec.NumContinuousActions, VectorActionSpec.NumDiscreteActions, VectorActionSpec.BranchSizes) }; } + + public void OnBeforeSerialize() + { + if (!hasUpgradedBrainParametersWithActionSpec) + { + if (VectorActionSpec.NumContinuousActions == 0 && VectorActionSpaceTypeDeprecated == SpaceType.Continuous) + { + VectorActionSpec.NumContinuousActions = VectorActionSizeDeprecated[0]; + } + if (VectorActionSpec.NumDiscreteActions == 0 && VectorActionSpaceTypeDeprecated == SpaceType.Discrete) + { + VectorActionSpec.NumDiscreteActions = VectorActionSizeDeprecated.Length; + VectorActionSpec.BranchSizes = VectorActionSizeDeprecated; + VectorActionSpec.SumOfDiscreteBranchSizes = VectorActionSizeDeprecated.Sum(); + } + hasUpgradedBrainParametersWithActionSpec = true; + } + } + + public void OnAfterDeserialize() + { + if (!hasUpgradedBrainParametersWithActionSpec) + { + if (VectorActionSpec.NumContinuousActions == 0 && VectorActionSpaceTypeDeprecated == SpaceType.Continuous) + { + VectorActionSpec.NumContinuousActions = VectorActionSizeDeprecated[0]; + } + if (VectorActionSpec.NumDiscreteActions == 0 && VectorActionSpaceTypeDeprecated == SpaceType.Discrete) + { + VectorActionSpec.NumDiscreteActions = VectorActionSizeDeprecated.Length; + VectorActionSpec.BranchSizes = VectorActionSizeDeprecated; + VectorActionSpec.SumOfDiscreteBranchSizes = VectorActionSizeDeprecated.Sum(); + } + hasUpgradedBrainParametersWithActionSpec = true; + } + } } } From 178e0950292b177152d8c36d1f556ada07333b71 Mon Sep 17 00:00:00 2001 From: Ruo-Ping Dong Date: Tue, 1 Dec 2020 13:06:30 -0800 Subject: [PATCH 05/36] add function to set ActionSpec --- com.unity.ml-agents/Runtime/Actuators/ActionSpec.cs | 12 ++++++++++++ .../Runtime/Policies/BrainParameters.cs | 12 ++++-------- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/com.unity.ml-agents/Runtime/Actuators/ActionSpec.cs b/com.unity.ml-agents/Runtime/Actuators/ActionSpec.cs index eb19a3a280..97091bf2c3 100644 --- a/com.unity.ml-agents/Runtime/Actuators/ActionSpec.cs +++ b/com.unity.ml-agents/Runtime/Actuators/ActionSpec.cs @@ -73,6 +73,18 @@ public static ActionSpec MakeDiscrete(params int[] branchSizes) return actuatorSpace; } + public void SetContinuous(int numContinuousActions) + { + m_NumContinuousActions = numContinuousActions; + } + + public void SetDiscrete(int numDiscreteActions, int[] branchSizes) + { + m_NumDiscreteActions = numDiscreteActions; + m_BranchSizes = branchSizes; + m_SumOfDiscreteBranchSizes = branchSizes.Sum(); + } + internal ActionSpec(int numContinuousActions, int numDiscreteActions, int[] branchSizes = null) { m_NumContinuousActions = numContinuousActions; diff --git a/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs b/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs index bcbbed3801..1bb6d50fc0 100644 --- a/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs +++ b/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs @@ -127,13 +127,11 @@ public void OnBeforeSerialize() { if (VectorActionSpec.NumContinuousActions == 0 && VectorActionSpaceTypeDeprecated == SpaceType.Continuous) { - VectorActionSpec.NumContinuousActions = VectorActionSizeDeprecated[0]; + VectorActionSpec.SetContinuous(VectorActionSizeDeprecated[0]); } if (VectorActionSpec.NumDiscreteActions == 0 && VectorActionSpaceTypeDeprecated == SpaceType.Discrete) { - VectorActionSpec.NumDiscreteActions = VectorActionSizeDeprecated.Length; - VectorActionSpec.BranchSizes = VectorActionSizeDeprecated; - VectorActionSpec.SumOfDiscreteBranchSizes = VectorActionSizeDeprecated.Sum(); + VectorActionSpec.SetDiscrete(VectorActionSizeDeprecated.Length, VectorActionSizeDeprecated); } hasUpgradedBrainParametersWithActionSpec = true; } @@ -145,13 +143,11 @@ public void OnAfterDeserialize() { if (VectorActionSpec.NumContinuousActions == 0 && VectorActionSpaceTypeDeprecated == SpaceType.Continuous) { - VectorActionSpec.NumContinuousActions = VectorActionSizeDeprecated[0]; + VectorActionSpec.SetContinuous(VectorActionSizeDeprecated[0]); } if (VectorActionSpec.NumDiscreteActions == 0 && VectorActionSpaceTypeDeprecated == SpaceType.Discrete) { - VectorActionSpec.NumDiscreteActions = VectorActionSizeDeprecated.Length; - VectorActionSpec.BranchSizes = VectorActionSizeDeprecated; - VectorActionSpec.SumOfDiscreteBranchSizes = VectorActionSizeDeprecated.Sum(); + VectorActionSpec.SetDiscrete(VectorActionSizeDeprecated.Length, VectorActionSizeDeprecated); } hasUpgradedBrainParametersWithActionSpec = true; } From accf0d50e1b0e698b5cce1bbfe9c8ac519aa14e5 Mon Sep 17 00:00:00 2001 From: Ruo-Ping Dong Date: Tue, 1 Dec 2020 13:38:45 -0800 Subject: [PATCH 06/36] update BrainParameters when changed in editor --- .../Editor/BehaviorParametersEditor.cs | 14 ++++++++++++++ .../Runtime/Policies/BrainParameters.cs | 6 ++++++ 2 files changed, 20 insertions(+) diff --git a/com.unity.ml-agents/Editor/BehaviorParametersEditor.cs b/com.unity.ml-agents/Editor/BehaviorParametersEditor.cs index c4c9841dd5..a1d398555b 100644 --- a/com.unity.ml-agents/Editor/BehaviorParametersEditor.cs +++ b/com.unity.ml-agents/Editor/BehaviorParametersEditor.cs @@ -27,6 +27,7 @@ public override void OnInspectorGUI() var so = serializedObject; so.Update(); bool needPolicyUpdate; // Whether the name, model, inference device, or BehaviorType changed. + bool needBrainParametersUpdate; // Drawing the Behavior Parameters EditorGUI.indentLevel++; @@ -38,11 +39,13 @@ public override void OnInspectorGUI() } needPolicyUpdate = EditorGUI.EndChangeCheck(); + EditorGUI.BeginChangeCheck(); EditorGUI.BeginDisabledGroup(!EditorUtilities.CanUpdateModelProperties()); { EditorGUILayout.PropertyField(so.FindProperty("m_BrainParameters"), true); } EditorGUI.EndDisabledGroup(); + needBrainParametersUpdate = EditorGUI.EndChangeCheck(); EditorGUI.BeginChangeCheck(); { @@ -76,6 +79,10 @@ public override void OnInspectorGUI() { UpdateAgentPolicy(); } + if (needBrainParametersUpdate) + { + UpdateBrainParameters(); + } } /// @@ -156,5 +163,12 @@ void UpdateAgentPolicy() var behaviorParameters = (BehaviorParameters)target; behaviorParameters.UpdateAgentPolicy(); } + + void UpdateBrainParameters() + { + var behaviorParameters = (BehaviorParameters)target; + var brainParameters = behaviorParameters.BrainParameters; + brainParameters.UpdateBrainParameters(); + } } } diff --git a/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs b/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs index 1bb6d50fc0..634c5bf27b 100644 --- a/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs +++ b/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs @@ -152,5 +152,11 @@ public void OnAfterDeserialize() hasUpgradedBrainParametersWithActionSpec = true; } } + + internal void UpdateBrainParameters() + { + VectorActionSpec.NumDiscreteActions = VectorActionSpec.BranchSizes.Length; + VectorActionSpec.SumOfDiscreteBranchSizes = VectorActionSpec.BranchSizes.Sum(); + } } } From 05a3c6b0833588ab8e4d6ca4f5a680903d060911 Mon Sep 17 00:00:00 2001 From: Ruo-Ping Dong Date: Tue, 1 Dec 2020 13:57:16 -0800 Subject: [PATCH 07/36] fix tests --- .../Editor/Actuators/VectorActuatorTests.cs | 10 +++---- .../Tests/Editor/DemonstrationTests.cs | 6 ++--- .../Tests/Editor/MLAgentsEditModeTest.cs | 27 +++++++------------ .../Tests/Editor/ParameterLoaderTest.cs | 14 +++++----- .../Tests/Runtime/RuntimeAPITest.cs | 3 +-- 5 files changed, 23 insertions(+), 37 deletions(-) diff --git a/com.unity.ml-agents/Tests/Editor/Actuators/VectorActuatorTests.cs b/com.unity.ml-agents/Tests/Editor/Actuators/VectorActuatorTests.cs index 7d998f3a8f..1c42f2d3c4 100644 --- a/com.unity.ml-agents/Tests/Editor/Actuators/VectorActuatorTests.cs +++ b/com.unity.ml-agents/Tests/Editor/Actuators/VectorActuatorTests.cs @@ -32,13 +32,13 @@ public void WriteDiscreteActionMask(IDiscreteActionMask actionMask) public void TestConstruct() { var ar = new TestActionReceiver(); - var va = new VectorActuator(ar, new[] { 1, 2, 3 }, SpaceType.Discrete, "name"); + var va = new VectorActuator(ar, ActionSpec.MakeDiscrete(new int[] { 1, 2, 3 }), "name"); Assert.IsTrue(va.ActionSpec.NumDiscreteActions == 3); Assert.IsTrue(va.ActionSpec.SumOfDiscreteBranchSizes == 6); Assert.IsTrue(va.ActionSpec.NumContinuousActions == 0); - var va1 = new VectorActuator(ar, new[] { 4 }, SpaceType.Continuous, "name"); + var va1 = new VectorActuator(ar, ActionSpec.MakeContinuous(4), "name"); Assert.IsTrue(va1.ActionSpec.NumContinuousActions == 4); Assert.IsTrue(va1.ActionSpec.SumOfDiscreteBranchSizes == 0); @@ -49,7 +49,7 @@ public void TestConstruct() public void TestOnActionReceived() { var ar = new TestActionReceiver(); - var va = new VectorActuator(ar, new[] { 1, 2, 3 }, SpaceType.Discrete, "name"); + var va = new VectorActuator(ar, ActionSpec.MakeDiscrete(new int[] { 1, 2, 3 }), "name"); var discreteActions = new[] { 0, 1, 1 }; var ab = new ActionBuffers(ActionSegment.Empty, @@ -67,7 +67,7 @@ public void TestOnActionReceived() public void TestResetData() { var ar = new TestActionReceiver(); - var va = new VectorActuator(ar, new[] { 1, 2, 3 }, SpaceType.Discrete, "name"); + var va = new VectorActuator(ar, ActionSpec.MakeDiscrete(new int[] { 1, 2, 3 }), "name"); var discreteActions = new[] { 0, 1, 1 }; var ab = new ActionBuffers(ActionSegment.Empty, @@ -80,7 +80,7 @@ public void TestResetData() public void TestWriteDiscreteActionMask() { var ar = new TestActionReceiver(); - var va = new VectorActuator(ar, new[] { 1, 2, 3 }, SpaceType.Discrete, "name"); + var va = new VectorActuator(ar, ActionSpec.MakeDiscrete(new int[] { 1, 2, 3 }), "name"); var bdam = new ActuatorDiscreteActionMask(new[] { va }, 6, 3); var groundTruthMask = new[] { false, true, false, false, true, true }; diff --git a/com.unity.ml-agents/Tests/Editor/DemonstrationTests.cs b/com.unity.ml-agents/Tests/Editor/DemonstrationTests.cs index fd31b649f0..72dabb97a0 100644 --- a/com.unity.ml-agents/Tests/Editor/DemonstrationTests.cs +++ b/com.unity.ml-agents/Tests/Editor/DemonstrationTests.cs @@ -47,8 +47,7 @@ public void TestStoreInitialize() bp.BrainParameters.VectorObservationSize = 3; bp.BrainParameters.NumStackedVectorObservations = 2; bp.BrainParameters.VectorActionDescriptions = new[] { "TestActionA", "TestActionB" }; - bp.BrainParameters.VectorActionSize = new[] { 2, 2 }; - bp.BrainParameters.VectorActionSpaceType = SpaceType.Discrete; + bp.BrainParameters.VectorActionSpec = ActionSpec.MakeDiscrete(new[] { 2, 2 }); gameobj.AddComponent(); @@ -104,8 +103,7 @@ public void TestAgentWrite() bpA.BrainParameters.VectorObservationSize = 3; bpA.BrainParameters.NumStackedVectorObservations = 1; bpA.BrainParameters.VectorActionDescriptions = new[] { "TestActionA", "TestActionB" }; - bpA.BrainParameters.VectorActionSize = new[] { 2, 2 }; - bpA.BrainParameters.VectorActionSpaceType = SpaceType.Discrete; + bpA.BrainParameters.VectorActionSpec = ActionSpec.MakeDiscrete(new[] { 2, 2 }); agentGo1.AddComponent(); var agent1 = agentGo1.GetComponent(); diff --git a/com.unity.ml-agents/Tests/Editor/MLAgentsEditModeTest.cs b/com.unity.ml-agents/Tests/Editor/MLAgentsEditModeTest.cs index 2f3dede67e..d01f3b0d49 100644 --- a/com.unity.ml-agents/Tests/Editor/MLAgentsEditModeTest.cs +++ b/com.unity.ml-agents/Tests/Editor/MLAgentsEditModeTest.cs @@ -340,14 +340,12 @@ public void TestAgent() { var agentGo1 = new GameObject("TestAgent"); var bp1 = agentGo1.AddComponent(); - bp1.BrainParameters.VectorActionSize = new[] { 1 }; - bp1.BrainParameters.VectorActionSpaceType = SpaceType.Continuous; + bp1.BrainParameters.VectorActionSpec = ActionSpec.MakeContinuous(1); agentGo1.AddComponent(); var agent1 = agentGo1.GetComponent(); var agentGo2 = new GameObject("TestAgent"); var bp2 = agentGo2.AddComponent(); - bp2.BrainParameters.VectorActionSize = new[] { 1 }; - bp2.BrainParameters.VectorActionSpaceType = SpaceType.Continuous; + bp2.BrainParameters.VectorActionSpec = ActionSpec.MakeContinuous(1); agentGo2.AddComponent(); var agent2 = agentGo2.GetComponent(); @@ -452,14 +450,12 @@ public void TestAgent() { var agentGo1 = new GameObject("TestAgent"); var bp1 = agentGo1.AddComponent(); - bp1.BrainParameters.VectorActionSize = new[] { 1 }; - bp1.BrainParameters.VectorActionSpaceType = SpaceType.Continuous; + bp1.BrainParameters.VectorActionSpec = ActionSpec.MakeContinuous(1); agentGo1.AddComponent(); var agent1 = agentGo1.GetComponent(); var agentGo2 = new GameObject("TestAgent"); var bp2 = agentGo2.AddComponent(); - bp2.BrainParameters.VectorActionSize = new[] { 1 }; - bp2.BrainParameters.VectorActionSpaceType = SpaceType.Continuous; + bp2.BrainParameters.VectorActionSpec = ActionSpec.MakeContinuous(1); agentGo2.AddComponent(); var agent2 = agentGo2.GetComponent(); @@ -538,8 +534,7 @@ public void AssertStackingReset() { var agentGo1 = new GameObject("TestAgent"); var bp1 = agentGo1.AddComponent(); - bp1.BrainParameters.VectorActionSize = new[] { 1 }; - bp1.BrainParameters.VectorActionSpaceType = SpaceType.Continuous; + bp1.BrainParameters.VectorActionSpec = ActionSpec.MakeContinuous(1); var agent1 = agentGo1.AddComponent(); var behaviorParameters = agentGo1.GetComponent(); behaviorParameters.BrainParameters.NumStackedVectorObservations = 3; @@ -588,13 +583,11 @@ public void TestCumulativeReward() { var agentGo1 = new GameObject("TestAgent"); var bp1 = agentGo1.AddComponent(); - bp1.BrainParameters.VectorActionSize = new[] { 1 }; - bp1.BrainParameters.VectorActionSpaceType = SpaceType.Continuous; + bp1.BrainParameters.VectorActionSpec = ActionSpec.MakeContinuous(1); var agent1 = agentGo1.AddComponent(); var agentGo2 = new GameObject("TestAgent"); var bp2 = agentGo2.AddComponent(); - bp2.BrainParameters.VectorActionSize = new[] { 1 }; - bp2.BrainParameters.VectorActionSpaceType = SpaceType.Continuous; + bp2.BrainParameters.VectorActionSpec = ActionSpec.MakeContinuous(1); var agent2 = agentGo2.AddComponent(); var aca = Academy.Instance; @@ -632,8 +625,7 @@ public void TestMaxStepsReset() { var agentGo1 = new GameObject("TestAgent"); var bp1 = agentGo1.AddComponent(); - bp1.BrainParameters.VectorActionSize = new[] { 1 }; - bp1.BrainParameters.VectorActionSpaceType = SpaceType.Continuous; + bp1.BrainParameters.VectorActionSpec = ActionSpec.MakeContinuous(1); agentGo1.AddComponent(); var agent1 = agentGo1.GetComponent(); var aca = Academy.Instance; @@ -698,8 +690,7 @@ public void TestHeuristicPolicyStepsSensors() // Make sure that Agents with HeuristicPolicies step their sensors each Academy step. var agentGo1 = new GameObject("TestAgent"); var bp1 = agentGo1.AddComponent(); - bp1.BrainParameters.VectorActionSize = new[] { 1 }; - bp1.BrainParameters.VectorActionSpaceType = SpaceType.Continuous; + bp1.BrainParameters.VectorActionSpec = ActionSpec.MakeContinuous(1); agentGo1.AddComponent(); var agent1 = agentGo1.GetComponent(); var aca = Academy.Instance; diff --git a/com.unity.ml-agents/Tests/Editor/ParameterLoaderTest.cs b/com.unity.ml-agents/Tests/Editor/ParameterLoaderTest.cs index f827dd570f..c7e4ff8952 100644 --- a/com.unity.ml-agents/Tests/Editor/ParameterLoaderTest.cs +++ b/com.unity.ml-agents/Tests/Editor/ParameterLoaderTest.cs @@ -95,9 +95,8 @@ BrainParameters GetContinuous2vis8vec2actionBrainParameters() { var validBrainParameters = new BrainParameters(); validBrainParameters.VectorObservationSize = 8; - validBrainParameters.VectorActionSize = new[] { 2 }; validBrainParameters.NumStackedVectorObservations = 1; - validBrainParameters.VectorActionSpaceType = SpaceType.Continuous; + validBrainParameters.VectorActionSpec = ActionSpec.MakeContinuous(2); return validBrainParameters; } @@ -105,9 +104,8 @@ BrainParameters GetDiscrete1vis0vec_2_3action_recurrModelBrainParameters() { var validBrainParameters = new BrainParameters(); validBrainParameters.VectorObservationSize = 0; - validBrainParameters.VectorActionSize = new[] { 2, 3 }; validBrainParameters.NumStackedVectorObservations = 1; - validBrainParameters.VectorActionSpaceType = SpaceType.Discrete; + validBrainParameters.VectorActionSpec = ActionSpec.MakeDiscrete(new[] { 2, 3 }); return validBrainParameters; } @@ -333,12 +331,12 @@ public void TestCheckModelThrowsActionContinuous(bool useDeprecatedNNModel) var model = useDeprecatedNNModel ? ModelLoader.Load(continuousNNModel) : ModelLoader.Load(continuousONNXModel); var brainParameters = GetContinuous2vis8vec2actionBrainParameters(); - brainParameters.VectorActionSize = new[] { 3 }; // Invalid action + brainParameters.VectorActionSpec.SetContinuous(3); // Invalid action var errors = BarracudaModelParamLoader.CheckModel(model, brainParameters, new SensorComponent[] { sensor_21_20_3, sensor_20_22_3 }, new ActuatorComponent[0]); Assert.Greater(errors.Count(), 0); brainParameters = GetContinuous2vis8vec2actionBrainParameters(); - brainParameters.VectorActionSpaceType = SpaceType.Discrete;// Invalid SpaceType + brainParameters.VectorActionSpec = ActionSpec.MakeDiscrete(new int[] { 3 }); // Invalid SpaceType errors = BarracudaModelParamLoader.CheckModel(model, brainParameters, new SensorComponent[] { sensor_21_20_3, sensor_20_22_3 }, new ActuatorComponent[0]); Assert.Greater(errors.Count(), 0); } @@ -350,12 +348,12 @@ public void TestCheckModelThrowsActionDiscrete(bool useDeprecatedNNModel) var model = useDeprecatedNNModel ? ModelLoader.Load(discreteNNModel) : ModelLoader.Load(discreteONNXModel); var brainParameters = GetDiscrete1vis0vec_2_3action_recurrModelBrainParameters(); - brainParameters.VectorActionSize = new[] { 3, 3 }; // Invalid action + brainParameters.VectorActionSpec = ActionSpec.MakeDiscrete(new[] { 3, 3 }); // Invalid action var errors = BarracudaModelParamLoader.CheckModel(model, brainParameters, new SensorComponent[] { sensor_21_20_3 }, new ActuatorComponent[0]); Assert.Greater(errors.Count(), 0); brainParameters = GetContinuous2vis8vec2actionBrainParameters(); - brainParameters.VectorActionSpaceType = SpaceType.Continuous;// Invalid SpaceType + brainParameters.VectorActionSpec = ActionSpec.MakeContinuous(2); // Invalid SpaceType errors = BarracudaModelParamLoader.CheckModel(model, brainParameters, new SensorComponent[] { sensor_21_20_3 }, new ActuatorComponent[0]); Assert.Greater(errors.Count(), 0); } diff --git a/com.unity.ml-agents/Tests/Runtime/RuntimeAPITest.cs b/com.unity.ml-agents/Tests/Runtime/RuntimeAPITest.cs index 62041f71f5..5d3e1f971d 100644 --- a/com.unity.ml-agents/Tests/Runtime/RuntimeAPITest.cs +++ b/com.unity.ml-agents/Tests/Runtime/RuntimeAPITest.cs @@ -72,8 +72,7 @@ public IEnumerator RuntimeApiTestWithEnumeratorPasses() behaviorParams.BrainParameters.VectorObservationSize = 3; behaviorParams.BrainParameters.NumStackedVectorObservations = 2; behaviorParams.BrainParameters.VectorActionDescriptions = new[] { "TestActionA", "TestActionB" }; - behaviorParams.BrainParameters.VectorActionSize = new[] { 2, 2 }; - behaviorParams.BrainParameters.VectorActionSpaceType = SpaceType.Discrete; + behaviorParams.BrainParameters.VectorActionSpec = ActionSpec.MakeDiscrete(new[] { 2, 2 }); behaviorParams.BehaviorName = "TestBehavior"; behaviorParams.TeamId = 42; behaviorParams.UseChildSensors = true; From 3e3ebaaa1c35ebe38d4d940a3ed7a90925641182 Mon Sep 17 00:00:00 2001 From: Ruo-Ping Dong Date: Tue, 1 Dec 2020 14:05:56 -0800 Subject: [PATCH 08/36] enable hybrid ParameterLoaderTest --- .../Tests/Editor/ParameterLoaderTest.cs | 115 +++++++++--------- 1 file changed, 55 insertions(+), 60 deletions(-) diff --git a/com.unity.ml-agents/Tests/Editor/ParameterLoaderTest.cs b/com.unity.ml-agents/Tests/Editor/ParameterLoaderTest.cs index c7e4ff8952..dfb894af29 100644 --- a/com.unity.ml-agents/Tests/Editor/ParameterLoaderTest.cs +++ b/com.unity.ml-agents/Tests/Editor/ParameterLoaderTest.cs @@ -109,16 +109,14 @@ BrainParameters GetDiscrete1vis0vec_2_3action_recurrModelBrainParameters() return validBrainParameters; } - // TODO: update and enable this after integrating action spec into BrainParameters - // BrainParameters GetHybridBrainParameters() - // { - // var validBrainParameters = new BrainParameters(); - // validBrainParameters.VectorObservationSize = 53; - // validBrainParameters.VectorActionSize = new[] { 2 }; - // validBrainParameters.NumStackedVectorObservations = 1; - // validBrainParameters.VectorActionSpaceType = SpaceType.Discrete; - // return validBrainParameters; - // } + BrainParameters GetHybridBrainParameters() + { + var validBrainParameters = new BrainParameters(); + validBrainParameters.VectorObservationSize = 53; + validBrainParameters.NumStackedVectorObservations = 1; + validBrainParameters.VectorActionSpec = new ActionSpec(3, 1, new int[] { 2 }); + return validBrainParameters; + } [SetUp] public void SetUp() @@ -252,19 +250,18 @@ public void TestCheckModelValidDiscrete(bool useDeprecatedNNModel) Assert.AreEqual(0, errors.Count()); // There should not be any errors } - // TODO: update and enable this test after integrating action spec into BrainParameters - // [Test] - // public void TestCheckModelValidHybrid() - // { - // var model = ModelLoader.Load(hybridModel); - // var validBrainParameters = GetHybridBrainParameters(); + [Test] + public void TestCheckModelValidHybrid() + { + var model = ModelLoader.Load(hybridONNXModel); + var validBrainParameters = GetHybridBrainParameters(); - // var errors = BarracudaModelParamLoader.CheckModel( - // model, validBrainParameters, - // new SensorComponent[] { }, new ActuatorComponent[0] - // ); - // Assert.AreEqual(0, errors.Count()); // There should not be any errors - // } + var errors = BarracudaModelParamLoader.CheckModel( + model, validBrainParameters, + new SensorComponent[] { }, new ActuatorComponent[0] + ); + Assert.AreEqual(0, errors.Count()); // There should not be any errors + } [TestCase(true)] [TestCase(false)] @@ -301,28 +298,27 @@ public void TestCheckModelThrowsVectorObservationDiscrete(bool useDeprecatedNNMo Assert.Greater(errors.Count(), 0); } - // TODO: update and enable this test after integrating action spec into BrainParameters - // [Test] - // public void TestCheckModelThrowsVectorObservationHybrid() - // { - // var model = ModelLoader.Load(hybridModel); - - // var brainParameters = GetHybridBrainParameters(); - // brainParameters.VectorObservationSize = 9; // Invalid observation - // var errors = BarracudaModelParamLoader.CheckModel( - // model, brainParameters, - // new SensorComponent[] { }, new ActuatorComponent[0] - // ); - // Assert.Greater(errors.Count(), 0); - - // brainParameters = GetContinuous2vis8vec2actionBrainParameters(); - // brainParameters.NumStackedVectorObservations = 2;// Invalid stacking - // errors = BarracudaModelParamLoader.CheckModel( - // model, brainParameters, - // new SensorComponent[] { }, new ActuatorComponent[0] - // ); - // Assert.Greater(errors.Count(), 0); - // } + [Test] + public void TestCheckModelThrowsVectorObservationHybrid() + { + var model = ModelLoader.Load(hybridONNXModel); + + var brainParameters = GetHybridBrainParameters(); + brainParameters.VectorObservationSize = 9; // Invalid observation + var errors = BarracudaModelParamLoader.CheckModel( + model, brainParameters, + new SensorComponent[] { }, new ActuatorComponent[0] + ); + Assert.Greater(errors.Count(), 0); + + brainParameters = GetContinuous2vis8vec2actionBrainParameters(); + brainParameters.NumStackedVectorObservations = 2;// Invalid stacking + errors = BarracudaModelParamLoader.CheckModel( + model, brainParameters, + new SensorComponent[] { }, new ActuatorComponent[0] + ); + Assert.Greater(errors.Count(), 0); + } [TestCase(true)] [TestCase(false)] @@ -358,22 +354,21 @@ public void TestCheckModelThrowsActionDiscrete(bool useDeprecatedNNModel) Assert.Greater(errors.Count(), 0); } - // TODO: update and enable this test after integrating action spec into BrainParameters - // [Test] - // public void TestCheckModelThrowsActionHybrid() - // { - // var model = ModelLoader.Load(hybridModel); - - // var brainParameters = GetHybridBrainParameters(); - // brainParameters.VectorActionSize = new[] { 3 }; // Invalid action - // var errors = BarracudaModelParamLoader.CheckModel(model, brainParameters, new SensorComponent[] { sensor_21_20_3, sensor_20_22_3 }, new ActuatorComponent[0]); - // Assert.Greater(errors.Count(), 0); - - // brainParameters = GetContinuous2vis8vec2actionBrainParameters(); - // brainParameters.VectorActionSpaceType = SpaceType.Discrete;// Invalid SpaceType - // errors = BarracudaModelParamLoader.CheckModel(model, brainParameters, new SensorComponent[] { sensor_21_20_3, sensor_20_22_3 }, new ActuatorComponent[0]); - // Assert.Greater(errors.Count(), 0); - // } + [Test] + public void TestCheckModelThrowsActionHybrid() + { + var model = ModelLoader.Load(hybridONNXModel); + + var brainParameters = GetHybridBrainParameters(); + brainParameters.VectorActionSpec = new ActionSpec(3, 1, new int[] { 3 }); ; // Invalid discrete action size + var errors = BarracudaModelParamLoader.CheckModel(model, brainParameters, new SensorComponent[] { sensor_21_20_3, sensor_20_22_3 }, new ActuatorComponent[0]); + Assert.Greater(errors.Count(), 0); + + brainParameters = GetContinuous2vis8vec2actionBrainParameters(); + brainParameters.VectorActionSpec = ActionSpec.MakeDiscrete(new int[] { 2 }); // Missing continuous action + errors = BarracudaModelParamLoader.CheckModel(model, brainParameters, new SensorComponent[] { sensor_21_20_3, sensor_20_22_3 }, new ActuatorComponent[0]); + Assert.Greater(errors.Count(), 0); + } [Test] public void TestCheckModelThrowsNoModel() From cc227fde5f71275dfce50dab89e3000c4ab8c290 Mon Sep 17 00:00:00 2001 From: Ruo-Ping Dong Date: Tue, 1 Dec 2020 14:07:47 -0800 Subject: [PATCH 09/36] fix tests --- .../Tests/Editor/Actuators/VectorActuatorTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/com.unity.ml-agents/Tests/Editor/Actuators/VectorActuatorTests.cs b/com.unity.ml-agents/Tests/Editor/Actuators/VectorActuatorTests.cs index 1c42f2d3c4..a5ca76156d 100644 --- a/com.unity.ml-agents/Tests/Editor/Actuators/VectorActuatorTests.cs +++ b/com.unity.ml-agents/Tests/Editor/Actuators/VectorActuatorTests.cs @@ -42,7 +42,7 @@ public void TestConstruct() Assert.IsTrue(va1.ActionSpec.NumContinuousActions == 4); Assert.IsTrue(va1.ActionSpec.SumOfDiscreteBranchSizes == 0); - Assert.AreEqual(va1.Name, "name-Continuous"); + Assert.AreEqual(va1.Name, "name-Continuous-4-Discrete-0"); } [Test] From 351a479925f12f0309e6db3f15a1ac5b3424620c Mon Sep 17 00:00:00 2001 From: Ruo-Ping Dong Date: Tue, 1 Dec 2020 14:34:37 -0800 Subject: [PATCH 10/36] fix demonstration drawer --- .../Editor/DemonstrationDrawer.cs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/com.unity.ml-agents/Editor/DemonstrationDrawer.cs b/com.unity.ml-agents/Editor/DemonstrationDrawer.cs index 7869c5d25a..9511ea28b7 100644 --- a/com.unity.ml-agents/Editor/DemonstrationDrawer.cs +++ b/com.unity.ml-agents/Editor/DemonstrationDrawer.cs @@ -72,17 +72,17 @@ static string BuildIntArrayLabel(SerializedProperty actionSizeProperty) /// void MakeActionsProperty(SerializedProperty property) { - // TODO: update deprecated fields to ActionSpec - var actSizeProperty = property.FindPropertyRelative("VectorActionSize"); - var actSpaceTypeProp = property.FindPropertyRelative("VectorActionSpaceType"); + var actSpecProperty = property.FindPropertyRelative("VectorActionSpec"); + var continuousSizeProperty = actSpecProperty.FindPropertyRelative("m_NumContinuousActions"); + var discreteSizeProperty = actSpecProperty.FindPropertyRelative("m_NumDiscreteActions"); - var vecActSizeLabel = - actSizeProperty.displayName + ": " + BuildIntArrayLabel(actSizeProperty); - var actSpaceTypeLabel = actSpaceTypeProp.displayName + ": " + - (SpaceType)actSpaceTypeProp.enumValueIndex; + var continuousSizeLabel = + continuousSizeProperty.displayName + ": " + continuousSizeProperty.intValue; + var discreteSizeLabel = discreteSizeProperty.displayName + ": " + + discreteSizeProperty.intValue; - EditorGUILayout.LabelField(vecActSizeLabel); - EditorGUILayout.LabelField(actSpaceTypeLabel); + EditorGUILayout.LabelField(continuousSizeLabel); + EditorGUILayout.LabelField(discreteSizeLabel); } /// From 68646860704f7c936093491cea36d351a8698c5d Mon Sep 17 00:00:00 2001 From: Ruo-Ping Dong Date: Tue, 1 Dec 2020 14:54:58 -0800 Subject: [PATCH 11/36] add comments --- .../Editor/BehaviorParametersEditor.cs | 5 +- .../Runtime/Actuators/ActionSpec.cs | 10 ++++ .../Runtime/Policies/BrainParameters.cs | 49 +++++++++++++++++-- 3 files changed, 58 insertions(+), 6 deletions(-) diff --git a/com.unity.ml-agents/Editor/BehaviorParametersEditor.cs b/com.unity.ml-agents/Editor/BehaviorParametersEditor.cs index a1d398555b..ace69c9cb0 100644 --- a/com.unity.ml-agents/Editor/BehaviorParametersEditor.cs +++ b/com.unity.ml-agents/Editor/BehaviorParametersEditor.cs @@ -27,7 +27,7 @@ public override void OnInspectorGUI() var so = serializedObject; so.Update(); bool needPolicyUpdate; // Whether the name, model, inference device, or BehaviorType changed. - bool needBrainParametersUpdate; + bool needBrainParametersUpdate; // Whether the brain parameters changed // Drawing the Behavior Parameters EditorGUI.indentLevel++; @@ -167,8 +167,7 @@ void UpdateAgentPolicy() void UpdateBrainParameters() { var behaviorParameters = (BehaviorParameters)target; - var brainParameters = behaviorParameters.BrainParameters; - brainParameters.UpdateBrainParameters(); + behaviorParameters.BrainParameters.UpdateBrainParameters(); } } } diff --git a/com.unity.ml-agents/Runtime/Actuators/ActionSpec.cs b/com.unity.ml-agents/Runtime/Actuators/ActionSpec.cs index 97091bf2c3..5dff2fb8e6 100644 --- a/com.unity.ml-agents/Runtime/Actuators/ActionSpec.cs +++ b/com.unity.ml-agents/Runtime/Actuators/ActionSpec.cs @@ -73,11 +73,21 @@ public static ActionSpec MakeDiscrete(params int[] branchSizes) return actuatorSpace; } + /// + /// Set action size fields related to continuous actions. + /// + /// Number of Continuous Actions public void SetContinuous(int numContinuousActions) { m_NumContinuousActions = numContinuousActions; } + /// + /// Set action size fields related to discrete actions. + /// + /// Number of Discrete Actions + /// The array of branch sizes for the discrete action space. Each index + /// contains the number of actions available for that branch. public void SetDiscrete(int numDiscreteActions, int[] branchSizes) { m_NumDiscreteActions = numDiscreteActions; diff --git a/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs b/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs index 634c5bf27b..7a395f5096 100644 --- a/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs +++ b/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs @@ -52,10 +52,13 @@ public class BrainParameters : ISerializationCallbackReceiver [FormerlySerializedAs("numStackedVectorObservations")] [Range(1, 50)] public int NumStackedVectorObservations = 1; + /// + /// The specification of the Action space for the BrainParameters. + /// public ActionSpec VectorActionSpec = new ActionSpec(0, 0, new int[] { }); /// - /// The size of the action space. + /// (Deprecated) The size of the action space. /// /// The size specified is interpreted differently depending on whether /// the agent uses the continuous or the discrete action space. @@ -75,7 +78,7 @@ public class BrainParameters : ISerializationCallbackReceiver public string[] VectorActionDescriptions; /// - /// Defines if the action is discrete or continuous. + /// (Deprecated) Defines if the action is discrete or continuous. /// [FormerlySerializedAs("vectorActionSpaceType")] [FormerlySerializedAs("VectorActionSpaceType")] @@ -86,7 +89,7 @@ public class BrainParameters : ISerializationCallbackReceiver internal bool hasUpgradedBrainParametersWithActionSpec; /// - /// The number of actions specified by this Brain. + /// (Deprecated) The number of actions specified by this Brain. /// public int NumActionsDeprecated { @@ -121,6 +124,26 @@ public BrainParameters Clone() }; } + /// + /// Called by Unity immediately before serializing this object. + /// + /// + /// The BrainParameter class uses OnBeforeSerialize() for internal housekeeping. Call the + /// base class implementation if you need your own custom serialization logic. + /// + /// See [OnBeforeSerialize] for more information. + /// + /// [OnBeforeSerialize]: https://docs.unity3d.com/ScriptReference/ISerializationCallbackReceiver.OnAfterDeserialize.html + /// + /// + /// + /// public new void OnBeforeSerialize() + /// { + /// base.OnBeforeSerialize(); + /// // additional serialization logic... + /// } + /// + /// public void OnBeforeSerialize() { if (!hasUpgradedBrainParametersWithActionSpec) @@ -137,6 +160,26 @@ public void OnBeforeSerialize() } } + /// + /// Called by Unity immediately after deserializing this object. + /// + /// + /// The BrainParameter class uses OnAfterDeserialize() for internal housekeeping. Call the + /// base class implementation if you need your own custom deserialization logic. + /// + /// See [OnAfterDeserialize] for more information. + /// + /// [OnAfterDeserialize]: https://docs.unity3d.com/ScriptReference/ISerializationCallbackReceiver.OnAfterDeserialize.html + /// + /// + /// + /// public new void OnAfterDeserialize() + /// { + /// base.OnAfterDeserialize(); + /// // additional deserialization logic... + /// } + /// + /// public void OnAfterDeserialize() { if (!hasUpgradedBrainParametersWithActionSpec) From 397fcf18dc663bba442596beb280ca274dab78f0 Mon Sep 17 00:00:00 2001 From: Ruo-Ping Dong Date: Wed, 2 Dec 2020 17:07:17 -0800 Subject: [PATCH 12/36] rename method --- com.unity.ml-agents/Editor/BehaviorParametersEditor.cs | 2 +- com.unity.ml-agents/Editor/BrainParametersDrawer.cs | 3 --- com.unity.ml-agents/Runtime/Policies/BrainParameters.cs | 2 +- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/com.unity.ml-agents/Editor/BehaviorParametersEditor.cs b/com.unity.ml-agents/Editor/BehaviorParametersEditor.cs index ace69c9cb0..182404d3f2 100644 --- a/com.unity.ml-agents/Editor/BehaviorParametersEditor.cs +++ b/com.unity.ml-agents/Editor/BehaviorParametersEditor.cs @@ -167,7 +167,7 @@ void UpdateAgentPolicy() void UpdateBrainParameters() { var behaviorParameters = (BehaviorParameters)target; - behaviorParameters.BrainParameters.UpdateBrainParameters(); + behaviorParameters.BrainParameters.UpdateDiscreteParameters(); } } } diff --git a/com.unity.ml-agents/Editor/BrainParametersDrawer.cs b/com.unity.ml-agents/Editor/BrainParametersDrawer.cs index 49bab1bcce..7dc14a2fe7 100644 --- a/com.unity.ml-agents/Editor/BrainParametersDrawer.cs +++ b/com.unity.ml-agents/Editor/BrainParametersDrawer.cs @@ -16,10 +16,7 @@ internal class BrainParametersDrawer : PropertyDrawer const int k_VecObsNumLine = 3; const string k_ActionSpecName = "VectorActionSpec"; const string k_ContinuousActionSizeName = "m_NumContinuousActions"; - const string k_DiscreteActionSizeName = "m_NumDiscreteActions"; const string k_DiscreteBranchSizeName = "m_BranchSizes"; - const string k_ActionSizePropName = "VectorActionSize"; - const string k_ActionTypePropName = "VectorActionSpaceType"; const string k_ActionDescriptionPropName = "VectorActionDescriptions"; const string k_VecObsPropName = "VectorObservationSize"; const string k_NumVecObsPropName = "NumStackedVectorObservations"; diff --git a/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs b/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs index 7a395f5096..c7e366cddd 100644 --- a/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs +++ b/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs @@ -196,7 +196,7 @@ public void OnAfterDeserialize() } } - internal void UpdateBrainParameters() + internal void UpdateDiscreteParameters() { VectorActionSpec.NumDiscreteActions = VectorActionSpec.BranchSizes.Length; VectorActionSpec.SumOfDiscreteBranchSizes = VectorActionSpec.BranchSizes.Sum(); From 0bf75e3da58212e8a978a6569b3a414b7e2c4783 Mon Sep 17 00:00:00 2001 From: Ruo-Ping Dong Date: Mon, 7 Dec 2020 17:24:12 -0800 Subject: [PATCH 13/36] check both continuous/discrete output shape regardless of model output format --- .../Editor/BehaviorParametersEditor.cs | 2 ++ .../Inference/BarracudaModelParamLoader.cs | 33 +++++++------------ 2 files changed, 13 insertions(+), 22 deletions(-) diff --git a/com.unity.ml-agents/Editor/BehaviorParametersEditor.cs b/com.unity.ml-agents/Editor/BehaviorParametersEditor.cs index 182404d3f2..2d233e7169 100644 --- a/com.unity.ml-agents/Editor/BehaviorParametersEditor.cs +++ b/com.unity.ml-agents/Editor/BehaviorParametersEditor.cs @@ -168,6 +168,8 @@ void UpdateBrainParameters() { var behaviorParameters = (BehaviorParameters)target; behaviorParameters.BrainParameters.UpdateDiscreteParameters(); + + DisplayFailedModelChecks(); } } } diff --git a/com.unity.ml-agents/Runtime/Inference/BarracudaModelParamLoader.cs b/com.unity.ml-agents/Runtime/Inference/BarracudaModelParamLoader.cs index d71edc2fef..19bd3dce6a 100644 --- a/com.unity.ml-agents/Runtime/Inference/BarracudaModelParamLoader.cs +++ b/com.unity.ml-agents/Runtime/Inference/BarracudaModelParamLoader.cs @@ -439,30 +439,18 @@ static IEnumerable CheckOutputTensorShape( { var failedModelChecks = new List(); - var tensorTester = new Dictionary>(); - if (model.HasContinuousOutputs()) - { - tensorTester[model.ContinuousOutputName()] = CheckContinuousActionOutputShape; - } - if (model.HasDiscreteOutputs()) - { - tensorTester[model.DiscreteOutputName()] = CheckDiscreteActionOutputShape; - } - // If the model expects an output but it is not in this list var modelContinuousActionSize = model.ContinuousOutputSize(); + var continuousError = CheckContinuousActionOutputShape(brainParameters, actuatorComponents, modelContinuousActionSize); + if (continuousError != null) + { + failedModelChecks.Add(continuousError); + } var modelSumDiscreteBranchSizes = model.DiscreteOutputSize(); - foreach (var name in model.outputs) + var discreteError = CheckDiscreteActionOutputShape(brainParameters, actuatorComponents, modelSumDiscreteBranchSizes); + if (discreteError != null) { - if (tensorTester.ContainsKey(name)) - { - var tester = tensorTester[name]; - var error = tester.Invoke(brainParameters, actuatorComponents, model.GetShapeByName(name), modelContinuousActionSize, modelSumDiscreteBranchSizes); - if (error != null) - { - failedModelChecks.Add(error); - } - } + failedModelChecks.Add(discreteError); } return failedModelChecks; } @@ -487,8 +475,9 @@ static IEnumerable CheckOutputTensorShape( /// check failed. If the check passed, returns null. /// static string CheckDiscreteActionOutputShape( - BrainParameters brainParameters, ActuatorComponent[] actuatorComponents, TensorShape? shape, int modelContinuousActionSize, int modelSumDiscreteBranchSizes) + BrainParameters brainParameters, ActuatorComponent[] actuatorComponents, int modelSumDiscreteBranchSizes) { + // TODO: check each branch size instead of sum of branch sizes var sumOfDiscreteBranchSizes = brainParameters.VectorActionSpec.SumOfDiscreteBranchSizes; foreach (var actuatorComponent in actuatorComponents) @@ -523,7 +512,7 @@ static string CheckDiscreteActionOutputShape( /// If the Check failed, returns a string containing information about why the /// check failed. If the check passed, returns null. static string CheckContinuousActionOutputShape( - BrainParameters brainParameters, ActuatorComponent[] actuatorComponents, TensorShape? shape, int modelContinuousActionSize, int modelSumDiscreteBranchSizes) + BrainParameters brainParameters, ActuatorComponent[] actuatorComponents, int modelContinuousActionSize) { var numContinuousActions = brainParameters.VectorActionSpec.NumContinuousActions; From 91d4125da5197c412e2f1c5bd0ac4fec71dc9753 Mon Sep 17 00:00:00 2001 From: Ruo-Ping Dong Date: Wed, 9 Dec 2020 11:43:37 -0800 Subject: [PATCH 14/36] simplify actionspec. put property names in const string --- .../Editor/BehaviorParametersEditor.cs | 38 +++++++++---------- .../Editor/BrainParametersDrawer.cs | 4 +- .../Editor/DemonstrationDrawer.cs | 34 +++++++++++------ .../Runtime/Actuators/ActionSpec.cs | 20 +++------- .../Runtime/Actuators/ActuatorManager.cs | 2 +- .../Runtime/Communicator/GrpcExtensions.cs | 11 +++--- .../Runtime/Policies/BrainParameters.cs | 14 ++----- .../EditModeTestInternalBrainTensorApplier.cs | 2 +- .../Tests/Editor/ModelRunnerTest.cs | 2 +- .../Tests/Editor/ParameterLoaderTest.cs | 4 +- 10 files changed, 62 insertions(+), 69 deletions(-) diff --git a/com.unity.ml-agents/Editor/BehaviorParametersEditor.cs b/com.unity.ml-agents/Editor/BehaviorParametersEditor.cs index 2d233e7169..1786f210ae 100644 --- a/com.unity.ml-agents/Editor/BehaviorParametersEditor.cs +++ b/com.unity.ml-agents/Editor/BehaviorParametersEditor.cs @@ -21,6 +21,14 @@ internal class BehaviorParametersEditor : UnityEditor.Editor float m_TimeSinceModelReload; // Whether or not the model needs to be reloaded bool m_RequireReload; + const string k_BehaviorName = "m_BehaviorName"; + const string k_BrainParametersName = "m_BrainParameters"; + const string k_ModelName = "m_Model"; + const string k_InferenceDeviceName = "m_InferenceDevice"; + const string k_BehaviorTypeName = "m_BehaviorType"; + const string k_TeamIdName = "TeamId"; + const string k_UseChildSensorsName = "m_UseChildSensors"; + const string k_ObservableAttributeHandlingName = "m_ObservableAttributeHandling"; public override void OnInspectorGUI() { @@ -35,38 +43,38 @@ public override void OnInspectorGUI() EditorGUI.BeginChangeCheck(); { - EditorGUILayout.PropertyField(so.FindProperty("m_BehaviorName")); + EditorGUILayout.PropertyField(so.FindProperty(k_BehaviorName)); } needPolicyUpdate = EditorGUI.EndChangeCheck(); EditorGUI.BeginChangeCheck(); EditorGUI.BeginDisabledGroup(!EditorUtilities.CanUpdateModelProperties()); { - EditorGUILayout.PropertyField(so.FindProperty("m_BrainParameters"), true); + EditorGUILayout.PropertyField(so.FindProperty(k_BrainParametersName), true); } EditorGUI.EndDisabledGroup(); needBrainParametersUpdate = EditorGUI.EndChangeCheck(); EditorGUI.BeginChangeCheck(); { - EditorGUILayout.PropertyField(so.FindProperty("m_Model"), true); + EditorGUILayout.PropertyField(so.FindProperty(k_ModelName), true); EditorGUI.indentLevel++; - EditorGUILayout.PropertyField(so.FindProperty("m_InferenceDevice"), true); + EditorGUILayout.PropertyField(so.FindProperty(k_InferenceDeviceName), true); EditorGUI.indentLevel--; } needPolicyUpdate = needPolicyUpdate || EditorGUI.EndChangeCheck(); EditorGUI.BeginChangeCheck(); { - EditorGUILayout.PropertyField(so.FindProperty("m_BehaviorType")); + EditorGUILayout.PropertyField(so.FindProperty(k_BehaviorTypeName)); } needPolicyUpdate = needPolicyUpdate || EditorGUI.EndChangeCheck(); - EditorGUILayout.PropertyField(so.FindProperty("TeamId")); + EditorGUILayout.PropertyField(so.FindProperty(k_TeamIdName)); EditorGUI.BeginDisabledGroup(!EditorUtilities.CanUpdateModelProperties()); { - EditorGUILayout.PropertyField(so.FindProperty("m_UseChildSensors"), true); - EditorGUILayout.PropertyField(so.FindProperty("m_ObservableAttributeHandling"), true); + EditorGUILayout.PropertyField(so.FindProperty(k_UseChildSensorsName), true); + EditorGUILayout.PropertyField(so.FindProperty(k_ObservableAttributeHandlingName), true); } EditorGUI.EndDisabledGroup(); @@ -79,10 +87,6 @@ public override void OnInspectorGUI() { UpdateAgentPolicy(); } - if (needBrainParametersUpdate) - { - UpdateBrainParameters(); - } } /// @@ -98,7 +102,7 @@ void DisplayFailedModelChecks() // Display all failed checks D.logEnabled = false; Model barracudaModel = null; - var model = (NNModel)serializedObject.FindProperty("m_Model").objectReferenceValue; + var model = (NNModel)serializedObject.FindProperty(k_ModelName).objectReferenceValue; var behaviorParameters = (BehaviorParameters)target; // Grab the sensor components, since we need them to determine the observation sizes. @@ -163,13 +167,5 @@ void UpdateAgentPolicy() var behaviorParameters = (BehaviorParameters)target; behaviorParameters.UpdateAgentPolicy(); } - - void UpdateBrainParameters() - { - var behaviorParameters = (BehaviorParameters)target; - behaviorParameters.BrainParameters.UpdateDiscreteParameters(); - - DisplayFailedModelChecks(); - } } } diff --git a/com.unity.ml-agents/Editor/BrainParametersDrawer.cs b/com.unity.ml-agents/Editor/BrainParametersDrawer.cs index 7dc14a2fe7..a0ef0cd212 100644 --- a/com.unity.ml-agents/Editor/BrainParametersDrawer.cs +++ b/com.unity.ml-agents/Editor/BrainParametersDrawer.cs @@ -165,8 +165,8 @@ static void DrawDiscreteVectorAction(Rect position, SerializedProperty property) static float GetHeightDrawVectorAction(SerializedProperty property) { var actionSpecProperty = property.FindPropertyRelative(k_ActionSpecName); - var actionSize = 3 + actionSpecProperty.FindPropertyRelative("m_BranchSizes").arraySize; - return actionSize * k_LineHeight; + var numActionLines = 3 + actionSpecProperty.FindPropertyRelative(k_DiscreteBranchSizeName).arraySize; + return numActionLines * k_LineHeight; } } } diff --git a/com.unity.ml-agents/Editor/DemonstrationDrawer.cs b/com.unity.ml-agents/Editor/DemonstrationDrawer.cs index 9511ea28b7..4fca85ac6d 100644 --- a/com.unity.ml-agents/Editor/DemonstrationDrawer.cs +++ b/com.unity.ml-agents/Editor/DemonstrationDrawer.cs @@ -17,12 +17,24 @@ internal class DemonstrationEditor : UnityEditor.Editor SerializedProperty m_BrainParameters; SerializedProperty m_DemoMetaData; SerializedProperty m_ObservationShapes; + const string k_BrainParametersName = "brainParameters"; + const string k_MetaDataName = "metaData"; + const string k_ObservationSummariesName = "observationSummaries"; + const string k_DemonstrationName = "demonstrationName"; + const string k_NumberStepsName = "numberSteps"; + const string k_NumberEpisodesName = "numberEpisodes"; + const string k_MeanRewardName = "meanReward"; + const string k_VectorActionSpecName = "VectorActionSpec"; + const string k_NumContinuousActionsName = "m_NumContinuousActions"; + const string k_NumDiscreteActionsName = "m_NumDiscreteActions"; + const string k_ShapeName = "shape"; + void OnEnable() { - m_BrainParameters = serializedObject.FindProperty("brainParameters"); - m_DemoMetaData = serializedObject.FindProperty("metaData"); - m_ObservationShapes = serializedObject.FindProperty("observationSummaries"); + m_BrainParameters = serializedObject.FindProperty(k_BrainParametersName); + m_DemoMetaData = serializedObject.FindProperty(k_MetaDataName); + m_ObservationShapes = serializedObject.FindProperty(k_ObservationSummariesName); } /// @@ -30,10 +42,10 @@ void OnEnable() /// void MakeMetaDataProperty(SerializedProperty property) { - var nameProp = property.FindPropertyRelative("demonstrationName"); - var experiencesProp = property.FindPropertyRelative("numberSteps"); - var episodesProp = property.FindPropertyRelative("numberEpisodes"); - var rewardsProp = property.FindPropertyRelative("meanReward"); + var nameProp = property.FindPropertyRelative(k_DemonstrationName); + var experiencesProp = property.FindPropertyRelative(k_NumberStepsName); + var episodesProp = property.FindPropertyRelative(k_NumberEpisodesName); + var rewardsProp = property.FindPropertyRelative(k_MeanRewardName); var nameLabel = nameProp.displayName + ": " + nameProp.stringValue; var experiencesLabel = experiencesProp.displayName + ": " + experiencesProp.intValue; @@ -72,9 +84,9 @@ static string BuildIntArrayLabel(SerializedProperty actionSizeProperty) /// void MakeActionsProperty(SerializedProperty property) { - var actSpecProperty = property.FindPropertyRelative("VectorActionSpec"); - var continuousSizeProperty = actSpecProperty.FindPropertyRelative("m_NumContinuousActions"); - var discreteSizeProperty = actSpecProperty.FindPropertyRelative("m_NumDiscreteActions"); + var actSpecProperty = property.FindPropertyRelative(k_VectorActionSpecName); + var continuousSizeProperty = actSpecProperty.FindPropertyRelative(k_NumContinuousActionsName); + var discreteSizeProperty = actSpecProperty.FindPropertyRelative(k_NumDiscreteActionsName); var continuousSizeLabel = continuousSizeProperty.displayName + ": " + continuousSizeProperty.intValue; @@ -96,7 +108,7 @@ void MakeObservationsProperty(SerializedProperty obsSummariesProperty) for (var i = 0; i < numObservations; i++) { var summary = obsSummariesProperty.GetArrayElementAtIndex(i); - var shapeProperty = summary.FindPropertyRelative("shape"); + var shapeProperty = summary.FindPropertyRelative(k_ShapeName); shapesLabels.Add(BuildIntArrayLabel(shapeProperty)); } diff --git a/com.unity.ml-agents/Runtime/Actuators/ActionSpec.cs b/com.unity.ml-agents/Runtime/Actuators/ActionSpec.cs index 5dff2fb8e6..0fce3e6082 100644 --- a/com.unity.ml-agents/Runtime/Actuators/ActionSpec.cs +++ b/com.unity.ml-agents/Runtime/Actuators/ActionSpec.cs @@ -15,10 +15,6 @@ public struct ActionSpec int[] m_BranchSizes; [SerializeField] int m_NumContinuousActions; - [SerializeField] - int m_NumDiscreteActions; - [SerializeField] - int m_SumOfDiscreteBranchSizes; /// /// An array of branch sizes for our action space. @@ -40,13 +36,13 @@ public struct ActionSpec /// /// The number of branches for a Discrete . /// - public int NumDiscreteActions { get { return m_NumDiscreteActions; } set { m_NumDiscreteActions = value; } } + public int NumDiscreteActions { get { return m_BranchSizes.Length; } } /// /// Get the total number of Discrete Actions that can be taken by calculating the Sum /// of all of the Discrete Action branch sizes. /// - public int SumOfDiscreteBranchSizes { get { return m_SumOfDiscreteBranchSizes; } set { m_SumOfDiscreteBranchSizes = value; } } + public int SumOfDiscreteBranchSizes { get { return m_BranchSizes.Sum(); } } /// /// Creates a Continuous with the number of actions available. @@ -55,7 +51,7 @@ public struct ActionSpec /// An Continuous ActionSpec initialized with the number of actions available. public static ActionSpec MakeContinuous(int numActions) { - var actuatorSpace = new ActionSpec(numActions, 0); + var actuatorSpace = new ActionSpec(numActions, null); return actuatorSpace; } @@ -69,7 +65,7 @@ public static ActionSpec MakeContinuous(int numActions) public static ActionSpec MakeDiscrete(params int[] branchSizes) { var numActions = branchSizes.Length; - var actuatorSpace = new ActionSpec(0, numActions, branchSizes); + var actuatorSpace = new ActionSpec(0, branchSizes); return actuatorSpace; } @@ -88,19 +84,15 @@ public void SetContinuous(int numContinuousActions) /// Number of Discrete Actions /// The array of branch sizes for the discrete action space. Each index /// contains the number of actions available for that branch. - public void SetDiscrete(int numDiscreteActions, int[] branchSizes) + public void SetDiscrete(int[] branchSizes) { - m_NumDiscreteActions = numDiscreteActions; m_BranchSizes = branchSizes; - m_SumOfDiscreteBranchSizes = branchSizes.Sum(); } - internal ActionSpec(int numContinuousActions, int numDiscreteActions, int[] branchSizes = null) + internal ActionSpec(int numContinuousActions, int[] branchSizes = null) { m_NumContinuousActions = numContinuousActions; - m_NumDiscreteActions = numDiscreteActions; m_BranchSizes = branchSizes; - m_SumOfDiscreteBranchSizes = branchSizes?.Sum() ?? 0; } /// diff --git a/com.unity.ml-agents/Runtime/Actuators/ActuatorManager.cs b/com.unity.ml-agents/Runtime/Actuators/ActuatorManager.cs index de5b644621..72e62dab4f 100644 --- a/com.unity.ml-agents/Runtime/Actuators/ActuatorManager.cs +++ b/com.unity.ml-agents/Runtime/Actuators/ActuatorManager.cs @@ -137,7 +137,7 @@ internal static ActionSpec CombineActionSpecs(IList actuators) } } - return new ActionSpec(numContinuousActions, numDiscreteActions, combinedBranchSizes); + return new ActionSpec(numContinuousActions, combinedBranchSizes); } /// diff --git a/com.unity.ml-agents/Runtime/Communicator/GrpcExtensions.cs b/com.unity.ml-agents/Runtime/Communicator/GrpcExtensions.cs index 4e2958d46a..4be6d20f1c 100644 --- a/com.unity.ml-agents/Runtime/Communicator/GrpcExtensions.cs +++ b/com.unity.ml-agents/Runtime/Communicator/GrpcExtensions.cs @@ -176,12 +176,11 @@ public static BrainParameters ToBrainParameters(this BrainParametersProto bpp) /// An ActionSpec struct. public static ActionSpec ToActionSpec(this ActionSpecProto actionSpecProto) { - var actionSpec = new ActionSpec - ( - actionSpecProto.NumContinuousActions, - actionSpecProto.NumDiscreteActions, - actionSpecProto.DiscreteBranchSizes.ToArray() - ); + var actionSpec = new ActionSpec(actionSpecProto.NumContinuousActions); + if (actionSpecProto.DiscreteBranchSizes != null) + { + actionSpec.BranchSizes = actionSpecProto.DiscreteBranchSizes.ToArray(); + } return actionSpec; } diff --git a/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs b/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs index c7e366cddd..626de95e4d 100644 --- a/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs +++ b/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs @@ -55,7 +55,7 @@ public class BrainParameters : ISerializationCallbackReceiver /// /// The specification of the Action space for the BrainParameters. /// - public ActionSpec VectorActionSpec = new ActionSpec(0, 0, new int[] { }); + public ActionSpec VectorActionSpec = new ActionSpec(0, new int[] { }); /// /// (Deprecated) The size of the action space. @@ -120,7 +120,7 @@ public BrainParameters Clone() VectorActionSizeDeprecated = (int[])VectorActionSizeDeprecated.Clone(), VectorActionDescriptions = (string[])VectorActionDescriptions.Clone(), VectorActionSpaceTypeDeprecated = VectorActionSpaceTypeDeprecated, - VectorActionSpec = new ActionSpec(VectorActionSpec.NumContinuousActions, VectorActionSpec.NumDiscreteActions, VectorActionSpec.BranchSizes) + VectorActionSpec = new ActionSpec(VectorActionSpec.NumContinuousActions, VectorActionSpec.BranchSizes) }; } @@ -154,7 +154,7 @@ public void OnBeforeSerialize() } if (VectorActionSpec.NumDiscreteActions == 0 && VectorActionSpaceTypeDeprecated == SpaceType.Discrete) { - VectorActionSpec.SetDiscrete(VectorActionSizeDeprecated.Length, VectorActionSizeDeprecated); + VectorActionSpec.SetDiscrete(VectorActionSizeDeprecated); } hasUpgradedBrainParametersWithActionSpec = true; } @@ -190,16 +190,10 @@ public void OnAfterDeserialize() } if (VectorActionSpec.NumDiscreteActions == 0 && VectorActionSpaceTypeDeprecated == SpaceType.Discrete) { - VectorActionSpec.SetDiscrete(VectorActionSizeDeprecated.Length, VectorActionSizeDeprecated); + VectorActionSpec.SetDiscrete(VectorActionSizeDeprecated); } hasUpgradedBrainParametersWithActionSpec = true; } } - - internal void UpdateDiscreteParameters() - { - VectorActionSpec.NumDiscreteActions = VectorActionSpec.BranchSizes.Length; - VectorActionSpec.SumOfDiscreteBranchSizes = VectorActionSpec.BranchSizes.Sum(); - } } } diff --git a/com.unity.ml-agents/Tests/Editor/EditModeTestInternalBrainTensorApplier.cs b/com.unity.ml-agents/Tests/Editor/EditModeTestInternalBrainTensorApplier.cs index 9bac28fbbf..a327e32956 100644 --- a/com.unity.ml-agents/Tests/Editor/EditModeTestInternalBrainTensorApplier.cs +++ b/com.unity.ml-agents/Tests/Editor/EditModeTestInternalBrainTensorApplier.cs @@ -84,7 +84,7 @@ public void ApplyDiscreteActionOutput() [Test] public void ApplyHybridActionOutput() { - var actionSpec = new ActionSpec(3, 2, new int[] { 2, 3 }); + var actionSpec = new ActionSpec(3, new int[] { 2, 3 }); var continuousInputTensor = new TensorProxy() { shape = new long[] { 2, 3 }, diff --git a/com.unity.ml-agents/Tests/Editor/ModelRunnerTest.cs b/com.unity.ml-agents/Tests/Editor/ModelRunnerTest.cs index 53a72f243f..522afd5254 100644 --- a/com.unity.ml-agents/Tests/Editor/ModelRunnerTest.cs +++ b/com.unity.ml-agents/Tests/Editor/ModelRunnerTest.cs @@ -37,7 +37,7 @@ ActionSpec GetDiscrete1vis0vec_2_3action_recurrModelActionSpec() ActionSpec GetHybrid0vis53vec_3c_2dActionSpec() { - return new ActionSpec(3, 1, new int[] { 2 }); + return new ActionSpec(3, new int[] { 2 }); } [SetUp] diff --git a/com.unity.ml-agents/Tests/Editor/ParameterLoaderTest.cs b/com.unity.ml-agents/Tests/Editor/ParameterLoaderTest.cs index dfb894af29..02c0cf2970 100644 --- a/com.unity.ml-agents/Tests/Editor/ParameterLoaderTest.cs +++ b/com.unity.ml-agents/Tests/Editor/ParameterLoaderTest.cs @@ -114,7 +114,7 @@ BrainParameters GetHybridBrainParameters() var validBrainParameters = new BrainParameters(); validBrainParameters.VectorObservationSize = 53; validBrainParameters.NumStackedVectorObservations = 1; - validBrainParameters.VectorActionSpec = new ActionSpec(3, 1, new int[] { 2 }); + validBrainParameters.VectorActionSpec = new ActionSpec(3, new int[] { 2 }); return validBrainParameters; } @@ -360,7 +360,7 @@ public void TestCheckModelThrowsActionHybrid() var model = ModelLoader.Load(hybridONNXModel); var brainParameters = GetHybridBrainParameters(); - brainParameters.VectorActionSpec = new ActionSpec(3, 1, new int[] { 3 }); ; // Invalid discrete action size + brainParameters.VectorActionSpec = new ActionSpec(3, new int[] { 3 }); ; // Invalid discrete action size var errors = BarracudaModelParamLoader.CheckModel(model, brainParameters, new SensorComponent[] { sensor_21_20_3, sensor_20_22_3 }, new ActuatorComponent[0]); Assert.Greater(errors.Count(), 0); From def4b0c06a7bf550ed4125f51efb25351c1bf9f4 Mon Sep 17 00:00:00 2001 From: Ruo-Ping Dong Date: Wed, 9 Dec 2020 14:37:20 -0800 Subject: [PATCH 15/36] fix merge --- ml-agents/mlagents/trainers/sac/optimizer_torch.py | 7 +++++-- .../mlagents/trainers/tests/tensorflow/test_simple_rl.py | 6 ++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/ml-agents/mlagents/trainers/sac/optimizer_torch.py b/ml-agents/mlagents/trainers/sac/optimizer_torch.py index f5f89d30ec..81a7997df6 100644 --- a/ml-agents/mlagents/trainers/sac/optimizer_torch.py +++ b/ml-agents/mlagents/trainers/sac/optimizer_torch.py @@ -159,7 +159,10 @@ def __init__(self, policy: TorchPolicy, trainer_params: TrainerSettings): requires_grad=True, ) _cont_log_ent_coef = torch.nn.Parameter( - torch.log(torch.as_tensor([self.init_entcoef])), requires_grad=True + torch.log( + torch.as_tensor([self.init_entcoef] * self._action_spec.continuous_size) + ), + requires_grad=True, ) self._log_ent_coef = TorchSACOptimizer.LogEntCoef( discrete=_disc_log_ent_coef, continuous=_cont_log_ent_coef @@ -423,7 +426,7 @@ def sac_entropy_loss( ) # We update all the _cont_ent_coef as one block entropy_loss += -1 * ModelUtils.masked_mean( - _cont_ent_coef * target_current_diff, loss_masks + torch.mean(_cont_ent_coef) * target_current_diff, loss_masks ) return entropy_loss diff --git a/ml-agents/mlagents/trainers/tests/tensorflow/test_simple_rl.py b/ml-agents/mlagents/trainers/tests/tensorflow/test_simple_rl.py index 8ba24c50ba..c397a2bd93 100644 --- a/ml-agents/mlagents/trainers/tests/tensorflow/test_simple_rl.py +++ b/ml-agents/mlagents/trainers/tests/tensorflow/test_simple_rl.py @@ -184,7 +184,7 @@ def test_visual_advanced_ppo(vis_encode_type, num_visual): PPO_TF_CONFIG, hyperparameters=new_hyperparams, network_settings=new_networksettings, - max_steps=400, + max_steps=300, summary_freq=100, framework=FrameworkType.TENSORFLOW, ) @@ -218,9 +218,7 @@ def test_recurrent_ppo(action_sizes): @pytest.mark.parametrize("action_sizes", [(0, 1), (1, 0)]) def test_simple_sac(action_sizes): env = SimpleEnvironment([BRAIN_NAME], action_sizes=action_sizes) - config = attr.evolve( - SAC_TF_CONFIG, framework=FrameworkType.TENSORFLOW, max_steps=900 - ) + config = attr.evolve(SAC_TF_CONFIG, framework=FrameworkType.TENSORFLOW) _check_environment_trains(env, {BRAIN_NAME: config}) From 8ed6ac6ff3f4fe435285890b9234eaa87b2f6e78 Mon Sep 17 00:00:00 2001 From: Ruo-Ping Dong Date: Wed, 9 Dec 2020 14:48:40 -0800 Subject: [PATCH 16/36] make SetContinuous/Discrete() internal --- com.unity.ml-agents/Runtime/Actuators/ActionSpec.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/com.unity.ml-agents/Runtime/Actuators/ActionSpec.cs b/com.unity.ml-agents/Runtime/Actuators/ActionSpec.cs index 0fce3e6082..77b3f33ad6 100644 --- a/com.unity.ml-agents/Runtime/Actuators/ActionSpec.cs +++ b/com.unity.ml-agents/Runtime/Actuators/ActionSpec.cs @@ -73,7 +73,7 @@ public static ActionSpec MakeDiscrete(params int[] branchSizes) /// Set action size fields related to continuous actions. /// /// Number of Continuous Actions - public void SetContinuous(int numContinuousActions) + internal void SetContinuous(int numContinuousActions) { m_NumContinuousActions = numContinuousActions; } @@ -84,7 +84,7 @@ public void SetContinuous(int numContinuousActions) /// Number of Discrete Actions /// The array of branch sizes for the discrete action space. Each index /// contains the number of actions available for that branch. - public void SetDiscrete(int[] branchSizes) + internal void SetDiscrete(int[] branchSizes) { m_BranchSizes = branchSizes; } From 2ae9473c806c97db3c6b926079ad974fb46de056 Mon Sep 17 00:00:00 2001 From: Ruo-Ping Dong Date: Wed, 9 Dec 2020 14:49:08 -0800 Subject: [PATCH 17/36] fix prev action check --- .../Runtime/Inference/BarracudaModelParamLoader.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/com.unity.ml-agents/Runtime/Inference/BarracudaModelParamLoader.cs b/com.unity.ml-agents/Runtime/Inference/BarracudaModelParamLoader.cs index 19bd3dce6a..db20edbee0 100644 --- a/com.unity.ml-agents/Runtime/Inference/BarracudaModelParamLoader.cs +++ b/com.unity.ml-agents/Runtime/Inference/BarracudaModelParamLoader.cs @@ -397,8 +397,7 @@ static string CheckPreviousActionShape( BrainParameters brainParameters, TensorProxy tensorProxy, SensorComponent[] sensorComponents, int observableAttributeTotalSize) { - var numberActionsBp = brainParameters.VectorActionSpec.NumContinuousActions + - brainParameters.VectorActionSpec.NumDiscreteActions; + var numberActionsBp = brainParameters.VectorActionSpec.NumDiscreteActions; var numberActionsT = tensorProxy.shape[tensorProxy.shape.Length - 1]; if (numberActionsBp != numberActionsT) { From d53a9fda2af3ec6dd71b30d36a44a2f78f5a7444 Mon Sep 17 00:00:00 2001 From: Ruo-Ping Dong Date: Wed, 9 Dec 2020 14:55:44 -0800 Subject: [PATCH 18/36] fix tests --- com.unity.ml-agents/Runtime/Actuators/ActionSpec.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/com.unity.ml-agents/Runtime/Actuators/ActionSpec.cs b/com.unity.ml-agents/Runtime/Actuators/ActionSpec.cs index 77b3f33ad6..d881e9c13e 100644 --- a/com.unity.ml-agents/Runtime/Actuators/ActionSpec.cs +++ b/com.unity.ml-agents/Runtime/Actuators/ActionSpec.cs @@ -36,13 +36,13 @@ public struct ActionSpec /// /// The number of branches for a Discrete . /// - public int NumDiscreteActions { get { return m_BranchSizes.Length; } } + public int NumDiscreteActions { get { return m_BranchSizes == null ? 0 : m_BranchSizes.Length; } } /// /// Get the total number of Discrete Actions that can be taken by calculating the Sum /// of all of the Discrete Action branch sizes. /// - public int SumOfDiscreteBranchSizes { get { return m_BranchSizes.Sum(); } } + public int SumOfDiscreteBranchSizes { get { return m_BranchSizes == null ? 0 : m_BranchSizes.Sum(); } } /// /// Creates a Continuous with the number of actions available. From ddde65303e890ba44e85569016bea593c91d2003 Mon Sep 17 00:00:00 2001 From: Ruo-Ping Dong Date: Wed, 9 Dec 2020 15:05:05 -0800 Subject: [PATCH 19/36] rename VectorActionSpec to ActionSpec --- .../Editor/BrainParametersDrawer.cs | 2 +- .../Editor/DemonstrationDrawer.cs | 4 ++-- com.unity.ml-agents/Runtime/Agent.cs | 2 +- .../Runtime/Communicator/GrpcExtensions.cs | 4 ++-- .../Inference/BarracudaModelParamLoader.cs | 6 +++--- .../Runtime/Policies/BrainParameters.cs | 20 +++++++++---------- .../Tests/Editor/DemonstrationTests.cs | 4 ++-- .../Tests/Editor/MLAgentsEditModeTest.cs | 18 ++++++++--------- .../Tests/Editor/ParameterLoaderTest.cs | 18 ++++++++--------- .../Tests/Runtime/RuntimeAPITest.cs | 2 +- 10 files changed, 40 insertions(+), 40 deletions(-) diff --git a/com.unity.ml-agents/Editor/BrainParametersDrawer.cs b/com.unity.ml-agents/Editor/BrainParametersDrawer.cs index a0ef0cd212..68c89a3043 100644 --- a/com.unity.ml-agents/Editor/BrainParametersDrawer.cs +++ b/com.unity.ml-agents/Editor/BrainParametersDrawer.cs @@ -14,7 +14,7 @@ internal class BrainParametersDrawer : PropertyDrawer // The height of a line in the Unity Inspectors const float k_LineHeight = 17f; const int k_VecObsNumLine = 3; - const string k_ActionSpecName = "VectorActionSpec"; + const string k_ActionSpecName = "ActionSpec"; const string k_ContinuousActionSizeName = "m_NumContinuousActions"; const string k_DiscreteBranchSizeName = "m_BranchSizes"; const string k_ActionDescriptionPropName = "VectorActionDescriptions"; diff --git a/com.unity.ml-agents/Editor/DemonstrationDrawer.cs b/com.unity.ml-agents/Editor/DemonstrationDrawer.cs index 4fca85ac6d..528ea7be90 100644 --- a/com.unity.ml-agents/Editor/DemonstrationDrawer.cs +++ b/com.unity.ml-agents/Editor/DemonstrationDrawer.cs @@ -24,7 +24,7 @@ internal class DemonstrationEditor : UnityEditor.Editor const string k_NumberStepsName = "numberSteps"; const string k_NumberEpisodesName = "numberEpisodes"; const string k_MeanRewardName = "meanReward"; - const string k_VectorActionSpecName = "VectorActionSpec"; + const string k_ActionSpecName = "ActionSpec"; const string k_NumContinuousActionsName = "m_NumContinuousActions"; const string k_NumDiscreteActionsName = "m_NumDiscreteActions"; const string k_ShapeName = "shape"; @@ -84,7 +84,7 @@ static string BuildIntArrayLabel(SerializedProperty actionSizeProperty) /// void MakeActionsProperty(SerializedProperty property) { - var actSpecProperty = property.FindPropertyRelative(k_VectorActionSpecName); + var actSpecProperty = property.FindPropertyRelative(k_ActionSpecName); var continuousSizeProperty = actSpecProperty.FindPropertyRelative(k_NumContinuousActionsName); var discreteSizeProperty = actSpecProperty.FindPropertyRelative(k_NumDiscreteActionsName); diff --git a/com.unity.ml-agents/Runtime/Agent.cs b/com.unity.ml-agents/Runtime/Agent.cs index 7622a76e86..8fd1597adb 100644 --- a/com.unity.ml-agents/Runtime/Agent.cs +++ b/com.unity.ml-agents/Runtime/Agent.cs @@ -988,7 +988,7 @@ void InitializeActuators() // Support legacy OnActionReceived // TODO don't set this up if the sizes are 0? var param = m_PolicyFactory.BrainParameters; - m_VectorActuator = new VectorActuator(this, param.VectorActionSpec); + m_VectorActuator = new VectorActuator(this, param.ActionSpec); m_ActuatorManager = new ActuatorManager(attachedActuators.Length + 1); m_LegacyActionCache = new float[m_VectorActuator.TotalNumberOfActions()]; diff --git a/com.unity.ml-agents/Runtime/Communicator/GrpcExtensions.cs b/com.unity.ml-agents/Runtime/Communicator/GrpcExtensions.cs index 4be6d20f1c..a7f64446d0 100644 --- a/com.unity.ml-agents/Runtime/Communicator/GrpcExtensions.cs +++ b/com.unity.ml-agents/Runtime/Communicator/GrpcExtensions.cs @@ -107,7 +107,7 @@ public static BrainParametersProto ToProto(this BrainParameters bp, string name, VectorActionSpaceTypeDeprecated = (SpaceTypeProto)bp.VectorActionSpaceTypeDeprecated, BrainName = name, IsTraining = isTraining, - ActionSpec = ToActionSpecProto(bp.VectorActionSpec), + ActionSpec = ToActionSpecProto(bp.ActionSpec), }; if (bp.VectorActionDescriptions != null) { @@ -164,7 +164,7 @@ public static BrainParameters ToBrainParameters(this BrainParametersProto bpp) VectorActionSizeDeprecated = bpp.VectorActionSizeDeprecated.ToArray(), VectorActionDescriptions = bpp.VectorActionDescriptionsDeprecated.ToArray(), VectorActionSpaceTypeDeprecated = (SpaceType)bpp.VectorActionSpaceTypeDeprecated, - VectorActionSpec = ToActionSpec(bpp.ActionSpec), + ActionSpec = ToActionSpec(bpp.ActionSpec), }; return bp; } diff --git a/com.unity.ml-agents/Runtime/Inference/BarracudaModelParamLoader.cs b/com.unity.ml-agents/Runtime/Inference/BarracudaModelParamLoader.cs index db20edbee0..f3f2c63012 100644 --- a/com.unity.ml-agents/Runtime/Inference/BarracudaModelParamLoader.cs +++ b/com.unity.ml-agents/Runtime/Inference/BarracudaModelParamLoader.cs @@ -397,7 +397,7 @@ static string CheckPreviousActionShape( BrainParameters brainParameters, TensorProxy tensorProxy, SensorComponent[] sensorComponents, int observableAttributeTotalSize) { - var numberActionsBp = brainParameters.VectorActionSpec.NumDiscreteActions; + var numberActionsBp = brainParameters.ActionSpec.NumDiscreteActions; var numberActionsT = tensorProxy.shape[tensorProxy.shape.Length - 1]; if (numberActionsBp != numberActionsT) { @@ -477,7 +477,7 @@ static string CheckDiscreteActionOutputShape( BrainParameters brainParameters, ActuatorComponent[] actuatorComponents, int modelSumDiscreteBranchSizes) { // TODO: check each branch size instead of sum of branch sizes - var sumOfDiscreteBranchSizes = brainParameters.VectorActionSpec.SumOfDiscreteBranchSizes; + var sumOfDiscreteBranchSizes = brainParameters.ActionSpec.SumOfDiscreteBranchSizes; foreach (var actuatorComponent in actuatorComponents) { @@ -513,7 +513,7 @@ static string CheckDiscreteActionOutputShape( static string CheckContinuousActionOutputShape( BrainParameters brainParameters, ActuatorComponent[] actuatorComponents, int modelContinuousActionSize) { - var numContinuousActions = brainParameters.VectorActionSpec.NumContinuousActions; + var numContinuousActions = brainParameters.ActionSpec.NumContinuousActions; foreach (var actuatorComponent in actuatorComponents) { diff --git a/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs b/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs index 626de95e4d..d4d6de65bf 100644 --- a/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs +++ b/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs @@ -55,7 +55,7 @@ public class BrainParameters : ISerializationCallbackReceiver /// /// The specification of the Action space for the BrainParameters. /// - public ActionSpec VectorActionSpec = new ActionSpec(0, new int[] { }); + public ActionSpec ActionSpec = new ActionSpec(0, new int[] { }); /// /// (Deprecated) The size of the action space. @@ -120,7 +120,7 @@ public BrainParameters Clone() VectorActionSizeDeprecated = (int[])VectorActionSizeDeprecated.Clone(), VectorActionDescriptions = (string[])VectorActionDescriptions.Clone(), VectorActionSpaceTypeDeprecated = VectorActionSpaceTypeDeprecated, - VectorActionSpec = new ActionSpec(VectorActionSpec.NumContinuousActions, VectorActionSpec.BranchSizes) + ActionSpec = new ActionSpec(ActionSpec.NumContinuousActions, ActionSpec.BranchSizes) }; } @@ -148,13 +148,13 @@ public void OnBeforeSerialize() { if (!hasUpgradedBrainParametersWithActionSpec) { - if (VectorActionSpec.NumContinuousActions == 0 && VectorActionSpaceTypeDeprecated == SpaceType.Continuous) + if (ActionSpec.NumContinuousActions == 0 && VectorActionSpaceTypeDeprecated == SpaceType.Continuous) { - VectorActionSpec.SetContinuous(VectorActionSizeDeprecated[0]); + ActionSpec.SetContinuous(VectorActionSizeDeprecated[0]); } - if (VectorActionSpec.NumDiscreteActions == 0 && VectorActionSpaceTypeDeprecated == SpaceType.Discrete) + if (ActionSpec.NumDiscreteActions == 0 && VectorActionSpaceTypeDeprecated == SpaceType.Discrete) { - VectorActionSpec.SetDiscrete(VectorActionSizeDeprecated); + ActionSpec.SetDiscrete(VectorActionSizeDeprecated); } hasUpgradedBrainParametersWithActionSpec = true; } @@ -184,13 +184,13 @@ public void OnAfterDeserialize() { if (!hasUpgradedBrainParametersWithActionSpec) { - if (VectorActionSpec.NumContinuousActions == 0 && VectorActionSpaceTypeDeprecated == SpaceType.Continuous) + if (ActionSpec.NumContinuousActions == 0 && VectorActionSpaceTypeDeprecated == SpaceType.Continuous) { - VectorActionSpec.SetContinuous(VectorActionSizeDeprecated[0]); + ActionSpec.SetContinuous(VectorActionSizeDeprecated[0]); } - if (VectorActionSpec.NumDiscreteActions == 0 && VectorActionSpaceTypeDeprecated == SpaceType.Discrete) + if (ActionSpec.NumDiscreteActions == 0 && VectorActionSpaceTypeDeprecated == SpaceType.Discrete) { - VectorActionSpec.SetDiscrete(VectorActionSizeDeprecated); + ActionSpec.SetDiscrete(VectorActionSizeDeprecated); } hasUpgradedBrainParametersWithActionSpec = true; } diff --git a/com.unity.ml-agents/Tests/Editor/DemonstrationTests.cs b/com.unity.ml-agents/Tests/Editor/DemonstrationTests.cs index 72dabb97a0..caacb94b64 100644 --- a/com.unity.ml-agents/Tests/Editor/DemonstrationTests.cs +++ b/com.unity.ml-agents/Tests/Editor/DemonstrationTests.cs @@ -47,7 +47,7 @@ public void TestStoreInitialize() bp.BrainParameters.VectorObservationSize = 3; bp.BrainParameters.NumStackedVectorObservations = 2; bp.BrainParameters.VectorActionDescriptions = new[] { "TestActionA", "TestActionB" }; - bp.BrainParameters.VectorActionSpec = ActionSpec.MakeDiscrete(new[] { 2, 2 }); + bp.BrainParameters.ActionSpec = ActionSpec.MakeDiscrete(new[] { 2, 2 }); gameobj.AddComponent(); @@ -103,7 +103,7 @@ public void TestAgentWrite() bpA.BrainParameters.VectorObservationSize = 3; bpA.BrainParameters.NumStackedVectorObservations = 1; bpA.BrainParameters.VectorActionDescriptions = new[] { "TestActionA", "TestActionB" }; - bpA.BrainParameters.VectorActionSpec = ActionSpec.MakeDiscrete(new[] { 2, 2 }); + bpA.BrainParameters.ActionSpec = ActionSpec.MakeDiscrete(new[] { 2, 2 }); agentGo1.AddComponent(); var agent1 = agentGo1.GetComponent(); diff --git a/com.unity.ml-agents/Tests/Editor/MLAgentsEditModeTest.cs b/com.unity.ml-agents/Tests/Editor/MLAgentsEditModeTest.cs index d01f3b0d49..b7b70f1c81 100644 --- a/com.unity.ml-agents/Tests/Editor/MLAgentsEditModeTest.cs +++ b/com.unity.ml-agents/Tests/Editor/MLAgentsEditModeTest.cs @@ -340,12 +340,12 @@ public void TestAgent() { var agentGo1 = new GameObject("TestAgent"); var bp1 = agentGo1.AddComponent(); - bp1.BrainParameters.VectorActionSpec = ActionSpec.MakeContinuous(1); + bp1.BrainParameters.ActionSpec = ActionSpec.MakeContinuous(1); agentGo1.AddComponent(); var agent1 = agentGo1.GetComponent(); var agentGo2 = new GameObject("TestAgent"); var bp2 = agentGo2.AddComponent(); - bp2.BrainParameters.VectorActionSpec = ActionSpec.MakeContinuous(1); + bp2.BrainParameters.ActionSpec = ActionSpec.MakeContinuous(1); agentGo2.AddComponent(); var agent2 = agentGo2.GetComponent(); @@ -450,12 +450,12 @@ public void TestAgent() { var agentGo1 = new GameObject("TestAgent"); var bp1 = agentGo1.AddComponent(); - bp1.BrainParameters.VectorActionSpec = ActionSpec.MakeContinuous(1); + bp1.BrainParameters.ActionSpec = ActionSpec.MakeContinuous(1); agentGo1.AddComponent(); var agent1 = agentGo1.GetComponent(); var agentGo2 = new GameObject("TestAgent"); var bp2 = agentGo2.AddComponent(); - bp2.BrainParameters.VectorActionSpec = ActionSpec.MakeContinuous(1); + bp2.BrainParameters.ActionSpec = ActionSpec.MakeContinuous(1); agentGo2.AddComponent(); var agent2 = agentGo2.GetComponent(); @@ -534,7 +534,7 @@ public void AssertStackingReset() { var agentGo1 = new GameObject("TestAgent"); var bp1 = agentGo1.AddComponent(); - bp1.BrainParameters.VectorActionSpec = ActionSpec.MakeContinuous(1); + bp1.BrainParameters.ActionSpec = ActionSpec.MakeContinuous(1); var agent1 = agentGo1.AddComponent(); var behaviorParameters = agentGo1.GetComponent(); behaviorParameters.BrainParameters.NumStackedVectorObservations = 3; @@ -583,11 +583,11 @@ public void TestCumulativeReward() { var agentGo1 = new GameObject("TestAgent"); var bp1 = agentGo1.AddComponent(); - bp1.BrainParameters.VectorActionSpec = ActionSpec.MakeContinuous(1); + bp1.BrainParameters.ActionSpec = ActionSpec.MakeContinuous(1); var agent1 = agentGo1.AddComponent(); var agentGo2 = new GameObject("TestAgent"); var bp2 = agentGo2.AddComponent(); - bp2.BrainParameters.VectorActionSpec = ActionSpec.MakeContinuous(1); + bp2.BrainParameters.ActionSpec = ActionSpec.MakeContinuous(1); var agent2 = agentGo2.AddComponent(); var aca = Academy.Instance; @@ -625,7 +625,7 @@ public void TestMaxStepsReset() { var agentGo1 = new GameObject("TestAgent"); var bp1 = agentGo1.AddComponent(); - bp1.BrainParameters.VectorActionSpec = ActionSpec.MakeContinuous(1); + bp1.BrainParameters.ActionSpec = ActionSpec.MakeContinuous(1); agentGo1.AddComponent(); var agent1 = agentGo1.GetComponent(); var aca = Academy.Instance; @@ -690,7 +690,7 @@ public void TestHeuristicPolicyStepsSensors() // Make sure that Agents with HeuristicPolicies step their sensors each Academy step. var agentGo1 = new GameObject("TestAgent"); var bp1 = agentGo1.AddComponent(); - bp1.BrainParameters.VectorActionSpec = ActionSpec.MakeContinuous(1); + bp1.BrainParameters.ActionSpec = ActionSpec.MakeContinuous(1); agentGo1.AddComponent(); var agent1 = agentGo1.GetComponent(); var aca = Academy.Instance; diff --git a/com.unity.ml-agents/Tests/Editor/ParameterLoaderTest.cs b/com.unity.ml-agents/Tests/Editor/ParameterLoaderTest.cs index 02c0cf2970..3e50a2f8fa 100644 --- a/com.unity.ml-agents/Tests/Editor/ParameterLoaderTest.cs +++ b/com.unity.ml-agents/Tests/Editor/ParameterLoaderTest.cs @@ -96,7 +96,7 @@ BrainParameters GetContinuous2vis8vec2actionBrainParameters() var validBrainParameters = new BrainParameters(); validBrainParameters.VectorObservationSize = 8; validBrainParameters.NumStackedVectorObservations = 1; - validBrainParameters.VectorActionSpec = ActionSpec.MakeContinuous(2); + validBrainParameters.ActionSpec = ActionSpec.MakeContinuous(2); return validBrainParameters; } @@ -105,7 +105,7 @@ BrainParameters GetDiscrete1vis0vec_2_3action_recurrModelBrainParameters() var validBrainParameters = new BrainParameters(); validBrainParameters.VectorObservationSize = 0; validBrainParameters.NumStackedVectorObservations = 1; - validBrainParameters.VectorActionSpec = ActionSpec.MakeDiscrete(new[] { 2, 3 }); + validBrainParameters.ActionSpec = ActionSpec.MakeDiscrete(new[] { 2, 3 }); return validBrainParameters; } @@ -114,7 +114,7 @@ BrainParameters GetHybridBrainParameters() var validBrainParameters = new BrainParameters(); validBrainParameters.VectorObservationSize = 53; validBrainParameters.NumStackedVectorObservations = 1; - validBrainParameters.VectorActionSpec = new ActionSpec(3, new int[] { 2 }); + validBrainParameters.ActionSpec = new ActionSpec(3, new int[] { 2 }); return validBrainParameters; } @@ -327,12 +327,12 @@ public void TestCheckModelThrowsActionContinuous(bool useDeprecatedNNModel) var model = useDeprecatedNNModel ? ModelLoader.Load(continuousNNModel) : ModelLoader.Load(continuousONNXModel); var brainParameters = GetContinuous2vis8vec2actionBrainParameters(); - brainParameters.VectorActionSpec.SetContinuous(3); // Invalid action + brainParameters.ActionSpec.SetContinuous(3); // Invalid action var errors = BarracudaModelParamLoader.CheckModel(model, brainParameters, new SensorComponent[] { sensor_21_20_3, sensor_20_22_3 }, new ActuatorComponent[0]); Assert.Greater(errors.Count(), 0); brainParameters = GetContinuous2vis8vec2actionBrainParameters(); - brainParameters.VectorActionSpec = ActionSpec.MakeDiscrete(new int[] { 3 }); // Invalid SpaceType + brainParameters.ActionSpec = ActionSpec.MakeDiscrete(new int[] { 3 }); // Invalid SpaceType errors = BarracudaModelParamLoader.CheckModel(model, brainParameters, new SensorComponent[] { sensor_21_20_3, sensor_20_22_3 }, new ActuatorComponent[0]); Assert.Greater(errors.Count(), 0); } @@ -344,12 +344,12 @@ public void TestCheckModelThrowsActionDiscrete(bool useDeprecatedNNModel) var model = useDeprecatedNNModel ? ModelLoader.Load(discreteNNModel) : ModelLoader.Load(discreteONNXModel); var brainParameters = GetDiscrete1vis0vec_2_3action_recurrModelBrainParameters(); - brainParameters.VectorActionSpec = ActionSpec.MakeDiscrete(new[] { 3, 3 }); // Invalid action + brainParameters.ActionSpec = ActionSpec.MakeDiscrete(new[] { 3, 3 }); // Invalid action var errors = BarracudaModelParamLoader.CheckModel(model, brainParameters, new SensorComponent[] { sensor_21_20_3 }, new ActuatorComponent[0]); Assert.Greater(errors.Count(), 0); brainParameters = GetContinuous2vis8vec2actionBrainParameters(); - brainParameters.VectorActionSpec = ActionSpec.MakeContinuous(2); // Invalid SpaceType + brainParameters.ActionSpec = ActionSpec.MakeContinuous(2); // Invalid SpaceType errors = BarracudaModelParamLoader.CheckModel(model, brainParameters, new SensorComponent[] { sensor_21_20_3 }, new ActuatorComponent[0]); Assert.Greater(errors.Count(), 0); } @@ -360,12 +360,12 @@ public void TestCheckModelThrowsActionHybrid() var model = ModelLoader.Load(hybridONNXModel); var brainParameters = GetHybridBrainParameters(); - brainParameters.VectorActionSpec = new ActionSpec(3, new int[] { 3 }); ; // Invalid discrete action size + brainParameters.ActionSpec = new ActionSpec(3, new int[] { 3 }); ; // Invalid discrete action size var errors = BarracudaModelParamLoader.CheckModel(model, brainParameters, new SensorComponent[] { sensor_21_20_3, sensor_20_22_3 }, new ActuatorComponent[0]); Assert.Greater(errors.Count(), 0); brainParameters = GetContinuous2vis8vec2actionBrainParameters(); - brainParameters.VectorActionSpec = ActionSpec.MakeDiscrete(new int[] { 2 }); // Missing continuous action + brainParameters.ActionSpec = ActionSpec.MakeDiscrete(new int[] { 2 }); // Missing continuous action errors = BarracudaModelParamLoader.CheckModel(model, brainParameters, new SensorComponent[] { sensor_21_20_3, sensor_20_22_3 }, new ActuatorComponent[0]); Assert.Greater(errors.Count(), 0); } diff --git a/com.unity.ml-agents/Tests/Runtime/RuntimeAPITest.cs b/com.unity.ml-agents/Tests/Runtime/RuntimeAPITest.cs index 5d3e1f971d..f132fc54c6 100644 --- a/com.unity.ml-agents/Tests/Runtime/RuntimeAPITest.cs +++ b/com.unity.ml-agents/Tests/Runtime/RuntimeAPITest.cs @@ -72,7 +72,7 @@ public IEnumerator RuntimeApiTestWithEnumeratorPasses() behaviorParams.BrainParameters.VectorObservationSize = 3; behaviorParams.BrainParameters.NumStackedVectorObservations = 2; behaviorParams.BrainParameters.VectorActionDescriptions = new[] { "TestActionA", "TestActionB" }; - behaviorParams.BrainParameters.VectorActionSpec = ActionSpec.MakeDiscrete(new[] { 2, 2 }); + behaviorParams.BrainParameters.ActionSpec = ActionSpec.MakeDiscrete(new[] { 2, 2 }); behaviorParams.BehaviorName = "TestBehavior"; behaviorParams.TeamId = 42; behaviorParams.UseChildSensors = true; From b42a7403bc64a031f8e6af542436df152883538e Mon Sep 17 00:00:00 2001 From: Ruo-Ping Dong Date: Wed, 9 Dec 2020 15:09:58 -0800 Subject: [PATCH 20/36] remove unused import --- com.unity.ml-agents/Editor/DemonstrationDrawer.cs | 1 - com.unity.ml-agents/Runtime/Agent.cs | 1 - com.unity.ml-agents/Runtime/Policies/BrainParameters.cs | 1 - 3 files changed, 3 deletions(-) diff --git a/com.unity.ml-agents/Editor/DemonstrationDrawer.cs b/com.unity.ml-agents/Editor/DemonstrationDrawer.cs index 528ea7be90..e6e037a3a7 100644 --- a/com.unity.ml-agents/Editor/DemonstrationDrawer.cs +++ b/com.unity.ml-agents/Editor/DemonstrationDrawer.cs @@ -2,7 +2,6 @@ using System.Text; using UnityEditor; using Unity.MLAgents.Demonstrations; -using Unity.MLAgents.Policies; namespace Unity.MLAgents.Editor diff --git a/com.unity.ml-agents/Runtime/Agent.cs b/com.unity.ml-agents/Runtime/Agent.cs index 8fd1597adb..512aba370e 100644 --- a/com.unity.ml-agents/Runtime/Agent.cs +++ b/com.unity.ml-agents/Runtime/Agent.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.Collections.ObjectModel; -using System.Linq; using UnityEngine; using Unity.Barracuda; using Unity.MLAgents.Actuators; diff --git a/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs b/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs index d4d6de65bf..6934a96800 100644 --- a/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs +++ b/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs @@ -1,5 +1,4 @@ using System; -using System.Linq; using UnityEngine; using UnityEngine.Serialization; using Unity.MLAgents.Actuators; From f9336ebbe7428c6a2203cf2ee2e7bdab85e9ba39 Mon Sep 17 00:00:00 2001 From: Ruo-Ping Dong Date: Wed, 9 Dec 2020 15:18:02 -0800 Subject: [PATCH 21/36] fix bad merge --- ml-agents/mlagents/trainers/sac/optimizer_torch.py | 7 ++----- .../mlagents/trainers/tests/tensorflow/test_simple_rl.py | 6 ++++-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/ml-agents/mlagents/trainers/sac/optimizer_torch.py b/ml-agents/mlagents/trainers/sac/optimizer_torch.py index 81a7997df6..f5f89d30ec 100644 --- a/ml-agents/mlagents/trainers/sac/optimizer_torch.py +++ b/ml-agents/mlagents/trainers/sac/optimizer_torch.py @@ -159,10 +159,7 @@ def __init__(self, policy: TorchPolicy, trainer_params: TrainerSettings): requires_grad=True, ) _cont_log_ent_coef = torch.nn.Parameter( - torch.log( - torch.as_tensor([self.init_entcoef] * self._action_spec.continuous_size) - ), - requires_grad=True, + torch.log(torch.as_tensor([self.init_entcoef])), requires_grad=True ) self._log_ent_coef = TorchSACOptimizer.LogEntCoef( discrete=_disc_log_ent_coef, continuous=_cont_log_ent_coef @@ -426,7 +423,7 @@ def sac_entropy_loss( ) # We update all the _cont_ent_coef as one block entropy_loss += -1 * ModelUtils.masked_mean( - torch.mean(_cont_ent_coef) * target_current_diff, loss_masks + _cont_ent_coef * target_current_diff, loss_masks ) return entropy_loss diff --git a/ml-agents/mlagents/trainers/tests/tensorflow/test_simple_rl.py b/ml-agents/mlagents/trainers/tests/tensorflow/test_simple_rl.py index c397a2bd93..8ba24c50ba 100644 --- a/ml-agents/mlagents/trainers/tests/tensorflow/test_simple_rl.py +++ b/ml-agents/mlagents/trainers/tests/tensorflow/test_simple_rl.py @@ -184,7 +184,7 @@ def test_visual_advanced_ppo(vis_encode_type, num_visual): PPO_TF_CONFIG, hyperparameters=new_hyperparams, network_settings=new_networksettings, - max_steps=300, + max_steps=400, summary_freq=100, framework=FrameworkType.TENSORFLOW, ) @@ -218,7 +218,9 @@ def test_recurrent_ppo(action_sizes): @pytest.mark.parametrize("action_sizes", [(0, 1), (1, 0)]) def test_simple_sac(action_sizes): env = SimpleEnvironment([BRAIN_NAME], action_sizes=action_sizes) - config = attr.evolve(SAC_TF_CONFIG, framework=FrameworkType.TENSORFLOW) + config = attr.evolve( + SAC_TF_CONFIG, framework=FrameworkType.TENSORFLOW, max_steps=900 + ) _check_environment_trains(env, {BRAIN_NAME: config}) From 2d0f9dae6bc5759deb6a11a5d4c0eea756d785e6 Mon Sep 17 00:00:00 2001 From: Ruo-Ping Dong Date: Wed, 9 Dec 2020 15:34:38 -0800 Subject: [PATCH 22/36] pass multiple args instead of new array to makeDiscrete() --- .../Tests/Editor/Actuators/VectorActuatorTests.cs | 8 ++++---- com.unity.ml-agents/Tests/Editor/DemonstrationTests.cs | 4 ++-- .../Editor/EditModeTestInternalBrainTensorApplier.cs | 2 +- com.unity.ml-agents/Tests/Editor/ParameterLoaderTest.cs | 8 ++++---- com.unity.ml-agents/Tests/Runtime/RuntimeAPITest.cs | 2 +- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/com.unity.ml-agents/Tests/Editor/Actuators/VectorActuatorTests.cs b/com.unity.ml-agents/Tests/Editor/Actuators/VectorActuatorTests.cs index a5ca76156d..8129e830d0 100644 --- a/com.unity.ml-agents/Tests/Editor/Actuators/VectorActuatorTests.cs +++ b/com.unity.ml-agents/Tests/Editor/Actuators/VectorActuatorTests.cs @@ -32,7 +32,7 @@ public void WriteDiscreteActionMask(IDiscreteActionMask actionMask) public void TestConstruct() { var ar = new TestActionReceiver(); - var va = new VectorActuator(ar, ActionSpec.MakeDiscrete(new int[] { 1, 2, 3 }), "name"); + var va = new VectorActuator(ar, ActionSpec.MakeDiscrete(1, 2, 3), "name"); Assert.IsTrue(va.ActionSpec.NumDiscreteActions == 3); Assert.IsTrue(va.ActionSpec.SumOfDiscreteBranchSizes == 6); @@ -49,7 +49,7 @@ public void TestConstruct() public void TestOnActionReceived() { var ar = new TestActionReceiver(); - var va = new VectorActuator(ar, ActionSpec.MakeDiscrete(new int[] { 1, 2, 3 }), "name"); + var va = new VectorActuator(ar, ActionSpec.MakeDiscrete(1, 2, 3), "name"); var discreteActions = new[] { 0, 1, 1 }; var ab = new ActionBuffers(ActionSegment.Empty, @@ -67,7 +67,7 @@ public void TestOnActionReceived() public void TestResetData() { var ar = new TestActionReceiver(); - var va = new VectorActuator(ar, ActionSpec.MakeDiscrete(new int[] { 1, 2, 3 }), "name"); + var va = new VectorActuator(ar, ActionSpec.MakeDiscrete(1, 2, 3), "name"); var discreteActions = new[] { 0, 1, 1 }; var ab = new ActionBuffers(ActionSegment.Empty, @@ -80,7 +80,7 @@ public void TestResetData() public void TestWriteDiscreteActionMask() { var ar = new TestActionReceiver(); - var va = new VectorActuator(ar, ActionSpec.MakeDiscrete(new int[] { 1, 2, 3 }), "name"); + var va = new VectorActuator(ar, ActionSpec.MakeDiscrete(1, 2, 3), "name"); var bdam = new ActuatorDiscreteActionMask(new[] { va }, 6, 3); var groundTruthMask = new[] { false, true, false, false, true, true }; diff --git a/com.unity.ml-agents/Tests/Editor/DemonstrationTests.cs b/com.unity.ml-agents/Tests/Editor/DemonstrationTests.cs index caacb94b64..34911e478a 100644 --- a/com.unity.ml-agents/Tests/Editor/DemonstrationTests.cs +++ b/com.unity.ml-agents/Tests/Editor/DemonstrationTests.cs @@ -47,7 +47,7 @@ public void TestStoreInitialize() bp.BrainParameters.VectorObservationSize = 3; bp.BrainParameters.NumStackedVectorObservations = 2; bp.BrainParameters.VectorActionDescriptions = new[] { "TestActionA", "TestActionB" }; - bp.BrainParameters.ActionSpec = ActionSpec.MakeDiscrete(new[] { 2, 2 }); + bp.BrainParameters.ActionSpec = ActionSpec.MakeDiscrete(2, 2); gameobj.AddComponent(); @@ -103,7 +103,7 @@ public void TestAgentWrite() bpA.BrainParameters.VectorObservationSize = 3; bpA.BrainParameters.NumStackedVectorObservations = 1; bpA.BrainParameters.VectorActionDescriptions = new[] { "TestActionA", "TestActionB" }; - bpA.BrainParameters.ActionSpec = ActionSpec.MakeDiscrete(new[] { 2, 2 }); + bpA.BrainParameters.ActionSpec = ActionSpec.MakeDiscrete(2, 2); agentGo1.AddComponent(); var agent1 = agentGo1.GetComponent(); diff --git a/com.unity.ml-agents/Tests/Editor/EditModeTestInternalBrainTensorApplier.cs b/com.unity.ml-agents/Tests/Editor/EditModeTestInternalBrainTensorApplier.cs index a327e32956..e82c402505 100644 --- a/com.unity.ml-agents/Tests/Editor/EditModeTestInternalBrainTensorApplier.cs +++ b/com.unity.ml-agents/Tests/Editor/EditModeTestInternalBrainTensorApplier.cs @@ -54,7 +54,7 @@ public void ApplyContinuousActionOutput() [Test] public void ApplyDiscreteActionOutput() { - var actionSpec = ActionSpec.MakeDiscrete(new int[] { 2, 3 }); + var actionSpec = ActionSpec.MakeDiscrete(2, 3); var inputTensor = new TensorProxy() { shape = new long[] { 2, 5 }, diff --git a/com.unity.ml-agents/Tests/Editor/ParameterLoaderTest.cs b/com.unity.ml-agents/Tests/Editor/ParameterLoaderTest.cs index 3e50a2f8fa..ed772e1152 100644 --- a/com.unity.ml-agents/Tests/Editor/ParameterLoaderTest.cs +++ b/com.unity.ml-agents/Tests/Editor/ParameterLoaderTest.cs @@ -105,7 +105,7 @@ BrainParameters GetDiscrete1vis0vec_2_3action_recurrModelBrainParameters() var validBrainParameters = new BrainParameters(); validBrainParameters.VectorObservationSize = 0; validBrainParameters.NumStackedVectorObservations = 1; - validBrainParameters.ActionSpec = ActionSpec.MakeDiscrete(new[] { 2, 3 }); + validBrainParameters.ActionSpec = ActionSpec.MakeDiscrete(2, 3); return validBrainParameters; } @@ -332,7 +332,7 @@ public void TestCheckModelThrowsActionContinuous(bool useDeprecatedNNModel) Assert.Greater(errors.Count(), 0); brainParameters = GetContinuous2vis8vec2actionBrainParameters(); - brainParameters.ActionSpec = ActionSpec.MakeDiscrete(new int[] { 3 }); // Invalid SpaceType + brainParameters.ActionSpec = ActionSpec.MakeDiscrete(3); // Invalid SpaceType errors = BarracudaModelParamLoader.CheckModel(model, brainParameters, new SensorComponent[] { sensor_21_20_3, sensor_20_22_3 }, new ActuatorComponent[0]); Assert.Greater(errors.Count(), 0); } @@ -344,7 +344,7 @@ public void TestCheckModelThrowsActionDiscrete(bool useDeprecatedNNModel) var model = useDeprecatedNNModel ? ModelLoader.Load(discreteNNModel) : ModelLoader.Load(discreteONNXModel); var brainParameters = GetDiscrete1vis0vec_2_3action_recurrModelBrainParameters(); - brainParameters.ActionSpec = ActionSpec.MakeDiscrete(new[] { 3, 3 }); // Invalid action + brainParameters.ActionSpec = ActionSpec.MakeDiscrete(3, 3); // Invalid action var errors = BarracudaModelParamLoader.CheckModel(model, brainParameters, new SensorComponent[] { sensor_21_20_3 }, new ActuatorComponent[0]); Assert.Greater(errors.Count(), 0); @@ -365,7 +365,7 @@ public void TestCheckModelThrowsActionHybrid() Assert.Greater(errors.Count(), 0); brainParameters = GetContinuous2vis8vec2actionBrainParameters(); - brainParameters.ActionSpec = ActionSpec.MakeDiscrete(new int[] { 2 }); // Missing continuous action + brainParameters.ActionSpec = ActionSpec.MakeDiscrete(2); // Missing continuous action errors = BarracudaModelParamLoader.CheckModel(model, brainParameters, new SensorComponent[] { sensor_21_20_3, sensor_20_22_3 }, new ActuatorComponent[0]); Assert.Greater(errors.Count(), 0); } diff --git a/com.unity.ml-agents/Tests/Runtime/RuntimeAPITest.cs b/com.unity.ml-agents/Tests/Runtime/RuntimeAPITest.cs index f132fc54c6..ea13050b54 100644 --- a/com.unity.ml-agents/Tests/Runtime/RuntimeAPITest.cs +++ b/com.unity.ml-agents/Tests/Runtime/RuntimeAPITest.cs @@ -72,7 +72,7 @@ public IEnumerator RuntimeApiTestWithEnumeratorPasses() behaviorParams.BrainParameters.VectorObservationSize = 3; behaviorParams.BrainParameters.NumStackedVectorObservations = 2; behaviorParams.BrainParameters.VectorActionDescriptions = new[] { "TestActionA", "TestActionB" }; - behaviorParams.BrainParameters.ActionSpec = ActionSpec.MakeDiscrete(new[] { 2, 2 }); + behaviorParams.BrainParameters.ActionSpec = ActionSpec.MakeDiscrete(2, 2); behaviorParams.BehaviorName = "TestBehavior"; behaviorParams.TeamId = 42; behaviorParams.UseChildSensors = true; From ca778e365d4a0fa88ffb36235af79d4c71c10d0e Mon Sep 17 00:00:00 2001 From: Ruo-Ping Dong Date: Wed, 9 Dec 2020 15:51:49 -0800 Subject: [PATCH 23/36] change back public fields' name --- .../Runtime/Policies/BrainParameters.cs | 107 ++++++------------ 1 file changed, 37 insertions(+), 70 deletions(-) diff --git a/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs b/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs index 6934a96800..4237e44771 100644 --- a/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs +++ b/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs @@ -56,6 +56,11 @@ public class BrainParameters : ISerializationCallbackReceiver /// public ActionSpec ActionSpec = new ActionSpec(0, new int[] { }); + + [FormerlySerializedAs("vectorActionSize")] + [FormerlySerializedAs("VectorActionSize")] + internal int[] m_VectorActionSizeDeprecated = new[] { 1 }; + /// /// (Deprecated) The size of the action space. /// @@ -66,9 +71,11 @@ public class BrainParameters : ISerializationCallbackReceiver /// the action. /// For the discrete action space: the number of branches in the action space. /// - [FormerlySerializedAs("vectorActionSize")] - [FormerlySerializedAs("VectorActionSize")] - public int[] VectorActionSizeDeprecated = new[] { 1 }; + /// [Obsolete("VectorActionSize has been deprecated, please use ActionSpec instead.")] + public int[] VectorActionSize + { + get { return ActionSpec.NumContinuousActions > 0 ? new int[] { ActionSpec.NumContinuousActions } : ActionSpec.BranchSizes; } + } /// /// The list of strings describing what the actions correspond to. @@ -76,12 +83,18 @@ public class BrainParameters : ISerializationCallbackReceiver [FormerlySerializedAs("vectorActionDescriptions")] public string[] VectorActionDescriptions; + [FormerlySerializedAs("vectorActionSpaceType")] + [FormerlySerializedAs("VectorActionSpaceType")] + internal SpaceType m_ActionSpaceTypeDeprecated = SpaceType.Discrete; + /// /// (Deprecated) Defines if the action is discrete or continuous. /// - [FormerlySerializedAs("vectorActionSpaceType")] - [FormerlySerializedAs("VectorActionSpaceType")] - public SpaceType VectorActionSpaceTypeDeprecated = SpaceType.Discrete; + /// [Obsolete("VectorActionSpaceType has been deprecated, please use ActionSpec instead.")] + public SpaceType VectorActionSpaceType + { + get { return ActionSpec.NumContinuousActions > 0 ? SpaceType.Continuous : SpaceType.Discrete; } + } [SerializeField] [HideInInspector] @@ -90,19 +103,12 @@ public class BrainParameters : ISerializationCallbackReceiver /// /// (Deprecated) The number of actions specified by this Brain. /// - public int NumActionsDeprecated + /// [Obsolete("NumActions has been deprecated, please use ActionSpec instead.")] + public int NumActions { get { - switch (VectorActionSpaceTypeDeprecated) - { - case SpaceType.Discrete: - return VectorActionSizeDeprecated.Length; - case SpaceType.Continuous: - return VectorActionSizeDeprecated[0]; - default: - return 0; - } + return ActionSpec.NumContinuousActions > 0 ? ActionSpec.NumContinuousActions : ActionSpec.NumDiscreteActions; } } @@ -116,83 +122,44 @@ public BrainParameters Clone() { VectorObservationSize = VectorObservationSize, NumStackedVectorObservations = NumStackedVectorObservations, - VectorActionSizeDeprecated = (int[])VectorActionSizeDeprecated.Clone(), VectorActionDescriptions = (string[])VectorActionDescriptions.Clone(), - VectorActionSpaceTypeDeprecated = VectorActionSpaceTypeDeprecated, ActionSpec = new ActionSpec(ActionSpec.NumContinuousActions, ActionSpec.BranchSizes) }; } /// - /// Called by Unity immediately before serializing this object. + /// Propogate ActionSpec fields from deprecated fields /// - /// - /// The BrainParameter class uses OnBeforeSerialize() for internal housekeeping. Call the - /// base class implementation if you need your own custom serialization logic. - /// - /// See [OnBeforeSerialize] for more information. - /// - /// [OnBeforeSerialize]: https://docs.unity3d.com/ScriptReference/ISerializationCallbackReceiver.OnAfterDeserialize.html - /// - /// - /// - /// public new void OnBeforeSerialize() - /// { - /// base.OnBeforeSerialize(); - /// // additional serialization logic... - /// } - /// - /// - public void OnBeforeSerialize() + private void UpdateToActionSpec() { if (!hasUpgradedBrainParametersWithActionSpec) { - if (ActionSpec.NumContinuousActions == 0 && VectorActionSpaceTypeDeprecated == SpaceType.Continuous) + if (ActionSpec.NumContinuousActions == 0 && m_ActionSpaceTypeDeprecated == SpaceType.Continuous) { - ActionSpec.SetContinuous(VectorActionSizeDeprecated[0]); + ActionSpec.SetContinuous(m_VectorActionSizeDeprecated[0]); } - if (ActionSpec.NumDiscreteActions == 0 && VectorActionSpaceTypeDeprecated == SpaceType.Discrete) + if (ActionSpec.NumDiscreteActions == 0 && m_ActionSpaceTypeDeprecated == SpaceType.Discrete) { - ActionSpec.SetDiscrete(VectorActionSizeDeprecated); + ActionSpec.SetDiscrete(m_VectorActionSizeDeprecated); } hasUpgradedBrainParametersWithActionSpec = true; } } + /// + /// Called by Unity immediately before serializing this object. + /// + public void OnBeforeSerialize() + { + UpdateToActionSpec(); + } + /// /// Called by Unity immediately after deserializing this object. /// - /// - /// The BrainParameter class uses OnAfterDeserialize() for internal housekeeping. Call the - /// base class implementation if you need your own custom deserialization logic. - /// - /// See [OnAfterDeserialize] for more information. - /// - /// [OnAfterDeserialize]: https://docs.unity3d.com/ScriptReference/ISerializationCallbackReceiver.OnAfterDeserialize.html - /// - /// - /// - /// public new void OnAfterDeserialize() - /// { - /// base.OnAfterDeserialize(); - /// // additional deserialization logic... - /// } - /// - /// public void OnAfterDeserialize() { - if (!hasUpgradedBrainParametersWithActionSpec) - { - if (ActionSpec.NumContinuousActions == 0 && VectorActionSpaceTypeDeprecated == SpaceType.Continuous) - { - ActionSpec.SetContinuous(VectorActionSizeDeprecated[0]); - } - if (ActionSpec.NumDiscreteActions == 0 && VectorActionSpaceTypeDeprecated == SpaceType.Discrete) - { - ActionSpec.SetDiscrete(VectorActionSizeDeprecated); - } - hasUpgradedBrainParametersWithActionSpec = true; - } + UpdateToActionSpec(); } } } From 8e9977b86f157c6088fe4fecbc6e35cf0fb56870 Mon Sep 17 00:00:00 2001 From: Ruo-Ping Dong Date: Wed, 9 Dec 2020 15:55:11 -0800 Subject: [PATCH 24/36] change back public fields' name --- com.unity.ml-agents/Runtime/Agent.cs | 2 +- com.unity.ml-agents/Runtime/Communicator/GrpcExtensions.cs | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/com.unity.ml-agents/Runtime/Agent.cs b/com.unity.ml-agents/Runtime/Agent.cs index 512aba370e..7f78726e22 100644 --- a/com.unity.ml-agents/Runtime/Agent.cs +++ b/com.unity.ml-agents/Runtime/Agent.cs @@ -887,7 +887,7 @@ public virtual void Initialize() { } public virtual void Heuristic(in ActionBuffers actionsOut) { // For backward compatibility - switch (m_PolicyFactory.BrainParameters.VectorActionSpaceTypeDeprecated) + switch (m_PolicyFactory.BrainParameters.VectorActionSpaceType) { case SpaceType.Continuous: Heuristic(actionsOut.ContinuousActions.Array); diff --git a/com.unity.ml-agents/Runtime/Communicator/GrpcExtensions.cs b/com.unity.ml-agents/Runtime/Communicator/GrpcExtensions.cs index a7f64446d0..e8b116c751 100644 --- a/com.unity.ml-agents/Runtime/Communicator/GrpcExtensions.cs +++ b/com.unity.ml-agents/Runtime/Communicator/GrpcExtensions.cs @@ -103,8 +103,8 @@ public static BrainParametersProto ToProto(this BrainParameters bp, string name, { var brainParametersProto = new BrainParametersProto { - VectorActionSizeDeprecated = { bp.VectorActionSizeDeprecated }, - VectorActionSpaceTypeDeprecated = (SpaceTypeProto)bp.VectorActionSpaceTypeDeprecated, + VectorActionSizeDeprecated = { bp.VectorActionSize }, + VectorActionSpaceTypeDeprecated = (SpaceTypeProto)bp.VectorActionSpaceType, BrainName = name, IsTraining = isTraining, ActionSpec = ToActionSpecProto(bp.ActionSpec), @@ -161,9 +161,7 @@ public static BrainParameters ToBrainParameters(this BrainParametersProto bpp) { var bp = new BrainParameters { - VectorActionSizeDeprecated = bpp.VectorActionSizeDeprecated.ToArray(), VectorActionDescriptions = bpp.VectorActionDescriptionsDeprecated.ToArray(), - VectorActionSpaceTypeDeprecated = (SpaceType)bpp.VectorActionSpaceTypeDeprecated, ActionSpec = ToActionSpec(bpp.ActionSpec), }; return bp; From a5b7302b75df0a5c2a2659c98f76ff6b43665ee8 Mon Sep 17 00:00:00 2001 From: Ruo-Ping Dong Date: Wed, 9 Dec 2020 17:03:42 -0800 Subject: [PATCH 25/36] fix clone brainParameter --- .../Runtime/Policies/BrainParameters.cs | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs b/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs index 4237e44771..590fa7ecc1 100644 --- a/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs +++ b/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs @@ -123,7 +123,9 @@ public BrainParameters Clone() VectorObservationSize = VectorObservationSize, NumStackedVectorObservations = NumStackedVectorObservations, VectorActionDescriptions = (string[])VectorActionDescriptions.Clone(), - ActionSpec = new ActionSpec(ActionSpec.NumContinuousActions, ActionSpec.BranchSizes) + ActionSpec = new ActionSpec(ActionSpec.NumContinuousActions, ActionSpec.BranchSizes), + m_VectorActionSizeDeprecated = (int[])m_VectorActionSizeDeprecated.Clone(), + m_ActionSpaceTypeDeprecated = m_ActionSpaceTypeDeprecated, }; } @@ -134,13 +136,16 @@ private void UpdateToActionSpec() { if (!hasUpgradedBrainParametersWithActionSpec) { - if (ActionSpec.NumContinuousActions == 0 && m_ActionSpaceTypeDeprecated == SpaceType.Continuous) + if (ActionSpec.NumContinuousActions == 0 && ActionSpec.NumDiscreteActions == 0) { - ActionSpec.SetContinuous(m_VectorActionSizeDeprecated[0]); - } - if (ActionSpec.NumDiscreteActions == 0 && m_ActionSpaceTypeDeprecated == SpaceType.Discrete) - { - ActionSpec.SetDiscrete(m_VectorActionSizeDeprecated); + if (ActionSpec.NumContinuousActions == 0 && m_ActionSpaceTypeDeprecated == SpaceType.Continuous) + { + ActionSpec = ActionSpec.MakeContinuous(m_VectorActionSizeDeprecated[0]); + } + if (ActionSpec.NumDiscreteActions == 0 && m_ActionSpaceTypeDeprecated == SpaceType.Discrete) + { + ActionSpec = ActionSpec.MakeDiscrete(m_VectorActionSizeDeprecated); + } } hasUpgradedBrainParametersWithActionSpec = true; } From ede61b2fcc5d605b449a65382c8614eaa5d15c42 Mon Sep 17 00:00:00 2001 From: Ruo-Ping Dong Date: Wed, 9 Dec 2020 17:10:35 -0800 Subject: [PATCH 26/36] remove setContinuous and setDiscrete --- .../Runtime/Actuators/ActionSpec.cs | 20 ------------------- .../Tests/Editor/ParameterLoaderTest.cs | 2 +- 2 files changed, 1 insertion(+), 21 deletions(-) diff --git a/com.unity.ml-agents/Runtime/Actuators/ActionSpec.cs b/com.unity.ml-agents/Runtime/Actuators/ActionSpec.cs index d881e9c13e..b3b48aacac 100644 --- a/com.unity.ml-agents/Runtime/Actuators/ActionSpec.cs +++ b/com.unity.ml-agents/Runtime/Actuators/ActionSpec.cs @@ -69,26 +69,6 @@ public static ActionSpec MakeDiscrete(params int[] branchSizes) return actuatorSpace; } - /// - /// Set action size fields related to continuous actions. - /// - /// Number of Continuous Actions - internal void SetContinuous(int numContinuousActions) - { - m_NumContinuousActions = numContinuousActions; - } - - /// - /// Set action size fields related to discrete actions. - /// - /// Number of Discrete Actions - /// The array of branch sizes for the discrete action space. Each index - /// contains the number of actions available for that branch. - internal void SetDiscrete(int[] branchSizes) - { - m_BranchSizes = branchSizes; - } - internal ActionSpec(int numContinuousActions, int[] branchSizes = null) { m_NumContinuousActions = numContinuousActions; diff --git a/com.unity.ml-agents/Tests/Editor/ParameterLoaderTest.cs b/com.unity.ml-agents/Tests/Editor/ParameterLoaderTest.cs index ed772e1152..ebeb3c5284 100644 --- a/com.unity.ml-agents/Tests/Editor/ParameterLoaderTest.cs +++ b/com.unity.ml-agents/Tests/Editor/ParameterLoaderTest.cs @@ -327,7 +327,7 @@ public void TestCheckModelThrowsActionContinuous(bool useDeprecatedNNModel) var model = useDeprecatedNNModel ? ModelLoader.Load(continuousNNModel) : ModelLoader.Load(continuousONNXModel); var brainParameters = GetContinuous2vis8vec2actionBrainParameters(); - brainParameters.ActionSpec.SetContinuous(3); // Invalid action + brainParameters.ActionSpec = ActionSpec.MakeContinuous(3); // Invalid action var errors = BarracudaModelParamLoader.CheckModel(model, brainParameters, new SensorComponent[] { sensor_21_20_3, sensor_20_22_3 }, new ActuatorComponent[0]); Assert.Greater(errors.Count(), 0); From 59285bdcf2f5b5181af56941a4209e09ba353fad Mon Sep 17 00:00:00 2001 From: Ruo-Ping Dong Date: Wed, 9 Dec 2020 17:22:05 -0800 Subject: [PATCH 27/36] update changelog --- com.unity.ml-agents/CHANGELOG.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/com.unity.ml-agents/CHANGELOG.md b/com.unity.ml-agents/CHANGELOG.md index 8732b2cf57..055723aab6 100755 --- a/com.unity.ml-agents/CHANGELOG.md +++ b/com.unity.ml-agents/CHANGELOG.md @@ -10,11 +10,10 @@ and this project adheres to ## [Unreleased] ### Major Changes #### com.unity.ml-agents (C#) +- Agent with both continuous and discrete actions is now supported. You can specify +continuous and discrete action sizes repectively in Behavior Parameters. #### ml-agents / ml-agents-envs / gym-unity (Python) - PyTorch trainers now support training agents with both continuous and discrete action spaces. -Currently, this can only be done with Actuators. Please see -[here](../Project/Assets/ML-Agents/Examples/Basic/Scripts/BasicActuatorComponent.cs) for an -example of how to use Actuators. (#4702) ### Minor Changes #### com.unity.ml-agents / com.unity.ml-agents.extensions (C#) From b24c013d9de9ada15e374a6b07413c566f6ced6d Mon Sep 17 00:00:00 2001 From: Ruo-Ping Dong Date: Wed, 9 Dec 2020 17:47:39 -0800 Subject: [PATCH 28/36] fix internal field serialization --- .../Runtime/Policies/BrainParameters.cs | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs b/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs index 590fa7ecc1..392fce3ed3 100644 --- a/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs +++ b/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs @@ -56,7 +56,7 @@ public class BrainParameters : ISerializationCallbackReceiver /// public ActionSpec ActionSpec = new ActionSpec(0, new int[] { }); - + [SerializeField] [FormerlySerializedAs("vectorActionSize")] [FormerlySerializedAs("VectorActionSize")] internal int[] m_VectorActionSizeDeprecated = new[] { 1 }; @@ -83,6 +83,7 @@ public int[] VectorActionSize [FormerlySerializedAs("vectorActionDescriptions")] public string[] VectorActionDescriptions; + [SerializeField] [FormerlySerializedAs("vectorActionSpaceType")] [FormerlySerializedAs("VectorActionSpaceType")] internal SpaceType m_ActionSpaceTypeDeprecated = SpaceType.Discrete; @@ -136,17 +137,16 @@ private void UpdateToActionSpec() { if (!hasUpgradedBrainParametersWithActionSpec) { - if (ActionSpec.NumContinuousActions == 0 && ActionSpec.NumDiscreteActions == 0) + + if (m_ActionSpaceTypeDeprecated == SpaceType.Continuous) { - if (ActionSpec.NumContinuousActions == 0 && m_ActionSpaceTypeDeprecated == SpaceType.Continuous) - { - ActionSpec = ActionSpec.MakeContinuous(m_VectorActionSizeDeprecated[0]); - } - if (ActionSpec.NumDiscreteActions == 0 && m_ActionSpaceTypeDeprecated == SpaceType.Discrete) - { - ActionSpec = ActionSpec.MakeDiscrete(m_VectorActionSizeDeprecated); - } + ActionSpec = ActionSpec.MakeContinuous(m_VectorActionSizeDeprecated[0]); } + if (m_ActionSpaceTypeDeprecated == SpaceType.Discrete) + { + ActionSpec = ActionSpec.MakeDiscrete(m_VectorActionSizeDeprecated); + } + hasUpgradedBrainParametersWithActionSpec = true; } } From 51587dee37fd5760204b547e40da1fe1395eca88 Mon Sep 17 00:00:00 2001 From: Ruo-Ping Dong Date: Thu, 10 Dec 2020 16:48:25 -0800 Subject: [PATCH 29/36] revert breaking change --- .../Editor/BrainParametersDrawer.cs | 2 +- .../Runtime/Actuators/ActionSpec.cs | 10 ++--- .../Runtime/Actuators/IActionReceiver.cs | 41 +++++++++++++++++++ .../Runtime/Communicator/GrpcExtensions.cs | 7 +++- .../Runtime/Policies/BrainParameters.cs | 35 +++++----------- 5 files changed, 61 insertions(+), 34 deletions(-) diff --git a/com.unity.ml-agents/Editor/BrainParametersDrawer.cs b/com.unity.ml-agents/Editor/BrainParametersDrawer.cs index 68c89a3043..969f5eda39 100644 --- a/com.unity.ml-agents/Editor/BrainParametersDrawer.cs +++ b/com.unity.ml-agents/Editor/BrainParametersDrawer.cs @@ -16,7 +16,7 @@ internal class BrainParametersDrawer : PropertyDrawer const int k_VecObsNumLine = 3; const string k_ActionSpecName = "ActionSpec"; const string k_ContinuousActionSizeName = "m_NumContinuousActions"; - const string k_DiscreteBranchSizeName = "m_BranchSizes"; + const string k_DiscreteBranchSizeName = "BranchSizes"; const string k_ActionDescriptionPropName = "VectorActionDescriptions"; const string k_VecObsPropName = "VectorObservationSize"; const string k_NumVecObsPropName = "NumStackedVectorObservations"; diff --git a/com.unity.ml-agents/Runtime/Actuators/ActionSpec.cs b/com.unity.ml-agents/Runtime/Actuators/ActionSpec.cs index b3b48aacac..c33dea2d22 100644 --- a/com.unity.ml-agents/Runtime/Actuators/ActionSpec.cs +++ b/com.unity.ml-agents/Runtime/Actuators/ActionSpec.cs @@ -11,8 +11,6 @@ namespace Unity.MLAgents.Actuators [Serializable] public struct ActionSpec { - [SerializeField] - int[] m_BranchSizes; [SerializeField] int m_NumContinuousActions; @@ -26,7 +24,7 @@ public struct ActionSpec /// /// For an IActuator with a Continuous it will be null. /// - public int[] BranchSizes { get { return m_BranchSizes; } set { m_BranchSizes = value; } } + public int[] BranchSizes; /// /// The number of actions for a Continuous . @@ -36,13 +34,13 @@ public struct ActionSpec /// /// The number of branches for a Discrete . /// - public int NumDiscreteActions { get { return m_BranchSizes == null ? 0 : m_BranchSizes.Length; } } + public int NumDiscreteActions { get { return BranchSizes == null ? 0 : BranchSizes.Length; } } /// /// Get the total number of Discrete Actions that can be taken by calculating the Sum /// of all of the Discrete Action branch sizes. /// - public int SumOfDiscreteBranchSizes { get { return m_BranchSizes == null ? 0 : m_BranchSizes.Sum(); } } + public int SumOfDiscreteBranchSizes { get { return BranchSizes == null ? 0 : BranchSizes.Sum(); } } /// /// Creates a Continuous with the number of actions available. @@ -72,7 +70,7 @@ public static ActionSpec MakeDiscrete(params int[] branchSizes) internal ActionSpec(int numContinuousActions, int[] branchSizes = null) { m_NumContinuousActions = numContinuousActions; - m_BranchSizes = branchSizes; + BranchSizes = branchSizes; } /// diff --git a/com.unity.ml-agents/Runtime/Actuators/IActionReceiver.cs b/com.unity.ml-agents/Runtime/Actuators/IActionReceiver.cs index 4abfb2b35f..f64100527e 100644 --- a/com.unity.ml-agents/Runtime/Actuators/IActionReceiver.cs +++ b/com.unity.ml-agents/Runtime/Actuators/IActionReceiver.cs @@ -147,6 +147,47 @@ public override int GetHashCode() return (ContinuousActions.GetHashCode() * 397) ^ DiscreteActions.GetHashCode(); } } + + /// + /// Packs the continuous and discrete actions into one float array. The array passed into this method + /// must have a Length that is greater than or equal to the sum of the Lengths of + /// and . + /// + /// A float array to pack actions into whose length is greater than or + /// equal to the addition of the Lengths of this objects and + /// segments. + /// [Obsolete("PackActions has been deprecated.")] + public void PackActions(in float[] destination) + { + Debug.Assert(destination.Length >= ContinuousActions.Length + DiscreteActions.Length, + $"argument '{nameof(destination)}' is not large enough to pack the actions into.\n" + + $"{nameof(destination)}.Length: {destination.Length}\n" + + $"{nameof(ContinuousActions)}.Length + {nameof(DiscreteActions)}.Length: {ContinuousActions.Length + DiscreteActions.Length}"); + + var start = 0; + if (ContinuousActions.Length > 0) + { + Array.Copy(ContinuousActions.Array, + ContinuousActions.Offset, + destination, + start, + ContinuousActions.Length); + start = ContinuousActions.Length; + } + if (start >= destination.Length) + { + return; + } + + if (DiscreteActions.Length > 0) + { + Array.Copy(DiscreteActions.Array, + DiscreteActions.Offset, + destination, + start, + DiscreteActions.Length); + } + } } /// diff --git a/com.unity.ml-agents/Runtime/Communicator/GrpcExtensions.cs b/com.unity.ml-agents/Runtime/Communicator/GrpcExtensions.cs index e8b116c751..ac64bb67a1 100644 --- a/com.unity.ml-agents/Runtime/Communicator/GrpcExtensions.cs +++ b/com.unity.ml-agents/Runtime/Communicator/GrpcExtensions.cs @@ -101,10 +101,12 @@ public static List GetObservationSummaries(this AgentInfoAct /// Whether or not the Brain is training. public static BrainParametersProto ToProto(this BrainParameters bp, string name, bool isTraining) { + var vectorActionSizeDeprecated = bp.ActionSpec.NumContinuousActions > 0 ? new int[] { bp.ActionSpec.NumContinuousActions } : bp.ActionSpec.BranchSizes; + var vectorActionSpaceTypeDeprecated = bp.ActionSpec.NumContinuousActions > 0 ? SpaceType.Continuous : SpaceType.Discrete; var brainParametersProto = new BrainParametersProto { - VectorActionSizeDeprecated = { bp.VectorActionSize }, - VectorActionSpaceTypeDeprecated = (SpaceTypeProto)bp.VectorActionSpaceType, + VectorActionSizeDeprecated = { vectorActionSizeDeprecated }, + VectorActionSpaceTypeDeprecated = (SpaceTypeProto)vectorActionSpaceTypeDeprecated, BrainName = name, IsTraining = isTraining, ActionSpec = ToActionSpecProto(bp.ActionSpec), @@ -200,6 +202,7 @@ public static ActionSpecProto ToActionSpecProto(this ActionSpec actionSpec) } return actionSpecProto; } + #endregion #region DemonstrationMetaData diff --git a/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs b/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs index 392fce3ed3..ce58732dc3 100644 --- a/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs +++ b/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs @@ -56,11 +56,6 @@ public class BrainParameters : ISerializationCallbackReceiver /// public ActionSpec ActionSpec = new ActionSpec(0, new int[] { }); - [SerializeField] - [FormerlySerializedAs("vectorActionSize")] - [FormerlySerializedAs("VectorActionSize")] - internal int[] m_VectorActionSizeDeprecated = new[] { 1 }; - /// /// (Deprecated) The size of the action space. /// @@ -72,10 +67,8 @@ public class BrainParameters : ISerializationCallbackReceiver /// For the discrete action space: the number of branches in the action space. /// /// [Obsolete("VectorActionSize has been deprecated, please use ActionSpec instead.")] - public int[] VectorActionSize - { - get { return ActionSpec.NumContinuousActions > 0 ? new int[] { ActionSpec.NumContinuousActions } : ActionSpec.BranchSizes; } - } + [FormerlySerializedAs("vectorActionSize")] + public int[] VectorActionSize = new[] { 1 }; /// /// The list of strings describing what the actions correspond to. @@ -83,19 +76,12 @@ public int[] VectorActionSize [FormerlySerializedAs("vectorActionDescriptions")] public string[] VectorActionDescriptions; - [SerializeField] - [FormerlySerializedAs("vectorActionSpaceType")] - [FormerlySerializedAs("VectorActionSpaceType")] - internal SpaceType m_ActionSpaceTypeDeprecated = SpaceType.Discrete; - /// /// (Deprecated) Defines if the action is discrete or continuous. /// /// [Obsolete("VectorActionSpaceType has been deprecated, please use ActionSpec instead.")] - public SpaceType VectorActionSpaceType - { - get { return ActionSpec.NumContinuousActions > 0 ? SpaceType.Continuous : SpaceType.Discrete; } - } + [FormerlySerializedAs("vectorActionSpaceType")] + public SpaceType VectorActionSpaceType = SpaceType.Discrete; [SerializeField] [HideInInspector] @@ -125,8 +111,8 @@ public BrainParameters Clone() NumStackedVectorObservations = NumStackedVectorObservations, VectorActionDescriptions = (string[])VectorActionDescriptions.Clone(), ActionSpec = new ActionSpec(ActionSpec.NumContinuousActions, ActionSpec.BranchSizes), - m_VectorActionSizeDeprecated = (int[])m_VectorActionSizeDeprecated.Clone(), - m_ActionSpaceTypeDeprecated = m_ActionSpaceTypeDeprecated, + VectorActionSize = (int[])VectorActionSize.Clone(), + VectorActionSpaceType = VectorActionSpaceType, }; } @@ -137,14 +123,13 @@ private void UpdateToActionSpec() { if (!hasUpgradedBrainParametersWithActionSpec) { - - if (m_ActionSpaceTypeDeprecated == SpaceType.Continuous) + if (VectorActionSpaceType == SpaceType.Continuous) { - ActionSpec = ActionSpec.MakeContinuous(m_VectorActionSizeDeprecated[0]); + ActionSpec = ActionSpec.MakeContinuous(VectorActionSize[0]); } - if (m_ActionSpaceTypeDeprecated == SpaceType.Discrete) + if (VectorActionSpaceType == SpaceType.Discrete) { - ActionSpec = ActionSpec.MakeDiscrete(m_VectorActionSizeDeprecated); + ActionSpec = ActionSpec.MakeDiscrete(VectorActionSize); } hasUpgradedBrainParametersWithActionSpec = true; From b6c62d844dd280ae44da46ea30d43c2b8edf8b1f Mon Sep 17 00:00:00 2001 From: Ruo-Ping Dong Date: Fri, 11 Dec 2020 12:51:45 -0800 Subject: [PATCH 30/36] sync deprecated fields --- .../Editor/BrainParametersDrawer.cs | 11 ++-- .../Runtime/Policies/BrainParameters.cs | 51 +++++++++++++++++-- 2 files changed, 52 insertions(+), 10 deletions(-) diff --git a/com.unity.ml-agents/Editor/BrainParametersDrawer.cs b/com.unity.ml-agents/Editor/BrainParametersDrawer.cs index 969f5eda39..9910188865 100644 --- a/com.unity.ml-agents/Editor/BrainParametersDrawer.cs +++ b/com.unity.ml-agents/Editor/BrainParametersDrawer.cs @@ -14,9 +14,8 @@ internal class BrainParametersDrawer : PropertyDrawer // The height of a line in the Unity Inspectors const float k_LineHeight = 17f; const int k_VecObsNumLine = 3; - const string k_ActionSpecName = "ActionSpec"; const string k_ContinuousActionSizeName = "m_NumContinuousActions"; - const string k_DiscreteBranchSizeName = "BranchSizes"; + const string k_DiscreteBranchSizeName = "m_DiscreteBranchSizes"; const string k_ActionDescriptionPropName = "VectorActionDescriptions"; const string k_VecObsPropName = "VectorObservationSize"; const string k_NumVecObsPropName = "NumStackedVectorObservations"; @@ -98,10 +97,9 @@ static void DrawVectorAction(Rect position, SerializedProperty property) EditorGUI.LabelField(position, "Vector Action"); position.y += k_LineHeight; EditorGUI.indentLevel++; - var actionSpecProperty = property.FindPropertyRelative(k_ActionSpecName); - DrawContinuousVectorAction(position, actionSpecProperty); + DrawContinuousVectorAction(position, property); position.y += k_LineHeight; - DrawDiscreteVectorAction(position, actionSpecProperty); + DrawDiscreteVectorAction(position, property); } /// @@ -164,8 +162,7 @@ static void DrawDiscreteVectorAction(Rect position, SerializedProperty property) /// The height of the drawer of the Vector Action. static float GetHeightDrawVectorAction(SerializedProperty property) { - var actionSpecProperty = property.FindPropertyRelative(k_ActionSpecName); - var numActionLines = 3 + actionSpecProperty.FindPropertyRelative(k_DiscreteBranchSizeName).arraySize; + var numActionLines = 3 + property.FindPropertyRelative(k_DiscreteBranchSizeName).arraySize; return numActionLines * k_LineHeight; } } diff --git a/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs b/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs index ce58732dc3..26d3a73b9c 100644 --- a/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs +++ b/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs @@ -51,10 +51,33 @@ public class BrainParameters : ISerializationCallbackReceiver [FormerlySerializedAs("numStackedVectorObservations")] [Range(1, 50)] public int NumStackedVectorObservations = 1; + [SerializeField] + internal int m_NumContinuousActions = 0; + [SerializeField] + internal int[] m_DiscreteBranchSizes = null; + /// /// The specification of the Action space for the BrainParameters. /// - public ActionSpec ActionSpec = new ActionSpec(0, new int[] { }); + public ActionSpec ActionSpec + { + get { return new ActionSpec(m_NumContinuousActions, m_DiscreteBranchSizes); } + set + { + m_NumContinuousActions = value.NumContinuousActions; + m_DiscreteBranchSizes = value.BranchSizes; + if (m_NumContinuousActions == 0) + { + VectorActionSize = value.BranchSizes; + VectorActionSpaceType = SpaceType.Discrete; + } + else if (m_DiscreteBranchSizes == null || m_DiscreteBranchSizes.Length == 0) + { + VectorActionSize = new[] { m_NumContinuousActions }; + VectorActionSpaceType = SpaceType.Continuous; + } + } + } /// /// (Deprecated) The size of the action space. @@ -125,23 +148,44 @@ private void UpdateToActionSpec() { if (VectorActionSpaceType == SpaceType.Continuous) { - ActionSpec = ActionSpec.MakeContinuous(VectorActionSize[0]); + m_NumContinuousActions = VectorActionSize[0]; + m_DiscreteBranchSizes = null; } if (VectorActionSpaceType == SpaceType.Discrete) { - ActionSpec = ActionSpec.MakeDiscrete(VectorActionSize); + m_NumContinuousActions = 0; + m_DiscreteBranchSizes = VectorActionSize; } hasUpgradedBrainParametersWithActionSpec = true; } } + private void SyncDeprecatedActionFields() + { + if (m_NumContinuousActions == 0) + { + VectorActionSize = ActionSpec.BranchSizes; + VectorActionSpaceType = SpaceType.Discrete; + } + else if (m_DiscreteBranchSizes == null || m_DiscreteBranchSizes.Length == 0) + { + VectorActionSize = new[] { m_NumContinuousActions }; + VectorActionSpaceType = SpaceType.Continuous; + } + else + { + VectorActionSize = null; + } + } + /// /// Called by Unity immediately before serializing this object. /// public void OnBeforeSerialize() { UpdateToActionSpec(); + SyncDeprecatedActionFields(); } /// @@ -150,6 +194,7 @@ public void OnBeforeSerialize() public void OnAfterDeserialize() { UpdateToActionSpec(); + SyncDeprecatedActionFields(); } } } From cd28e02497c3edd51b4c7d2c296ceed667af76ae Mon Sep 17 00:00:00 2001 From: Ruo-Ping Dong Date: Fri, 11 Dec 2020 14:00:31 -0800 Subject: [PATCH 31/36] fix BrainParameterToProto --- com.unity.ml-agents/Runtime/Communicator/GrpcExtensions.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/com.unity.ml-agents/Runtime/Communicator/GrpcExtensions.cs b/com.unity.ml-agents/Runtime/Communicator/GrpcExtensions.cs index ac64bb67a1..2c3ac2600a 100644 --- a/com.unity.ml-agents/Runtime/Communicator/GrpcExtensions.cs +++ b/com.unity.ml-agents/Runtime/Communicator/GrpcExtensions.cs @@ -105,12 +105,15 @@ public static BrainParametersProto ToProto(this BrainParameters bp, string name, var vectorActionSpaceTypeDeprecated = bp.ActionSpec.NumContinuousActions > 0 ? SpaceType.Continuous : SpaceType.Discrete; var brainParametersProto = new BrainParametersProto { - VectorActionSizeDeprecated = { vectorActionSizeDeprecated }, VectorActionSpaceTypeDeprecated = (SpaceTypeProto)vectorActionSpaceTypeDeprecated, BrainName = name, IsTraining = isTraining, ActionSpec = ToActionSpecProto(bp.ActionSpec), }; + if (vectorActionSizeDeprecated != null) + { + brainParametersProto.VectorActionSizeDeprecated.AddRange(vectorActionSizeDeprecated); + } if (bp.VectorActionDescriptions != null) { brainParametersProto.VectorActionDescriptionsDeprecated.AddRange(bp.VectorActionDescriptions); From b1d04b59d8d63b25c251c0d1ed2edbdf54bff37b Mon Sep 17 00:00:00 2001 From: Ruo-Ping Dong Date: Fri, 11 Dec 2020 14:44:46 -0800 Subject: [PATCH 32/36] change back VectorActuator suffix --- .../Runtime/Actuators/VectorActuator.cs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/com.unity.ml-agents/Runtime/Actuators/VectorActuator.cs b/com.unity.ml-agents/Runtime/Actuators/VectorActuator.cs index aa4d60a05e..a82a417455 100644 --- a/com.unity.ml-agents/Runtime/Actuators/VectorActuator.cs +++ b/com.unity.ml-agents/Runtime/Actuators/VectorActuator.cs @@ -30,7 +30,19 @@ public VectorActuator(IActionReceiver actionReceiver, { m_ActionReceiver = actionReceiver; ActionSpec = actionSpec; - string suffix = $"-Continuous-{actionSpec.NumContinuousActions}-Discrete-{actionSpec.NumDiscreteActions}"; + string suffix; + if (actionSpec.NumContinuousActions == 0) + { + suffix = "-Continuous"; + } + else if (actionSpec.NumDiscreteActions == 0) + { + suffix = "-Discrete"; + } + else + { + suffix = $"-Continuous-{actionSpec.NumContinuousActions}-Discrete-{actionSpec.NumDiscreteActions}"; + } Name = name + suffix; } From 4219e5ce1c55bcc41883fa24fb95042ed5f0a5af Mon Sep 17 00:00:00 2001 From: Ruo-Ping Dong Date: Fri, 11 Dec 2020 15:34:47 -0800 Subject: [PATCH 33/36] add back ActionSpec --- .../Editor/BrainParametersDrawer.cs | 11 ++++--- .../Runtime/Actuators/VectorActuator.cs | 4 +-- .../Runtime/Communicator/GrpcExtensions.cs | 8 ++--- .../Runtime/Policies/BrainParameters.cs | 30 +++++++++---------- .../Editor/Actuators/VectorActuatorTests.cs | 2 +- 5 files changed, 27 insertions(+), 28 deletions(-) diff --git a/com.unity.ml-agents/Editor/BrainParametersDrawer.cs b/com.unity.ml-agents/Editor/BrainParametersDrawer.cs index 9910188865..1e09a5f291 100644 --- a/com.unity.ml-agents/Editor/BrainParametersDrawer.cs +++ b/com.unity.ml-agents/Editor/BrainParametersDrawer.cs @@ -14,8 +14,9 @@ internal class BrainParametersDrawer : PropertyDrawer // The height of a line in the Unity Inspectors const float k_LineHeight = 17f; const int k_VecObsNumLine = 3; + const string k_ActionSpecName = "m_ActionSpec"; const string k_ContinuousActionSizeName = "m_NumContinuousActions"; - const string k_DiscreteBranchSizeName = "m_DiscreteBranchSizes"; + const string k_DiscreteBranchSizeName = "BranchSizes"; const string k_ActionDescriptionPropName = "VectorActionDescriptions"; const string k_VecObsPropName = "VectorObservationSize"; const string k_NumVecObsPropName = "NumStackedVectorObservations"; @@ -97,9 +98,10 @@ static void DrawVectorAction(Rect position, SerializedProperty property) EditorGUI.LabelField(position, "Vector Action"); position.y += k_LineHeight; EditorGUI.indentLevel++; - DrawContinuousVectorAction(position, property); + var actionSpecProperty = property.FindPropertyRelative(k_ActionSpecName); + DrawContinuousVectorAction(position, actionSpecProperty); position.y += k_LineHeight; - DrawDiscreteVectorAction(position, property); + DrawDiscreteVectorAction(position, actionSpecProperty); } /// @@ -162,7 +164,8 @@ static void DrawDiscreteVectorAction(Rect position, SerializedProperty property) /// The height of the drawer of the Vector Action. static float GetHeightDrawVectorAction(SerializedProperty property) { - var numActionLines = 3 + property.FindPropertyRelative(k_DiscreteBranchSizeName).arraySize; + var actionSpecProperty = property.FindPropertyRelative(k_ActionSpecName); + var numActionLines = 3 + actionSpecProperty.FindPropertyRelative(k_DiscreteBranchSizeName).arraySize; return numActionLines * k_LineHeight; } } diff --git a/com.unity.ml-agents/Runtime/Actuators/VectorActuator.cs b/com.unity.ml-agents/Runtime/Actuators/VectorActuator.cs index a82a417455..fd2e77a5ea 100644 --- a/com.unity.ml-agents/Runtime/Actuators/VectorActuator.cs +++ b/com.unity.ml-agents/Runtime/Actuators/VectorActuator.cs @@ -33,11 +33,11 @@ public VectorActuator(IActionReceiver actionReceiver, string suffix; if (actionSpec.NumContinuousActions == 0) { - suffix = "-Continuous"; + suffix = "-Discrete"; } else if (actionSpec.NumDiscreteActions == 0) { - suffix = "-Discrete"; + suffix = "-Continuous"; } else { diff --git a/com.unity.ml-agents/Runtime/Communicator/GrpcExtensions.cs b/com.unity.ml-agents/Runtime/Communicator/GrpcExtensions.cs index 2c3ac2600a..6bc493c91f 100644 --- a/com.unity.ml-agents/Runtime/Communicator/GrpcExtensions.cs +++ b/com.unity.ml-agents/Runtime/Communicator/GrpcExtensions.cs @@ -101,18 +101,16 @@ public static List GetObservationSummaries(this AgentInfoAct /// Whether or not the Brain is training. public static BrainParametersProto ToProto(this BrainParameters bp, string name, bool isTraining) { - var vectorActionSizeDeprecated = bp.ActionSpec.NumContinuousActions > 0 ? new int[] { bp.ActionSpec.NumContinuousActions } : bp.ActionSpec.BranchSizes; - var vectorActionSpaceTypeDeprecated = bp.ActionSpec.NumContinuousActions > 0 ? SpaceType.Continuous : SpaceType.Discrete; var brainParametersProto = new BrainParametersProto { - VectorActionSpaceTypeDeprecated = (SpaceTypeProto)vectorActionSpaceTypeDeprecated, + VectorActionSpaceTypeDeprecated = (SpaceTypeProto)bp.VectorActionSpaceType, BrainName = name, IsTraining = isTraining, ActionSpec = ToActionSpecProto(bp.ActionSpec), }; - if (vectorActionSizeDeprecated != null) + if (bp.VectorActionSize != null) { - brainParametersProto.VectorActionSizeDeprecated.AddRange(vectorActionSizeDeprecated); + brainParametersProto.VectorActionSizeDeprecated.AddRange(bp.VectorActionSize); } if (bp.VectorActionDescriptions != null) { diff --git a/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs b/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs index 26d3a73b9c..02b838f2ea 100644 --- a/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs +++ b/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs @@ -52,28 +52,26 @@ public class BrainParameters : ISerializationCallbackReceiver [Range(1, 50)] public int NumStackedVectorObservations = 1; [SerializeField] - internal int m_NumContinuousActions = 0; - [SerializeField] - internal int[] m_DiscreteBranchSizes = null; + internal ActionSpec m_ActionSpec = new ActionSpec(0, null); /// /// The specification of the Action space for the BrainParameters. /// public ActionSpec ActionSpec { - get { return new ActionSpec(m_NumContinuousActions, m_DiscreteBranchSizes); } + get { return m_ActionSpec; } set { - m_NumContinuousActions = value.NumContinuousActions; - m_DiscreteBranchSizes = value.BranchSizes; - if (m_NumContinuousActions == 0) + m_ActionSpec.NumContinuousActions = value.NumContinuousActions; + m_ActionSpec.BranchSizes = value.BranchSizes; + if (m_ActionSpec.NumContinuousActions == 0) { VectorActionSize = value.BranchSizes; VectorActionSpaceType = SpaceType.Discrete; } - else if (m_DiscreteBranchSizes == null || m_DiscreteBranchSizes.Length == 0) + else if (m_ActionSpec.NumDiscreteActions == 0) { - VectorActionSize = new[] { m_NumContinuousActions }; + VectorActionSize = new[] { m_ActionSpec.NumContinuousActions }; VectorActionSpaceType = SpaceType.Continuous; } } @@ -148,13 +146,13 @@ private void UpdateToActionSpec() { if (VectorActionSpaceType == SpaceType.Continuous) { - m_NumContinuousActions = VectorActionSize[0]; - m_DiscreteBranchSizes = null; + m_ActionSpec.NumContinuousActions = VectorActionSize[0]; + m_ActionSpec.BranchSizes = null; } if (VectorActionSpaceType == SpaceType.Discrete) { - m_NumContinuousActions = 0; - m_DiscreteBranchSizes = VectorActionSize; + m_ActionSpec.NumContinuousActions = 0; + m_ActionSpec.BranchSizes = VectorActionSize; } hasUpgradedBrainParametersWithActionSpec = true; @@ -163,14 +161,14 @@ private void UpdateToActionSpec() private void SyncDeprecatedActionFields() { - if (m_NumContinuousActions == 0) + if (m_ActionSpec.NumContinuousActions == 0) { VectorActionSize = ActionSpec.BranchSizes; VectorActionSpaceType = SpaceType.Discrete; } - else if (m_DiscreteBranchSizes == null || m_DiscreteBranchSizes.Length == 0) + else if (m_ActionSpec.NumDiscreteActions == 0) { - VectorActionSize = new[] { m_NumContinuousActions }; + VectorActionSize = new[] { m_ActionSpec.NumContinuousActions }; VectorActionSpaceType = SpaceType.Continuous; } else diff --git a/com.unity.ml-agents/Tests/Editor/Actuators/VectorActuatorTests.cs b/com.unity.ml-agents/Tests/Editor/Actuators/VectorActuatorTests.cs index 8129e830d0..38969b3a00 100644 --- a/com.unity.ml-agents/Tests/Editor/Actuators/VectorActuatorTests.cs +++ b/com.unity.ml-agents/Tests/Editor/Actuators/VectorActuatorTests.cs @@ -42,7 +42,7 @@ public void TestConstruct() Assert.IsTrue(va1.ActionSpec.NumContinuousActions == 4); Assert.IsTrue(va1.ActionSpec.SumOfDiscreteBranchSizes == 0); - Assert.AreEqual(va1.Name, "name-Continuous-4-Discrete-0"); + Assert.AreEqual(va1.Name, "name-Continuous"); } [Test] From aa3d799eea048335b9337a00430537c428d493d2 Mon Sep 17 00:00:00 2001 From: Ruo-Ping Dong Date: Fri, 11 Dec 2020 15:43:56 -0800 Subject: [PATCH 34/36] set deprecated to null --- com.unity.ml-agents/Runtime/Policies/BrainParameters.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs b/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs index 02b838f2ea..92b27f700c 100644 --- a/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs +++ b/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs @@ -74,6 +74,10 @@ public ActionSpec ActionSpec VectorActionSize = new[] { m_ActionSpec.NumContinuousActions }; VectorActionSpaceType = SpaceType.Continuous; } + else + { + VectorActionSize = null; + } } } From b64d85a18bbdc141aa595cc807c72b6d47b9bc97 Mon Sep 17 00:00:00 2001 From: Ruo-Ping Dong Date: Fri, 11 Dec 2020 15:57:01 -0800 Subject: [PATCH 35/36] replace with SyncDeprecatedActionFields --- .../Runtime/Policies/BrainParameters.cs | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs b/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs index 92b27f700c..987e42c186 100644 --- a/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs +++ b/com.unity.ml-agents/Runtime/Policies/BrainParameters.cs @@ -64,20 +64,7 @@ public ActionSpec ActionSpec { m_ActionSpec.NumContinuousActions = value.NumContinuousActions; m_ActionSpec.BranchSizes = value.BranchSizes; - if (m_ActionSpec.NumContinuousActions == 0) - { - VectorActionSize = value.BranchSizes; - VectorActionSpaceType = SpaceType.Discrete; - } - else if (m_ActionSpec.NumDiscreteActions == 0) - { - VectorActionSize = new[] { m_ActionSpec.NumContinuousActions }; - VectorActionSpaceType = SpaceType.Continuous; - } - else - { - VectorActionSize = null; - } + SyncDeprecatedActionFields(); } } @@ -163,6 +150,9 @@ private void UpdateToActionSpec() } } + /// + /// Sync values in ActionSpec fields to deprecated fields + /// private void SyncDeprecatedActionFields() { if (m_ActionSpec.NumContinuousActions == 0) From d7c82988347972d2248ae4dada9b65e471621fa0 Mon Sep 17 00:00:00 2001 From: Ruo-Ping Dong Date: Fri, 11 Dec 2020 16:15:44 -0800 Subject: [PATCH 36/36] update changelog --- com.unity.ml-agents/CHANGELOG.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/com.unity.ml-agents/CHANGELOG.md b/com.unity.ml-agents/CHANGELOG.md index 055723aab6..effcab1da1 100755 --- a/com.unity.ml-agents/CHANGELOG.md +++ b/com.unity.ml-agents/CHANGELOG.md @@ -10,13 +10,12 @@ and this project adheres to ## [Unreleased] ### Major Changes #### com.unity.ml-agents (C#) -- Agent with both continuous and discrete actions is now supported. You can specify -continuous and discrete action sizes repectively in Behavior Parameters. #### ml-agents / ml-agents-envs / gym-unity (Python) -- PyTorch trainers now support training agents with both continuous and discrete action spaces. - +- PyTorch trainers now support training agents with both continuous and discrete action spaces. (#4702) ### Minor Changes #### com.unity.ml-agents / com.unity.ml-agents.extensions (C#) +- Agent with both continuous and discrete actions is now supported. You can specify +continuous and discrete action sizes repectively in Behavior Parameters. (#4702, #4718) #### ml-agents / ml-agents-envs / gym-unity (Python) - `ActionSpec.validate_action()` now enforces that `UnityEnvironment.set_action_for_agent()` receives a 1D `np.array`.