fix the reward wont add correctly

Author: ChenYuSean

Assets/ML-Agents/Timers/MainGame_timers.json

@@ -1 +1 @@
-{"count":1,"self":37.4832064,"total":37.9379595,"children":{"InitializeActuators":{"count":1,"self":0.0009954,"total":0.0009954,"children":null},"InitializeSensors":{"count":1,"self":0.0019997,"total":0.0019997,"children":null},"AgentSendState":{"count":1657,"self":0.0103012,"total":0.0253481,"children":{"CollectObservations":{"count":310,"self":0.0039689999999999994,"total":0.0039689999999999994,"children":null},"WriteActionMask":{"count":310,"self":0,"total":0,"children":null},"RequestDecision":{"count":310,"self":0.0110779,"total":0.0110779,"children":null}}},"DecideAction":{"count":1657,"self":0.4179885,"total":0.4179885,"children":null},"AgentAct":{"count":1657,"self":0.0068666,"total":0.0068666,"children":null}},"gauges":{"Player.CumulativeReward":{"count":18,"max":-1,"min":-1,"runningAverage":-1,"value":-1,"weightedAverage":-1}},"metadata":{"timer_format_version":"0.1.0","start_time_seconds":"1668950585","unity_version":"2020.3.25f1","command_line_arguments":"D:\\Unity3D\\2020.3.25f1\\Editor\\Unity.exe -projectpath D:\/Unity Hub\/Project\/CrossyRoadEX -useHub -hubIPC -cloudEnvironment production -hubSessionId ede81690-68d0-11ed-8bd6-190a1d4b9e9f -accessToken XWH1YHOH09f5fs1Iahj5ELeeE_bmQEEUm5FZ_yenRl001ef","communication_protocol_version":"1.5.0","com.unity.ml-agents_version":"2.2.1-exp.1","scene_name":"MainGame","end_time_seconds":"1668950622"}}
+{"count":1,"self":34.9703776,"total":35.2413671,"children":{"InitializeActuators":{"count":1,"self":0.002992,"total":0.002992,"children":null},"InitializeSensors":{"count":1,"self":0.0041308,"total":0.0041308,"children":null},"AgentSendState":{"count":1508,"self":0.0060555999999999995,"total":0.0209852,"children":{"CollectObservations":{"count":893,"self":0.00881,"total":0.00881,"children":null},"WriteActionMask":{"count":893,"self":0.0019311999999999999,"total":0.0019311999999999999,"children":null},"RequestDecision":{"count":893,"self":0.0041884,"total":0.0041884,"children":null}}},"DecideAction":{"count":1508,"self":0.0066473,"total":0.0066473,"children":null},"AgentAct":{"count":1508,"self":0.23395559999999999,"total":0.23395559999999999,"children":null}},"gauges":{"Player.CumulativeReward":{"count":4,"max":14,"min":-1,"runningAverage":2.75,"value":-1,"weightedAverage":1.8125}},"metadata":{"timer_format_version":"0.1.0","start_time_seconds":"1669445190","unity_version":"2020.3.25f1","command_line_arguments":"D:\\Unity3D\\2020.3.25f1\\Editor\\Unity.exe -projectpath D:\\Unity Hub\\Project\\CrossyRoadEX -useHub -hubIPC -cloudEnvironment production -licensingIpc LicenseClient-seanc -hubSessionId bae391b0-6d54-11ed-acd1-1b4f3f0a746b -accessToken ojuXIW3TZExkvNSRgMkty-jXpRnJfQkVKVmaGvKDHno01ef","communication_protocol_version":"1.5.0","com.unity.ml-agents_version":"2.2.1-exp.1","scene_name":"MainGame","end_time_seconds":"1669445225"}}

Assets/Scripts/Controller/PlayerMovementScript.cs

@@ -105,8 +105,7 @@ public float ActionHandle(int actionIndex)
         float reward = 0f;
         if (IsMoving || !canMove || Time.time - st < input_delay) return 0f;
         reward = InputTransfer(actionIndex);
-        if ((int)current.z > score) return 1f;
-        return 0f;
+        return reward;
     }
 
     /// <summary>
@@ -122,6 +121,7 @@ private float InputTransfer(int code)
         {
             case 1:
                 successCheck = Move(new Vector3(0, 0, 3));
+                if (successCheck == 1 && (int)current.z + 3 > score) reward = 1.0f;
                 break;
             case 2:
                 successCheck = Move(new Vector3(0, 0, -3));

Assets/Scripts/TrainingAgent/PlayerAgent.cs

@@ -70,7 +70,7 @@ private void LevelObservation(VectorSensor sensor)
             }
             else
             {
-                sensor.AddObservation(0);
+                sensor.AddObservation(-1);
                 ObjectsObservation(sensor, emptyList);
             }
         }
@@ -103,7 +103,6 @@ private void ObjectsObservation(VectorSensor sensor,List<GameObject> olist)
 
     public override void OnActionReceived(ActionBuffers actions)
     {
-        //Debug.Log(actions.DiscreteActions[0]);
         var reward = PMScript.ActionHandle(actions.DiscreteActions[0]);
         SetReward(reward);
     }

Executable/v1.0/EnviromentTest.ipynb

@@ -15,25 +15,321 @@
     "```"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Enviroment\n",
+    "Get the enviroment by calling `UnityEnviroment()` to get the corresponding file name env.<br />\n",
+    "- Close: `env.close()` close the enviroment. Release the port of commutator. <br />\n",
+    "- Reset: `env.reset()` resets the enviroment. <br />\n",
+    "- Set Action: `env.set_actions(behavior_name: str, action: ActionTuple)` set up the actions for next step.More Info down below <br />\n",
+    "- Step: `env.step()` move the simulation forward. Pass the action to Unity. <br />\n",
+    "- Get Step: `env.get_steps(behavior_name: str)` get the decision step from Unity.More Info down below <br />\n",
+    " "
+   ]
+  },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 35,
    "metadata": {},
    "outputs": [],
    "source": [
     "from mlagents_envs.environment import UnityEnvironment\n",
+    "import numpy as np\n",
     "# This is a non-blocking call that only loads the environment.\n",
     "env = UnityEnvironment(file_name=\"CRML\", seed=1, side_channels=[])\n",
     "# Start interacting with the environment.\n",
-    "env.reset()\n",
-    "behavior_names = env.behavior_specs.keys()"
+    "env.reset()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Behavior Spec\n",
+    "Contains the specs of the behavior. Including `ActionSpec` and `Observation Spec`<br />\n",
+    "`env.behavior_specs` is a dictionary of (Name: str, Spec: Behavior_Spec) <br />\n",
+    "Get the names of Behavior Spec by  `list(env.behavior_specs)` or `list(env.behavior_specs.keys())` <br />\n",
+    "Get the corresonding Behavior Spec by `env.behavior_specs[behavior_name]` <br />"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 36,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Name of the behavior : Player?team=0\n",
+      "Behavior spec of Player?team=0 : BehaviorSpec(observation_specs=[ObservationSpec(shape=(60,), dimension_property=(<DimensionProperty.NONE: 1>,), observation_type=<ObservationType.DEFAULT: 0>, name='StackingSensor_size2_VectorSensor_size30')], action_spec=ActionSpec(continuous_size=0, discrete_branches=(5,)))\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Since there's only one behavior, get the first on the list\n",
+    "behavior_name = list(env.behavior_specs)[0]\n",
+    "print(f\"Name of the behavior : {behavior_name}\")\n",
+    "spec = env.behavior_specs[behavior_name]\n",
+    "print(f\"Behavior spec of {behavior_name} : {spec}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Observation Spec\n",
+    "Note: this is NOT the Observation Space that observed for agent, but an info spec of it <br />\n",
+    "Get the Observation Spec by `spec.observation_specs`\n",
+    "- Shape: numbers of observation\n",
+    "- Dimension Property:\n",
+    "- Observation Type:\n",
+    "- Name:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 58,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Number of observations :  1\n",
+      "Spec Info : ObservationSpec(shape=(60,), dimension_property=(<DimensionProperty.NONE: 1>,), observation_type=<ObservationType.DEFAULT: 0>, name='StackingSensor_size2_VectorSensor_size30')\n",
+      "Is there a visual observation ? False\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Examine the number of observations per Agent\n",
+    "print(\"Number of observations : \", len(spec.observation_specs))\n",
+    "print(f\"Spec Info : {spec.observation_specs[0]}\")\n",
+    "# Is there a visual observation ?\n",
+    "# Visual observation have 3 dimensions: Height, Width and number of channels\n",
+    "vis_obs = any(len(spec.shape) == 3 for spec in spec.observation_specs)\n",
+    "print(\"Is there a visual observation ?\", vis_obs)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Action Spec\n",
+    "Note: this is NOT the Action Space that set for agent, but an info spec of it <br />\n",
+    "Get the Action Spec by `spec.action_specs` <br />\n",
+    "Random Action: `action_spec.random_action(n_agent: int)` create an random action for n agents <br />\n",
+    "Empty Action: `action_spec.empty_action(n_agent: int)` create an empty action for n agents <br />"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "There are 1 discrete actions\n",
+      "Action number 0 has 5 different options\n",
+      "Continuous: 0, Discrete: (5,)\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Is the Action continuous or multi-discrete ?\n",
+    "if spec.action_spec.continuous_size > 0:\n",
+    "  print(f\"There are {spec.action_spec.continuous_size} continuous actions\")\n",
+    "if spec.action_spec.is_discrete():\n",
+    "  print(f\"There are {spec.action_spec.discrete_size} discrete actions\")\n",
+    "\n",
+    "# For discrete actions only : How many different options does each action has ?\n",
+    "if spec.action_spec.discrete_size > 0:\n",
+    "  for action, branch_size in enumerate(spec.action_spec.discrete_branches):\n",
+    "    print(f\"Action number {action} has {branch_size} different options\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Action Tuple\n",
+    "Class that's pack NamedTuple as Action\n",
+    "- `action.discrete`: get the discrete actions\n",
+    "- `action.continuous`: get the continuous actions\n",
+    "- `action.add_discrete`: add the discrete actions\n",
+    "- `action.add_continous`: add the continuous actions\n",
+    "\n",
+    "Axis 0(Rows): Different Agents actions value <br />\n",
+    "Axis 1(Columns): Different Actions on Same agent<br />"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 32,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[[1 2]\n",
+      " [3 4]]\n",
+      "[[0.5]]\n",
+      "[[0]\n",
+      " [3]]\n"
+     ]
+    }
+   ],
+   "source": [
+    "from mlagents_envs.environment import ActionTuple\n",
+    "action = ActionTuple()\n",
+    "action.add_discrete(np.array([[1,2],[3,4]])) # [1,2] actions on Agent 1, [3,4] actions on Agent 2\n",
+    "print(action.discrete)\n",
+    "action.add_continuous(np.array([[0.5]]))\n",
+    "print(action.continuous)\n",
+    "print(spec.action_spec.random_action(2).discrete) # Get 1 random action under Action Spec for 2 agents"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Decision Steps / Terminal Steps\n",
+    "Decision Steps and Terminal Steps are the list that agents called for the need of decision.<br />\n",
+    "Difference between Decision Steps and Terminal Steps is that terminal step only calls on episode end, while \n",
+    "decision step can be called at anytime.\n",
+    "- Decision Steps: `env.get_steps(Behavior_Name:str)` get the steps from agents requested of the behavior<br />\n",
+    "- Agent ID: `steps.agent_id()` get the agents id corresponding to the step <br />\n",
+    "- `len(DecisionSteps)`: Returns the number of agents requesting a decision since the last call to env.step()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 37,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[0]\n"
+     ]
+    }
+   ],
+   "source": [
+    "decision_steps, terminal_steps = env.get_steps(behavior_name) \n",
+    "print(decision_steps.agent_id)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Observation and Reward of Steps\n",
+    "Observation of an agent: `steps[agent_id].obs` <br />\n",
+    "Reward of an agent: `steps[agent_id].reward` <br /> <br/>\n",
+    "Observation of all agent: `steps.obs` <br />\n",
+    "Reward of all agent: `steps.reward` <br />"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 56,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Observation of Agent 0: [array([   0.,    0.,    0.,    0.,    0.,    0.,    0.,    0.,    0.,\n",
+      "          0.,    0.,    0.,    0.,    0.,    0.,    0.,    0.,    0.,\n",
+      "          0.,    0.,    0.,    0.,    0.,    0.,    0.,    0.,    0.,\n",
+      "          0.,    0.,    0.,    0.,    0.,    0., -100., -100., -100.,\n",
+      "       -100., -100., -100.,    0., -100., -100., -100., -100., -100.,\n",
+      "       -100.,    0., -100., -100., -100., -100., -100., -100.,    0.,\n",
+      "       -100., -100., -100., -100., -100., -100.], dtype=float32)]\n",
+      "Reward of Agent 0: 0.0\n",
+      "0.0\n"
+     ]
+    }
+   ],
+   "source": [
+    "agent_id = decision_steps.agent_id[0]\n",
+    "print(f\"Observation of Agent {agent_id}: {decision_steps[agent_id].obs}\")\n",
+    "print(f\"Reward of Agent {agent_id}: {decision_steps[agent_id].reward}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Set the Action and Run\n",
+    "- `env.set_actions(behavior_name: str, action: ActionTuple)` : Setup Action for next step\n",
+    "- `env.step()` : Foward to next step"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 57,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "action = spec.action_spec.random_action(len(decision_steps))\n",
+    "env.set_actions(behavior_name, action)\n",
+    "env.step()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Run the enviroment for serval episode"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
    "metadata": {},
    "outputs": [],
+   "source": [
+    "ep = 2\n",
+    "for episode in range(ep):\n",
+    "  env.reset()\n",
+    "  decision_steps, terminal_steps = env.get_steps(behavior_name) # Get the first step\n",
+    "  tracked_agent = decision_steps.agent_id[0] # Track the agent (Since there's only one)\n",
+    "  done = False # For the tracked_agent\n",
+    "  episode_rewards = 0 # For the tracked_agent\n",
+    "  while not done:\n",
+    "    # Generate an action for all agents\n",
+    "    action = spec.action_spec.random_action(len(decision_steps))\n",
+    "    # Set the actions\n",
+    "    env.set_actions(behavior_name, action)\n",
+    "\n",
+    "    # Move the simulation forward\n",
+    "    env.step()\n",
+    "\n",
+    "    # Get the new simulation results\n",
+    "    decision_steps, terminal_steps = env.get_steps(behavior_name)\n",
+    "    if tracked_agent in decision_steps: # The agent requested a decision\n",
+    "      episode_rewards += decision_steps[tracked_agent].reward\n",
+    "      print(f\"step reward:{decision_steps[tracked_agent].reward}\")\n",
+    "    if tracked_agent in terminal_steps: # The agent terminated its episode\n",
+    "      episode_rewards += terminal_steps[tracked_agent].reward\n",
+    "      print(f\"step reward:{terminal_steps[tracked_agent].reward}\")\n",
+    "      done = True\n",
+    "  print(f\"Total rewards for episode {episode} is {episode_rewards}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [],
    "source": [
     "env.close()"
    ]