examples/quickstart.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "MAVA_Quickstart.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "WEAq7x7ff1fE",
        "7SGFGmWnhuI2",
        "ohA5m0REjhu-",
        "avvSeVahk_Nt",
        "5i3tj4h-lTm4",
        "qBWiibHIleQk",
        "iU-jePUo0Odg",
        "CxFY0WOtNYfJ",
        "c3O9mqkINYfJ",
        "5TwjhQ0K4_yd",
        "XKne7VqnOI_O"
      ],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8uCEQLS3zZUn"
      },
      "source": [
        "# MAVA Quickstart Notebook\n",
        "<img src=\"https://raw.githubusercontent.com/instadeepai/Mava/develop/docs/images/mava.png\" />\n",
        "\n",
        "### Guide to installing Mava, creating and training your first Multi-Agent System. \n",
        "\n",
        "For more details about Mava and an overview of its design/features, please visit our [repo](https://github.com/instadeepai/Mava). \n",
        "\n",
        "<a href=\"https://colab.research.google.com/github/instadeepai/Mava/blob/develop/examples/quickstart.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WEAq7x7ff1fE"
      },
      "source": [
        "## 1. Installation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Pl4ed6X22tZq"
      },
      "source": [
        "#@title Install Mava and Some Supported Environments (Run Cell)\n",
        "%%capture\n",
        "!pip install id-mava[reverb,tf,launchpad,envs]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aDYrT7BVw7Dx"
      },
      "source": [
        "#@title Installs and Imports for Agent Visualization (Run Cell)\n",
        "%%capture\n",
        "!pip install id-mava[record_episode]\n",
        "! apt-get update -y &&  apt-get install -y xvfb &&  apt-get install -y python-opengl && apt-get install ffmpeg && apt-get install python-opengl -y && apt install xvfb -y && pip install pyvirtualdisplay \n",
        "\n",
        "import os\n",
        "from IPython.display import HTML\n",
        "from pyvirtualdisplay import Display\n",
        "\n",
        "display = Display(visible=0, size=(1024, 768))\n",
        "display.start()\n",
        "os.environ[\"DISPLAY\"] = \":\" + str(display.display)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7SGFGmWnhuI2"
      },
      "source": [
        "## 2. Import Modules"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8SvWrsWExz31"
      },
      "source": [
        "#@title Imports Modules (Run Cell)\n",
        "import functools\n",
        "from datetime import datetime\n",
        "from typing import Any, Dict, Mapping, Sequence, Union\n",
        "\n",
        "import launchpad as lp\n",
        "import numpy as np\n",
        "import sonnet as snt\n",
        "import tensorflow as tf\n",
        "from absl import app, flags\n",
        "from acme import types\n",
        "from acme.tf import networks\n",
        "from acme.tf import utils as tf2_utils\n",
        "from launchpad.nodes.python.local_multi_processing import PythonProcess\n",
        "\n",
        "from mava import specs as mava_specs\n",
        "from mava.systems.tf import maddpg\n",
        "from mava.utils import lp_utils\n",
        "from mava.utils.environments import debugging_utils\n",
        "from mava.wrappers import MonitorParallelEnvironmentLoop\n",
        "from mava.components.tf import architectures\n",
        "from mava.utils.loggers import logger_utils"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ul_phKL7h4Vq"
      },
      "source": [
        "## 3. Train a Multi-Agent Reinforcement Learning (MARL) `DDPG` System"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "l8XqA9M2iyK_"
      },
      "source": [
        "### Define Agent Networks\n",
        "We will use the default agent networks for the `maddpg` system."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UJ4-cN2dkXjq"
      },
      "source": [
        "network_factory = lp_utils.partial_kwargs(maddpg.make_default_networks)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ohA5m0REjhu-"
      },
      "source": [
        "### Select Environment\n",
        "We will use our [debug environment](https://github.com/instadeepai/Mava#debugging)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fw_4dR1jj-Wv"
      },
      "source": [
        "env_name = \"simple_spread\"\n",
        "action_space = \"continuous\"\n",
        "\n",
        "environment_factory = functools.partial(\n",
        "    debugging_utils.make_environment,\n",
        "    env_name=env_name,\n",
        "    action_space=action_space,\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lcZKJhnyk45C"
      },
      "source": [
        "### Create MARL System"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "avvSeVahk_Nt"
      },
      "source": [
        "#### Specify logging and checkpointing config. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u8J05yDlk-ya"
      },
      "source": [
        "# Directory to store checkpoints and log data. \n",
        "base_dir = \"~/mava\"\n",
        "\n",
        "# File name \n",
        "mava_id = datetime.now().strftime(\"%Y-%m-%d_%H:%M:%S\")\n",
        "\n",
        "# Log every [log_every] seconds\n",
        "log_every = 15\n",
        "logger_factory = functools.partial(\n",
        "    logger_utils.make_logger,\n",
        "    directory=base_dir,\n",
        "    to_terminal=True,\n",
        "    to_tensorboard=True,\n",
        "    time_stamp=mava_id,\n",
        "    time_delta=log_every,\n",
        ")\n",
        "\n",
        "# Checkpointer appends \"Checkpoints\" to checkpoint_dir\n",
        "checkpoint_dir = f\"{base_dir}/{mava_id}\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5i3tj4h-lTm4"
      },
      "source": [
        "#### Create Multi-Agent DDPG System."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CS618jAtxM1h"
      },
      "source": [
        "system = maddpg.MADDPG(\n",
        "    environment_factory=environment_factory,\n",
        "    network_factory=network_factory,\n",
        "    logger_factory=logger_factory,\n",
        "    num_executors=1,\n",
        "    policy_optimizer=snt.optimizers.Adam(learning_rate=1e-4),\n",
        "    critic_optimizer=snt.optimizers.Adam(learning_rate=1e-4),\n",
        "    checkpoint_subpath=checkpoint_dir,\n",
        "    max_gradient_norm=40.0,\n",
        "    checkpoint=False,\n",
        "    batch_size=1024,\n",
        "\n",
        "    # Record agents in environment. \n",
        "    eval_loop_fn=MonitorParallelEnvironmentLoop,\n",
        "    eval_loop_fn_kwargs={\"path\": checkpoint_dir, \"record_every\": 10, \"fps\": 5},\n",
        ").build()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qBWiibHIleQk"
      },
      "source": [
        "### Run Multi-Agent DDPG System."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gsoLWPTClnMt"
      },
      "source": [
        "# Ensure only trainer runs on gpu, while other processes run on cpu. \n",
        "gpu_id = -1\n",
        "env_vars = {\"CUDA_VISIBLE_DEVICES\": str(gpu_id)}\n",
        "local_resources = {\n",
        "    \"trainer\": [],\n",
        "    \"evaluator\": PythonProcess(env=env_vars),\n",
        "    \"executor\": PythonProcess(env=env_vars),\n",
        "}\n",
        "\n",
        "lp.launch(\n",
        "    system,\n",
        "    lp.LaunchType.LOCAL_MULTI_PROCESSING,\n",
        "    terminal=\"output_to_files\",\n",
        "    local_resources=local_resources,\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uN2KNO5V11E1"
      },
      "source": [
        "### Logs and Outputs"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JfI2fNFeltBm"
      },
      "source": [
        "#### View outputs from the evaluator process.\n",
        "*You might need to wait a few moments after launching the run.*\n",
        "The `CUDA_ERROR_NO_DEVICE` error is expected since the GPU is only used by the trainer. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5OchHHlv-dqv"
      },
      "source": [
        "!cat /tmp/launchpad_out/evaluator/0"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XHf3jDe3ySk7"
      },
      "source": [
        "#### View Stored Data \n",
        "*You might need to wait a few moments after launching the run.*"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IPahKjTnqBAO"
      },
      "source": [
        "! ls ~/mava/$mava_id"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SHygoBPW-3KV"
      },
      "source": [
        "### Tensorboard\n",
        "*You might need to wait a few moments after launching the run.*"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l181SBwtBo9M"
      },
      "source": [
        "# Load the TensorBoard notebook extension\n",
        "%load_ext tensorboard"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BJl7LKmHAOk-"
      },
      "source": [
        "To view training results, start tensorboard and filter for the `evaluator/RawEpisodeReturn` tag.\n",
        "\n",
        "A good score is a `RawEpisodeReturn` between 30-40. Although this system is stochastic, it should reach that score atleast by 100 evaluator episodes.    "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3fU3yEhdFx1O"
      },
      "source": [
        "%tensorboard --logdir ~/mava/$mava_id/tensorboard/evaluator"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zDlUXGltyVhM"
      },
      "source": [
        "### View Agent Recording\n",
        "Once a good score is reached, you can view intelligent multi-agent behaviour by viewing the agent recordings."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-2l8o2zDBbuN"
      },
      "source": [
        "#### Check if any agent recordings are available. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HXB1IKfysMT6"
      },
      "source": [
        "! ls ~/mava/$mava_id/recordings"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HjcnXbl7BfJc"
      },
      "source": [
        "#### View the latest agent recording. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DEEshoXd2K1S"
      },
      "source": [
        "import glob\n",
        "import os \n",
        "import IPython\n",
        "\n",
        "# Recordings\n",
        "list_of_files = glob.glob(f\"/root/mava/{mava_id}/recordings/*.html\")\n",
        "\n",
        "if(list_of_files == 0):\n",
        "  print(\"No recordings are available yet. Please wait or run the 'Run Multi-Agent DDPG System.' cell if you haven't already done this.\")\n",
        "else:\n",
        "  latest_file = max(list_of_files, key=os.path.getctime)\n",
        "  print(\"Run the next cell to visualize your agents!\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WJ33l0uIJ9xB"
      },
      "source": [
        "If the agents are trained, they should move to assigned landmarks."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "95GOv5vc5z5Q"
      },
      "source": [
        "# Latest file needs to point to the latest recording\n",
        "IPython.display.HTML(filename=latest_file)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KYekMtHB26yL"
      },
      "source": [
        "## 4. What's next?\n",
        "- Run MARL System with custom agent networks.\n",
        "- Try Different Architectures.\n",
        "- Scaling. "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YnKx6MRKYYAP"
      },
      "source": [
        "### Run MARL System with custom agent networks"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iU-jePUo0Odg"
      },
      "source": [
        "#### Build your own custom networks"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o64UWlhttvl5"
      },
      "source": [
        "def make_custom_network(environment_spec,shared_weights):\n",
        "\n",
        "  \"\"\"Creates networks used by the agents.\"\"\"\n",
        "  specs = environment_spec.get_agent_specs()\n",
        "\n",
        "\n",
        "  # Create agent_type specs\n",
        "  if shared_weights:\n",
        "    specs = {key.split(\"_\")[0]: specs[key] for key in specs.keys()}\n",
        "\n",
        "  observation_networks = {}\n",
        "  policy_networks = {}\n",
        "  critic_networks = {}\n",
        "\n",
        "  for agent in specs.keys():\n",
        "    \n",
        "    agent_act_spec = specs[agent].actions\n",
        "\n",
        "    # Get total number of action dimensions from action spec.\n",
        "    num_dimensions = np.prod(agent_act_spec.shape, dtype=int)\n",
        "    \n",
        "    # Create policy network\n",
        "    policy_network = snt.Sequential([\n",
        "        snt.Linear(output_size=100),\n",
        "        tf.nn.relu,\n",
        "        snt.Linear(output_size=num_dimensions),\n",
        "        tf.nn.relu,\n",
        "        networks.TanhToSpec(agent_act_spec)\n",
        "    ])\n",
        "\n",
        "    # Create the critic network.\n",
        "    critic_network = snt.Sequential([\n",
        "         # The multiplexer concatenates the observations/actions.\n",
        "        networks.CriticMultiplexer(),\n",
        "        snt.Linear(output_size=256),\n",
        "        tf.nn.relu,\n",
        "        snt.Linear(output_size=256),\n",
        "        tf.nn.relu,\n",
        "        snt.Linear(1)\n",
        "    ])\n",
        "\n",
        "    # An optional network to process observations\n",
        "    observation_network = tf2_utils.to_sonnet_module(tf.identity)\n",
        "\n",
        "    observation_networks[agent] = observation_network\n",
        "    policy_networks[agent] = policy_network\n",
        "    critic_networks[agent] = critic_network\n",
        "\n",
        "  return {\n",
        "      \"policies\": policy_networks,\n",
        "      \"critics\": critic_networks,\n",
        "      \"observations\": observation_networks,\n",
        "  }\n",
        "\n",
        "network_factory = lp_utils.partial_kwargs(make_custom_network)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "l76NFhGkz_8K"
      },
      "source": [
        "#### Run System with custom networks\n",
        "Let build our own custom agent networks. "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZRNfWuJD0cTw"
      },
      "source": [
        "##### Run System"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "form",
        "id": "LE_135VP0I0x"
      },
      "source": [
        "%%capture\n",
        "#@title Kill old runs. (Run Cell)\n",
        "!ps aux  |  grep -i launchpad  |  awk '{print $2}'  |  xargs sudo kill -9"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "form",
        "id": "7SjXoC1RL4uk"
      },
      "source": [
        "#@title Logging config. (Run Cell)\n",
        "# Directory to store checkpoints and log data. \n",
        "base_dir = \"~/mava/\"\n",
        "\n",
        "# File name \n",
        "mava_id = datetime.now().strftime(\"%Y-%m-%d_%H:%M:%S\")\n",
        "\n",
        "# Log every [log_every] seconds\n",
        "log_every = 15\n",
        "logger_factory = functools.partial(\n",
        "    logger_utils.make_logger,\n",
        "    directory=base_dir,\n",
        "    to_terminal=True,\n",
        "    to_tensorboard=True,\n",
        "    time_stamp=mava_id,\n",
        "    time_delta=log_every,\n",
        ")\n",
        "\n",
        "# Checkpointer appends \"Checkpoints\" to checkpoint_dir\n",
        "checkpoint_dir = f\"{base_dir}/{mava_id}\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s-6fwzxxbLIk",
        "cellView": "form"
      },
      "source": [
        "#@title Run system with custom networks. (Run Cell)\n",
        "\n",
        "# System\n",
        "system = maddpg.MADDPG(\n",
        "    environment_factory=environment_factory,\n",
        "    network_factory=network_factory,\n",
        "    logger_factory=logger_factory,\n",
        "    num_executors=1,\n",
        "    policy_optimizer=snt.optimizers.Adam(learning_rate=1e-4),\n",
        "    critic_optimizer=snt.optimizers.Adam(learning_rate=1e-4),\n",
        "    checkpoint_subpath=checkpoint_dir,\n",
        "    max_gradient_norm=40.0,\n",
        "    checkpoint=False,\n",
        "\n",
        "    # Record agents in environment. \n",
        "    eval_loop_fn=MonitorParallelEnvironmentLoop,\n",
        "    eval_loop_fn_kwargs={\"path\": checkpoint_dir, \"record_every\": 10, \"fps\": 5},\n",
        ").build()\n",
        "\n",
        "# Ensure only trainer runs on gpu, while other processes run on cpu. \n",
        "gpu_id = -1\n",
        "env_vars = {\"CUDA_VISIBLE_DEVICES\": str(gpu_id)}\n",
        "local_resources = {\n",
        "    \"trainer\": [],\n",
        "    \"evaluator\": PythonProcess(env=env_vars),\n",
        "    \"executor\": PythonProcess(env=env_vars),\n",
        "}\n",
        "\n",
        "lp.launch(\n",
        "    system,\n",
        "    lp.LaunchType.LOCAL_MULTI_PROCESSING,\n",
        "    terminal=\"output_to_files\",\n",
        "    local_resources=local_resources,\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xepCdLHN0jhA"
      },
      "source": [
        "##### View logs\n",
        "*You might need to wait a few moments after launching the run.*"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qhKKXZiwwG_Z"
      },
      "source": [
        "cat /tmp/launchpad_out/evaluator/0"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VzGIPIbrMn94"
      },
      "source": [
        "#### Tensorboard\n",
        "You might need to wait a few moments after launching the run."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OVMrr79dwJax"
      },
      "source": [
        "%tensorboard --logdir ~/mava/$mava_id/tensorboard/evaluator "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iDkVxBsw00VK"
      },
      "source": [
        "### Try Different Architectures\n",
        "Mava provides several components to support the design of MARL systems such as different system architectures and modules. For more information on different architectures, please have a look at our [components](https://github.com/instadeepai/Mava#components), visit [here](https://github.com/instadeepai/Mava/tree/develop/mava/components/tf/architectures) or view our [examples](https://github.com/instadeepai/Mava/tree/develop/examples).\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "form",
        "id": "7au2jUneMy-k"
      },
      "source": [
        "%%capture\n",
        "#@title Kill old runs. (Run Cell)\n",
        "!ps aux  |  grep -i launchpad  |  awk '{print $2}'  |  xargs sudo kill -9"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "form",
        "id": "3WV1d8rmMy-l"
      },
      "source": [
        "#@title Logging config. (Run Cell)\n",
        "# Directory to store checkpoints and log data. \n",
        "base_dir = \"~/mava/\"\n",
        "\n",
        "# File name \n",
        "mava_id = datetime.now().strftime(\"%Y-%m-%d_%H:%M:%S\")\n",
        "\n",
        "# Log every [log_every] seconds\n",
        "log_every = 15\n",
        "logger_factory = functools.partial(\n",
        "    logger_utils.make_logger,\n",
        "    directory=base_dir,\n",
        "    to_terminal=True,\n",
        "    to_tensorboard=True,\n",
        "    time_stamp=mava_id,\n",
        "    time_delta=log_every,\n",
        ")\n",
        "\n",
        "# Checkpointer appends \"Checkpoints\" to checkpoint_dir\n",
        "checkpoint_dir = f\"{base_dir}/{mava_id}\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JoFD5FfuLQhX"
      },
      "source": [
        "Let try switch from **Decentralised** (default) to **Centralised** architecture. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "29bNf4WEpu9H"
      },
      "source": [
        "# networks\n",
        "network_factory = lp_utils.partial_kwargs(maddpg.make_default_networks,shared_weights=False)\n",
        "\n",
        "# distributed program\n",
        "program = maddpg.MADDPG(\n",
        "    environment_factory=environment_factory,\n",
        "    network_factory=network_factory,\n",
        "    logger_factory=logger_factory,\n",
        "    num_executors=1,\n",
        "    policy_optimizer=snt.optimizers.Adam(learning_rate=1e-4),\n",
        "    critic_optimizer=snt.optimizers.Adam(learning_rate=1e-4),\n",
        "    checkpoint_subpath=checkpoint_dir,\n",
        "    max_gradient_norm=40.0,\n",
        "    checkpoint=False,\n",
        "\n",
        "    # Record agents in environment. \n",
        "    eval_loop_fn=MonitorParallelEnvironmentLoop,\n",
        "    eval_loop_fn_kwargs={\"path\": checkpoint_dir, \"record_every\": 10, \"fps\": 5},\n",
        "\n",
        "    # Centralised architecture and training. \n",
        "    architecture=architectures.CentralisedQValueCritic,\n",
        "    trainer_fn=maddpg.MADDPGCentralisedTrainer,\n",
        "    shared_weights=False,\n",
        ").build()\n",
        "\n",
        "# Ensure only trainer runs on gpu, while other processes run on cpu. \n",
        "gpu_id = -1\n",
        "env_vars = {\"CUDA_VISIBLE_DEVICES\": str(gpu_id)}\n",
        "local_resources = {\n",
        "    \"trainer\": [],\n",
        "    \"evaluator\": PythonProcess(env=env_vars),\n",
        "    \"executor\": PythonProcess(env=env_vars),\n",
        "}\n",
        "\n",
        "lp.launch(\n",
        "    program,\n",
        "    lp.LaunchType.LOCAL_MULTI_PROCESSING,\n",
        "    terminal=\"output_to_files\",\n",
        "    local_resources=local_resources,\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CxFY0WOtNYfJ"
      },
      "source": [
        "##### View logs\n",
        "*You might need to wait a few moments after launching the run.*"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "piVPE-WuNYfJ"
      },
      "source": [
        "cat /tmp/launchpad_out/evaluator/0"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "c3O9mqkINYfJ"
      },
      "source": [
        "#### Tensorboard\n",
        "You might need to wait a few moments after launching the run."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4VCNkcrqNYfJ"
      },
      "source": [
        "%tensorboard --logdir ~/mava/$mava_id/tensorboard/evaluator "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5TwjhQ0K4_yd"
      },
      "source": [
        "### Scaling\n",
        "Mava allows for simple scaling of MARL systems. \n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "form",
        "id": "mAe8wVIVNnIh"
      },
      "source": [
        "%%capture\n",
        "#@title Kill old runs. (Run Cell)\n",
        "!ps aux  |  grep -i launchpad  |  awk '{print $2}'  |  xargs sudo kill -9"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "form",
        "id": "hNJpEGUONnIi"
      },
      "source": [
        "#@title Logging config. (Run Cell)\n",
        "# Directory to store checkpoints and log data. \n",
        "base_dir = \"~/mava/\"\n",
        "\n",
        "# File name \n",
        "mava_id = datetime.now().strftime(\"%Y-%m-%d_%H:%M:%S\")\n",
        "\n",
        "# Log every [log_every] seconds\n",
        "log_every = 15\n",
        "logger_factory = functools.partial(\n",
        "    logger_utils.make_logger,\n",
        "    directory=base_dir,\n",
        "    to_terminal=True,\n",
        "    to_tensorboard=True,\n",
        "    time_stamp=mava_id,\n",
        "    time_delta=log_every,\n",
        ")\n",
        "\n",
        "# Checkpointer appends \"Checkpoints\" to checkpoint_dir\n",
        "checkpoint_dir = f\"{base_dir}/{mava_id}\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gsyoRbRHNwdN"
      },
      "source": [
        "Simply increase the **num_executors**. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0eD2R8yo5YBf"
      },
      "source": [
        "# networks\n",
        "network_factory = lp_utils.partial_kwargs(maddpg.make_default_networks)\n",
        "\n",
        "# distributed program\n",
        "program = maddpg.MADDPG(\n",
        "    environment_factory=environment_factory,\n",
        "    network_factory=network_factory,\n",
        "    logger_factory=logger_factory,\n",
        "    num_executors=4,\n",
        "    policy_optimizer=snt.optimizers.Adam(learning_rate=1e-4),\n",
        "    critic_optimizer=snt.optimizers.Adam(learning_rate=1e-4),\n",
        "    checkpoint_subpath=checkpoint_dir,\n",
        "    max_gradient_norm=40.0,\n",
        "    checkpoint=False,\n",
        "\n",
        "    # Record agents in environment. \n",
        "    eval_loop_fn=MonitorParallelEnvironmentLoop,\n",
        "    eval_loop_fn_kwargs={\"path\": checkpoint_dir, \"record_every\": 10, \"fps\": 5},\n",
        ").build()\n",
        "\n",
        "# Ensure only trainer runs on gpu, while other processes run on cpu. \n",
        "gpu_id = -1\n",
        "env_vars = {\"CUDA_VISIBLE_DEVICES\": str(gpu_id)}\n",
        "local_resources = {\n",
        "    \"trainer\": [],\n",
        "    \"evaluator\": PythonProcess(env=env_vars),\n",
        "    \"executor\": PythonProcess(env=env_vars),\n",
        "}\n",
        "\n",
        "lp.launch(\n",
        "    program,\n",
        "    lp.LaunchType.LOCAL_MULTI_PROCESSING,\n",
        "    terminal=\"output_to_files\",\n",
        "    local_resources=local_resources,\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XKne7VqnOI_O"
      },
      "source": [
        "##### View logs\n",
        "*You might need to wait a few moments after launching the run.*"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0tsn_0AzOI_O"
      },
      "source": [
        "cat /tmp/launchpad_out/evaluator/0"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PmkPnQalOI_T"
      },
      "source": [
        "#### Tensorboard\n",
        "You might need to wait a few moments after launching the run."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G0ttIuwvOI_T"
      },
      "source": [
        "%tensorboard --logdir ~/mava/$mava_id/tensorboard/evaluator "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_PmsI_-55Y9p"
      },
      "source": [
        "## For more examples using different systems, environments and architectures, visit our [github page](https://github.com/instadeepai/Mava/tree/develop/examples)."
      ]
    }
  ]
}