ALAE_Inference_Wandb.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "ALAE Inference Wandb.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/garfieldbart/chrome-ibood-tracker/blob/master/ALAE_Inference_Wandb.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aXpc2zY2PJOA",
        "colab_type": "text"
      },
      "source": [
        "## Clone the Repo"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oJmmpXfLNEig",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "f116f4ba-86f5-419e-c6ca-fa6c0fa54067"
      },
      "source": [
        "!git clone https://github.com/podgorskiy/ALAE.git"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "fatal: destination path 'ALAE' already exists and is not an empty directory.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jv8aM0eKPMps",
        "colab_type": "text"
      },
      "source": [
        "## Set Path Variables"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E4hFx1ZHNOms",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "359b759e-3ad7-485c-90c3-83349b64dc97"
      },
      "source": [
        "%cd ALAE\n",
        "%set_env PYTHONPATH=/project/pylib/src:/env/python"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/ALAE\n",
            "env: PYTHONPATH=/project/pylib/src:/env/python\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KRdwKrMsPST9",
        "colab_type": "text"
      },
      "source": [
        "## Install Requirements"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nNNj88ayNULF",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 432
        },
        "outputId": "059f2040-4a1e-4b61-f6d9-9c70d17f8c46"
      },
      "source": [
        "%pip install -r requirements.txt\n",
        "%pip install dareblopy\n",
        "%pip install wandb -q --upgrade"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 1)) (20.4)\n",
            "Requirement already satisfied: imageio in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 2)) (2.4.1)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 3)) (1.18.5)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 4)) (1.4.1)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 5)) (4.41.1)\n",
            "Requirement already satisfied: dlutils in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 6)) (0.0.12)\n",
            "Requirement already satisfied: bimpy in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 7)) (0.0.13)\n",
            "Requirement already satisfied: torch>=1.3 in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 8)) (1.5.0+cu101)\n",
            "Requirement already satisfied: torchvision in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 9)) (0.6.0+cu101)\n",
            "Requirement already satisfied: sklearn in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 10)) (0.0)\n",
            "Requirement already satisfied: yacs in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 11)) (0.1.7)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 12)) (3.2.1)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->-r requirements.txt (line 1)) (2.4.7)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from packaging->-r requirements.txt (line 1)) (1.12.0)\n",
            "Requirement already satisfied: pillow in /usr/local/lib/python3.6/dist-packages (from imageio->-r requirements.txt (line 2)) (7.0.0)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from torch>=1.3->-r requirements.txt (line 8)) (0.16.0)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.6/dist-packages (from sklearn->-r requirements.txt (line 10)) (0.22.2.post1)\n",
            "Requirement already satisfied: PyYAML in /usr/local/lib/python3.6/dist-packages (from yacs->-r requirements.txt (line 11)) (3.13)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib->-r requirements.txt (line 12)) (0.10.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->-r requirements.txt (line 12)) (1.2.0)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->-r requirements.txt (line 12)) (2.8.1)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn->sklearn->-r requirements.txt (line 10)) (0.15.1)\n",
            "Requirement already satisfied: dareblopy in /usr/local/lib/python3.6/dist-packages (0.0.3)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from dareblopy) (1.18.5)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nXK9r6S8KCbV",
        "colab_type": "text"
      },
      "source": [
        "## Download Pre-trained Models"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MoV5FwfbNzDy",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 259
        },
        "outputId": "00999e47-eec4-483d-a105-e1245856b61e"
      },
      "source": [
        "!python training_artifacts/download_all.py"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading: model_submitted.pth\n",
            "File training_artifacts/ffhq/model_submitted.pth already exists, skipping\n",
            "Downloading: model_157.pth\n",
            "File training_artifacts/ffhq/model_157.pth already exists, skipping\n",
            "Downloading: model_194.pth\n",
            "File training_artifacts/ffhq/model_194.pth already exists, skipping\n",
            "Downloading: model_final.pth\n",
            "File training_artifacts/celeba/model_final.pth already exists, skipping\n",
            "Downloading: model_final.pth\n",
            "File training_artifacts/bedroom/model_final.pth already exists, skipping\n",
            "Downloading: model_262r.pth\n",
            "File training_artifacts/celeba-hq256/model_262r.pth already exists, skipping\n",
            "Downloading: model_580r.pth\n",
            "File training_artifacts/celeba-hq256/model_580r.pth already exists, skipping\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4hSz6CfxMFnW",
        "colab_type": "text"
      },
      "source": [
        "## Imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8UNZtBlMRZXk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%load_ext autoreload\n",
        "%autoreload 2\n",
        "\n",
        "import os\n",
        "\n",
        "import torch.utils.data\n",
        "from torchvision.utils import make_grid\n",
        "from net import *\n",
        "from model import Model\n",
        "from launcher import run\n",
        "from checkpointer import Checkpointer\n",
        "from dlutils.pytorch import count_parameters\n",
        "from defaults import get_cfg_defaults\n",
        "import lreq\n",
        "import logging\n",
        "from PIL import Image\n",
        "import PIL\n",
        "import bimpy\n",
        "import cv2\n",
        "import random \n",
        "from skimage.transform import resize\n",
        "# Ignore excessive warnings\n",
        "import logging\n",
        "logging.propagate = False \n",
        "logging.getLogger().setLevel(logging.ERROR)\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "\n",
        "lreq.use_implicit_lreq.set(True)\n",
        "\n",
        "# WandB – Import the wandb library\n",
        "import wandb"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Pdhl6YRBLbqf",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "87980132-cfaf-4e4b-fbab-2dec07ffd746"
      },
      "source": [
        "# WandB – Login to your wandb account so you can log all your metrics\n",
        "wandb.login()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f0jnVjGjLcYd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# WandB – Initialize a new run\n",
        "wandb.init(project=\"alae\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8r6Lbjf3Nolr",
        "colab_type": "text"
      },
      "source": [
        "## General Disclaimer\n",
        "I've taken the code from individual files in the repo and modified them here in this notebook to enable logging with Weights and Biases. No other changes have been made"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pCIy0on9u7Nh",
        "colab_type": "text"
      },
      "source": [
        "### Helper methods to parse stuff :)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sXb1awkvaxF3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def parse_configs(config_file='configs/ffhq.yaml', write_log=False):\n",
        "    import sys\n",
        "    cfg = get_cfg_defaults()\n",
        "    world_size = 1\n",
        "    if len(os.path.splitext(config_file)[1]) == 0:\n",
        "        config_file += '.yaml'\n",
        "    if not os.path.exists(config_file) and os.path.exists(os.path.join('configs', config_file)):\n",
        "        config_file = os.path.join('configs', config_file)\n",
        "    cfg.merge_from_file(config_file)\n",
        "    cfg.freeze()\n",
        "\n",
        "    logger = logging.getLogger(\"logger\")\n",
        "    logger.setLevel(logging.DEBUG)\n",
        "\n",
        "    output_dir = cfg.OUTPUT_DIR\n",
        "    os.makedirs(output_dir, exist_ok=True)\n",
        "\n",
        "    ch = logging.StreamHandler(stream=sys.stdout)\n",
        "    ch.setLevel(logging.DEBUG)\n",
        "    formatter = logging.Formatter(\"%(asctime)s %(name)s %(levelname)s: %(message)s\")\n",
        "    ch.setFormatter(formatter)\n",
        "    logger.addHandler(ch)\n",
        "\n",
        "    if write_log:\n",
        "        filepath = os.path.join(output_dir, 'log.txt')\n",
        "        if isinstance(write_log, str):\n",
        "            filepath = write_log\n",
        "        fh = logging.FileHandler(filepath)\n",
        "        fh.setLevel(logging.DEBUG)\n",
        "        fh.setFormatter(formatter)\n",
        "        logger.addHandler(fh)\n",
        "\n",
        "    logger.info(\"World size: {}\".format(world_size))\n",
        "\n",
        "    logger.info(\"Loaded configuration file {}\".format(config_file))\n",
        "    with open(config_file, \"r\") as cf:\n",
        "        config_str = \"\\n\" + cf.read()\n",
        "        logger.info(config_str)\n",
        "    logger.info(\"Running with config:\\n{}\".format(cfg))\n",
        "\n",
        "    \n",
        "    torch.set_default_tensor_type('torch.cuda.FloatTensor')\n",
        "    device_ = torch.cuda.current_device()\n",
        "    print(\"Running on \", torch.cuda.get_device_name(device_))\n",
        "\n",
        "    return cfg, logger, device_"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6WYnKE7xMIl7",
        "colab_type": "text"
      },
      "source": [
        "## Style Mixing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m0uv661xSSS-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "src_len = 5\n",
        "dst_len = 6\n",
        "\n",
        "\n",
        "def place(canvas, image, x, y):\n",
        "    image = image.cpu().detach().numpy()\n",
        "    im_size = image.shape[1]\n",
        "    canvas[:, y * im_size: (y + 1) * im_size, x * im_size: (x + 1) * im_size] = image * 0.5 + 0.5\n",
        "\n",
        "\n",
        "def mix(cfg, logger):\n",
        "    with torch.no_grad():\n",
        "        _main(cfg, logger)\n",
        "\n",
        "\n",
        "def _main(cfg, logger):\n",
        "    torch.cuda.set_device(0)\n",
        "    model = Model(\n",
        "        startf=cfg.MODEL.START_CHANNEL_COUNT,\n",
        "        layer_count=cfg.MODEL.LAYER_COUNT,\n",
        "        maxf=cfg.MODEL.MAX_CHANNEL_COUNT,\n",
        "        latent_size=cfg.MODEL.LATENT_SPACE_SIZE,\n",
        "        truncation_psi=cfg.MODEL.TRUNCATIOM_PSI,\n",
        "        truncation_cutoff=cfg.MODEL.TRUNCATIOM_CUTOFF,\n",
        "        mapping_layers=cfg.MODEL.MAPPING_LAYERS,\n",
        "        channels=cfg.MODEL.CHANNELS,\n",
        "        generator=cfg.MODEL.GENERATOR,\n",
        "        encoder=cfg.MODEL.ENCODER)\n",
        "    model.cuda(0)\n",
        "    model.eval()\n",
        "    model.requires_grad_(False)\n",
        "\n",
        "    decoder = model.decoder\n",
        "    encoder = model.encoder\n",
        "    mapping_tl = model.mapping_tl\n",
        "    mapping_fl = model.mapping_fl\n",
        "    dlatent_avg = model.dlatent_avg\n",
        "\n",
        "    logger.info(\"Trainable parameters generator:\")\n",
        "    count_parameters(decoder)\n",
        "\n",
        "    logger.info(\"Trainable parameters discriminator:\")\n",
        "    count_parameters(encoder)\n",
        "\n",
        "    arguments = dict()\n",
        "    arguments[\"iteration\"] = 0\n",
        "\n",
        "    model_dict = {\n",
        "        'discriminator_s': encoder,\n",
        "        'generator_s': decoder,\n",
        "        'mapping_tl_s': mapping_tl,\n",
        "        'mapping_fl_s': mapping_fl,\n",
        "        'dlatent_avg': dlatent_avg\n",
        "    }\n",
        "\n",
        "    checkpointer = Checkpointer(cfg,\n",
        "                                model_dict,\n",
        "                                {},\n",
        "                                logger=logger,\n",
        "                                save=False)\n",
        "\n",
        "    extra_checkpoint_data = checkpointer.load()\n",
        "    last_epoch = list(extra_checkpoint_data['auxiliary']['scheduler'].values())[0]['last_epoch']\n",
        "    logger.info(\"Model trained for %d epochs\" % last_epoch)\n",
        "\n",
        "    model.eval()\n",
        "\n",
        "    layer_count = cfg.MODEL.LAYER_COUNT\n",
        "\n",
        "    def encode(x):\n",
        "        layer_count = cfg.MODEL.LAYER_COUNT\n",
        "\n",
        "        zlist = []\n",
        "        for i in range(x.shape[0]):\n",
        "            Z, _ = model.encode(x[i][None, ...], layer_count - 1, 1)\n",
        "            zlist.append(Z)\n",
        "        Z = torch.cat(zlist)\n",
        "        Z = Z.repeat(1, model.mapping_fl.num_layers, 1)\n",
        "        return Z\n",
        "\n",
        "    def decode(x):\n",
        "        decoded = []\n",
        "        for i in range(x.shape[0]):\n",
        "            r = model.decoder(x[i][None, ...], layer_count - 1, 1, noise=True)\n",
        "            decoded.append(r)\n",
        "        return torch.cat(decoded)\n",
        "\n",
        "    path = cfg.DATASET.STYLE_MIX_PATH\n",
        "    im_size = 2 ** (cfg.MODEL.LAYER_COUNT + 1)\n",
        "\n",
        "    src_originals = []\n",
        "    for i in range(src_len):\n",
        "        try:\n",
        "            im = np.asarray(Image.open(os.path.join(path, 'src/%d.png' % i)))\n",
        "        except FileNotFoundError:\n",
        "            im = np.asarray(Image.open(os.path.join(path, 'src/%d.jpg' % i)))\n",
        "        im = im.transpose((2, 0, 1))\n",
        "        x = torch.tensor(np.asarray(im, dtype=np.float32), requires_grad=True).cuda() / 127.5 - 1.\n",
        "        if x.shape[0] == 4:\n",
        "            x = x[:3]\n",
        "        factor = x.shape[2] // im_size\n",
        "        if factor != 1:\n",
        "            x = torch.nn.functional.avg_pool2d(x[None, ...], factor, factor)[0]\n",
        "        assert x.shape[2] == im_size\n",
        "        src_originals.append(x)\n",
        "    src_originals = torch.stack([x for x in src_originals])\n",
        "    dst_originals = []\n",
        "    for i in range(dst_len):\n",
        "        try:\n",
        "            im = np.asarray(Image.open(os.path.join(path, 'dst/%d.png' % i)))\n",
        "        except FileNotFoundError:\n",
        "            im = np.asarray(Image.open(os.path.join(path, 'dst/%d.jpg' % i)))\n",
        "        im = im.transpose((2, 0, 1))\n",
        "        x = torch.tensor(np.asarray(im, dtype=np.float32), requires_grad=True).cuda() / 127.5 - 1.\n",
        "        if x.shape[0] == 4:\n",
        "            x = x[:3]\n",
        "        factor = x.shape[2] // im_size\n",
        "        if factor != 1:\n",
        "            x = torch.nn.functional.avg_pool2d(x[None, ...], factor, factor)[0]\n",
        "        assert x.shape[2] == im_size\n",
        "        dst_originals.append(x)\n",
        "    dst_originals = torch.stack([x for x in dst_originals])\n",
        "\n",
        "    src_latents = encode(src_originals)\n",
        "    src_images = decode(src_latents)\n",
        "\n",
        "    dst_latents = encode(dst_originals)\n",
        "    dst_images = decode(dst_latents)\n",
        "\n",
        "    canvas = np.zeros([3, im_size * (dst_len + 1), im_size * (src_len + 1)])\n",
        "\n",
        "    os.makedirs('style_mixing/output/%s/' % cfg.NAME, exist_ok=True)\n",
        "\n",
        "    # Lists for Wandb logging\n",
        "    source_images = []\n",
        "    coarse_images = []\n",
        "    recons_images = []\n",
        "\n",
        "\n",
        "    for i in range(src_len):\n",
        "        img = src_originals[i] * 0.5 + 0.5\n",
        "        source_images.append(wandb.Image(img,caption=\"{} Source_{}\".format(cfg.NAME, i)))\n",
        "        place(canvas, src_originals[i], 1 + i, 0)\n",
        "\n",
        "    for i in range(dst_len):\n",
        "        img = dst_originals[i] * 0.5 + 0.5\n",
        "        coarse_images.append(wandb.Image(img,caption=\"{} Coarse_{}\".format(cfg.NAME, i)))\n",
        "        place(canvas, dst_originals[i], 0, 1 + i)\n",
        "\n",
        "    style_ranges = [range(0, 4)] * 3 + [range(4, 8)] * 2 + [range(8, layer_count * 2)]\n",
        "\n",
        "    def mix_styles(style_src, style_dst, r):\n",
        "        style = style_dst.clone()\n",
        "        style[:, r] = style_src[:, r]\n",
        "        return style\n",
        "\n",
        "    for row in range(dst_len):\n",
        "        row_latents = torch.stack([dst_latents[row]] * src_len)\n",
        "        style = mix_styles(src_latents, row_latents, style_ranges[row])\n",
        "        rec = model.decoder(style, layer_count - 1, 1, noise=True)\n",
        "        for j in range(rec.shape[0]):\n",
        "            img = rec[j] * 0.5 + 0.5\n",
        "            recons_images.append(wandb.Image(img,caption=\"{} Source_{}_{}\".format(cfg.NAME, row, j)))\n",
        "            place(canvas, rec[j], 1 + j, 1 + row)\n",
        "\n",
        "    #wandb.log({\"Source {} Images\".format(cfg.NAME): source_images})\n",
        "    #wandb.log({\"Coarse {} Images\".format(cfg.NAME): coarse_images})\n",
        "    #wandb.log({\"Reconstructed {} Images\".format(cfg.NAME): recons_images})\n",
        "    wandb.log({\"Style Mixed Output from {} Images\".format(cfg.NAME): [wandb.Image(torch.Tensor(canvas),caption=\"{} Style Mixed Output\".format(cfg.NAME))]})"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A0LQqeoAT_p3",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "764d31b0-bd57-4524-8b29-171bf88d881c"
      },
      "source": [
        "all_configs = ['configs/bedroom.yaml','configs/celeba.yaml','configs/ffhq.yaml']\n",
        "for config in all_configs[1:]:\n",
        "  cfg, logger, device = parse_configs(config)\n",
        "  mix(cfg, logger)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2020-06-11 18:48:49,892 logger INFO: World size: 1\n",
            "2020-06-11 18:48:49,893 logger INFO: Loaded configuration file configs/celeba.yaml\n",
            "2020-06-11 18:48:49,893 logger INFO: \n",
            "# Config for training ALAE on CelebA at resolution 128x128\n",
            "\n",
            "NAME: celeba\n",
            "PPL_CELEBA_ADJUSTMENT: True\n",
            "DATASET:\n",
            "  PART_COUNT: 16\n",
            "  SIZE: 182637\n",
            "  SIZE_TEST: 202576 - 182637\n",
            "  PATH: /data/datasets/celeba/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/celeba-test/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  MAX_RESOLUTION_LEVEL: 7\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign128x128\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_celeba\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 256\n",
            "  LAYER_COUNT: 6\n",
            "  MAX_CHANNEL_COUNT: 256\n",
            "  START_CHANNEL_COUNT: 64\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/celeba\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 6\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 80\n",
            "  #                    4       8       16       32       64       128        256       512       1024\n",
            "  LOD_2_BATCH_8GPU: [512,    256,     128,      64,      32,       32,        32,       32,        32]\n",
            "  LOD_2_BATCH_4GPU: [512,    256,     128,      64,      32,       32,        32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [128,    128,     128,      64,      32,       32,        16]\n",
            "  LOD_2_BATCH_1GPU: [128,    128,     128,      64,      32,       16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:48:49,894 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 7\n",
            "  PART_COUNT: 16\n",
            "  PART_COUNT_TEST: 1\n",
            "  PATH: /data/datasets/celeba/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/celeba-test/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign128x128\n",
            "  SIZE: 182637\n",
            "  SIZE_TEST: 19939\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_celeba\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 256\n",
            "  LAYER_COUNT: 6\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 256\n",
            "  START_CHANNEL_COUNT: 64\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: celeba\n",
            "OUTPUT_DIR: training_artifacts/celeba\n",
            "PPL_CELEBA_ADJUSTMENT: True\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 6\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [128, 128, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [128, 128, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 80\n",
            "Running on  Tesla P100-PCIE-16GB\n",
            "2020-06-11 18:48:52,688 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:48:52,689 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:48:52,691 logger INFO: Loading checkpoint from training_artifacts/celeba/model_final.pth\n",
            "2020-06-11 18:48:52,801 logger INFO: Model trained for 74 epochs\n",
            "2020-06-11 18:48:55,097 logger INFO: World size: 1\n",
            "2020-06-11 18:48:55,097 logger INFO: World size: 1\n",
            "2020-06-11 18:48:55,099 logger INFO: Loaded configuration file configs/ffhq.yaml\n",
            "2020-06-11 18:48:55,099 logger INFO: Loaded configuration file configs/ffhq.yaml\n",
            "2020-06-11 18:48:55,105 logger INFO: \n",
            " # Config for training ALAE on FFHQ at resolution 1024x1024\n",
            "\n",
            "NAME: ffhq\n",
            "DATASET:\n",
            "  PART_COUNT: 16\n",
            "  SIZE: 60000\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 300\n",
            "  #                    4    8   16    32    64    128    256\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128,   64,   32,    32,    32,       32,        32] # If GPU memory ~16GB reduce last number from 32 to 24\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128,   64,   32,    32,    32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128,   64,   32,    32,    16]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128,   64,   32,    16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:48:55,105 logger INFO: \n",
            " # Config for training ALAE on FFHQ at resolution 1024x1024\n",
            "\n",
            "NAME: ffhq\n",
            "DATASET:\n",
            "  PART_COUNT: 16\n",
            "  SIZE: 60000\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 300\n",
            "  #                    4    8   16    32    64    128    256\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128,   64,   32,    32,    32,       32,        32] # If GPU memory ~16GB reduce last number from 32 to 24\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128,   64,   32,    32,    32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128,   64,   32,    32,    16]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128,   64,   32,    16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:48:55,113 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "  PART_COUNT: 16\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  SIZE: 60000\n",
            "  SIZE_TEST: 10000\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: ffhq\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "PPL_CELEBA_ADJUSTMENT: False\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 300\n",
            "2020-06-11 18:48:55,113 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "  PART_COUNT: 16\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  SIZE: 60000\n",
            "  SIZE_TEST: 10000\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: ffhq\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "PPL_CELEBA_ADJUSTMENT: False\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 300\n",
            "Running on  Tesla P100-PCIE-16GB\n",
            "2020-06-11 18:48:55,144 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:48:55,144 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:48:55,146 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:48:55,146 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:48:55,149 logger INFO: Loading checkpoint from training_artifacts/ffhq/model_157.pth\n",
            "2020-06-11 18:48:55,149 logger INFO: Loading checkpoint from training_artifacts/ffhq/model_157.pth\n",
            "2020-06-11 18:48:55,615 logger INFO: Model trained for 157 epochs\n",
            "2020-06-11 18:48:55,615 logger INFO: Model trained for 157 epochs\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HzQALWTSSPYx",
        "colab_type": "text"
      },
      "source": [
        "## Multi Scale Reconstruction"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vDRn8KD7ZA6k",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def place(canvas, image, x, y):\n",
        "    im_size = image.shape[2]\n",
        "    if len(image.shape) == 4:\n",
        "        image = image[0]\n",
        "    canvas[:, y: y + im_size, x: x + im_size] = image * 0.5 + 0.5\n",
        "\n",
        "def sample(cfg, logger):\n",
        "    torch.cuda.set_device(0)\n",
        "    model = Model(\n",
        "        startf=cfg.MODEL.START_CHANNEL_COUNT,\n",
        "        layer_count=cfg.MODEL.LAYER_COUNT,\n",
        "        maxf=cfg.MODEL.MAX_CHANNEL_COUNT,\n",
        "        latent_size=cfg.MODEL.LATENT_SPACE_SIZE,\n",
        "        truncation_psi=cfg.MODEL.TRUNCATIOM_PSI,\n",
        "        truncation_cutoff=cfg.MODEL.TRUNCATIOM_CUTOFF,\n",
        "        mapping_layers=cfg.MODEL.MAPPING_LAYERS,\n",
        "        channels=cfg.MODEL.CHANNELS,\n",
        "        generator=cfg.MODEL.GENERATOR,\n",
        "        encoder=cfg.MODEL.ENCODER)\n",
        "    model.cuda(0)\n",
        "    model.eval()\n",
        "    model.requires_grad_(False)\n",
        "\n",
        "    decoder = model.decoder\n",
        "    encoder = model.encoder\n",
        "    mapping_tl = model.mapping_tl\n",
        "    mapping_fl = model.mapping_fl\n",
        "    dlatent_avg = model.dlatent_avg\n",
        "\n",
        "    logger.info(\"Trainable parameters generator:\")\n",
        "    count_parameters(decoder)\n",
        "\n",
        "    logger.info(\"Trainable parameters discriminator:\")\n",
        "    count_parameters(encoder)\n",
        "\n",
        "    arguments = dict()\n",
        "    arguments[\"iteration\"] = 0\n",
        "\n",
        "    model_dict = {\n",
        "        'discriminator_s': encoder,\n",
        "        'generator_s': decoder,\n",
        "        'mapping_tl_s': mapping_tl,\n",
        "        'mapping_fl_s': mapping_fl,\n",
        "        'dlatent_avg': dlatent_avg\n",
        "    }\n",
        "\n",
        "    checkpointer = Checkpointer(cfg,\n",
        "                                model_dict,\n",
        "                                {},\n",
        "                                logger=logger,\n",
        "                                save=False)\n",
        "\n",
        "    extra_checkpoint_data = checkpointer.load()\n",
        "\n",
        "    model.eval()\n",
        "\n",
        "    layer_count = cfg.MODEL.LAYER_COUNT\n",
        "\n",
        "    def encode(x):\n",
        "        Z, _ = model.encode(x, layer_count - 1, 1)\n",
        "        Z = Z.repeat(1, model.mapping_fl.num_layers, 1)\n",
        "        return Z\n",
        "\n",
        "    def decode(x):\n",
        "        layer_idx = torch.arange(2 * cfg.MODEL.LAYER_COUNT)[np.newaxis, :, np.newaxis]\n",
        "        ones = torch.ones(layer_idx.shape, dtype=torch.float32)\n",
        "        coefs = torch.where(layer_idx < model.truncation_cutoff, 1.0 * ones, ones)\n",
        "        # x = torch.lerp(model.dlatent_avg.buff.data, x, coefs)\n",
        "        return model.decoder(x, layer_count - 1, 1, noise=True)\n",
        "\n",
        "    path = cfg.DATASET.SAMPLES_PATH\n",
        "    # path = 'dataset_samples/faces/realign1024x1024_paper'\n",
        "\n",
        "    im_size = 2 ** (cfg.MODEL.LAYER_COUNT + 1)\n",
        "\n",
        "    paths = list(os.listdir(path))\n",
        "\n",
        "    paths = sorted(paths)\n",
        "    random.seed(5)\n",
        "    random.shuffle(paths)\n",
        "\n",
        "    def move_to(list, item, new_index):\n",
        "        list.remove(item)\n",
        "        list.insert(new_index, item)\n",
        "\n",
        "    # move_to(paths, '00026.png', 0)\n",
        "    # move_to(paths, '00074.png', 1)\n",
        "    # move_to(paths, '00134.png', 2)\n",
        "    # move_to(paths, '00036.png', 3)\n",
        "\n",
        "    def make(paths):\n",
        "        src = []\n",
        "        for filename in paths:\n",
        "            img = np.asarray(Image.open(path + '/' + filename))\n",
        "            if img.shape[2] == 4:\n",
        "                img = img[:, :, :3]\n",
        "            im = img.transpose((2, 0, 1))\n",
        "            x = torch.tensor(np.asarray(im, dtype=np.float32), requires_grad=True).cuda() / 127.5 - 1.\n",
        "            if x.shape[0] == 4:\n",
        "                x = x[:3]\n",
        "            factor = x.shape[2] // im_size\n",
        "            if factor != 1:\n",
        "                x = torch.nn.functional.avg_pool2d(x[None, ...], factor, factor)[0]\n",
        "            assert x.shape[2] == im_size\n",
        "            src.append(x)\n",
        "\n",
        "        with torch.no_grad():\n",
        "            reconstructions = []\n",
        "            for s in src:\n",
        "                latents = encode(s[None, ...])\n",
        "                reconstructions.append(decode(latents).cpu().detach().numpy())\n",
        "        return src, reconstructions\n",
        "\n",
        "    def chunker_list(seq, size):\n",
        "        return list((seq[i::size] for i in range(size)))\n",
        "\n",
        "    final = chunker_list(paths, 4)\n",
        "    path0, path1, path2, path3 = final\n",
        "\n",
        "    path0.reverse()\n",
        "    path1.reverse()\n",
        "    path2.reverse()\n",
        "    path3.reverse()\n",
        "\n",
        "    src0, rec0 = make(path0)\n",
        "    src1, rec1 = make(path1)\n",
        "    src2, rec2 = make(path2)\n",
        "    src3, rec3 = make(path3)\n",
        "\n",
        "    initial_resolution = im_size\n",
        "\n",
        "    lods_down = 1\n",
        "    padding_step = 4\n",
        "\n",
        "    width = 0\n",
        "    height = 0\n",
        "\n",
        "    current_padding = 0\n",
        "\n",
        "    final_resolution = initial_resolution\n",
        "    for _ in range(lods_down):\n",
        "        final_resolution /= 2\n",
        "\n",
        "    for i in range(lods_down + 1):\n",
        "        width += current_padding * 2 ** (lods_down - i)\n",
        "        height += current_padding * 2 ** (lods_down - i)\n",
        "        current_padding += padding_step\n",
        "\n",
        "    width += 2 ** (lods_down + 1) * final_resolution\n",
        "    height += (lods_down + 1) * initial_resolution\n",
        "\n",
        "    width = int(width)\n",
        "    height = int(height)\n",
        "\n",
        "    def make_part(current_padding, src, rec):\n",
        "        canvas = np.ones([3, height + 20, width + 10])\n",
        "\n",
        "        padd = 0\n",
        "\n",
        "        initial_padding = current_padding\n",
        "\n",
        "        height_padding = 0\n",
        "\n",
        "        for i in range(lods_down + 1):\n",
        "            for x in range(2 ** i):\n",
        "                for y in range(2 ** i):\n",
        "                    try:\n",
        "                        ims = src.pop()\n",
        "                        imr = rec.pop()[0]\n",
        "                        ims = ims.cpu().detach().numpy()\n",
        "                        imr = imr\n",
        "\n",
        "                        res = int(initial_resolution / 2 ** i)\n",
        "\n",
        "                        ims = resize(ims, (3, initial_resolution / 2 ** i, initial_resolution / 2 ** i))\n",
        "                        imr = resize(imr, (3, initial_resolution / 2 ** i, initial_resolution / 2 ** i))\n",
        "\n",
        "                        place(canvas, ims,\n",
        "                              current_padding + x * (2 * res + current_padding),\n",
        "                              i * initial_resolution + height_padding + y * (res + current_padding))\n",
        "\n",
        "                        place(canvas, imr,\n",
        "                              current_padding + res + x * (2 * res + current_padding),\n",
        "                              i * initial_resolution + height_padding + y * (res + current_padding))\n",
        "\n",
        "                    except IndexError:\n",
        "                        return canvas\n",
        "\n",
        "            height_padding += initial_padding * 2\n",
        "\n",
        "            current_padding -= padding_step\n",
        "            padd += padding_step\n",
        "        return canvas\n",
        "\n",
        "    canvas = [make_part(current_padding, src0, rec0), make_part(current_padding, src1, rec1),\n",
        "              make_part(current_padding, src2, rec2), make_part(current_padding, src3, rec3)]\n",
        "\n",
        "    canvas = np.concatenate(canvas, axis=2)\n",
        "\n",
        "    #print('Saving image')\n",
        "    #save_path = 'make_figures/output/%s/reconstructions_multiresolution.png' % cfg.NAME\n",
        "    #os.makedirs(os.path.dirname(save_path), exist_ok=True)\n",
        "    #save_image(torch.Tensor(canvas), save_path)\n",
        "    wandb.log({\"Multiresolution Reconstruction from {} Images\".format(cfg.NAME): [wandb.Image(torch.Tensor(canvas),caption=\"{} Multi Res Output\".format(cfg.NAME))]})"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R1q0iWN-dw6s",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "94d5e81b-54ae-4eef-be42-935647bac1cf"
      },
      "source": [
        "all_configs = ['configs/bedroom.yaml','configs/celeba.yaml']\n",
        "for config in all_configs:\n",
        "  cfg, logger, device = parse_configs(config)\n",
        "  sample(cfg, logger)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2020-06-11 18:50:10,639 logger INFO: World size: 1\n",
            "2020-06-11 18:50:10,639 logger INFO: World size: 1\n",
            "2020-06-11 18:50:10,639 logger INFO: World size: 1\n",
            "2020-06-11 18:50:10,641 logger INFO: Loaded configuration file configs/bedroom.yaml\n",
            "2020-06-11 18:50:10,641 logger INFO: Loaded configuration file configs/bedroom.yaml\n",
            "2020-06-11 18:50:10,641 logger INFO: Loaded configuration file configs/bedroom.yaml\n",
            "2020-06-11 18:50:10,643 logger INFO: \n",
            " # Config for training ALAE on lsun-bedroom at resolution 256x256\n",
            "\n",
            "NAME: bedroom\n",
            "DATASET:\n",
            "  PART_COUNT: 4\n",
            "  SIZE: 758260\n",
            "  FFHQ_SOURCE: /data/datasets/lsun-bedroom-full/lsun-bedroom-full-r%02d.tfrecords\n",
            "  PATH: /data/datasets/lsun-bedroom-full/splitted/lsun-bedroom-full-r%02d.tfrecords.%03d\n",
            "  MAX_RESOLUTION_LEVEL: 8\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/bedroom256x256\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_bedroom\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 7\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 32\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/bedroom\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 2\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 112\n",
            "  #                    4    8   16    32    64    128    256\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128,   64,   32,    32,    32,       32,        32]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128,   64,   32,    32,    32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128,   64,   32,    32,    16]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128,   64,   32,    16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:50:10,643 logger INFO: \n",
            " # Config for training ALAE on lsun-bedroom at resolution 256x256\n",
            "\n",
            "NAME: bedroom\n",
            "DATASET:\n",
            "  PART_COUNT: 4\n",
            "  SIZE: 758260\n",
            "  FFHQ_SOURCE: /data/datasets/lsun-bedroom-full/lsun-bedroom-full-r%02d.tfrecords\n",
            "  PATH: /data/datasets/lsun-bedroom-full/splitted/lsun-bedroom-full-r%02d.tfrecords.%03d\n",
            "  MAX_RESOLUTION_LEVEL: 8\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/bedroom256x256\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_bedroom\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 7\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 32\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/bedroom\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 2\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 112\n",
            "  #                    4    8   16    32    64    128    256\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128,   64,   32,    32,    32,       32,        32]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128,   64,   32,    32,    32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128,   64,   32,    32,    16]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128,   64,   32,    16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:50:10,643 logger INFO: \n",
            " # Config for training ALAE on lsun-bedroom at resolution 256x256\n",
            "\n",
            "NAME: bedroom\n",
            "DATASET:\n",
            "  PART_COUNT: 4\n",
            "  SIZE: 758260\n",
            "  FFHQ_SOURCE: /data/datasets/lsun-bedroom-full/lsun-bedroom-full-r%02d.tfrecords\n",
            "  PATH: /data/datasets/lsun-bedroom-full/splitted/lsun-bedroom-full-r%02d.tfrecords.%03d\n",
            "  MAX_RESOLUTION_LEVEL: 8\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/bedroom256x256\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_bedroom\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 7\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 32\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/bedroom\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 2\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 112\n",
            "  #                    4    8   16    32    64    128    256\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128,   64,   32,    32,    32,       32,        32]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128,   64,   32,    32,    32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128,   64,   32,    32,    16]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128,   64,   32,    16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:50:10,645 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/lsun-bedroom-full/lsun-bedroom-full-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 8\n",
            "  PART_COUNT: 4\n",
            "  PART_COUNT_TEST: 1\n",
            "  PATH: /data/datasets/lsun-bedroom-full/splitted/lsun-bedroom-full-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: \n",
            "  SAMPLES_PATH: dataset_samples/bedroom256x256\n",
            "  SIZE: 758260\n",
            "  SIZE_TEST: 10000\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_bedroom\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 7\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 32\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: bedroom\n",
            "OUTPUT_DIR: training_artifacts/bedroom\n",
            "PPL_CELEBA_ADJUSTMENT: False\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 2\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 112\n",
            "2020-06-11 18:50:10,645 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/lsun-bedroom-full/lsun-bedroom-full-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 8\n",
            "  PART_COUNT: 4\n",
            "  PART_COUNT_TEST: 1\n",
            "  PATH: /data/datasets/lsun-bedroom-full/splitted/lsun-bedroom-full-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: \n",
            "  SAMPLES_PATH: dataset_samples/bedroom256x256\n",
            "  SIZE: 758260\n",
            "  SIZE_TEST: 10000\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_bedroom\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 7\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 32\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: bedroom\n",
            "OUTPUT_DIR: training_artifacts/bedroom\n",
            "PPL_CELEBA_ADJUSTMENT: False\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 2\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 112\n",
            "2020-06-11 18:50:10,645 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/lsun-bedroom-full/lsun-bedroom-full-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 8\n",
            "  PART_COUNT: 4\n",
            "  PART_COUNT_TEST: 1\n",
            "  PATH: /data/datasets/lsun-bedroom-full/splitted/lsun-bedroom-full-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: \n",
            "  SAMPLES_PATH: dataset_samples/bedroom256x256\n",
            "  SIZE: 758260\n",
            "  SIZE_TEST: 10000\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_bedroom\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 7\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 32\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: bedroom\n",
            "OUTPUT_DIR: training_artifacts/bedroom\n",
            "PPL_CELEBA_ADJUSTMENT: False\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 2\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 112\n",
            "Running on  Tesla P100-PCIE-16GB\n",
            "2020-06-11 18:50:10,681 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:50:10,681 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:50:10,681 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:50:10,683 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:50:10,683 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:50:10,683 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:50:10,687 logger INFO: Loading checkpoint from training_artifacts/bedroom/model_final.pth\n",
            "2020-06-11 18:50:10,687 logger INFO: Loading checkpoint from training_artifacts/bedroom/model_final.pth\n",
            "2020-06-11 18:50:10,687 logger INFO: Loading checkpoint from training_artifacts/bedroom/model_final.pth\n",
            "2020-06-11 18:50:17,555 logger INFO: World size: 1\n",
            "2020-06-11 18:50:17,555 logger INFO: World size: 1\n",
            "2020-06-11 18:50:17,555 logger INFO: World size: 1\n",
            "2020-06-11 18:50:17,555 logger INFO: World size: 1\n",
            "2020-06-11 18:50:17,559 logger INFO: Loaded configuration file configs/celeba.yaml\n",
            "2020-06-11 18:50:17,559 logger INFO: Loaded configuration file configs/celeba.yaml\n",
            "2020-06-11 18:50:17,559 logger INFO: Loaded configuration file configs/celeba.yaml\n",
            "2020-06-11 18:50:17,559 logger INFO: Loaded configuration file configs/celeba.yaml\n",
            "2020-06-11 18:50:17,564 logger INFO: \n",
            "# Config for training ALAE on CelebA at resolution 128x128\n",
            "\n",
            "NAME: celeba\n",
            "PPL_CELEBA_ADJUSTMENT: True\n",
            "DATASET:\n",
            "  PART_COUNT: 16\n",
            "  SIZE: 182637\n",
            "  SIZE_TEST: 202576 - 182637\n",
            "  PATH: /data/datasets/celeba/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/celeba-test/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  MAX_RESOLUTION_LEVEL: 7\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign128x128\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_celeba\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 256\n",
            "  LAYER_COUNT: 6\n",
            "  MAX_CHANNEL_COUNT: 256\n",
            "  START_CHANNEL_COUNT: 64\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/celeba\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 6\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 80\n",
            "  #                    4       8       16       32       64       128        256       512       1024\n",
            "  LOD_2_BATCH_8GPU: [512,    256,     128,      64,      32,       32,        32,       32,        32]\n",
            "  LOD_2_BATCH_4GPU: [512,    256,     128,      64,      32,       32,        32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [128,    128,     128,      64,      32,       32,        16]\n",
            "  LOD_2_BATCH_1GPU: [128,    128,     128,      64,      32,       16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:50:17,564 logger INFO: \n",
            "# Config for training ALAE on CelebA at resolution 128x128\n",
            "\n",
            "NAME: celeba\n",
            "PPL_CELEBA_ADJUSTMENT: True\n",
            "DATASET:\n",
            "  PART_COUNT: 16\n",
            "  SIZE: 182637\n",
            "  SIZE_TEST: 202576 - 182637\n",
            "  PATH: /data/datasets/celeba/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/celeba-test/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  MAX_RESOLUTION_LEVEL: 7\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign128x128\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_celeba\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 256\n",
            "  LAYER_COUNT: 6\n",
            "  MAX_CHANNEL_COUNT: 256\n",
            "  START_CHANNEL_COUNT: 64\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/celeba\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 6\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 80\n",
            "  #                    4       8       16       32       64       128        256       512       1024\n",
            "  LOD_2_BATCH_8GPU: [512,    256,     128,      64,      32,       32,        32,       32,        32]\n",
            "  LOD_2_BATCH_4GPU: [512,    256,     128,      64,      32,       32,        32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [128,    128,     128,      64,      32,       32,        16]\n",
            "  LOD_2_BATCH_1GPU: [128,    128,     128,      64,      32,       16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:50:17,564 logger INFO: \n",
            "# Config for training ALAE on CelebA at resolution 128x128\n",
            "\n",
            "NAME: celeba\n",
            "PPL_CELEBA_ADJUSTMENT: True\n",
            "DATASET:\n",
            "  PART_COUNT: 16\n",
            "  SIZE: 182637\n",
            "  SIZE_TEST: 202576 - 182637\n",
            "  PATH: /data/datasets/celeba/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/celeba-test/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  MAX_RESOLUTION_LEVEL: 7\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign128x128\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_celeba\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 256\n",
            "  LAYER_COUNT: 6\n",
            "  MAX_CHANNEL_COUNT: 256\n",
            "  START_CHANNEL_COUNT: 64\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/celeba\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 6\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 80\n",
            "  #                    4       8       16       32       64       128        256       512       1024\n",
            "  LOD_2_BATCH_8GPU: [512,    256,     128,      64,      32,       32,        32,       32,        32]\n",
            "  LOD_2_BATCH_4GPU: [512,    256,     128,      64,      32,       32,        32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [128,    128,     128,      64,      32,       32,        16]\n",
            "  LOD_2_BATCH_1GPU: [128,    128,     128,      64,      32,       16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:50:17,564 logger INFO: \n",
            "# Config for training ALAE on CelebA at resolution 128x128\n",
            "\n",
            "NAME: celeba\n",
            "PPL_CELEBA_ADJUSTMENT: True\n",
            "DATASET:\n",
            "  PART_COUNT: 16\n",
            "  SIZE: 182637\n",
            "  SIZE_TEST: 202576 - 182637\n",
            "  PATH: /data/datasets/celeba/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/celeba-test/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  MAX_RESOLUTION_LEVEL: 7\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign128x128\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_celeba\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 256\n",
            "  LAYER_COUNT: 6\n",
            "  MAX_CHANNEL_COUNT: 256\n",
            "  START_CHANNEL_COUNT: 64\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/celeba\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 6\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 80\n",
            "  #                    4       8       16       32       64       128        256       512       1024\n",
            "  LOD_2_BATCH_8GPU: [512,    256,     128,      64,      32,       32,        32,       32,        32]\n",
            "  LOD_2_BATCH_4GPU: [512,    256,     128,      64,      32,       32,        32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [128,    128,     128,      64,      32,       32,        16]\n",
            "  LOD_2_BATCH_1GPU: [128,    128,     128,      64,      32,       16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:50:17,573 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 7\n",
            "  PART_COUNT: 16\n",
            "  PART_COUNT_TEST: 1\n",
            "  PATH: /data/datasets/celeba/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/celeba-test/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign128x128\n",
            "  SIZE: 182637\n",
            "  SIZE_TEST: 19939\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_celeba\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 256\n",
            "  LAYER_COUNT: 6\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 256\n",
            "  START_CHANNEL_COUNT: 64\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: celeba\n",
            "OUTPUT_DIR: training_artifacts/celeba\n",
            "PPL_CELEBA_ADJUSTMENT: True\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 6\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [128, 128, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [128, 128, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 80\n",
            "2020-06-11 18:50:17,573 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 7\n",
            "  PART_COUNT: 16\n",
            "  PART_COUNT_TEST: 1\n",
            "  PATH: /data/datasets/celeba/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/celeba-test/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign128x128\n",
            "  SIZE: 182637\n",
            "  SIZE_TEST: 19939\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_celeba\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 256\n",
            "  LAYER_COUNT: 6\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 256\n",
            "  START_CHANNEL_COUNT: 64\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: celeba\n",
            "OUTPUT_DIR: training_artifacts/celeba\n",
            "PPL_CELEBA_ADJUSTMENT: True\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 6\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [128, 128, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [128, 128, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 80\n",
            "2020-06-11 18:50:17,573 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 7\n",
            "  PART_COUNT: 16\n",
            "  PART_COUNT_TEST: 1\n",
            "  PATH: /data/datasets/celeba/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/celeba-test/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign128x128\n",
            "  SIZE: 182637\n",
            "  SIZE_TEST: 19939\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_celeba\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 256\n",
            "  LAYER_COUNT: 6\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 256\n",
            "  START_CHANNEL_COUNT: 64\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: celeba\n",
            "OUTPUT_DIR: training_artifacts/celeba\n",
            "PPL_CELEBA_ADJUSTMENT: True\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 6\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [128, 128, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [128, 128, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 80\n",
            "2020-06-11 18:50:17,573 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 7\n",
            "  PART_COUNT: 16\n",
            "  PART_COUNT_TEST: 1\n",
            "  PATH: /data/datasets/celeba/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/celeba-test/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign128x128\n",
            "  SIZE: 182637\n",
            "  SIZE_TEST: 19939\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_celeba\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 256\n",
            "  LAYER_COUNT: 6\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 256\n",
            "  START_CHANNEL_COUNT: 64\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: celeba\n",
            "OUTPUT_DIR: training_artifacts/celeba\n",
            "PPL_CELEBA_ADJUSTMENT: True\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 6\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [128, 128, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [128, 128, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 80\n",
            "Running on  Tesla P100-PCIE-16GB\n",
            "2020-06-11 18:50:17,609 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:50:17,609 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:50:17,609 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:50:17,609 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:50:17,613 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:50:17,613 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:50:17,613 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:50:17,613 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:50:17,619 logger INFO: Loading checkpoint from training_artifacts/celeba/model_final.pth\n",
            "2020-06-11 18:50:17,619 logger INFO: Loading checkpoint from training_artifacts/celeba/model_final.pth\n",
            "2020-06-11 18:50:17,619 logger INFO: Loading checkpoint from training_artifacts/celeba/model_final.pth\n",
            "2020-06-11 18:50:17,619 logger INFO: Loading checkpoint from training_artifacts/celeba/model_final.pth\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tivR8hpDFjaN",
        "colab_type": "text"
      },
      "source": [
        "## Interpolation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HDWWSNTnZ619",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "lreq.use_implicit_lreq.set(True)\n",
        "\n",
        "\n",
        "def place(canvas, image, x, y):\n",
        "    im_size = image.shape[2]\n",
        "    if len(image.shape) == 4:\n",
        "        image = image[0]\n",
        "    canvas[:, y: y + im_size, x: x + im_size] = image * 0.5 + 0.5\n",
        "\n",
        "def sample(cfg, logger):\n",
        "    torch.cuda.set_device(0)\n",
        "    model = Model(\n",
        "        startf=cfg.MODEL.START_CHANNEL_COUNT,\n",
        "        layer_count=cfg.MODEL.LAYER_COUNT,\n",
        "        maxf=cfg.MODEL.MAX_CHANNEL_COUNT,\n",
        "        latent_size=cfg.MODEL.LATENT_SPACE_SIZE,\n",
        "        truncation_psi=cfg.MODEL.TRUNCATIOM_PSI,\n",
        "        truncation_cutoff=cfg.MODEL.TRUNCATIOM_CUTOFF,\n",
        "        mapping_layers=cfg.MODEL.MAPPING_LAYERS,\n",
        "        channels=cfg.MODEL.CHANNELS,\n",
        "        generator=cfg.MODEL.GENERATOR,\n",
        "        encoder=cfg.MODEL.ENCODER)\n",
        "    model.cuda(0)\n",
        "    model.eval()\n",
        "    model.requires_grad_(False)\n",
        "\n",
        "    decoder = model.decoder\n",
        "    encoder = model.encoder\n",
        "    mapping_tl = model.mapping_tl\n",
        "    mapping_fl = model.mapping_fl\n",
        "    dlatent_avg = model.dlatent_avg\n",
        "\n",
        "    logger.info(\"Trainable parameters generator:\")\n",
        "    count_parameters(decoder)\n",
        "\n",
        "    logger.info(\"Trainable parameters discriminator:\")\n",
        "    count_parameters(encoder)\n",
        "\n",
        "    arguments = dict()\n",
        "    arguments[\"iteration\"] = 0\n",
        "\n",
        "    model_dict = {\n",
        "        'discriminator_s': encoder,\n",
        "        'generator_s': decoder,\n",
        "        'mapping_tl_s': mapping_tl,\n",
        "        'mapping_fl_s': mapping_fl,\n",
        "        'dlatent_avg': dlatent_avg\n",
        "    }\n",
        "\n",
        "    checkpointer = Checkpointer(cfg,\n",
        "                                model_dict,\n",
        "                                {},\n",
        "                                logger=logger,\n",
        "                                save=False)\n",
        "\n",
        "    extra_checkpoint_data = checkpointer.load()\n",
        "\n",
        "    model.eval()\n",
        "\n",
        "    layer_count = cfg.MODEL.LAYER_COUNT\n",
        "\n",
        "    def encode(x):\n",
        "        Z, _ = model.encode(x, layer_count - 1, 1)\n",
        "        Z = Z.repeat(1, model.mapping_fl.num_layers, 1)\n",
        "        return Z\n",
        "\n",
        "    def decode(x):\n",
        "        layer_idx = torch.arange(2 * cfg.MODEL.LAYER_COUNT)[np.newaxis, :, np.newaxis]\n",
        "        ones = torch.ones(layer_idx.shape, dtype=torch.float32)\n",
        "        coefs = torch.where(layer_idx < model.truncation_cutoff, ones, ones)\n",
        "        # x = torch.lerp(model.dlatent_avg.buff.data, x, coefs)\n",
        "        return model.decoder(x, layer_count - 1, 1, noise=True)\n",
        "\n",
        "    rnd = np.random.RandomState(4)\n",
        "    latents = rnd.randn(1, cfg.MODEL.LATENT_SPACE_SIZE)\n",
        "\n",
        "    path = cfg.DATASET.SAMPLES_PATH\n",
        "    im_size = 2 ** (cfg.MODEL.LAYER_COUNT + 1)\n",
        "\n",
        "    pathA = '00001.png'\n",
        "    pathB = '00022.png'\n",
        "    pathC = '00077.png'\n",
        "    pathD = '00016.png'\n",
        "\n",
        "    def open_image(filename):\n",
        "        img = np.asarray(Image.open(path + '/' + filename))\n",
        "        if img.shape[2] == 4:\n",
        "            img = img[:, :, :3]\n",
        "        im = img.transpose((2, 0, 1))\n",
        "        x = torch.tensor(np.asarray(im, dtype=np.float32), device='cpu', requires_grad=True).cuda() / 127.5 - 1.\n",
        "        if x.shape[0] == 4:\n",
        "            x = x[:3]\n",
        "        factor = x.shape[2] // im_size\n",
        "        if factor != 1:\n",
        "            x = torch.nn.functional.avg_pool2d(x[None, ...], factor, factor)[0]\n",
        "        assert x.shape[2] == im_size\n",
        "        _latents = encode(x[None, ...].cuda())\n",
        "        latents = _latents[0, 0]\n",
        "        return latents\n",
        "\n",
        "    def make(w):\n",
        "        with torch.no_grad():\n",
        "            w = w[None, None, ...].repeat(1, model.mapping_fl.num_layers, 1)\n",
        "            x_rec = decode(w)\n",
        "            return x_rec\n",
        "\n",
        "    wa = open_image(pathA)\n",
        "    wb = open_image(pathB)\n",
        "    wc = open_image(pathC)\n",
        "    wd = open_image(pathD)\n",
        "\n",
        "    height = 7\n",
        "    width = 7\n",
        "\n",
        "    images = []\n",
        "    for i in range(height):\n",
        "        for j in range(width):\n",
        "            kv = i / (height - 1.0)\n",
        "            kh = j / (width - 1.0)\n",
        "\n",
        "            ka = (1.0 - kh) * (1.0 - kv)\n",
        "            kb = kh * (1.0 - kv)\n",
        "            kc = (1.0 - kh) * kv\n",
        "            kd = kh * kv\n",
        "\n",
        "            w = ka * wa + kb * wb + kc * wc + kd * wd\n",
        "\n",
        "            interpolated = make(w)\n",
        "            images.append(interpolated)\n",
        " \n",
        "    images = torch.cat(images)\n",
        "    grid_ = make_grid(images, nrow=width)\n",
        "    wandb.log({\"Interpolations from {} Images\".format(cfg.NAME): [wandb.Image(torch.Tensor(grid_ *0.5 + 0.5),caption=\"{} Interpolation Output\".format(cfg.NAME))]})\n",
        "    #save_image(images * 0.5 + 0.5, 'make_figures/output/%s/interpolations.png' % cfg.NAME, nrow=width)\n",
        "    #save_image(images * 0.5 + 0.5, 'make_figures/output/%s/interpolations.jpg' % cfg.NAME, nrow=width)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DTYw-GpEZ8tp",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "86bc9519-4891-44d4-bd70-0dbbd2e70c12"
      },
      "source": [
        "all_configs = ['configs/bedroom.yaml','configs/celeba.yaml','configs/ffhq.yaml']\n",
        "for config in all_configs[1:]:\n",
        "  cfg, logger, device = parse_configs(config)\n",
        "  sample(cfg, logger)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2020-06-11 18:50:23,398 logger INFO: World size: 1\n",
            "2020-06-11 18:50:23,398 logger INFO: World size: 1\n",
            "2020-06-11 18:50:23,398 logger INFO: World size: 1\n",
            "2020-06-11 18:50:23,398 logger INFO: World size: 1\n",
            "2020-06-11 18:50:23,398 logger INFO: World size: 1\n",
            "2020-06-11 18:50:23,401 logger INFO: Loaded configuration file configs/celeba.yaml\n",
            "2020-06-11 18:50:23,401 logger INFO: Loaded configuration file configs/celeba.yaml\n",
            "2020-06-11 18:50:23,401 logger INFO: Loaded configuration file configs/celeba.yaml\n",
            "2020-06-11 18:50:23,401 logger INFO: Loaded configuration file configs/celeba.yaml\n",
            "2020-06-11 18:50:23,401 logger INFO: Loaded configuration file configs/celeba.yaml\n",
            "2020-06-11 18:50:23,405 logger INFO: \n",
            "# Config for training ALAE on CelebA at resolution 128x128\n",
            "\n",
            "NAME: celeba\n",
            "PPL_CELEBA_ADJUSTMENT: True\n",
            "DATASET:\n",
            "  PART_COUNT: 16\n",
            "  SIZE: 182637\n",
            "  SIZE_TEST: 202576 - 182637\n",
            "  PATH: /data/datasets/celeba/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/celeba-test/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  MAX_RESOLUTION_LEVEL: 7\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign128x128\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_celeba\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 256\n",
            "  LAYER_COUNT: 6\n",
            "  MAX_CHANNEL_COUNT: 256\n",
            "  START_CHANNEL_COUNT: 64\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/celeba\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 6\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 80\n",
            "  #                    4       8       16       32       64       128        256       512       1024\n",
            "  LOD_2_BATCH_8GPU: [512,    256,     128,      64,      32,       32,        32,       32,        32]\n",
            "  LOD_2_BATCH_4GPU: [512,    256,     128,      64,      32,       32,        32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [128,    128,     128,      64,      32,       32,        16]\n",
            "  LOD_2_BATCH_1GPU: [128,    128,     128,      64,      32,       16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:50:23,405 logger INFO: \n",
            "# Config for training ALAE on CelebA at resolution 128x128\n",
            "\n",
            "NAME: celeba\n",
            "PPL_CELEBA_ADJUSTMENT: True\n",
            "DATASET:\n",
            "  PART_COUNT: 16\n",
            "  SIZE: 182637\n",
            "  SIZE_TEST: 202576 - 182637\n",
            "  PATH: /data/datasets/celeba/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/celeba-test/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  MAX_RESOLUTION_LEVEL: 7\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign128x128\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_celeba\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 256\n",
            "  LAYER_COUNT: 6\n",
            "  MAX_CHANNEL_COUNT: 256\n",
            "  START_CHANNEL_COUNT: 64\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/celeba\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 6\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 80\n",
            "  #                    4       8       16       32       64       128        256       512       1024\n",
            "  LOD_2_BATCH_8GPU: [512,    256,     128,      64,      32,       32,        32,       32,        32]\n",
            "  LOD_2_BATCH_4GPU: [512,    256,     128,      64,      32,       32,        32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [128,    128,     128,      64,      32,       32,        16]\n",
            "  LOD_2_BATCH_1GPU: [128,    128,     128,      64,      32,       16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:50:23,405 logger INFO: \n",
            "# Config for training ALAE on CelebA at resolution 128x128\n",
            "\n",
            "NAME: celeba\n",
            "PPL_CELEBA_ADJUSTMENT: True\n",
            "DATASET:\n",
            "  PART_COUNT: 16\n",
            "  SIZE: 182637\n",
            "  SIZE_TEST: 202576 - 182637\n",
            "  PATH: /data/datasets/celeba/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/celeba-test/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  MAX_RESOLUTION_LEVEL: 7\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign128x128\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_celeba\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 256\n",
            "  LAYER_COUNT: 6\n",
            "  MAX_CHANNEL_COUNT: 256\n",
            "  START_CHANNEL_COUNT: 64\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/celeba\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 6\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 80\n",
            "  #                    4       8       16       32       64       128        256       512       1024\n",
            "  LOD_2_BATCH_8GPU: [512,    256,     128,      64,      32,       32,        32,       32,        32]\n",
            "  LOD_2_BATCH_4GPU: [512,    256,     128,      64,      32,       32,        32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [128,    128,     128,      64,      32,       32,        16]\n",
            "  LOD_2_BATCH_1GPU: [128,    128,     128,      64,      32,       16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:50:23,405 logger INFO: \n",
            "# Config for training ALAE on CelebA at resolution 128x128\n",
            "\n",
            "NAME: celeba\n",
            "PPL_CELEBA_ADJUSTMENT: True\n",
            "DATASET:\n",
            "  PART_COUNT: 16\n",
            "  SIZE: 182637\n",
            "  SIZE_TEST: 202576 - 182637\n",
            "  PATH: /data/datasets/celeba/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/celeba-test/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  MAX_RESOLUTION_LEVEL: 7\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign128x128\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_celeba\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 256\n",
            "  LAYER_COUNT: 6\n",
            "  MAX_CHANNEL_COUNT: 256\n",
            "  START_CHANNEL_COUNT: 64\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/celeba\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 6\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 80\n",
            "  #                    4       8       16       32       64       128        256       512       1024\n",
            "  LOD_2_BATCH_8GPU: [512,    256,     128,      64,      32,       32,        32,       32,        32]\n",
            "  LOD_2_BATCH_4GPU: [512,    256,     128,      64,      32,       32,        32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [128,    128,     128,      64,      32,       32,        16]\n",
            "  LOD_2_BATCH_1GPU: [128,    128,     128,      64,      32,       16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:50:23,405 logger INFO: \n",
            "# Config for training ALAE on CelebA at resolution 128x128\n",
            "\n",
            "NAME: celeba\n",
            "PPL_CELEBA_ADJUSTMENT: True\n",
            "DATASET:\n",
            "  PART_COUNT: 16\n",
            "  SIZE: 182637\n",
            "  SIZE_TEST: 202576 - 182637\n",
            "  PATH: /data/datasets/celeba/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/celeba-test/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  MAX_RESOLUTION_LEVEL: 7\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign128x128\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_celeba\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 256\n",
            "  LAYER_COUNT: 6\n",
            "  MAX_CHANNEL_COUNT: 256\n",
            "  START_CHANNEL_COUNT: 64\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/celeba\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 6\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 80\n",
            "  #                    4       8       16       32       64       128        256       512       1024\n",
            "  LOD_2_BATCH_8GPU: [512,    256,     128,      64,      32,       32,        32,       32,        32]\n",
            "  LOD_2_BATCH_4GPU: [512,    256,     128,      64,      32,       32,        32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [128,    128,     128,      64,      32,       32,        16]\n",
            "  LOD_2_BATCH_1GPU: [128,    128,     128,      64,      32,       16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:50:23,411 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 7\n",
            "  PART_COUNT: 16\n",
            "  PART_COUNT_TEST: 1\n",
            "  PATH: /data/datasets/celeba/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/celeba-test/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign128x128\n",
            "  SIZE: 182637\n",
            "  SIZE_TEST: 19939\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_celeba\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 256\n",
            "  LAYER_COUNT: 6\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 256\n",
            "  START_CHANNEL_COUNT: 64\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: celeba\n",
            "OUTPUT_DIR: training_artifacts/celeba\n",
            "PPL_CELEBA_ADJUSTMENT: True\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 6\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [128, 128, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [128, 128, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 80\n",
            "2020-06-11 18:50:23,411 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 7\n",
            "  PART_COUNT: 16\n",
            "  PART_COUNT_TEST: 1\n",
            "  PATH: /data/datasets/celeba/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/celeba-test/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign128x128\n",
            "  SIZE: 182637\n",
            "  SIZE_TEST: 19939\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_celeba\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 256\n",
            "  LAYER_COUNT: 6\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 256\n",
            "  START_CHANNEL_COUNT: 64\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: celeba\n",
            "OUTPUT_DIR: training_artifacts/celeba\n",
            "PPL_CELEBA_ADJUSTMENT: True\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 6\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [128, 128, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [128, 128, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 80\n",
            "2020-06-11 18:50:23,411 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 7\n",
            "  PART_COUNT: 16\n",
            "  PART_COUNT_TEST: 1\n",
            "  PATH: /data/datasets/celeba/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/celeba-test/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign128x128\n",
            "  SIZE: 182637\n",
            "  SIZE_TEST: 19939\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_celeba\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 256\n",
            "  LAYER_COUNT: 6\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 256\n",
            "  START_CHANNEL_COUNT: 64\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: celeba\n",
            "OUTPUT_DIR: training_artifacts/celeba\n",
            "PPL_CELEBA_ADJUSTMENT: True\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 6\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [128, 128, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [128, 128, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 80\n",
            "2020-06-11 18:50:23,411 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 7\n",
            "  PART_COUNT: 16\n",
            "  PART_COUNT_TEST: 1\n",
            "  PATH: /data/datasets/celeba/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/celeba-test/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign128x128\n",
            "  SIZE: 182637\n",
            "  SIZE_TEST: 19939\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_celeba\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 256\n",
            "  LAYER_COUNT: 6\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 256\n",
            "  START_CHANNEL_COUNT: 64\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: celeba\n",
            "OUTPUT_DIR: training_artifacts/celeba\n",
            "PPL_CELEBA_ADJUSTMENT: True\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 6\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [128, 128, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [128, 128, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 80\n",
            "2020-06-11 18:50:23,411 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 7\n",
            "  PART_COUNT: 16\n",
            "  PART_COUNT_TEST: 1\n",
            "  PATH: /data/datasets/celeba/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/celeba-test/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign128x128\n",
            "  SIZE: 182637\n",
            "  SIZE_TEST: 19939\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_celeba\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 256\n",
            "  LAYER_COUNT: 6\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 256\n",
            "  START_CHANNEL_COUNT: 64\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: celeba\n",
            "OUTPUT_DIR: training_artifacts/celeba\n",
            "PPL_CELEBA_ADJUSTMENT: True\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 6\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [128, 128, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [128, 128, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 80\n",
            "Running on  Tesla P100-PCIE-16GB\n",
            "2020-06-11 18:50:23,457 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:50:23,457 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:50:23,457 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:50:23,457 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:50:23,457 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:50:23,462 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:50:23,462 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:50:23,462 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:50:23,462 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:50:23,462 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:50:23,466 logger INFO: Loading checkpoint from training_artifacts/celeba/model_final.pth\n",
            "2020-06-11 18:50:23,466 logger INFO: Loading checkpoint from training_artifacts/celeba/model_final.pth\n",
            "2020-06-11 18:50:23,466 logger INFO: Loading checkpoint from training_artifacts/celeba/model_final.pth\n",
            "2020-06-11 18:50:23,466 logger INFO: Loading checkpoint from training_artifacts/celeba/model_final.pth\n",
            "2020-06-11 18:50:23,466 logger INFO: Loading checkpoint from training_artifacts/celeba/model_final.pth\n",
            "2020-06-11 18:50:25,979 logger INFO: World size: 1\n",
            "2020-06-11 18:50:25,979 logger INFO: World size: 1\n",
            "2020-06-11 18:50:25,979 logger INFO: World size: 1\n",
            "2020-06-11 18:50:25,979 logger INFO: World size: 1\n",
            "2020-06-11 18:50:25,979 logger INFO: World size: 1\n",
            "2020-06-11 18:50:25,979 logger INFO: World size: 1\n",
            "2020-06-11 18:50:25,990 logger INFO: Loaded configuration file configs/ffhq.yaml\n",
            "2020-06-11 18:50:25,990 logger INFO: Loaded configuration file configs/ffhq.yaml\n",
            "2020-06-11 18:50:25,990 logger INFO: Loaded configuration file configs/ffhq.yaml\n",
            "2020-06-11 18:50:25,990 logger INFO: Loaded configuration file configs/ffhq.yaml\n",
            "2020-06-11 18:50:25,990 logger INFO: Loaded configuration file configs/ffhq.yaml\n",
            "2020-06-11 18:50:25,990 logger INFO: Loaded configuration file configs/ffhq.yaml\n",
            "2020-06-11 18:50:25,995 logger INFO: \n",
            " # Config for training ALAE on FFHQ at resolution 1024x1024\n",
            "\n",
            "NAME: ffhq\n",
            "DATASET:\n",
            "  PART_COUNT: 16\n",
            "  SIZE: 60000\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 300\n",
            "  #                    4    8   16    32    64    128    256\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128,   64,   32,    32,    32,       32,        32] # If GPU memory ~16GB reduce last number from 32 to 24\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128,   64,   32,    32,    32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128,   64,   32,    32,    16]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128,   64,   32,    16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:50:25,995 logger INFO: \n",
            " # Config for training ALAE on FFHQ at resolution 1024x1024\n",
            "\n",
            "NAME: ffhq\n",
            "DATASET:\n",
            "  PART_COUNT: 16\n",
            "  SIZE: 60000\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 300\n",
            "  #                    4    8   16    32    64    128    256\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128,   64,   32,    32,    32,       32,        32] # If GPU memory ~16GB reduce last number from 32 to 24\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128,   64,   32,    32,    32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128,   64,   32,    32,    16]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128,   64,   32,    16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:50:25,995 logger INFO: \n",
            " # Config for training ALAE on FFHQ at resolution 1024x1024\n",
            "\n",
            "NAME: ffhq\n",
            "DATASET:\n",
            "  PART_COUNT: 16\n",
            "  SIZE: 60000\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 300\n",
            "  #                    4    8   16    32    64    128    256\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128,   64,   32,    32,    32,       32,        32] # If GPU memory ~16GB reduce last number from 32 to 24\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128,   64,   32,    32,    32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128,   64,   32,    32,    16]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128,   64,   32,    16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:50:25,995 logger INFO: \n",
            " # Config for training ALAE on FFHQ at resolution 1024x1024\n",
            "\n",
            "NAME: ffhq\n",
            "DATASET:\n",
            "  PART_COUNT: 16\n",
            "  SIZE: 60000\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 300\n",
            "  #                    4    8   16    32    64    128    256\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128,   64,   32,    32,    32,       32,        32] # If GPU memory ~16GB reduce last number from 32 to 24\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128,   64,   32,    32,    32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128,   64,   32,    32,    16]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128,   64,   32,    16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:50:25,995 logger INFO: \n",
            " # Config for training ALAE on FFHQ at resolution 1024x1024\n",
            "\n",
            "NAME: ffhq\n",
            "DATASET:\n",
            "  PART_COUNT: 16\n",
            "  SIZE: 60000\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 300\n",
            "  #                    4    8   16    32    64    128    256\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128,   64,   32,    32,    32,       32,        32] # If GPU memory ~16GB reduce last number from 32 to 24\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128,   64,   32,    32,    32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128,   64,   32,    32,    16]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128,   64,   32,    16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:50:25,995 logger INFO: \n",
            " # Config for training ALAE on FFHQ at resolution 1024x1024\n",
            "\n",
            "NAME: ffhq\n",
            "DATASET:\n",
            "  PART_COUNT: 16\n",
            "  SIZE: 60000\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 300\n",
            "  #                    4    8   16    32    64    128    256\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128,   64,   32,    32,    32,       32,        32] # If GPU memory ~16GB reduce last number from 32 to 24\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128,   64,   32,    32,    32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128,   64,   32,    32,    16]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128,   64,   32,    16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:50:26,008 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "  PART_COUNT: 16\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  SIZE: 60000\n",
            "  SIZE_TEST: 10000\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: ffhq\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "PPL_CELEBA_ADJUSTMENT: False\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 300\n",
            "2020-06-11 18:50:26,008 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "  PART_COUNT: 16\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  SIZE: 60000\n",
            "  SIZE_TEST: 10000\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: ffhq\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "PPL_CELEBA_ADJUSTMENT: False\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 300\n",
            "2020-06-11 18:50:26,008 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "  PART_COUNT: 16\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  SIZE: 60000\n",
            "  SIZE_TEST: 10000\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: ffhq\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "PPL_CELEBA_ADJUSTMENT: False\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 300\n",
            "2020-06-11 18:50:26,008 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "  PART_COUNT: 16\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  SIZE: 60000\n",
            "  SIZE_TEST: 10000\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: ffhq\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "PPL_CELEBA_ADJUSTMENT: False\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 300\n",
            "2020-06-11 18:50:26,008 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "  PART_COUNT: 16\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  SIZE: 60000\n",
            "  SIZE_TEST: 10000\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: ffhq\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "PPL_CELEBA_ADJUSTMENT: False\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 300\n",
            "2020-06-11 18:50:26,008 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "  PART_COUNT: 16\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  SIZE: 60000\n",
            "  SIZE_TEST: 10000\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: ffhq\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "PPL_CELEBA_ADJUSTMENT: False\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 300\n",
            "Running on  Tesla P100-PCIE-16GB\n",
            "2020-06-11 18:50:26,067 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:50:26,067 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:50:26,067 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:50:26,067 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:50:26,067 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:50:26,067 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:50:26,075 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:50:26,075 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:50:26,075 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:50:26,075 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:50:26,075 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:50:26,075 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:50:26,082 logger INFO: Loading checkpoint from training_artifacts/ffhq/model_157.pth\n",
            "2020-06-11 18:50:26,082 logger INFO: Loading checkpoint from training_artifacts/ffhq/model_157.pth\n",
            "2020-06-11 18:50:26,082 logger INFO: Loading checkpoint from training_artifacts/ffhq/model_157.pth\n",
            "2020-06-11 18:50:26,082 logger INFO: Loading checkpoint from training_artifacts/ffhq/model_157.pth\n",
            "2020-06-11 18:50:26,082 logger INFO: Loading checkpoint from training_artifacts/ffhq/model_157.pth\n",
            "2020-06-11 18:50:26,082 logger INFO: Loading checkpoint from training_artifacts/ffhq/model_157.pth\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m Error uploading \"media/images/Interpolations from ffhq Images_5.png\": CommError, File /tmp/tmpyqo4168kwandb/fda8oh2v-media/images/Interpolations from ffhq Images_5.png size shrank from 20909841 to 0 while it was being uploaded.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m Error uploading \"media/images/Interpolations from ffhq Images_5.png\": CommError, File /tmp/tmpyqo4168kwandb/2294s9q9-media/images/Interpolations from ffhq Images_5.png size shrank from 42409585 to 0 while it was being uploaded.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m Error uploading \"media/images/Interpolations from ffhq Images_5.png\": CommError, File /tmp/tmpyqo4168kwandb/4tl5ywdy-media/images/Interpolations from ffhq Images_5.png size shrank from 61615149 to 0 while it was being uploaded.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SUM9JC_sdGZW",
        "colab_type": "text"
      },
      "source": [
        "## Traversal\n",
        "Only available for FFHQ (Need to compute principal directions for others)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gMCp3JJydOoe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "lreq.use_implicit_lreq.set(True)\n",
        "\n",
        "# Generate GIF\n",
        "def log_gif(ims, fname, log_name):\n",
        "        ims[0].save(fname, save_all=True, append_images=ims[1:], duration=150, fps=1, loop=0)\n",
        "        wandb.log({\"{}\".format(log_name): wandb.Video(fname, fps=4, format=\"gif\")})\n",
        "\n",
        "# Helper to generate Images for GIF generation\n",
        "def prep_list(torch_list):\n",
        "  \"\"\"\n",
        "    Given a B x C X H X W list of tensors,\n",
        "    returns a PIL.Image list \n",
        "  \"\"\"\n",
        "  ims = []\n",
        "  for im in torch_list:\n",
        "    im = im * 0.5 + 0.5\n",
        "    arr = im.permute(2,3,1,0).squeeze().cpu().numpy()\n",
        "    # normalize the data to 0 - 1\n",
        "    arr = (arr - np.min(arr))/np.ptp(arr)\n",
        "\n",
        "    arr = 255 * arr # Now scale by 255\n",
        "    arr = arr.astype(np.uint8)\n",
        "    dims = (512,512)\n",
        "    img = Image.fromarray(arr, 'RGB')\n",
        "    img = img.resize(dims)\n",
        "    ims.append(img)\n",
        "    ims.append(img)\n",
        "\n",
        "  return ims\n",
        "\n",
        "def sample(cfg, logger):\n",
        "    torch.cuda.set_device(0)\n",
        "    model = Model(\n",
        "        startf=cfg.MODEL.START_CHANNEL_COUNT,\n",
        "        layer_count=cfg.MODEL.LAYER_COUNT,\n",
        "        maxf=cfg.MODEL.MAX_CHANNEL_COUNT,\n",
        "        latent_size=cfg.MODEL.LATENT_SPACE_SIZE,\n",
        "        truncation_psi=cfg.MODEL.TRUNCATIOM_PSI,\n",
        "        truncation_cutoff=cfg.MODEL.TRUNCATIOM_CUTOFF,\n",
        "        mapping_layers=cfg.MODEL.MAPPING_LAYERS,\n",
        "        channels=cfg.MODEL.CHANNELS,\n",
        "        generator=cfg.MODEL.GENERATOR,\n",
        "        encoder=cfg.MODEL.ENCODER)\n",
        "    model.cuda(0)\n",
        "    model.eval()\n",
        "    model.requires_grad_(False)\n",
        "\n",
        "    decoder = model.decoder\n",
        "    encoder = model.encoder\n",
        "    mapping_tl = model.mapping_tl\n",
        "    mapping_fl = model.mapping_fl\n",
        "    dlatent_avg = model.dlatent_avg\n",
        "\n",
        "    logger.info(\"Trainable parameters generator:\")\n",
        "    count_parameters(decoder)\n",
        "\n",
        "    logger.info(\"Trainable parameters discriminator:\")\n",
        "    count_parameters(encoder)\n",
        "\n",
        "    arguments = dict()\n",
        "    arguments[\"iteration\"] = 0\n",
        "\n",
        "    model_dict = {\n",
        "        'discriminator_s': encoder,\n",
        "        'generator_s': decoder,\n",
        "        'mapping_tl_s': mapping_tl,\n",
        "        'mapping_fl_s': mapping_fl,\n",
        "        'dlatent_avg': dlatent_avg\n",
        "    }\n",
        "\n",
        "    checkpointer = Checkpointer(cfg,\n",
        "                                model_dict,\n",
        "                                {},\n",
        "                                logger=logger,\n",
        "                                save=False)\n",
        "\n",
        "    extra_checkpoint_data = checkpointer.load()\n",
        "\n",
        "    model.eval()\n",
        "\n",
        "    layer_count = cfg.MODEL.LAYER_COUNT\n",
        "\n",
        "    def encode(x):\n",
        "        Z, _ = model.encode(x, layer_count - 1, 1)\n",
        "        Z = Z.repeat(1, model.mapping_fl.num_layers, 1)\n",
        "        return Z\n",
        "\n",
        "    def decode(x):\n",
        "        layer_idx = torch.arange(2 * cfg.MODEL.LAYER_COUNT)[np.newaxis, :, np.newaxis]\n",
        "        ones = torch.ones(layer_idx.shape, dtype=torch.float32)\n",
        "        coefs = torch.where(layer_idx < model.truncation_cutoff, ones, ones)\n",
        "        # x = torch.lerp(model.dlatent_avg.buff.data, x, coefs)\n",
        "        return model.decoder(x, layer_count - 1, 1, noise=True)\n",
        "\n",
        "    path = cfg.DATASET.SAMPLES_PATH\n",
        "    im_size = 2 ** (cfg.MODEL.LAYER_COUNT + 1)\n",
        "\n",
        "    def do_attribute_traversal(path, attrib_idx, start, end):\n",
        "        img = np.asarray(Image.open(path))\n",
        "        if img.shape[2] == 4:\n",
        "            img = img[:, :, :3]\n",
        "        im = img.transpose((2, 0, 1))\n",
        "        x = torch.tensor(np.asarray(im, dtype=np.float32), device='cpu', requires_grad=True).cuda() / 127.5 - 1.\n",
        "        if x.shape[0] == 4:\n",
        "            x = x[:3]\n",
        "        factor = x.shape[2] // im_size\n",
        "        if factor != 1:\n",
        "            x = torch.nn.functional.avg_pool2d(x[None, ...], factor, factor)[0]\n",
        "        assert x.shape[2] == im_size\n",
        "        _latents = encode(x[None, ...].cuda())\n",
        "        latents = _latents[0, 0]\n",
        "\n",
        "        latents -= model.dlatent_avg.buff.data[0]\n",
        "\n",
        "        w0 = torch.tensor(np.load(\"principal_directions/direction_%d.npy\" % attrib_idx), dtype=torch.float32)\n",
        "\n",
        "        attr0 = (latents * w0).sum()\n",
        "\n",
        "        latents = latents - attr0 * w0\n",
        "\n",
        "        def update_image(w):\n",
        "            with torch.no_grad():\n",
        "                w = w + model.dlatent_avg.buff.data[0]\n",
        "                w = w[None, None, ...].repeat(1, model.mapping_fl.num_layers, 1)\n",
        "\n",
        "                layer_idx = torch.arange(model.mapping_fl.num_layers)[np.newaxis, :, np.newaxis]\n",
        "                cur_layers = (7 + 1) * 2\n",
        "                mixing_cutoff = cur_layers\n",
        "                styles = torch.where(layer_idx < mixing_cutoff, w, _latents[0])\n",
        "\n",
        "                x_rec = decode(styles)\n",
        "                return x_rec\n",
        "\n",
        "        traversal = []\n",
        "\n",
        "        r = 7\n",
        "        inc = (end - start) / (r - 1)\n",
        "\n",
        "        for i in range(r):\n",
        "            W = latents + w0 * (attr0 + start)\n",
        "            im = update_image(W)\n",
        "\n",
        "            traversal.append(im)\n",
        "            attr0 += inc\n",
        "        res = torch.cat(traversal)\n",
        "\n",
        "        indices = [0, 1, 2, 3, 4, 10, 11, 17, 19]\n",
        "        labels = [\"gender\",\n",
        "                  \"smile\",\n",
        "                  \"attractive\",\n",
        "                  \"wavy-hair\",\n",
        "                  \"young\",\n",
        "                  \"big_lips\",\n",
        "                  \"big_nose\",\n",
        "                  \"chubby\",\n",
        "                  \"glasses\",\n",
        "                  ]\n",
        "        label_ = labels[indices.index(attrib_idx)]\n",
        "        return res * 0.5 + 0.5, label_, traversal\n",
        "        \n",
        "        #save_image(res * 0.5 + 0.5, \"make_figures/output/%s/traversal_%s.jpg\" % (\n",
        "        #    cfg.NAME, labels[indices.index(attrib_idx)]), pad_value=1)\n",
        "\n",
        "    res, label_, traversal = do_attribute_traversal(path + '/00049.png', 0, 0.6, -34)\n",
        "    ims = prep_list(traversal)\n",
        "    log_gif(ims, '{}.gif'.format(label_), \"Traversals of the {} Attribute\".format(label_))\n",
        "\n",
        "    res, label_, traversal = do_attribute_traversal(path + '/00125.png', 1, -3, 15.0)\n",
        "    ims = prep_list(traversal)\n",
        "    log_gif(ims, '{}.gif'.format(label_), \"Traversals of the {} Attribute\".format(label_))\n",
        "\n",
        "    res, label_, traversal = do_attribute_traversal(path + '/00057.png', 3, -2, 30.0)\n",
        "    ims = prep_list(traversal)\n",
        "    log_gif(ims, '{}.gif'.format(label_), \"Traversals of the {} Attribute\".format(label_))\n",
        "\n",
        "    res, label_, traversal = do_attribute_traversal(path + '/00031.png', 4, -10, 30.0)\n",
        "    ims = prep_list(traversal)\n",
        "    log_gif(ims, '{}.gif'.format(label_), \"Traversals of the {} Attribute\".format(label_))\n",
        "\n",
        "    res, label_, traversal = do_attribute_traversal(path + '/00088.png', 10, -0.3, 30.0)\n",
        "    ims = prep_list(traversal)\n",
        "    log_gif(ims, '{}.gif'.format(label_), \"Traversals of the {} Attribute\".format(label_))\n",
        "\n",
        "    res, label_, traversal = do_attribute_traversal(path + '/00004.png', 11, -25, 20.0)\n",
        "    ims = prep_list(traversal)\n",
        "    log_gif(ims, '{}.gif'.format(label_), \"Traversals of the {} Attribute\".format(label_))\n",
        "\n",
        "    res, label_, traversal = do_attribute_traversal(path + '/00012.png', 17, -40, 40.0)\n",
        "    ims = prep_list(traversal)\n",
        "    log_gif(ims, '{}.gif'.format(label_), \"Traversals of the {} Attribute\".format(label_))\n",
        "\n",
        "    res, label_, traversal = do_attribute_traversal(path + '/00017.png', 19, 0, 30.0)\n",
        "    ims = prep_list(traversal)\n",
        "    log_gif(ims, '{}.gif'.format(label_), \"Traversals of the {} Attribute\".format(label_))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6Ji7qDApdjUq",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "44969b34-b666-4ac6-ad3a-afa24563d5be"
      },
      "source": [
        "all_configs = ['configs/ffhq.yaml']\n",
        "for config in all_configs:\n",
        "  cfg, logger, device = parse_configs(config)\n",
        "  sample(cfg, logger)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2020-06-11 18:51:43,830 logger INFO: World size: 1\n",
            "2020-06-11 18:51:43,830 logger INFO: World size: 1\n",
            "2020-06-11 18:51:43,830 logger INFO: World size: 1\n",
            "2020-06-11 18:51:43,830 logger INFO: World size: 1\n",
            "2020-06-11 18:51:43,830 logger INFO: World size: 1\n",
            "2020-06-11 18:51:43,830 logger INFO: World size: 1\n",
            "2020-06-11 18:51:43,830 logger INFO: World size: 1\n",
            "2020-06-11 18:51:43,841 logger INFO: Loaded configuration file configs/ffhq.yaml\n",
            "2020-06-11 18:51:43,841 logger INFO: Loaded configuration file configs/ffhq.yaml\n",
            "2020-06-11 18:51:43,841 logger INFO: Loaded configuration file configs/ffhq.yaml\n",
            "2020-06-11 18:51:43,841 logger INFO: Loaded configuration file configs/ffhq.yaml\n",
            "2020-06-11 18:51:43,841 logger INFO: Loaded configuration file configs/ffhq.yaml\n",
            "2020-06-11 18:51:43,841 logger INFO: Loaded configuration file configs/ffhq.yaml\n",
            "2020-06-11 18:51:43,841 logger INFO: Loaded configuration file configs/ffhq.yaml\n",
            "2020-06-11 18:51:43,855 logger INFO: \n",
            " # Config for training ALAE on FFHQ at resolution 1024x1024\n",
            "\n",
            "NAME: ffhq\n",
            "DATASET:\n",
            "  PART_COUNT: 16\n",
            "  SIZE: 60000\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 300\n",
            "  #                    4    8   16    32    64    128    256\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128,   64,   32,    32,    32,       32,        32] # If GPU memory ~16GB reduce last number from 32 to 24\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128,   64,   32,    32,    32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128,   64,   32,    32,    16]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128,   64,   32,    16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:51:43,855 logger INFO: \n",
            " # Config for training ALAE on FFHQ at resolution 1024x1024\n",
            "\n",
            "NAME: ffhq\n",
            "DATASET:\n",
            "  PART_COUNT: 16\n",
            "  SIZE: 60000\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 300\n",
            "  #                    4    8   16    32    64    128    256\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128,   64,   32,    32,    32,       32,        32] # If GPU memory ~16GB reduce last number from 32 to 24\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128,   64,   32,    32,    32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128,   64,   32,    32,    16]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128,   64,   32,    16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:51:43,855 logger INFO: \n",
            " # Config for training ALAE on FFHQ at resolution 1024x1024\n",
            "\n",
            "NAME: ffhq\n",
            "DATASET:\n",
            "  PART_COUNT: 16\n",
            "  SIZE: 60000\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 300\n",
            "  #                    4    8   16    32    64    128    256\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128,   64,   32,    32,    32,       32,        32] # If GPU memory ~16GB reduce last number from 32 to 24\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128,   64,   32,    32,    32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128,   64,   32,    32,    16]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128,   64,   32,    16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:51:43,855 logger INFO: \n",
            " # Config for training ALAE on FFHQ at resolution 1024x1024\n",
            "\n",
            "NAME: ffhq\n",
            "DATASET:\n",
            "  PART_COUNT: 16\n",
            "  SIZE: 60000\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 300\n",
            "  #                    4    8   16    32    64    128    256\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128,   64,   32,    32,    32,       32,        32] # If GPU memory ~16GB reduce last number from 32 to 24\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128,   64,   32,    32,    32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128,   64,   32,    32,    16]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128,   64,   32,    16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:51:43,855 logger INFO: \n",
            " # Config for training ALAE on FFHQ at resolution 1024x1024\n",
            "\n",
            "NAME: ffhq\n",
            "DATASET:\n",
            "  PART_COUNT: 16\n",
            "  SIZE: 60000\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 300\n",
            "  #                    4    8   16    32    64    128    256\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128,   64,   32,    32,    32,       32,        32] # If GPU memory ~16GB reduce last number from 32 to 24\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128,   64,   32,    32,    32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128,   64,   32,    32,    16]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128,   64,   32,    16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:51:43,855 logger INFO: \n",
            " # Config for training ALAE on FFHQ at resolution 1024x1024\n",
            "\n",
            "NAME: ffhq\n",
            "DATASET:\n",
            "  PART_COUNT: 16\n",
            "  SIZE: 60000\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 300\n",
            "  #                    4    8   16    32    64    128    256\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128,   64,   32,    32,    32,       32,        32] # If GPU memory ~16GB reduce last number from 32 to 24\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128,   64,   32,    32,    32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128,   64,   32,    32,    16]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128,   64,   32,    16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:51:43,855 logger INFO: \n",
            " # Config for training ALAE on FFHQ at resolution 1024x1024\n",
            "\n",
            "NAME: ffhq\n",
            "DATASET:\n",
            "  PART_COUNT: 16\n",
            "  SIZE: 60000\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 300\n",
            "  #                    4    8   16    32    64    128    256\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128,   64,   32,    32,    32,       32,        32] # If GPU memory ~16GB reduce last number from 32 to 24\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128,   64,   32,    32,    32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128,   64,   32,    32,    16]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128,   64,   32,    16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:51:43,861 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "  PART_COUNT: 16\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  SIZE: 60000\n",
            "  SIZE_TEST: 10000\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: ffhq\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "PPL_CELEBA_ADJUSTMENT: False\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 300\n",
            "2020-06-11 18:51:43,861 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "  PART_COUNT: 16\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  SIZE: 60000\n",
            "  SIZE_TEST: 10000\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: ffhq\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "PPL_CELEBA_ADJUSTMENT: False\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 300\n",
            "2020-06-11 18:51:43,861 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "  PART_COUNT: 16\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  SIZE: 60000\n",
            "  SIZE_TEST: 10000\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: ffhq\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "PPL_CELEBA_ADJUSTMENT: False\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 300\n",
            "2020-06-11 18:51:43,861 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "  PART_COUNT: 16\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  SIZE: 60000\n",
            "  SIZE_TEST: 10000\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: ffhq\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "PPL_CELEBA_ADJUSTMENT: False\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 300\n",
            "2020-06-11 18:51:43,861 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "  PART_COUNT: 16\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  SIZE: 60000\n",
            "  SIZE_TEST: 10000\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: ffhq\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "PPL_CELEBA_ADJUSTMENT: False\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 300\n",
            "2020-06-11 18:51:43,861 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "  PART_COUNT: 16\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  SIZE: 60000\n",
            "  SIZE_TEST: 10000\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: ffhq\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "PPL_CELEBA_ADJUSTMENT: False\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 300\n",
            "2020-06-11 18:51:43,861 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "  PART_COUNT: 16\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  SIZE: 60000\n",
            "  SIZE_TEST: 10000\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: ffhq\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "PPL_CELEBA_ADJUSTMENT: False\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 300\n",
            "Running on  Tesla P100-PCIE-16GB\n",
            "2020-06-11 18:51:43,944 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:51:43,944 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:51:43,944 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:51:43,944 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:51:43,944 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:51:43,944 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:51:43,944 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:51:43,952 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:51:43,952 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:51:43,952 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:51:43,952 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:51:43,952 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:51:43,952 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:51:43,952 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:51:43,960 logger INFO: Loading checkpoint from training_artifacts/ffhq/model_157.pth\n",
            "2020-06-11 18:51:43,960 logger INFO: Loading checkpoint from training_artifacts/ffhq/model_157.pth\n",
            "2020-06-11 18:51:43,960 logger INFO: Loading checkpoint from training_artifacts/ffhq/model_157.pth\n",
            "2020-06-11 18:51:43,960 logger INFO: Loading checkpoint from training_artifacts/ffhq/model_157.pth\n",
            "2020-06-11 18:51:43,960 logger INFO: Loading checkpoint from training_artifacts/ffhq/model_157.pth\n",
            "2020-06-11 18:51:43,960 logger INFO: Loading checkpoint from training_artifacts/ffhq/model_157.pth\n",
            "2020-06-11 18:51:43,960 logger INFO: Loading checkpoint from training_artifacts/ffhq/model_157.pth\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bgFLDve7lLs5",
        "colab_type": "text"
      },
      "source": [
        "## Generation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LyMNn70PemM-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def draw_uncurated_result_figure(cfg, png, model, cx, cy, cw, ch, rows, lods, seed):\n",
        "    print(png)\n",
        "    N = sum(rows * 2**lod for lod in lods)\n",
        "    images = []\n",
        "\n",
        "    rnd = np.random.RandomState(5)\n",
        "    for i in range(N):\n",
        "        latents = rnd.randn(1, cfg.MODEL.LATENT_SPACE_SIZE)\n",
        "        samplez = torch.tensor(latents).float().cuda()\n",
        "        image = model.generate(cfg.DATASET.MAX_RESOLUTION_LEVEL-2, 1, samplez, 1, mixing=True)\n",
        "        images.append(image[0])\n",
        "\n",
        "    canvas = PIL.Image.new('RGB', (sum(cw // 2**lod for lod in lods), ch * rows), 'white')\n",
        "    image_iter = iter(list(images))\n",
        "    for col, lod in enumerate(lods):\n",
        "        for row in range(rows * 2**lod):\n",
        "            im = next(image_iter).cpu().numpy()\n",
        "            im = im.transpose(1, 2, 0)\n",
        "            im = im * 0.5 + 0.5\n",
        "            image = PIL.Image.fromarray(np.clip(im * 255, 0, 255).astype(np.uint8), 'RGB')\n",
        "            image = image.crop((cx, cy, cx + cw, cy + ch))\n",
        "            image = image.resize((cw // 2**lod, ch // 2**lod), PIL.Image.ANTIALIAS)\n",
        "            canvas.paste(image, (sum(cw // 2**lod for lod in lods[:col]), row * ch // 2**lod))\n",
        "    #canvas.save(png)\n",
        "    return canvas\n",
        "\n",
        "def sample(cfg, logger):\n",
        "    torch.cuda.set_device(0)\n",
        "    model = Model(\n",
        "        startf=cfg.MODEL.START_CHANNEL_COUNT,\n",
        "        layer_count=cfg.MODEL.LAYER_COUNT,\n",
        "        maxf=cfg.MODEL.MAX_CHANNEL_COUNT,\n",
        "        latent_size=cfg.MODEL.LATENT_SPACE_SIZE,\n",
        "        truncation_psi=cfg.MODEL.TRUNCATIOM_PSI,\n",
        "        truncation_cutoff=cfg.MODEL.TRUNCATIOM_CUTOFF,\n",
        "        style_mixing_prob=cfg.MODEL.STYLE_MIXING_PROB,\n",
        "        mapping_layers=cfg.MODEL.MAPPING_LAYERS,\n",
        "        channels=cfg.MODEL.CHANNELS,\n",
        "        generator=cfg.MODEL.GENERATOR,\n",
        "        encoder=cfg.MODEL.ENCODER)\n",
        "\n",
        "    model.cuda(0)\n",
        "    model.eval()\n",
        "    model.requires_grad_(False)\n",
        "\n",
        "    decoder = model.decoder\n",
        "    encoder = model.encoder\n",
        "    mapping_tl = model.mapping_tl\n",
        "    mapping_fl = model.mapping_fl\n",
        "\n",
        "    dlatent_avg = model.dlatent_avg\n",
        "\n",
        "    logger.info(\"Trainable parameters generator:\")\n",
        "    count_parameters(decoder)\n",
        "\n",
        "    logger.info(\"Trainable parameters discriminator:\")\n",
        "    count_parameters(encoder)\n",
        "\n",
        "    arguments = dict()\n",
        "    arguments[\"iteration\"] = 0\n",
        "\n",
        "    model_dict = {\n",
        "        'discriminator_s': encoder,\n",
        "        'generator_s': decoder,\n",
        "        'mapping_tl_s': mapping_tl,\n",
        "        'mapping_fl_s': mapping_fl,\n",
        "        'dlatent_avg': dlatent_avg\n",
        "    }\n",
        "\n",
        "    checkpointer = Checkpointer(cfg,\n",
        "                                model_dict,\n",
        "                                {},\n",
        "                                logger=logger,\n",
        "                                save=False)\n",
        "\n",
        "    checkpointer.load()\n",
        "\n",
        "    model.eval()\n",
        "\n",
        "    layer_count = cfg.MODEL.LAYER_COUNT\n",
        "\n",
        "    decoder = nn.DataParallel(decoder)\n",
        "\n",
        "    im_size = 2 ** (cfg.MODEL.LAYER_COUNT + 1)\n",
        "    with torch.no_grad():\n",
        "        res = draw_uncurated_result_figure(cfg, 'make_figures/output/%s/generations.jpg' % cfg.NAME,\n",
        "                                     model, cx=0, cy=0, cw=im_size, ch=im_size, rows=6, lods=[0, 0, 0, 1, 1, 2], seed=5)\n",
        "        wandb.log({\"Generation from {} Images\".format(cfg.NAME): [wandb.Image(res)]})"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E9myCMWxo6zX",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "8a1fccbb-8d90-479a-efe5-4936e212f47a"
      },
      "source": [
        "all_configs = ['configs/bedroom.yaml','configs/celeba.yaml','configs/ffhq.yaml']\n",
        "for config in all_configs:\n",
        "  cfg, logger, device = parse_configs(config)\n",
        "  sample(cfg, logger)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2020-06-11 18:52:02,790 logger INFO: World size: 1\n",
            "2020-06-11 18:52:02,790 logger INFO: World size: 1\n",
            "2020-06-11 18:52:02,790 logger INFO: World size: 1\n",
            "2020-06-11 18:52:02,790 logger INFO: World size: 1\n",
            "2020-06-11 18:52:02,790 logger INFO: World size: 1\n",
            "2020-06-11 18:52:02,790 logger INFO: World size: 1\n",
            "2020-06-11 18:52:02,790 logger INFO: World size: 1\n",
            "2020-06-11 18:52:02,790 logger INFO: World size: 1\n",
            "2020-06-11 18:52:02,798 logger INFO: Loaded configuration file configs/bedroom.yaml\n",
            "2020-06-11 18:52:02,798 logger INFO: Loaded configuration file configs/bedroom.yaml\n",
            "2020-06-11 18:52:02,798 logger INFO: Loaded configuration file configs/bedroom.yaml\n",
            "2020-06-11 18:52:02,798 logger INFO: Loaded configuration file configs/bedroom.yaml\n",
            "2020-06-11 18:52:02,798 logger INFO: Loaded configuration file configs/bedroom.yaml\n",
            "2020-06-11 18:52:02,798 logger INFO: Loaded configuration file configs/bedroom.yaml\n",
            "2020-06-11 18:52:02,798 logger INFO: Loaded configuration file configs/bedroom.yaml\n",
            "2020-06-11 18:52:02,798 logger INFO: Loaded configuration file configs/bedroom.yaml\n",
            "2020-06-11 18:52:02,806 logger INFO: \n",
            " # Config for training ALAE on lsun-bedroom at resolution 256x256\n",
            "\n",
            "NAME: bedroom\n",
            "DATASET:\n",
            "  PART_COUNT: 4\n",
            "  SIZE: 758260\n",
            "  FFHQ_SOURCE: /data/datasets/lsun-bedroom-full/lsun-bedroom-full-r%02d.tfrecords\n",
            "  PATH: /data/datasets/lsun-bedroom-full/splitted/lsun-bedroom-full-r%02d.tfrecords.%03d\n",
            "  MAX_RESOLUTION_LEVEL: 8\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/bedroom256x256\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_bedroom\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 7\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 32\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/bedroom\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 2\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 112\n",
            "  #                    4    8   16    32    64    128    256\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128,   64,   32,    32,    32,       32,        32]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128,   64,   32,    32,    32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128,   64,   32,    32,    16]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128,   64,   32,    16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:52:02,806 logger INFO: \n",
            " # Config for training ALAE on lsun-bedroom at resolution 256x256\n",
            "\n",
            "NAME: bedroom\n",
            "DATASET:\n",
            "  PART_COUNT: 4\n",
            "  SIZE: 758260\n",
            "  FFHQ_SOURCE: /data/datasets/lsun-bedroom-full/lsun-bedroom-full-r%02d.tfrecords\n",
            "  PATH: /data/datasets/lsun-bedroom-full/splitted/lsun-bedroom-full-r%02d.tfrecords.%03d\n",
            "  MAX_RESOLUTION_LEVEL: 8\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/bedroom256x256\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_bedroom\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 7\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 32\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/bedroom\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 2\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 112\n",
            "  #                    4    8   16    32    64    128    256\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128,   64,   32,    32,    32,       32,        32]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128,   64,   32,    32,    32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128,   64,   32,    32,    16]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128,   64,   32,    16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:52:02,806 logger INFO: \n",
            " # Config for training ALAE on lsun-bedroom at resolution 256x256\n",
            "\n",
            "NAME: bedroom\n",
            "DATASET:\n",
            "  PART_COUNT: 4\n",
            "  SIZE: 758260\n",
            "  FFHQ_SOURCE: /data/datasets/lsun-bedroom-full/lsun-bedroom-full-r%02d.tfrecords\n",
            "  PATH: /data/datasets/lsun-bedroom-full/splitted/lsun-bedroom-full-r%02d.tfrecords.%03d\n",
            "  MAX_RESOLUTION_LEVEL: 8\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/bedroom256x256\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_bedroom\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 7\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 32\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/bedroom\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 2\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 112\n",
            "  #                    4    8   16    32    64    128    256\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128,   64,   32,    32,    32,       32,        32]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128,   64,   32,    32,    32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128,   64,   32,    32,    16]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128,   64,   32,    16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:52:02,806 logger INFO: \n",
            " # Config for training ALAE on lsun-bedroom at resolution 256x256\n",
            "\n",
            "NAME: bedroom\n",
            "DATASET:\n",
            "  PART_COUNT: 4\n",
            "  SIZE: 758260\n",
            "  FFHQ_SOURCE: /data/datasets/lsun-bedroom-full/lsun-bedroom-full-r%02d.tfrecords\n",
            "  PATH: /data/datasets/lsun-bedroom-full/splitted/lsun-bedroom-full-r%02d.tfrecords.%03d\n",
            "  MAX_RESOLUTION_LEVEL: 8\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/bedroom256x256\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_bedroom\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 7\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 32\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/bedroom\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 2\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 112\n",
            "  #                    4    8   16    32    64    128    256\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128,   64,   32,    32,    32,       32,        32]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128,   64,   32,    32,    32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128,   64,   32,    32,    16]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128,   64,   32,    16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:52:02,806 logger INFO: \n",
            " # Config for training ALAE on lsun-bedroom at resolution 256x256\n",
            "\n",
            "NAME: bedroom\n",
            "DATASET:\n",
            "  PART_COUNT: 4\n",
            "  SIZE: 758260\n",
            "  FFHQ_SOURCE: /data/datasets/lsun-bedroom-full/lsun-bedroom-full-r%02d.tfrecords\n",
            "  PATH: /data/datasets/lsun-bedroom-full/splitted/lsun-bedroom-full-r%02d.tfrecords.%03d\n",
            "  MAX_RESOLUTION_LEVEL: 8\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/bedroom256x256\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_bedroom\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 7\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 32\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/bedroom\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 2\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 112\n",
            "  #                    4    8   16    32    64    128    256\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128,   64,   32,    32,    32,       32,        32]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128,   64,   32,    32,    32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128,   64,   32,    32,    16]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128,   64,   32,    16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:52:02,806 logger INFO: \n",
            " # Config for training ALAE on lsun-bedroom at resolution 256x256\n",
            "\n",
            "NAME: bedroom\n",
            "DATASET:\n",
            "  PART_COUNT: 4\n",
            "  SIZE: 758260\n",
            "  FFHQ_SOURCE: /data/datasets/lsun-bedroom-full/lsun-bedroom-full-r%02d.tfrecords\n",
            "  PATH: /data/datasets/lsun-bedroom-full/splitted/lsun-bedroom-full-r%02d.tfrecords.%03d\n",
            "  MAX_RESOLUTION_LEVEL: 8\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/bedroom256x256\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_bedroom\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 7\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 32\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/bedroom\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 2\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 112\n",
            "  #                    4    8   16    32    64    128    256\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128,   64,   32,    32,    32,       32,        32]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128,   64,   32,    32,    32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128,   64,   32,    32,    16]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128,   64,   32,    16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:52:02,806 logger INFO: \n",
            " # Config for training ALAE on lsun-bedroom at resolution 256x256\n",
            "\n",
            "NAME: bedroom\n",
            "DATASET:\n",
            "  PART_COUNT: 4\n",
            "  SIZE: 758260\n",
            "  FFHQ_SOURCE: /data/datasets/lsun-bedroom-full/lsun-bedroom-full-r%02d.tfrecords\n",
            "  PATH: /data/datasets/lsun-bedroom-full/splitted/lsun-bedroom-full-r%02d.tfrecords.%03d\n",
            "  MAX_RESOLUTION_LEVEL: 8\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/bedroom256x256\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_bedroom\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 7\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 32\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/bedroom\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 2\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 112\n",
            "  #                    4    8   16    32    64    128    256\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128,   64,   32,    32,    32,       32,        32]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128,   64,   32,    32,    32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128,   64,   32,    32,    16]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128,   64,   32,    16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:52:02,806 logger INFO: \n",
            " # Config for training ALAE on lsun-bedroom at resolution 256x256\n",
            "\n",
            "NAME: bedroom\n",
            "DATASET:\n",
            "  PART_COUNT: 4\n",
            "  SIZE: 758260\n",
            "  FFHQ_SOURCE: /data/datasets/lsun-bedroom-full/lsun-bedroom-full-r%02d.tfrecords\n",
            "  PATH: /data/datasets/lsun-bedroom-full/splitted/lsun-bedroom-full-r%02d.tfrecords.%03d\n",
            "  MAX_RESOLUTION_LEVEL: 8\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/bedroom256x256\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_bedroom\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 7\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 32\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/bedroom\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 2\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 112\n",
            "  #                    4    8   16    32    64    128    256\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128,   64,   32,    32,    32,       32,        32]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128,   64,   32,    32,    32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128,   64,   32,    32,    16]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128,   64,   32,    16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:52:02,815 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/lsun-bedroom-full/lsun-bedroom-full-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 8\n",
            "  PART_COUNT: 4\n",
            "  PART_COUNT_TEST: 1\n",
            "  PATH: /data/datasets/lsun-bedroom-full/splitted/lsun-bedroom-full-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: \n",
            "  SAMPLES_PATH: dataset_samples/bedroom256x256\n",
            "  SIZE: 758260\n",
            "  SIZE_TEST: 10000\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_bedroom\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 7\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 32\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: bedroom\n",
            "OUTPUT_DIR: training_artifacts/bedroom\n",
            "PPL_CELEBA_ADJUSTMENT: False\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 2\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 112\n",
            "2020-06-11 18:52:02,815 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/lsun-bedroom-full/lsun-bedroom-full-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 8\n",
            "  PART_COUNT: 4\n",
            "  PART_COUNT_TEST: 1\n",
            "  PATH: /data/datasets/lsun-bedroom-full/splitted/lsun-bedroom-full-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: \n",
            "  SAMPLES_PATH: dataset_samples/bedroom256x256\n",
            "  SIZE: 758260\n",
            "  SIZE_TEST: 10000\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_bedroom\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 7\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 32\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: bedroom\n",
            "OUTPUT_DIR: training_artifacts/bedroom\n",
            "PPL_CELEBA_ADJUSTMENT: False\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 2\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 112\n",
            "2020-06-11 18:52:02,815 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/lsun-bedroom-full/lsun-bedroom-full-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 8\n",
            "  PART_COUNT: 4\n",
            "  PART_COUNT_TEST: 1\n",
            "  PATH: /data/datasets/lsun-bedroom-full/splitted/lsun-bedroom-full-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: \n",
            "  SAMPLES_PATH: dataset_samples/bedroom256x256\n",
            "  SIZE: 758260\n",
            "  SIZE_TEST: 10000\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_bedroom\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 7\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 32\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: bedroom\n",
            "OUTPUT_DIR: training_artifacts/bedroom\n",
            "PPL_CELEBA_ADJUSTMENT: False\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 2\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 112\n",
            "2020-06-11 18:52:02,815 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/lsun-bedroom-full/lsun-bedroom-full-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 8\n",
            "  PART_COUNT: 4\n",
            "  PART_COUNT_TEST: 1\n",
            "  PATH: /data/datasets/lsun-bedroom-full/splitted/lsun-bedroom-full-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: \n",
            "  SAMPLES_PATH: dataset_samples/bedroom256x256\n",
            "  SIZE: 758260\n",
            "  SIZE_TEST: 10000\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_bedroom\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 7\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 32\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: bedroom\n",
            "OUTPUT_DIR: training_artifacts/bedroom\n",
            "PPL_CELEBA_ADJUSTMENT: False\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 2\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 112\n",
            "2020-06-11 18:52:02,815 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/lsun-bedroom-full/lsun-bedroom-full-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 8\n",
            "  PART_COUNT: 4\n",
            "  PART_COUNT_TEST: 1\n",
            "  PATH: /data/datasets/lsun-bedroom-full/splitted/lsun-bedroom-full-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: \n",
            "  SAMPLES_PATH: dataset_samples/bedroom256x256\n",
            "  SIZE: 758260\n",
            "  SIZE_TEST: 10000\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_bedroom\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 7\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 32\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: bedroom\n",
            "OUTPUT_DIR: training_artifacts/bedroom\n",
            "PPL_CELEBA_ADJUSTMENT: False\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 2\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 112\n",
            "2020-06-11 18:52:02,815 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/lsun-bedroom-full/lsun-bedroom-full-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 8\n",
            "  PART_COUNT: 4\n",
            "  PART_COUNT_TEST: 1\n",
            "  PATH: /data/datasets/lsun-bedroom-full/splitted/lsun-bedroom-full-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: \n",
            "  SAMPLES_PATH: dataset_samples/bedroom256x256\n",
            "  SIZE: 758260\n",
            "  SIZE_TEST: 10000\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_bedroom\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 7\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 32\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: bedroom\n",
            "OUTPUT_DIR: training_artifacts/bedroom\n",
            "PPL_CELEBA_ADJUSTMENT: False\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 2\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 112\n",
            "2020-06-11 18:52:02,815 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/lsun-bedroom-full/lsun-bedroom-full-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 8\n",
            "  PART_COUNT: 4\n",
            "  PART_COUNT_TEST: 1\n",
            "  PATH: /data/datasets/lsun-bedroom-full/splitted/lsun-bedroom-full-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: \n",
            "  SAMPLES_PATH: dataset_samples/bedroom256x256\n",
            "  SIZE: 758260\n",
            "  SIZE_TEST: 10000\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_bedroom\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 7\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 32\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: bedroom\n",
            "OUTPUT_DIR: training_artifacts/bedroom\n",
            "PPL_CELEBA_ADJUSTMENT: False\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 2\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 112\n",
            "2020-06-11 18:52:02,815 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/lsun-bedroom-full/lsun-bedroom-full-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 8\n",
            "  PART_COUNT: 4\n",
            "  PART_COUNT_TEST: 1\n",
            "  PATH: /data/datasets/lsun-bedroom-full/splitted/lsun-bedroom-full-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: \n",
            "  SAMPLES_PATH: dataset_samples/bedroom256x256\n",
            "  SIZE: 758260\n",
            "  SIZE_TEST: 10000\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_bedroom\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 7\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 32\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: bedroom\n",
            "OUTPUT_DIR: training_artifacts/bedroom\n",
            "PPL_CELEBA_ADJUSTMENT: False\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 2\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 112\n",
            "Running on  Tesla P100-PCIE-16GB\n",
            "2020-06-11 18:52:02,871 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:52:02,871 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:52:02,871 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:52:02,871 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:52:02,871 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:52:02,871 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:52:02,871 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:52:02,871 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:52:02,878 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:52:02,878 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:52:02,878 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:52:02,878 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:52:02,878 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:52:02,878 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:52:02,878 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:52:02,878 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:52:02,884 logger INFO: Loading checkpoint from training_artifacts/bedroom/model_final.pth\n",
            "2020-06-11 18:52:02,884 logger INFO: Loading checkpoint from training_artifacts/bedroom/model_final.pth\n",
            "2020-06-11 18:52:02,884 logger INFO: Loading checkpoint from training_artifacts/bedroom/model_final.pth\n",
            "2020-06-11 18:52:02,884 logger INFO: Loading checkpoint from training_artifacts/bedroom/model_final.pth\n",
            "2020-06-11 18:52:02,884 logger INFO: Loading checkpoint from training_artifacts/bedroom/model_final.pth\n",
            "2020-06-11 18:52:02,884 logger INFO: Loading checkpoint from training_artifacts/bedroom/model_final.pth\n",
            "2020-06-11 18:52:02,884 logger INFO: Loading checkpoint from training_artifacts/bedroom/model_final.pth\n",
            "2020-06-11 18:52:02,884 logger INFO: Loading checkpoint from training_artifacts/bedroom/model_final.pth\n",
            "make_figures/output/bedroom/generations.jpg\n",
            "2020-06-11 18:52:15,089 logger INFO: World size: 1\n",
            "2020-06-11 18:52:15,089 logger INFO: World size: 1\n",
            "2020-06-11 18:52:15,089 logger INFO: World size: 1\n",
            "2020-06-11 18:52:15,089 logger INFO: World size: 1\n",
            "2020-06-11 18:52:15,089 logger INFO: World size: 1\n",
            "2020-06-11 18:52:15,089 logger INFO: World size: 1\n",
            "2020-06-11 18:52:15,089 logger INFO: World size: 1\n",
            "2020-06-11 18:52:15,089 logger INFO: World size: 1\n",
            "2020-06-11 18:52:15,089 logger INFO: World size: 1\n",
            "2020-06-11 18:52:15,102 logger INFO: Loaded configuration file configs/celeba.yaml\n",
            "2020-06-11 18:52:15,102 logger INFO: Loaded configuration file configs/celeba.yaml\n",
            "2020-06-11 18:52:15,102 logger INFO: Loaded configuration file configs/celeba.yaml\n",
            "2020-06-11 18:52:15,102 logger INFO: Loaded configuration file configs/celeba.yaml\n",
            "2020-06-11 18:52:15,102 logger INFO: Loaded configuration file configs/celeba.yaml\n",
            "2020-06-11 18:52:15,102 logger INFO: Loaded configuration file configs/celeba.yaml\n",
            "2020-06-11 18:52:15,102 logger INFO: Loaded configuration file configs/celeba.yaml\n",
            "2020-06-11 18:52:15,102 logger INFO: Loaded configuration file configs/celeba.yaml\n",
            "2020-06-11 18:52:15,102 logger INFO: Loaded configuration file configs/celeba.yaml\n",
            "2020-06-11 18:52:15,112 logger INFO: \n",
            "# Config for training ALAE on CelebA at resolution 128x128\n",
            "\n",
            "NAME: celeba\n",
            "PPL_CELEBA_ADJUSTMENT: True\n",
            "DATASET:\n",
            "  PART_COUNT: 16\n",
            "  SIZE: 182637\n",
            "  SIZE_TEST: 202576 - 182637\n",
            "  PATH: /data/datasets/celeba/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/celeba-test/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  MAX_RESOLUTION_LEVEL: 7\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign128x128\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_celeba\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 256\n",
            "  LAYER_COUNT: 6\n",
            "  MAX_CHANNEL_COUNT: 256\n",
            "  START_CHANNEL_COUNT: 64\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/celeba\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 6\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 80\n",
            "  #                    4       8       16       32       64       128        256       512       1024\n",
            "  LOD_2_BATCH_8GPU: [512,    256,     128,      64,      32,       32,        32,       32,        32]\n",
            "  LOD_2_BATCH_4GPU: [512,    256,     128,      64,      32,       32,        32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [128,    128,     128,      64,      32,       32,        16]\n",
            "  LOD_2_BATCH_1GPU: [128,    128,     128,      64,      32,       16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:52:15,112 logger INFO: \n",
            "# Config for training ALAE on CelebA at resolution 128x128\n",
            "\n",
            "NAME: celeba\n",
            "PPL_CELEBA_ADJUSTMENT: True\n",
            "DATASET:\n",
            "  PART_COUNT: 16\n",
            "  SIZE: 182637\n",
            "  SIZE_TEST: 202576 - 182637\n",
            "  PATH: /data/datasets/celeba/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/celeba-test/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  MAX_RESOLUTION_LEVEL: 7\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign128x128\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_celeba\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 256\n",
            "  LAYER_COUNT: 6\n",
            "  MAX_CHANNEL_COUNT: 256\n",
            "  START_CHANNEL_COUNT: 64\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/celeba\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 6\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 80\n",
            "  #                    4       8       16       32       64       128        256       512       1024\n",
            "  LOD_2_BATCH_8GPU: [512,    256,     128,      64,      32,       32,        32,       32,        32]\n",
            "  LOD_2_BATCH_4GPU: [512,    256,     128,      64,      32,       32,        32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [128,    128,     128,      64,      32,       32,        16]\n",
            "  LOD_2_BATCH_1GPU: [128,    128,     128,      64,      32,       16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:52:15,112 logger INFO: \n",
            "# Config for training ALAE on CelebA at resolution 128x128\n",
            "\n",
            "NAME: celeba\n",
            "PPL_CELEBA_ADJUSTMENT: True\n",
            "DATASET:\n",
            "  PART_COUNT: 16\n",
            "  SIZE: 182637\n",
            "  SIZE_TEST: 202576 - 182637\n",
            "  PATH: /data/datasets/celeba/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/celeba-test/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  MAX_RESOLUTION_LEVEL: 7\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign128x128\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_celeba\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 256\n",
            "  LAYER_COUNT: 6\n",
            "  MAX_CHANNEL_COUNT: 256\n",
            "  START_CHANNEL_COUNT: 64\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/celeba\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 6\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 80\n",
            "  #                    4       8       16       32       64       128        256       512       1024\n",
            "  LOD_2_BATCH_8GPU: [512,    256,     128,      64,      32,       32,        32,       32,        32]\n",
            "  LOD_2_BATCH_4GPU: [512,    256,     128,      64,      32,       32,        32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [128,    128,     128,      64,      32,       32,        16]\n",
            "  LOD_2_BATCH_1GPU: [128,    128,     128,      64,      32,       16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:52:15,112 logger INFO: \n",
            "# Config for training ALAE on CelebA at resolution 128x128\n",
            "\n",
            "NAME: celeba\n",
            "PPL_CELEBA_ADJUSTMENT: True\n",
            "DATASET:\n",
            "  PART_COUNT: 16\n",
            "  SIZE: 182637\n",
            "  SIZE_TEST: 202576 - 182637\n",
            "  PATH: /data/datasets/celeba/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/celeba-test/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  MAX_RESOLUTION_LEVEL: 7\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign128x128\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_celeba\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 256\n",
            "  LAYER_COUNT: 6\n",
            "  MAX_CHANNEL_COUNT: 256\n",
            "  START_CHANNEL_COUNT: 64\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/celeba\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 6\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 80\n",
            "  #                    4       8       16       32       64       128        256       512       1024\n",
            "  LOD_2_BATCH_8GPU: [512,    256,     128,      64,      32,       32,        32,       32,        32]\n",
            "  LOD_2_BATCH_4GPU: [512,    256,     128,      64,      32,       32,        32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [128,    128,     128,      64,      32,       32,        16]\n",
            "  LOD_2_BATCH_1GPU: [128,    128,     128,      64,      32,       16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:52:15,112 logger INFO: \n",
            "# Config for training ALAE on CelebA at resolution 128x128\n",
            "\n",
            "NAME: celeba\n",
            "PPL_CELEBA_ADJUSTMENT: True\n",
            "DATASET:\n",
            "  PART_COUNT: 16\n",
            "  SIZE: 182637\n",
            "  SIZE_TEST: 202576 - 182637\n",
            "  PATH: /data/datasets/celeba/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/celeba-test/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  MAX_RESOLUTION_LEVEL: 7\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign128x128\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_celeba\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 256\n",
            "  LAYER_COUNT: 6\n",
            "  MAX_CHANNEL_COUNT: 256\n",
            "  START_CHANNEL_COUNT: 64\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/celeba\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 6\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 80\n",
            "  #                    4       8       16       32       64       128        256       512       1024\n",
            "  LOD_2_BATCH_8GPU: [512,    256,     128,      64,      32,       32,        32,       32,        32]\n",
            "  LOD_2_BATCH_4GPU: [512,    256,     128,      64,      32,       32,        32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [128,    128,     128,      64,      32,       32,        16]\n",
            "  LOD_2_BATCH_1GPU: [128,    128,     128,      64,      32,       16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:52:15,112 logger INFO: \n",
            "# Config for training ALAE on CelebA at resolution 128x128\n",
            "\n",
            "NAME: celeba\n",
            "PPL_CELEBA_ADJUSTMENT: True\n",
            "DATASET:\n",
            "  PART_COUNT: 16\n",
            "  SIZE: 182637\n",
            "  SIZE_TEST: 202576 - 182637\n",
            "  PATH: /data/datasets/celeba/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/celeba-test/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  MAX_RESOLUTION_LEVEL: 7\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign128x128\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_celeba\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 256\n",
            "  LAYER_COUNT: 6\n",
            "  MAX_CHANNEL_COUNT: 256\n",
            "  START_CHANNEL_COUNT: 64\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/celeba\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 6\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 80\n",
            "  #                    4       8       16       32       64       128        256       512       1024\n",
            "  LOD_2_BATCH_8GPU: [512,    256,     128,      64,      32,       32,        32,       32,        32]\n",
            "  LOD_2_BATCH_4GPU: [512,    256,     128,      64,      32,       32,        32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [128,    128,     128,      64,      32,       32,        16]\n",
            "  LOD_2_BATCH_1GPU: [128,    128,     128,      64,      32,       16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:52:15,112 logger INFO: \n",
            "# Config for training ALAE on CelebA at resolution 128x128\n",
            "\n",
            "NAME: celeba\n",
            "PPL_CELEBA_ADJUSTMENT: True\n",
            "DATASET:\n",
            "  PART_COUNT: 16\n",
            "  SIZE: 182637\n",
            "  SIZE_TEST: 202576 - 182637\n",
            "  PATH: /data/datasets/celeba/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/celeba-test/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  MAX_RESOLUTION_LEVEL: 7\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign128x128\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_celeba\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 256\n",
            "  LAYER_COUNT: 6\n",
            "  MAX_CHANNEL_COUNT: 256\n",
            "  START_CHANNEL_COUNT: 64\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/celeba\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 6\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 80\n",
            "  #                    4       8       16       32       64       128        256       512       1024\n",
            "  LOD_2_BATCH_8GPU: [512,    256,     128,      64,      32,       32,        32,       32,        32]\n",
            "  LOD_2_BATCH_4GPU: [512,    256,     128,      64,      32,       32,        32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [128,    128,     128,      64,      32,       32,        16]\n",
            "  LOD_2_BATCH_1GPU: [128,    128,     128,      64,      32,       16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:52:15,112 logger INFO: \n",
            "# Config for training ALAE on CelebA at resolution 128x128\n",
            "\n",
            "NAME: celeba\n",
            "PPL_CELEBA_ADJUSTMENT: True\n",
            "DATASET:\n",
            "  PART_COUNT: 16\n",
            "  SIZE: 182637\n",
            "  SIZE_TEST: 202576 - 182637\n",
            "  PATH: /data/datasets/celeba/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/celeba-test/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  MAX_RESOLUTION_LEVEL: 7\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign128x128\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_celeba\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 256\n",
            "  LAYER_COUNT: 6\n",
            "  MAX_CHANNEL_COUNT: 256\n",
            "  START_CHANNEL_COUNT: 64\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/celeba\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 6\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 80\n",
            "  #                    4       8       16       32       64       128        256       512       1024\n",
            "  LOD_2_BATCH_8GPU: [512,    256,     128,      64,      32,       32,        32,       32,        32]\n",
            "  LOD_2_BATCH_4GPU: [512,    256,     128,      64,      32,       32,        32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [128,    128,     128,      64,      32,       32,        16]\n",
            "  LOD_2_BATCH_1GPU: [128,    128,     128,      64,      32,       16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:52:15,112 logger INFO: \n",
            "# Config for training ALAE on CelebA at resolution 128x128\n",
            "\n",
            "NAME: celeba\n",
            "PPL_CELEBA_ADJUSTMENT: True\n",
            "DATASET:\n",
            "  PART_COUNT: 16\n",
            "  SIZE: 182637\n",
            "  SIZE_TEST: 202576 - 182637\n",
            "  PATH: /data/datasets/celeba/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/celeba-test/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  MAX_RESOLUTION_LEVEL: 7\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign128x128\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_celeba\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 256\n",
            "  LAYER_COUNT: 6\n",
            "  MAX_CHANNEL_COUNT: 256\n",
            "  START_CHANNEL_COUNT: 64\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/celeba\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 6\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 80\n",
            "  #                    4       8       16       32       64       128        256       512       1024\n",
            "  LOD_2_BATCH_8GPU: [512,    256,     128,      64,      32,       32,        32,       32,        32]\n",
            "  LOD_2_BATCH_4GPU: [512,    256,     128,      64,      32,       32,        32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [128,    128,     128,      64,      32,       32,        16]\n",
            "  LOD_2_BATCH_1GPU: [128,    128,     128,      64,      32,       16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:52:15,132 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 7\n",
            "  PART_COUNT: 16\n",
            "  PART_COUNT_TEST: 1\n",
            "  PATH: /data/datasets/celeba/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/celeba-test/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign128x128\n",
            "  SIZE: 182637\n",
            "  SIZE_TEST: 19939\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_celeba\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 256\n",
            "  LAYER_COUNT: 6\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 256\n",
            "  START_CHANNEL_COUNT: 64\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: celeba\n",
            "OUTPUT_DIR: training_artifacts/celeba\n",
            "PPL_CELEBA_ADJUSTMENT: True\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 6\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [128, 128, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [128, 128, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 80\n",
            "2020-06-11 18:52:15,132 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 7\n",
            "  PART_COUNT: 16\n",
            "  PART_COUNT_TEST: 1\n",
            "  PATH: /data/datasets/celeba/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/celeba-test/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign128x128\n",
            "  SIZE: 182637\n",
            "  SIZE_TEST: 19939\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_celeba\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 256\n",
            "  LAYER_COUNT: 6\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 256\n",
            "  START_CHANNEL_COUNT: 64\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: celeba\n",
            "OUTPUT_DIR: training_artifacts/celeba\n",
            "PPL_CELEBA_ADJUSTMENT: True\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 6\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [128, 128, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [128, 128, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 80\n",
            "2020-06-11 18:52:15,132 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 7\n",
            "  PART_COUNT: 16\n",
            "  PART_COUNT_TEST: 1\n",
            "  PATH: /data/datasets/celeba/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/celeba-test/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign128x128\n",
            "  SIZE: 182637\n",
            "  SIZE_TEST: 19939\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_celeba\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 256\n",
            "  LAYER_COUNT: 6\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 256\n",
            "  START_CHANNEL_COUNT: 64\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: celeba\n",
            "OUTPUT_DIR: training_artifacts/celeba\n",
            "PPL_CELEBA_ADJUSTMENT: True\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 6\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [128, 128, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [128, 128, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 80\n",
            "2020-06-11 18:52:15,132 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 7\n",
            "  PART_COUNT: 16\n",
            "  PART_COUNT_TEST: 1\n",
            "  PATH: /data/datasets/celeba/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/celeba-test/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign128x128\n",
            "  SIZE: 182637\n",
            "  SIZE_TEST: 19939\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_celeba\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 256\n",
            "  LAYER_COUNT: 6\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 256\n",
            "  START_CHANNEL_COUNT: 64\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: celeba\n",
            "OUTPUT_DIR: training_artifacts/celeba\n",
            "PPL_CELEBA_ADJUSTMENT: True\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 6\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [128, 128, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [128, 128, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 80\n",
            "2020-06-11 18:52:15,132 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 7\n",
            "  PART_COUNT: 16\n",
            "  PART_COUNT_TEST: 1\n",
            "  PATH: /data/datasets/celeba/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/celeba-test/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign128x128\n",
            "  SIZE: 182637\n",
            "  SIZE_TEST: 19939\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_celeba\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 256\n",
            "  LAYER_COUNT: 6\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 256\n",
            "  START_CHANNEL_COUNT: 64\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: celeba\n",
            "OUTPUT_DIR: training_artifacts/celeba\n",
            "PPL_CELEBA_ADJUSTMENT: True\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 6\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [128, 128, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [128, 128, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 80\n",
            "2020-06-11 18:52:15,132 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 7\n",
            "  PART_COUNT: 16\n",
            "  PART_COUNT_TEST: 1\n",
            "  PATH: /data/datasets/celeba/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/celeba-test/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign128x128\n",
            "  SIZE: 182637\n",
            "  SIZE_TEST: 19939\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_celeba\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 256\n",
            "  LAYER_COUNT: 6\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 256\n",
            "  START_CHANNEL_COUNT: 64\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: celeba\n",
            "OUTPUT_DIR: training_artifacts/celeba\n",
            "PPL_CELEBA_ADJUSTMENT: True\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 6\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [128, 128, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [128, 128, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 80\n",
            "2020-06-11 18:52:15,132 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 7\n",
            "  PART_COUNT: 16\n",
            "  PART_COUNT_TEST: 1\n",
            "  PATH: /data/datasets/celeba/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/celeba-test/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign128x128\n",
            "  SIZE: 182637\n",
            "  SIZE_TEST: 19939\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_celeba\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 256\n",
            "  LAYER_COUNT: 6\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 256\n",
            "  START_CHANNEL_COUNT: 64\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: celeba\n",
            "OUTPUT_DIR: training_artifacts/celeba\n",
            "PPL_CELEBA_ADJUSTMENT: True\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 6\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [128, 128, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [128, 128, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 80\n",
            "2020-06-11 18:52:15,132 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 7\n",
            "  PART_COUNT: 16\n",
            "  PART_COUNT_TEST: 1\n",
            "  PATH: /data/datasets/celeba/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/celeba-test/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign128x128\n",
            "  SIZE: 182637\n",
            "  SIZE_TEST: 19939\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_celeba\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 256\n",
            "  LAYER_COUNT: 6\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 256\n",
            "  START_CHANNEL_COUNT: 64\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: celeba\n",
            "OUTPUT_DIR: training_artifacts/celeba\n",
            "PPL_CELEBA_ADJUSTMENT: True\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 6\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [128, 128, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [128, 128, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 80\n",
            "2020-06-11 18:52:15,132 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 7\n",
            "  PART_COUNT: 16\n",
            "  PART_COUNT_TEST: 1\n",
            "  PATH: /data/datasets/celeba/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/celeba-test/tfrecords/celeba-r%02d.tfrecords.%03d\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign128x128\n",
            "  SIZE: 182637\n",
            "  SIZE_TEST: 19939\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_celeba\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 256\n",
            "  LAYER_COUNT: 6\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 256\n",
            "  START_CHANNEL_COUNT: 64\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: celeba\n",
            "OUTPUT_DIR: training_artifacts/celeba\n",
            "PPL_CELEBA_ADJUSTMENT: True\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 6\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [128, 128, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [128, 128, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 80\n",
            "Running on  Tesla P100-PCIE-16GB\n",
            "2020-06-11 18:52:15,211 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:52:15,211 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:52:15,211 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:52:15,211 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:52:15,211 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:52:15,211 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:52:15,211 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:52:15,211 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:52:15,211 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:52:15,221 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:52:15,221 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:52:15,221 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:52:15,221 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:52:15,221 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:52:15,221 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:52:15,221 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:52:15,221 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:52:15,221 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:52:15,231 logger INFO: Loading checkpoint from training_artifacts/celeba/model_final.pth\n",
            "2020-06-11 18:52:15,231 logger INFO: Loading checkpoint from training_artifacts/celeba/model_final.pth\n",
            "2020-06-11 18:52:15,231 logger INFO: Loading checkpoint from training_artifacts/celeba/model_final.pth\n",
            "2020-06-11 18:52:15,231 logger INFO: Loading checkpoint from training_artifacts/celeba/model_final.pth\n",
            "2020-06-11 18:52:15,231 logger INFO: Loading checkpoint from training_artifacts/celeba/model_final.pth\n",
            "2020-06-11 18:52:15,231 logger INFO: Loading checkpoint from training_artifacts/celeba/model_final.pth\n",
            "2020-06-11 18:52:15,231 logger INFO: Loading checkpoint from training_artifacts/celeba/model_final.pth\n",
            "2020-06-11 18:52:15,231 logger INFO: Loading checkpoint from training_artifacts/celeba/model_final.pth\n",
            "2020-06-11 18:52:15,231 logger INFO: Loading checkpoint from training_artifacts/celeba/model_final.pth\n",
            "make_figures/output/celeba/generations.jpg\n",
            "2020-06-11 18:52:16,158 logger INFO: World size: 1\n",
            "2020-06-11 18:52:16,158 logger INFO: World size: 1\n",
            "2020-06-11 18:52:16,158 logger INFO: World size: 1\n",
            "2020-06-11 18:52:16,158 logger INFO: World size: 1\n",
            "2020-06-11 18:52:16,158 logger INFO: World size: 1\n",
            "2020-06-11 18:52:16,158 logger INFO: World size: 1\n",
            "2020-06-11 18:52:16,158 logger INFO: World size: 1\n",
            "2020-06-11 18:52:16,158 logger INFO: World size: 1\n",
            "2020-06-11 18:52:16,158 logger INFO: World size: 1\n",
            "2020-06-11 18:52:16,158 logger INFO: World size: 1\n",
            "2020-06-11 18:52:16,178 logger INFO: Loaded configuration file configs/ffhq.yaml\n",
            "2020-06-11 18:52:16,178 logger INFO: Loaded configuration file configs/ffhq.yaml\n",
            "2020-06-11 18:52:16,178 logger INFO: Loaded configuration file configs/ffhq.yaml\n",
            "2020-06-11 18:52:16,178 logger INFO: Loaded configuration file configs/ffhq.yaml\n",
            "2020-06-11 18:52:16,178 logger INFO: Loaded configuration file configs/ffhq.yaml\n",
            "2020-06-11 18:52:16,178 logger INFO: Loaded configuration file configs/ffhq.yaml\n",
            "2020-06-11 18:52:16,178 logger INFO: Loaded configuration file configs/ffhq.yaml\n",
            "2020-06-11 18:52:16,178 logger INFO: Loaded configuration file configs/ffhq.yaml\n",
            "2020-06-11 18:52:16,178 logger INFO: Loaded configuration file configs/ffhq.yaml\n",
            "2020-06-11 18:52:16,178 logger INFO: Loaded configuration file configs/ffhq.yaml\n",
            "2020-06-11 18:52:16,198 logger INFO: \n",
            " # Config for training ALAE on FFHQ at resolution 1024x1024\n",
            "\n",
            "NAME: ffhq\n",
            "DATASET:\n",
            "  PART_COUNT: 16\n",
            "  SIZE: 60000\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 300\n",
            "  #                    4    8   16    32    64    128    256\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128,   64,   32,    32,    32,       32,        32] # If GPU memory ~16GB reduce last number from 32 to 24\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128,   64,   32,    32,    32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128,   64,   32,    32,    16]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128,   64,   32,    16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:52:16,198 logger INFO: \n",
            " # Config for training ALAE on FFHQ at resolution 1024x1024\n",
            "\n",
            "NAME: ffhq\n",
            "DATASET:\n",
            "  PART_COUNT: 16\n",
            "  SIZE: 60000\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 300\n",
            "  #                    4    8   16    32    64    128    256\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128,   64,   32,    32,    32,       32,        32] # If GPU memory ~16GB reduce last number from 32 to 24\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128,   64,   32,    32,    32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128,   64,   32,    32,    16]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128,   64,   32,    16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:52:16,198 logger INFO: \n",
            " # Config for training ALAE on FFHQ at resolution 1024x1024\n",
            "\n",
            "NAME: ffhq\n",
            "DATASET:\n",
            "  PART_COUNT: 16\n",
            "  SIZE: 60000\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 300\n",
            "  #                    4    8   16    32    64    128    256\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128,   64,   32,    32,    32,       32,        32] # If GPU memory ~16GB reduce last number from 32 to 24\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128,   64,   32,    32,    32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128,   64,   32,    32,    16]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128,   64,   32,    16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:52:16,198 logger INFO: \n",
            " # Config for training ALAE on FFHQ at resolution 1024x1024\n",
            "\n",
            "NAME: ffhq\n",
            "DATASET:\n",
            "  PART_COUNT: 16\n",
            "  SIZE: 60000\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 300\n",
            "  #                    4    8   16    32    64    128    256\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128,   64,   32,    32,    32,       32,        32] # If GPU memory ~16GB reduce last number from 32 to 24\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128,   64,   32,    32,    32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128,   64,   32,    32,    16]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128,   64,   32,    16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:52:16,198 logger INFO: \n",
            " # Config for training ALAE on FFHQ at resolution 1024x1024\n",
            "\n",
            "NAME: ffhq\n",
            "DATASET:\n",
            "  PART_COUNT: 16\n",
            "  SIZE: 60000\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 300\n",
            "  #                    4    8   16    32    64    128    256\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128,   64,   32,    32,    32,       32,        32] # If GPU memory ~16GB reduce last number from 32 to 24\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128,   64,   32,    32,    32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128,   64,   32,    32,    16]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128,   64,   32,    16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:52:16,198 logger INFO: \n",
            " # Config for training ALAE on FFHQ at resolution 1024x1024\n",
            "\n",
            "NAME: ffhq\n",
            "DATASET:\n",
            "  PART_COUNT: 16\n",
            "  SIZE: 60000\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 300\n",
            "  #                    4    8   16    32    64    128    256\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128,   64,   32,    32,    32,       32,        32] # If GPU memory ~16GB reduce last number from 32 to 24\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128,   64,   32,    32,    32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128,   64,   32,    32,    16]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128,   64,   32,    16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:52:16,198 logger INFO: \n",
            " # Config for training ALAE on FFHQ at resolution 1024x1024\n",
            "\n",
            "NAME: ffhq\n",
            "DATASET:\n",
            "  PART_COUNT: 16\n",
            "  SIZE: 60000\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 300\n",
            "  #                    4    8   16    32    64    128    256\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128,   64,   32,    32,    32,       32,        32] # If GPU memory ~16GB reduce last number from 32 to 24\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128,   64,   32,    32,    32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128,   64,   32,    32,    16]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128,   64,   32,    16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:52:16,198 logger INFO: \n",
            " # Config for training ALAE on FFHQ at resolution 1024x1024\n",
            "\n",
            "NAME: ffhq\n",
            "DATASET:\n",
            "  PART_COUNT: 16\n",
            "  SIZE: 60000\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 300\n",
            "  #                    4    8   16    32    64    128    256\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128,   64,   32,    32,    32,       32,        32] # If GPU memory ~16GB reduce last number from 32 to 24\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128,   64,   32,    32,    32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128,   64,   32,    32,    16]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128,   64,   32,    16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:52:16,198 logger INFO: \n",
            " # Config for training ALAE on FFHQ at resolution 1024x1024\n",
            "\n",
            "NAME: ffhq\n",
            "DATASET:\n",
            "  PART_COUNT: 16\n",
            "  SIZE: 60000\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 300\n",
            "  #                    4    8   16    32    64    128    256\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128,   64,   32,    32,    32,       32,        32] # If GPU memory ~16GB reduce last number from 32 to 24\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128,   64,   32,    32,    32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128,   64,   32,    32,    16]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128,   64,   32,    16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:52:16,198 logger INFO: \n",
            " # Config for training ALAE on FFHQ at resolution 1024x1024\n",
            "\n",
            "NAME: ffhq\n",
            "DATASET:\n",
            "  PART_COUNT: 16\n",
            "  SIZE: 60000\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "MODEL:\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  MAPPING_LAYERS: 8\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "TRAIN:\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  TRAIN_EPOCHS: 300\n",
            "  #                    4    8   16    32    64    128    256\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128,   64,   32,    32,    32,       32,        32] # If GPU memory ~16GB reduce last number from 32 to 24\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128,   64,   32,    32,    32,       32,        16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128,   64,   32,    32,    16]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128,   64,   32,    16]\n",
            "\n",
            "  LEARNING_RATES: [0.0015,  0.0015,   0.0015,   0.0015,  0.0015,   0.0015,     0.002,     0.003,    0.003]\n",
            "\n",
            "2020-06-11 18:52:16,220 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "  PART_COUNT: 16\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  SIZE: 60000\n",
            "  SIZE_TEST: 10000\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: ffhq\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "PPL_CELEBA_ADJUSTMENT: False\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 300\n",
            "2020-06-11 18:52:16,220 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "  PART_COUNT: 16\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  SIZE: 60000\n",
            "  SIZE_TEST: 10000\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: ffhq\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "PPL_CELEBA_ADJUSTMENT: False\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 300\n",
            "2020-06-11 18:52:16,220 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "  PART_COUNT: 16\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  SIZE: 60000\n",
            "  SIZE_TEST: 10000\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: ffhq\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "PPL_CELEBA_ADJUSTMENT: False\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 300\n",
            "2020-06-11 18:52:16,220 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "  PART_COUNT: 16\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  SIZE: 60000\n",
            "  SIZE_TEST: 10000\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: ffhq\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "PPL_CELEBA_ADJUSTMENT: False\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 300\n",
            "2020-06-11 18:52:16,220 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "  PART_COUNT: 16\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  SIZE: 60000\n",
            "  SIZE_TEST: 10000\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: ffhq\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "PPL_CELEBA_ADJUSTMENT: False\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 300\n",
            "2020-06-11 18:52:16,220 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "  PART_COUNT: 16\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  SIZE: 60000\n",
            "  SIZE_TEST: 10000\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: ffhq\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "PPL_CELEBA_ADJUSTMENT: False\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 300\n",
            "2020-06-11 18:52:16,220 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "  PART_COUNT: 16\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  SIZE: 60000\n",
            "  SIZE_TEST: 10000\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: ffhq\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "PPL_CELEBA_ADJUSTMENT: False\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 300\n",
            "2020-06-11 18:52:16,220 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "  PART_COUNT: 16\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  SIZE: 60000\n",
            "  SIZE_TEST: 10000\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: ffhq\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "PPL_CELEBA_ADJUSTMENT: False\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 300\n",
            "2020-06-11 18:52:16,220 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "  PART_COUNT: 16\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  SIZE: 60000\n",
            "  SIZE_TEST: 10000\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: ffhq\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "PPL_CELEBA_ADJUSTMENT: False\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 300\n",
            "2020-06-11 18:52:16,220 logger INFO: Running with config:\n",
            "DATASET:\n",
            "  FFHQ_SOURCE: /data/datasets/ffhq-dataset/tfrecords/ffhq/ffhq-r%02d.tfrecords\n",
            "  FLIP_IMAGES: True\n",
            "  MAX_RESOLUTION_LEVEL: 10\n",
            "  PART_COUNT: 16\n",
            "  PART_COUNT_TEST: 2\n",
            "  PATH: /data/datasets/ffhq-dataset_new/tfrecords/ffhq/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  PATH_TEST: /data/datasets/ffhq-dataset_new/tfrecords/ffhq-test/splitted/ffhq-r%02d.tfrecords.%03d\n",
            "  SAMPLES_PATH: dataset_samples/faces/realign1024x1024\n",
            "  SIZE: 60000\n",
            "  SIZE_TEST: 10000\n",
            "  STYLE_MIX_PATH: style_mixing/test_images/set_ffhq\n",
            "MODEL:\n",
            "  CHANNELS: 3\n",
            "  DLATENT_AVG_BETA: 0.995\n",
            "  ENCODER: EncoderDefault\n",
            "  GENERATOR: GeneratorDefault\n",
            "  LATENT_SPACE_SIZE: 512\n",
            "  LAYER_COUNT: 9\n",
            "  MAPPING_FROM_LATENT: MappingFromLatent\n",
            "  MAPPING_LAYERS: 8\n",
            "  MAPPING_TO_LATENT: MappingToLatent\n",
            "  MAX_CHANNEL_COUNT: 512\n",
            "  START_CHANNEL_COUNT: 16\n",
            "  STYLE_MIXING_PROB: 0.9\n",
            "  TRUNCATIOM_CUTOFF: 8\n",
            "  TRUNCATIOM_PSI: 0.7\n",
            "  Z_REGRESSION: False\n",
            "NAME: ffhq\n",
            "OUTPUT_DIR: training_artifacts/ffhq\n",
            "PPL_CELEBA_ADJUSTMENT: False\n",
            "TRAIN:\n",
            "  ADAM_BETA_0: 0.0\n",
            "  ADAM_BETA_1: 0.99\n",
            "  BASE_LEARNING_RATE: 0.002\n",
            "  EPOCHS_PER_LOD: 16\n",
            "  LEARNING_DECAY_RATE: 0.1\n",
            "  LEARNING_DECAY_STEPS: []\n",
            "  LEARNING_RATES: [0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.002, 0.003, 0.003]\n",
            "  LOD_2_BATCH_1GPU: [512, 256, 128, 64, 32, 16]\n",
            "  LOD_2_BATCH_2GPU: [512, 256, 128, 64, 32, 32, 16]\n",
            "  LOD_2_BATCH_4GPU: [512, 256, 128, 64, 32, 32, 32, 32, 16]\n",
            "  LOD_2_BATCH_8GPU: [512, 256, 128, 64, 32, 32, 32, 32, 32]\n",
            "  REPORT_FREQ: [100, 80, 60, 30, 20, 10, 10, 5, 5]\n",
            "  SNAPSHOT_FREQ: [300, 300, 300, 100, 50, 30, 20, 20, 10]\n",
            "  TRAIN_EPOCHS: 300\n",
            "Running on  Tesla P100-PCIE-16GB\n",
            "2020-06-11 18:52:16,284 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:52:16,284 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:52:16,284 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:52:16,284 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:52:16,284 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:52:16,284 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:52:16,284 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:52:16,284 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:52:16,284 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:52:16,284 logger INFO: Trainable parameters generator:\n",
            "2020-06-11 18:52:16,294 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:52:16,294 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:52:16,294 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:52:16,294 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:52:16,294 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:52:16,294 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:52:16,294 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:52:16,294 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:52:16,294 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:52:16,294 logger INFO: Trainable parameters discriminator:\n",
            "2020-06-11 18:52:16,308 logger INFO: Loading checkpoint from training_artifacts/ffhq/model_157.pth\n",
            "2020-06-11 18:52:16,308 logger INFO: Loading checkpoint from training_artifacts/ffhq/model_157.pth\n",
            "2020-06-11 18:52:16,308 logger INFO: Loading checkpoint from training_artifacts/ffhq/model_157.pth\n",
            "2020-06-11 18:52:16,308 logger INFO: Loading checkpoint from training_artifacts/ffhq/model_157.pth\n",
            "2020-06-11 18:52:16,308 logger INFO: Loading checkpoint from training_artifacts/ffhq/model_157.pth\n",
            "2020-06-11 18:52:16,308 logger INFO: Loading checkpoint from training_artifacts/ffhq/model_157.pth\n",
            "2020-06-11 18:52:16,308 logger INFO: Loading checkpoint from training_artifacts/ffhq/model_157.pth\n",
            "2020-06-11 18:52:16,308 logger INFO: Loading checkpoint from training_artifacts/ffhq/model_157.pth\n",
            "2020-06-11 18:52:16,308 logger INFO: Loading checkpoint from training_artifacts/ffhq/model_157.pth\n",
            "2020-06-11 18:52:16,308 logger INFO: Loading checkpoint from training_artifacts/ffhq/model_157.pth\n",
            "make_figures/output/ffhq/generations.jpg\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m Error uploading \"media/images/Generation from ffhq Images_16.png\": CommError, File /tmp/tmpb0npts29wandb/3qn3br0y-media/images/Generation from ffhq Images_16.png size shrank from 27661285 to 0 while it was being uploaded.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m Error uploading \"media/images/Generation from ffhq Images_16.png\": CommError, File /tmp/tmpb0npts29wandb/2tov9hpa-media/images/Generation from ffhq Images_16.png size shrank from 32774029 to 0 while it was being uploaded.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iV-XT0Lvo9dF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}