nmt_pytorch.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "nmt_pytorch.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyMJGSc6ZW1LFuu3JK0bxMkM",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "ca13eec63b404132b57eeedcee7a18c1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_19637672d0a74438955a834a134c99fc",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_6650b64e6b194ce397ea4ec78aa6ac5d",
              "IPY_MODEL_9655e7944c1340319d5eb57a54c5f44d"
            ]
          }
        },
        "19637672d0a74438955a834a134c99fc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6650b64e6b194ce397ea4ec78aa6ac5d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_677537f40efd40c4bc9e5c7a7423953e",
            "_dom_classes": [],
            "description": "loss: 0.239, acc: 1.000:   6%",
            "_model_name": "FloatProgressModel",
            "bar_style": "danger",
            "max": 200,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 13,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_bb144009793c4ce0a4b89535db878da1"
          }
        },
        "9655e7944c1340319d5eb57a54c5f44d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_60ee027ec6a14eb9b72157063a19eb39",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 13/200 [00:14&lt;00:16, 11.41it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0d409d7d00274b4a95420a47c8494eff"
          }
        },
        "677537f40efd40c4bc9e5c7a7423953e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "bb144009793c4ce0a4b89535db878da1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "60ee027ec6a14eb9b72157063a19eb39": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0d409d7d00274b4a95420a47c8494eff": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "519e9ca6deb74bb0881767954b2dd90b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_98be3fe6b67c41368f37afba327006c2",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_3e79c38fbd4d4a9fad9119a9d9071bcd",
              "IPY_MODEL_0c942e99434a4066b0ec1f40534e6059"
            ]
          }
        },
        "98be3fe6b67c41368f37afba327006c2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3e79c38fbd4d4a9fad9119a9d9071bcd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_01f6a7c8ab2349e3909e678343cce8af",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 9,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 9,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5170c2ba2ed64733b02b0908345daf58"
          }
        },
        "0c942e99434a4066b0ec1f40534e6059": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d55e0902596f49399241bbfc8b35813f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 9/9 [00:40&lt;00:00,  4.52s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d9496f1fe4ae496fb51074a0985e9fa6"
          }
        },
        "01f6a7c8ab2349e3909e678343cce8af": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5170c2ba2ed64733b02b0908345daf58": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d55e0902596f49399241bbfc8b35813f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d9496f1fe4ae496fb51074a0985e9fa6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "015ac36378d44aaebe5aa2f6f387a83a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_bab2bfdb6cb84a729931e0193084c5b5",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_4afd226c31704ed7b34cd1574b8d3df4",
              "IPY_MODEL_deffdd5bfbb9484ba9f42a5cddbca137"
            ]
          }
        },
        "bab2bfdb6cb84a729931e0193084c5b5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4afd226c31704ed7b34cd1574b8d3df4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_df7c0db7d117416fb9c4e3b10933eb4c",
            "_dom_classes": [],
            "description": "loss: 0.456, acc: 0.852 vloss: 1.907, vacc: 0.699: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 10,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 10,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_99648b7c36a44b9793d6b099ff2fb4de"
          }
        },
        "deffdd5bfbb9484ba9f42a5cddbca137": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_126ae19a0ea5426dabf56f6a7a0f0356",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 10/10 [3:00:36&lt;00:00, 1083.61s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ecfd340a2a8e4857bca0da7c59d9509e"
          }
        },
        "df7c0db7d117416fb9c4e3b10933eb4c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "99648b7c36a44b9793d6b099ff2fb4de": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "126ae19a0ea5426dabf56f6a7a0f0356": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ecfd340a2a8e4857bca0da7c59d9509e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7d0b4fb6343d416a9cf891bf318dbaab": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_48ecbcd3078c4c72881ee4382b0f59d3",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_1708e394423e48d7b733f7da1a8f4bc6",
              "IPY_MODEL_2fcbaf92d6a14791911cf6a02612838e"
            ]
          }
        },
        "48ecbcd3078c4c72881ee4382b0f59d3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1708e394423e48d7b733f7da1a8f4bc6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_c45a79677b1d469abac9785d972ba4fd",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 10,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 10,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a3be5489afc248f4a14dca6206b632ba"
          }
        },
        "2fcbaf92d6a14791911cf6a02612838e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_2e010f9e3bb94003867b09e7b283755e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 10/10 [3:00:36&lt;00:00, 1083.61s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1780d171a010439ba49adff481b2b0c2"
          }
        },
        "c45a79677b1d469abac9785d972ba4fd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a3be5489afc248f4a14dca6206b632ba": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2e010f9e3bb94003867b09e7b283755e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1780d171a010439ba49adff481b2b0c2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d61a6502d8474c918aea958da29b45ad": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_9831a2be48034da88c3fd20bcb3ab4dc",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_65dc6f0f37f840d7ad0d27a941ce8a10",
              "IPY_MODEL_8b137f34066e46beaf99dde3845fa73c"
            ]
          }
        },
        "9831a2be48034da88c3fd20bcb3ab4dc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "65dc6f0f37f840d7ad0d27a941ce8a10": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_61f365be6e544858a3a8fade0fb143b8",
            "_dom_classes": [],
            "description": "epoch: 1, loss: 2.969, acc: 0.552: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 234,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 234,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3baaa9de3e3b44109b8a65726a60e085"
          }
        },
        "8b137f34066e46beaf99dde3845fa73c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ff3103c168c349bfb0a26b052da363ae",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 234/234 [3:00:36&lt;00:00, 46.31s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4520da03f07340049c636d6889652f92"
          }
        },
        "61f365be6e544858a3a8fade0fb143b8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3baaa9de3e3b44109b8a65726a60e085": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ff3103c168c349bfb0a26b052da363ae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4520da03f07340049c636d6889652f92": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e06ce5d183324e92a1524ae457d0d048": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_17d17be4c1264b8badcc0b664e96e30e",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_3189c09632924f2980250eba99a9a69b",
              "IPY_MODEL_b5f41475999747aaba54ffab36d1db68"
            ]
          }
        },
        "17d17be4c1264b8badcc0b664e96e30e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3189c09632924f2980250eba99a9a69b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_c6862702472d4575b713639017423b7f",
            "_dom_classes": [],
            "description": "epoch: 1, val_loss: 2.591, val_acc: 0.604: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 34,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 34,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3d2cb22327f3429ea91aa2fe2a91fd77"
          }
        },
        "b5f41475999747aaba54ffab36d1db68": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_1ed053867b0b4125979ef8dd056b33bc",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 34/34 [2:43:22&lt;00:00, 288.31s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3bf1b4e9e0b54507aece7ab7fd506eb7"
          }
        },
        "c6862702472d4575b713639017423b7f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3d2cb22327f3429ea91aa2fe2a91fd77": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1ed053867b0b4125979ef8dd056b33bc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3bf1b4e9e0b54507aece7ab7fd506eb7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8c9fe6addbbb4654b6624fcae7589ad7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_cfb0c4abc0fd4986a111d964318b4df0",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_1b16b7b173794bbab385f4ff298c8fba",
              "IPY_MODEL_48e8f8ad22dd4591aad70059920d11ad"
            ]
          }
        },
        "cfb0c4abc0fd4986a111d964318b4df0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1b16b7b173794bbab385f4ff298c8fba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_6bacb7f5c9b34dd0a70ad3d75e4adb85",
            "_dom_classes": [],
            "description": "epoch: 2, loss: 1.788, acc: 0.657: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 234,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 234,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6f68228b808743d68bf780390b469d36"
          }
        },
        "48e8f8ad22dd4591aad70059920d11ad": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_0f6ac97bcbf34e4b8b08b9d7441ca5ec",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 234/234 [28:36&lt;00:00,  7.33s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_068f67f21dd145ec987c8f2f2f6357c9"
          }
        },
        "6bacb7f5c9b34dd0a70ad3d75e4adb85": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6f68228b808743d68bf780390b469d36": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0f6ac97bcbf34e4b8b08b9d7441ca5ec": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "068f67f21dd145ec987c8f2f2f6357c9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "373122c5949a42d9a62953fb284af278": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_2a70104a08394afc9a182e3babf387eb",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_750ad57244944e2f83a9f76856031d78",
              "IPY_MODEL_c819cdc4aaec4bfd9c484cb18e863c1a"
            ]
          }
        },
        "2a70104a08394afc9a182e3babf387eb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "750ad57244944e2f83a9f76856031d78": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_78eff642913f42479ebb244f86b65207",
            "_dom_classes": [],
            "description": "epoch: 2, val_loss: 1.914, val_acc: 0.680: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 34,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 34,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a872d6a68abb46e79e395d238493603a"
          }
        },
        "c819cdc4aaec4bfd9c484cb18e863c1a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_faa1d4b6034f4476b4495bf11bf9ef71",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 34/34 [2:25:31&lt;00:00, 256.81s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0e63da934c1c4587aa1aba0e62848305"
          }
        },
        "78eff642913f42479ebb244f86b65207": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a872d6a68abb46e79e395d238493603a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "faa1d4b6034f4476b4495bf11bf9ef71": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0e63da934c1c4587aa1aba0e62848305": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5c0a23a8a805473abc158859075cb78e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_55a9218b9bb44328b317b1faddbd385c",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_dbaad36f71db41a08f52be8ab636f9a7",
              "IPY_MODEL_b517d40d169341348dd34ffa7cfe2960"
            ]
          }
        },
        "55a9218b9bb44328b317b1faddbd385c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "dbaad36f71db41a08f52be8ab636f9a7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_7b4aa11e2d324c07af530263b2ccb911",
            "_dom_classes": [],
            "description": "epoch: 3, loss: 1.228, acc: 0.717: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 234,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 234,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0dc3f5951000406fb3c5c5c665ceac99"
          }
        },
        "b517d40d169341348dd34ffa7cfe2960": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_34666b0ed2904ce681b21c3e9d02b87f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 234/234 [2:24:41&lt;00:00, 37.10s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_96681c65f9174f6a97ff5177fc260ac6"
          }
        },
        "7b4aa11e2d324c07af530263b2ccb911": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0dc3f5951000406fb3c5c5c665ceac99": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "34666b0ed2904ce681b21c3e9d02b87f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "96681c65f9174f6a97ff5177fc260ac6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "82c49e4beaf14eaab296f9143fc7f340": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_ad6452eb47a44be3849769e68e105ef1",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_d6fd82fae8c34bcb84b42e8511edb719",
              "IPY_MODEL_91beac0824194108811c44068b6fca10"
            ]
          }
        },
        "ad6452eb47a44be3849769e68e105ef1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d6fd82fae8c34bcb84b42e8511edb719": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a662435c188e4864a26e1b2f8fa6820b",
            "_dom_classes": [],
            "description": "epoch: 3, val_loss: 1.736, val_acc: 0.682: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 34,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 34,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4e7f073fc3ee4371a48d3b4be456b099"
          }
        },
        "91beac0824194108811c44068b6fca10": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_2930419a04ea49188b6469ed1e8b530c",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 34/34 [39:19&lt;00:00, 69.39s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1484181ba2ad4250a1ca10a5f09d5413"
          }
        },
        "a662435c188e4864a26e1b2f8fa6820b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4e7f073fc3ee4371a48d3b4be456b099": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2930419a04ea49188b6469ed1e8b530c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1484181ba2ad4250a1ca10a5f09d5413": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "93dcd822bb2447e69e21bf43affa816d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_17fb22c2c78d4a91a38311b3ec7bdc90",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_4b3c524893ff44c7bebe86c65930bb31",
              "IPY_MODEL_d880ec03a8df443fb33b9a7f37182829"
            ]
          }
        },
        "17fb22c2c78d4a91a38311b3ec7bdc90": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4b3c524893ff44c7bebe86c65930bb31": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_37db82650beb4c4b95205b2c60a203ab",
            "_dom_classes": [],
            "description": "epoch: 4, loss: 1.031, acc: 0.735: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 234,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 234,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_03d95c59027f49be9e92e87230e43a30"
          }
        },
        "d880ec03a8df443fb33b9a7f37182829": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_0f2847dd9a884dc4a349b815565109bc",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 234/234 [38:28&lt;00:00,  9.87s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d9d39ddbf03e4109aaf41da4c1c4d546"
          }
        },
        "37db82650beb4c4b95205b2c60a203ab": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "03d95c59027f49be9e92e87230e43a30": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0f2847dd9a884dc4a349b815565109bc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d9d39ddbf03e4109aaf41da4c1c4d546": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0d70985d8cc94aada083b1ac1b925ddd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_abfcf98ad01b41ffbd9e8e5af869d0d5",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_2d0de4e5193f4da1ab5b050f66304739",
              "IPY_MODEL_f8cd7299e2ce423aa425f3906e27dbb2"
            ]
          }
        },
        "abfcf98ad01b41ffbd9e8e5af869d0d5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2d0de4e5193f4da1ab5b050f66304739": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_2574b247944f4ce69a308f0be4108096",
            "_dom_classes": [],
            "description": "epoch: 4, val_loss: 1.750, val_acc: 0.677: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 34,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 34,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_156c8294b6f84ed3b4d3e693512d3e57"
          }
        },
        "f8cd7299e2ce423aa425f3906e27dbb2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f64ddc92232a46bdb9c9e0e5cf25b451",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 34/34 [21:25&lt;00:00, 37.81s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_255c023290d940559c03cf6b761f7792"
          }
        },
        "2574b247944f4ce69a308f0be4108096": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "156c8294b6f84ed3b4d3e693512d3e57": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f64ddc92232a46bdb9c9e0e5cf25b451": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "255c023290d940559c03cf6b761f7792": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9d6c86104fe6430fbe553389424e402d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_de8047f896c34756a5e7b97c22455ce1",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_85acccda2f53404da4f221cbb109aebf",
              "IPY_MODEL_1b5b403cc1a24404afb96c759586abcb"
            ]
          }
        },
        "de8047f896c34756a5e7b97c22455ce1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "85acccda2f53404da4f221cbb109aebf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_4d805263d5334f01b7057561804514d6",
            "_dom_classes": [],
            "description": "epoch: 5, loss: 0.888, acc: 0.780: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 234,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 234,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_64d24441f4864391a1e42627a140c7ab"
          }
        },
        "1b5b403cc1a24404afb96c759586abcb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_95c078f338a24c638c764ea1fc486029",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 234/234 [20:35&lt;00:00,  5.28s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a80497abd09a4448a0bba1a9f50498da"
          }
        },
        "4d805263d5334f01b7057561804514d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "64d24441f4864391a1e42627a140c7ab": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "95c078f338a24c638c764ea1fc486029": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a80497abd09a4448a0bba1a9f50498da": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8cb1f3fcbcc04ad6abbd1f039ab01db1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_4754d4d10689464caa72a47660ec75a9",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_a3cdb7c5a98645b492ae02f23dd1f7f0",
              "IPY_MODEL_573f248058164c78b154e8dbb83d345e"
            ]
          }
        },
        "4754d4d10689464caa72a47660ec75a9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a3cdb7c5a98645b492ae02f23dd1f7f0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_5acc80d9223f453f851f55aa1ec6557c",
            "_dom_classes": [],
            "description": "epoch: 5, val_loss: 1.967, val_acc: 0.643: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 34,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 34,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b9d924d166c44507b2cc3fc1174e90a1"
          }
        },
        "573f248058164c78b154e8dbb83d345e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_47bafed9151249319dfb57d31985a118",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 34/34 [1:31:48&lt;00:00, 162.02s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e9eec820d84648c0ab0a7ee069c8f6d1"
          }
        },
        "5acc80d9223f453f851f55aa1ec6557c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b9d924d166c44507b2cc3fc1174e90a1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "47bafed9151249319dfb57d31985a118": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e9eec820d84648c0ab0a7ee069c8f6d1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d7474ea88a6f4b628753a1dcbbe1a15a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_4b2aec714e744b0da21ef15f754c9f4d",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_5a3ad7f393794dd7b025453f86e5e7b6",
              "IPY_MODEL_9f7a27ded9d842fa93b8fac020c87cbd"
            ]
          }
        },
        "4b2aec714e744b0da21ef15f754c9f4d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5a3ad7f393794dd7b025453f86e5e7b6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_cd6491cc2df64906af4115d13c2bf158",
            "_dom_classes": [],
            "description": "epoch: 6, loss: 0.747, acc: 0.827: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 234,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 234,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d506026e72e04c6683bad9fda717f3e0"
          }
        },
        "9f7a27ded9d842fa93b8fac020c87cbd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_dd019cb93f0d402a86c33c31b333d9ae",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 234/234 [1:31:00&lt;00:00, 23.33s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_66c1fba6371344dfbc30d5ec77ed43b7"
          }
        },
        "cd6491cc2df64906af4115d13c2bf158": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d506026e72e04c6683bad9fda717f3e0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "dd019cb93f0d402a86c33c31b333d9ae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "66c1fba6371344dfbc30d5ec77ed43b7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1df69c2620a3422caf56a1d8eaf17f95": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_c13c9c3c6cf34f6ebdca372539619bca",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_02568abcc8704e5c87d906150782e01e",
              "IPY_MODEL_d1e4c6f479ca49329e3a9184b5616f6f"
            ]
          }
        },
        "c13c9c3c6cf34f6ebdca372539619bca": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "02568abcc8704e5c87d906150782e01e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_1e6e56bc01934f538d01fbdae0888204",
            "_dom_classes": [],
            "description": "epoch: 6, val_loss: 1.893, val_acc: 0.664: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 34,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 34,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ac0c96db065c4d0d83bbe8ca3ae279d3"
          }
        },
        "d1e4c6f479ca49329e3a9184b5616f6f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_7440f41720f54852a6ac62baedaa5ff0",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 34/34 [39:23&lt;00:00, 69.52s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_929ebc32715c4755a195a53bfebcfc7f"
          }
        },
        "1e6e56bc01934f538d01fbdae0888204": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ac0c96db065c4d0d83bbe8ca3ae279d3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7440f41720f54852a6ac62baedaa5ff0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "929ebc32715c4755a195a53bfebcfc7f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c9a88645abd5447289eb1710cfb842f8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f9d5f1ea366c4fb98fe5d05e645750de",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_042d4b6aad1a4f6d83a0f74d31e072a6",
              "IPY_MODEL_72b7c97bc13842fd88fa5f7e6e3c1d4a"
            ]
          }
        },
        "f9d5f1ea366c4fb98fe5d05e645750de": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "042d4b6aad1a4f6d83a0f74d31e072a6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_dd3f314aa84b450fbf05b43f2cabcf6c",
            "_dom_classes": [],
            "description": "epoch: 7, loss: 0.738, acc: 0.794: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 234,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 234,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_653b98d3cfe647c88b5cb0a8aac08756"
          }
        },
        "72b7c97bc13842fd88fa5f7e6e3c1d4a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_39555e4075aa43b4bcc0aeb358b83428",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 234/234 [38:35&lt;00:00,  9.89s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0a8b844d0b6f4335b87c7cf845f6df79"
          }
        },
        "dd3f314aa84b450fbf05b43f2cabcf6c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "653b98d3cfe647c88b5cb0a8aac08756": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "39555e4075aa43b4bcc0aeb358b83428": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0a8b844d0b6f4335b87c7cf845f6df79": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9a8c19870c8845458802c45ab4dddc54": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_e47a683c97b341d2a9126223eea05951",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_b617eca4cc954e37aec4726ea749efa9",
              "IPY_MODEL_a41518b2e4784b3385b7b8bfb9c1a0a5"
            ]
          }
        },
        "e47a683c97b341d2a9126223eea05951": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b617eca4cc954e37aec4726ea749efa9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_fa06d4da6f0740b4816cd3109f7da059",
            "_dom_classes": [],
            "description": "epoch: 7, val_loss: 1.855, val_acc: 0.703: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 34,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 34,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_96a40a9bd9ae4e9cb0c6ef701b694047"
          }
        },
        "a41518b2e4784b3385b7b8bfb9c1a0a5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_4e7c18fe8a7e432681f9f6ca301ef905",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 34/34 [21:35&lt;00:00, 38.09s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_71811b687f0847aaafa9780298a95359"
          }
        },
        "fa06d4da6f0740b4816cd3109f7da059": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "96a40a9bd9ae4e9cb0c6ef701b694047": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4e7c18fe8a7e432681f9f6ca301ef905": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "71811b687f0847aaafa9780298a95359": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fd3aa0e21e2f46e485a2ff2b74cb56db": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_16a9d6d77e634fd29234a8874d4ab8fe",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_2137a5531e3f44c9ad0b02bae1a89ee8",
              "IPY_MODEL_fb2fe3226ca24d599b6fb35ed686b109"
            ]
          }
        },
        "16a9d6d77e634fd29234a8874d4ab8fe": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2137a5531e3f44c9ad0b02bae1a89ee8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_0725e09492e24567afc215c93bb32182",
            "_dom_classes": [],
            "description": "epoch: 8, loss: 0.605, acc: 0.825: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 234,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 234,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c9c485f438bf48d3ad512272e513ac4e"
          }
        },
        "fb2fe3226ca24d599b6fb35ed686b109": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d71f94f6f1864c8a979d85e082d2eb74",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 234/234 [20:46&lt;00:00,  5.33s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ada8ea18a0f943b78b36340e766dab1f"
          }
        },
        "0725e09492e24567afc215c93bb32182": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c9c485f438bf48d3ad512272e513ac4e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d71f94f6f1864c8a979d85e082d2eb74": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ada8ea18a0f943b78b36340e766dab1f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c5593ed3fbed4c06992bea0e95114779": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_722d13a7be7b4198ab54c5b6fd18a68e",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_263cb4095f44405496e0bb4868da9444",
              "IPY_MODEL_9027ed42304a4baa866779377ace6374"
            ]
          }
        },
        "722d13a7be7b4198ab54c5b6fd18a68e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "263cb4095f44405496e0bb4868da9444": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_649e699b66c84628aea8d005d87ea528",
            "_dom_classes": [],
            "description": "epoch: 8, val_loss: 2.313, val_acc: 0.694: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 34,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 34,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3a3679ca93b6484dadcc054f0683244b"
          }
        },
        "9027ed42304a4baa866779377ace6374": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_17d9e8ade7a84b74ae44b8e9ff7b33f6",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 34/34 [38:15&lt;00:00, 67.50s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e61c35a8e41d48a8ac0327acf0b4fea7"
          }
        },
        "649e699b66c84628aea8d005d87ea528": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3a3679ca93b6484dadcc054f0683244b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "17d9e8ade7a84b74ae44b8e9ff7b33f6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e61c35a8e41d48a8ac0327acf0b4fea7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9978568b073f414db4e2add4f4e69000": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_32be12261b0a4dca8952cc0adc7efd20",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_f4ad4e23bb544c819495c9eb0c90bd3f",
              "IPY_MODEL_a56adeb8bfd04e3a861e02ad35a31cd6"
            ]
          }
        },
        "32be12261b0a4dca8952cc0adc7efd20": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f4ad4e23bb544c819495c9eb0c90bd3f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_36ab7eac9cbd49f78fe290acd35661e0",
            "_dom_classes": [],
            "description": "epoch: 9, loss: 0.657, acc: 0.829: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 234,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 234,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_82502c2ac5644a04a5b25228e9ed7925"
          }
        },
        "a56adeb8bfd04e3a861e02ad35a31cd6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_8d28b519d2c242afbbc4fd4f5bfdd329",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 234/234 [37:26&lt;00:00,  9.60s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d4e27f51b17c4311aab7d23518c52eb9"
          }
        },
        "36ab7eac9cbd49f78fe290acd35661e0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "82502c2ac5644a04a5b25228e9ed7925": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8d28b519d2c242afbbc4fd4f5bfdd329": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d4e27f51b17c4311aab7d23518c52eb9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f303363789194905b1006ae323f45de0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_e5c5c622c2cf4fe7903ed1fb08442bb0",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_2f652b1c68d6475eaaf3eecf95a99b84",
              "IPY_MODEL_a9c48f3e4e8a4ae0a6cf7ca5503258f7"
            ]
          }
        },
        "e5c5c622c2cf4fe7903ed1fb08442bb0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2f652b1c68d6475eaaf3eecf95a99b84": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_1e619ab06022480c8ae02b883b2f8c4e",
            "_dom_classes": [],
            "description": "epoch: 9, val_loss: 1.942, val_acc: 0.668: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 34,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 34,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_23ee6ba51659460aaa61ba91fdced2ef"
          }
        },
        "a9c48f3e4e8a4ae0a6cf7ca5503258f7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_c206f6e381c64d94b281b51b5e542af9",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 34/34 [20:14&lt;00:00, 35.72s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c601af698a5344b7a84d7ed4e53706be"
          }
        },
        "1e619ab06022480c8ae02b883b2f8c4e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "23ee6ba51659460aaa61ba91fdced2ef": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c206f6e381c64d94b281b51b5e542af9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c601af698a5344b7a84d7ed4e53706be": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b69971ddccdb440183bf5abc621328b9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_66ee725b7bea4466bc35fbf58cf62b4c",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_1a32c2340ee04d40bdabf3d20f0c7bf2",
              "IPY_MODEL_00cd5c0c49e34e0fb7af8e881be0b582"
            ]
          }
        },
        "66ee725b7bea4466bc35fbf58cf62b4c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1a32c2340ee04d40bdabf3d20f0c7bf2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a419d3d60aac4362b8c1dbecd489015a",
            "_dom_classes": [],
            "description": "epoch: 10, loss: 0.456, acc: 0.853: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 234,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 234,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_fcf0a524ed854f2683e868ae9632d7e6"
          }
        },
        "00cd5c0c49e34e0fb7af8e881be0b582": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_81c3c19aa8724df7bd80ceda6f936668",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 234/234 [19:26&lt;00:00,  4.98s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_50a2a78b647740aeb934340b07a6d59d"
          }
        },
        "a419d3d60aac4362b8c1dbecd489015a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "fcf0a524ed854f2683e868ae9632d7e6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "81c3c19aa8724df7bd80ceda6f936668": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "50a2a78b647740aeb934340b07a6d59d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ccccd3dddc264dc8b1a29e1fb9102c2d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_6b120ea53769410c84baf5a5a3b9d4d4",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_09dc7963b2c54a9bad34bce95988dbdb",
              "IPY_MODEL_7f7d5cef62484cf1b48aac85216e9152"
            ]
          }
        },
        "6b120ea53769410c84baf5a5a3b9d4d4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "09dc7963b2c54a9bad34bce95988dbdb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_15a6c804b5774afab7b6574c84d55669",
            "_dom_classes": [],
            "description": "epoch: 10, test_loss: 2.118, test_acc: 0.692: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 34,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 34,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b2c75d4357f74a009548f91b1271755b"
          }
        },
        "7f7d5cef62484cf1b48aac85216e9152": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_561b660bc3604b1e91ad0a9828773cac",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 34/34 [02:24&lt;00:00,  4.25s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f7dd271f73b449c6adf722a4bf02693e"
          }
        },
        "15a6c804b5774afab7b6574c84d55669": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b2c75d4357f74a009548f91b1271755b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "561b660bc3604b1e91ad0a9828773cac": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f7dd271f73b449c6adf722a4bf02693e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9260723c67ed4b81bee3f8c7fc2b8575": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_6de6e8d77a9d416d9733d646ca09dc11",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_ed0f864400974c8f918706f137c24ddd",
              "IPY_MODEL_72ef77085f2b4f7984961cdb71b0fb5f"
            ]
          }
        },
        "6de6e8d77a9d416d9733d646ca09dc11": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ed0f864400974c8f918706f137c24ddd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a33b3064035646e0b2c8d05534d9c19d",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 67,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 67,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_bcbba3ae11c548b88c675691486a5d81"
          }
        },
        "72ef77085f2b4f7984961cdb71b0fb5f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_844286ca26024a1695b00a0e064c9a6d",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 67/67 [01:34&lt;00:00,  1.41s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b32c43fa3a9b48c6813033ea003f71a0"
          }
        },
        "a33b3064035646e0b2c8d05534d9c19d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "bcbba3ae11c548b88c675691486a5d81": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "844286ca26024a1695b00a0e064c9a6d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b32c43fa3a9b48c6813033ea003f71a0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/DmitriyValetov/nlp_course_project/blob/master/nmt_pytorch.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gc79ePRuJ1cu",
        "colab_type": "text"
      },
      "source": [
        "# TODO\n",
        "1. Check embeddings by dimension reduction (Are they adequate?) (Maybe use pre-trained?) \n",
        "2. Check attention matrix visually (Is it adequate?) (Maybe change attention algorithm to local or add context vector to decoder input?) \n",
        "3. Add cos similarity distance as attention type.\n",
        "4. If already have attention maybe we don't need to add encoder's hidden state (h1) to decoder input?\n",
        "5. Try summarization task. I think we need to use embedded (How?) sequence of sentences to encoder input, maybe simple sentences words embeddings sum.\n",
        "6. Try using subword vocabulary\n",
        "7. Try external attention (e.g. simple dict)\n",
        "8. Try multi attention (e.g. cos + dist or external + dist)\n",
        "\n",
        "How to improve baseline model?\n",
        "\n",
        "* Use `pad_packed_sequence` and `pack_padded_sequence` methods in Encoder and Decoder.\n",
        "* Use attention! In `src/models.py` file you can find class for GlobalAttention. I **STRONGLY RECOMMEND** to read this [article](https://lilianweng.github.io/lil-log/2018/06/24/attention-attention.html). (**DO NOT FORGET ABOUT ATTENTION MASKS FOR PAD TOKENS!**)\n",
        "* The baseline model is trained by [teacher forcing](https://machinelearningmastery.com/teacher-forcing-for-recurrent-neural-networks/) method, you can also read about [Professor forcing](https://arxiv.org/abs/1610.09038).\n",
        "* You can find something interesting [here](https://arxiv.org/abs/1409.3215) and [here](https://arxiv.org/abs/1409.0473).\n",
        "* Implement bidirectional GRU/LSTM in encoder.\n",
        "* You can try to use self-attention in encoder or decoder (**DO NOT FORGET ABOUT ATTENTION MASKS FOR PAD TOKENS!**).\n",
        "* Write validation loop (for example, check loss on validation dataset):)\n",
        "* You can try to implement [beam-search](https://machinelearningmastery.com/beam-search-decoder-natural-language-processing/)/[nucleus sampling](https://arxiv.org/abs/1904.09751).\n",
        "* Hyper-parameter tuning.\n",
        "* Read about BLEU metric and realize, how you can score it better (see the last cell).\n",
        "\n",
        "Baseline model\n",
        "\n",
        "Encoder baseline model contains: Embedding -> GRU -> SpatialDropout  \n",
        "Decoder: Embedding -> GRU -> pre-head layer -> head layer (with [weight tying](https://arxiv.org/abs/1608.05859))\n",
        "\n",
        "You can edit everything you want, your main task is get the highest BLEU score.\n",
        "\n",
        "![alt text](https://docs.google.com/drawings/u/0/d/sZ5dsXuaAP0zVG82SctfeKA/image?w=602&h=388&rev=6&ac=1&parent=1gURxb15No18tqqRDd36t4bzT0XghhDJPgzXRggbj0VU)\n",
        "\n",
        "![alt text](https://docs.google.com/drawings/u/0/d/scpWiFNRIINBzLR-Tgulruw/image?w=602&h=369&rev=350&ac=1&parent=1gURxb15No18tqqRDd36t4bzT0XghhDJPgzXRggbj0VU)\n",
        "\n",
        "|idx|token|\n",
        "|:-:|---|\n",
        "| 0 | \\<PAD\\> |\n",
        "| 1 | \\<GO\\> |\n",
        "| 2 | \\<EOS\\> |\n",
        "| 3 | \\<UNK\\> |\n",
        "|...| words/subwords/multiwords |\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RvgJWxDjqezW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# !pip install torchviz\n",
        "# from torchviz import make_dot"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AfI1NnKX9c1o",
        "colab_type": "code",
        "cellView": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 371,
          "referenced_widgets": [
            "ca13eec63b404132b57eeedcee7a18c1",
            "19637672d0a74438955a834a134c99fc",
            "6650b64e6b194ce397ea4ec78aa6ac5d",
            "9655e7944c1340319d5eb57a54c5f44d",
            "677537f40efd40c4bc9e5c7a7423953e",
            "bb144009793c4ce0a4b89535db878da1",
            "60ee027ec6a14eb9b72157063a19eb39",
            "0d409d7d00274b4a95420a47c8494eff"
          ]
        },
        "outputId": "7a13a868-5426-4491-dd54-6dc20cf8115f"
      },
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence\n",
        "\n",
        "class EncoderRNN(nn.Module):\n",
        "  def __init__(self, num_embeddings, embedding_dim, padding_idx,\n",
        "               rnn_type, hidden_size, num_layers=1, rnn_dropout=0,\n",
        "               bidirectional=False, dropout=0, pack=False):\n",
        "    super(EncoderRNN, self).__init__()\n",
        "    rnn_map = {'RNN': nn.RNN, 'GRU': nn.GRU, 'LSTM': nn.LSTM}\n",
        "    self.embedding = nn.Embedding(num_embeddings=num_embeddings, \n",
        "                                  embedding_dim=embedding_dim,\n",
        "                                  padding_idx=padding_idx)\n",
        "    self.rnn = rnn_map[rnn_type](input_size=embedding_dim, \n",
        "                                 hidden_size=hidden_size,\n",
        "                                 num_layers=num_layers,\n",
        "                                 batch_first=True,\n",
        "                                 dropout=rnn_dropout,\n",
        "                                 bidirectional=bidirectional)\n",
        "    self.dropout = nn.Dropout(dropout)\n",
        "    self.pack = pack\n",
        "\n",
        "  def forward(self, x1):  # [B, L]\n",
        "    x = self.embedding(x1)  # [B, L] -> [B, L, E]\n",
        "    if self.pack:  # [B, L, E] -> Packed [B, L, E]\n",
        "      ls = torch.sum(x1 != 0, dim=1)\n",
        "      x = pack_padded_sequence(x, ls, batch_first=True, enforce_sorted=False)\n",
        "    # [B, L, E] -> (Packed) [B, L, ND*H], ([NL*ND, B, H], ([NL*ND, B, H]))\n",
        "    if isinstance(self.rnn, nn.LSTM):\n",
        "      ht, (hn, cn) = self.rnn(x)\n",
        "      hn, cn = self.dropout(hn), self.dropout(cn)\n",
        "      return ht, (hn, cn)\n",
        "    else:\n",
        "      ht, hn = self.rnn(x)\n",
        "      hn = self.dropout(hn)\n",
        "      return ht, hn\n",
        "\n",
        "\n",
        "class AttentionDecoderRNN(nn.Module):\n",
        "  def __init__(self, num_embeddings, embedding_dim, padding_idx,\n",
        "               rnn_type, hidden_size, num_layers=1, rnn_dropout=0,\n",
        "               bidirectional=False, dropout=0, out_hidden=0,\n",
        "               attn_type='soft_dot', pack=False):\n",
        "    super(AttentionDecoderRNN, self).__init__()\n",
        "    rnn_map = {'RNN': nn.RNN, 'GRU': nn.GRU, 'LSTM': nn.LSTM}\n",
        "    attn_types = ['dot', 'cos', 'dist', 'soft_dot', 'soft_cos', 'soft_dist', 'none']\n",
        "    self.attn_type = attn_type\n",
        "    self.embedding = nn.Embedding(num_embeddings=num_embeddings, \n",
        "                                  embedding_dim=embedding_dim,\n",
        "                                  padding_idx=padding_idx)\n",
        "    self.rnn = rnn_map[rnn_type](input_size=embedding_dim, \n",
        "                                 hidden_size=hidden_size,\n",
        "                                 num_layers=num_layers,\n",
        "                                 batch_first=True,\n",
        "                                 dropout=rnn_dropout,\n",
        "                                 bidirectional=bidirectional)\n",
        "    out_input = hidden_size if self.attn_type == 'none' else 2*hidden_size\n",
        "    if out_hidden > 0:\n",
        "      self.out_hidden = nn.Linear(out_input, out_hidden)\n",
        "      self.out = nn.Linear(out_hidden, num_embeddings)\n",
        "    else:\n",
        "      self.out_hidden = None\n",
        "      self.out = nn.Linear(out_input, num_embeddings)\n",
        "    self.dropout = nn.Dropout(dropout)\n",
        "    self.softmax = nn.LogSoftmax(dim=2)\n",
        "    self.pack = pack\n",
        "\n",
        "  def forward(self, x2, h1, x1):  # [B, L], [B, L, ND*H], ([NL*ND, B, H], [NL*ND, B, H])\n",
        "    x = self.embedding(x2)  # [B, L] -> [B, L, E]\n",
        "    if self.pack:  # [B, L, E] -> Packed [B, L, E]\n",
        "      lengths = torch.sum(x2 != 0, dim=1)\n",
        "      x = pack_padded_sequence(x, lengths, batch_first=True, enforce_sorted=False)\n",
        "    ht1, hns1 = h1  # [B, L, ND*H], ([NL*ND, B, H], [NL*ND, B, H])\n",
        "    # [B, L, D], ([NL*ND, B, H], [NL*ND, B, H]) -> [B, L, ND*H], ([NL*ND, B, H], [NL*ND, B, H])\n",
        "    ht2, hns2 = self.rnn(x, hns1)\n",
        "    if self.pack:  # Packed [B, L, E] -> [B, L, E]\n",
        "      ht1, _ = pad_packed_sequence(ht1, batch_first=True)\n",
        "      ht2, _ = pad_packed_sequence(ht2, batch_first=True)\n",
        "    if self.attn_type == 'none':\n",
        "      if self.rnn.bidirectional:\n",
        "        B2, L2, NDH2 = ht2.size()\n",
        "        # [B, L, ND*H] -> [B, L, ND, H] -> [B, L, H]\n",
        "        ht2 = ht2.view(B2, L2, 2, int(NDH2/2)).mean(2)\n",
        "      x = self.dropout(ht2)\n",
        "      if self.out_hidden is not None:\n",
        "        x = self.out_hidden(x)\n",
        "      x = self.out(x)\n",
        "      y2 = self.softmax(x)\n",
        "      return y2, None\n",
        "    else:\n",
        "      if self.rnn.bidirectional:\n",
        "        B1, L1, NDH1 = ht1.size()\n",
        "        B2, L2, NDH2 = ht2.size()  # B1 == B2, NDH1 == NDH2\n",
        "        # [B, L, ND*H] -> [B, L, ND, H] -> [B, L, H]\n",
        "        ht1 = ht1.view(B1, L1, 2, int(NDH1/2)).mean(2)\n",
        "        ht2 = ht2.view(B2, L2, 2, int(NDH2/2)).mean(2)\n",
        "      # [L2, H], [L1, H] -> [L2, L1]\n",
        "      # mask where x1 and x2 token is <PAD>\n",
        "      # pad_mask = torch.einsum('bi,bj->bij', x2, x1) == 0\n",
        "      # mask where only x1 token is <PAD>\n",
        "      pad_mask = torch.einsum('bi,bj->bij', torch.ones_like(x2), x1) == 0\n",
        "      # [B, L2, H], [B, L1, H] -> [B, L2, L1]  # attention\n",
        "      # [B, L2, L1], [B, L1, H] -> [B, L2, H]  # weighted h1\n",
        "      if self.attn_type == 'dot':\n",
        "        attn = torch.einsum('bih,bjh->bij', ht2, ht1)  # dot product\n",
        "        attn[pad_mask] = 0\n",
        "      elif self.attn_type == 'soft_dot':\n",
        "        attn = torch.einsum('bih,bjh->bij', ht2, ht1)  # dot product\n",
        "        attn[pad_mask] = float('-inf')\n",
        "        attn = F.softmax(attn, 2)\n",
        "      elif self.attn_type == 'dist':\n",
        "        ht1, ht2 = ht1.contiguous(), ht2.contiguous()\n",
        "        attn = torch.cdist(ht2, ht1)  # euclidian distance\n",
        "        attn = torch.masked_fill(attn, pad_mask, float('inf'))\n",
        "        attn = F.threshold(attn, threshold=1e-6, value=1e-6)  # short dist\n",
        "        attn = 1/attn  # inverse\n",
        "        attn = F.normalize(attn, p=1, dim=2)  # to [0, 1]\n",
        "      elif self.attn_type == 'soft_dist':\n",
        "        ht1, ht2 = ht1.contiguous(), ht2.contiguous()\n",
        "        attn = torch.cdist(ht2, ht1)  # euclidian distance\n",
        "        attn = torch.masked_fill(attn, pad_mask, float('inf'))\n",
        "        attn = F.softmin(attn, 2)\n",
        "      elif self.attn_type == 'cos':\n",
        "        ht1n = F.normalize(ht1, p=2, dim=2)  # normalize to length = 1\n",
        "        ht2n = F.normalize(ht2, p=2, dim=2)  # normalize to length = 1\n",
        "        attn = torch.einsum('bih,bjh->bij', ht2n, ht1n)  # dot product\n",
        "        attn[pad_mask] = 0\n",
        "      elif self.attn_type == 'soft_cos':\n",
        "        ht1n = F.normalize(ht1, p=2, dim=2)  # normalize to length = 1\n",
        "        ht2n = F.normalize(ht2, p=2, dim=2)  # normalize to length = 1\n",
        "        attn = torch.einsum('bih,bjh->bij', ht2n, ht1n)  # dot product\n",
        "        attn[pad_mask] = float('-inf')\n",
        "        attn = F.softmax(attn, 2)\n",
        "      hw1 = torch.einsum('bij,bjh->bih', attn, ht1)  # weighted h1\n",
        "      ha = torch.cat((ht2, hw1), 2)  # [B, L2, H], [B, L2, H] -> [B, L2, H+H]\n",
        "      x = self.dropout(ha)  # [B, L2, H+H] -> [B, L2, H+H]\n",
        "      if self.out_hidden is not None:\n",
        "        x = self.out_hidden(x)   # [B, L2, H+H] -> [B, L2, OH]\n",
        "      x = self.out(x)  # [B, L2, H+H] or [B, L2, OH] -> [B, L2, D2]\n",
        "      y2 = self.softmax(x)  # [B, L2, D2] -> [B, L2, D2]\n",
        "      return y2, attn\n",
        "\n",
        "\n",
        "class EncoderDecoder(nn.Module):\n",
        "  def __init__(self, \n",
        "               enc_num_embeddings, enc_embedding_dim, enc_padding_idx,\n",
        "               dec_num_embeddings, dec_embedding_dim, dec_padding_idx,\n",
        "               rnn_type, hidden_size, num_layers=1, out_hidden=0,\n",
        "               enc_rnn_dropout=0, dec_rnn_dropout=0,\n",
        "               bidirectional=False, enc_dropout=0, dec_dropout=0, \n",
        "               attn_type='dot', pack=False):\n",
        "    super(EncoderDecoder, self).__init__()\n",
        "    self.encoder = EncoderRNN(num_embeddings=enc_num_embeddings, \n",
        "                              embedding_dim=enc_embedding_dim, \n",
        "                              padding_idx=enc_padding_idx,\n",
        "                              hidden_size=hidden_size, \n",
        "                              rnn_type=rnn_type, \n",
        "                              bidirectional=bidirectional,\n",
        "                              num_layers=num_layers,\n",
        "                              dropout=enc_dropout,\n",
        "                              rnn_dropout=enc_rnn_dropout,\n",
        "                              pack=pack)\n",
        "    self.decoder = AttentionDecoderRNN(num_embeddings=dec_num_embeddings, \n",
        "                                       embedding_dim=dec_embedding_dim, \n",
        "                                       padding_idx=dec_padding_idx,\n",
        "                                       hidden_size=hidden_size, \n",
        "                                       rnn_type=rnn_type, \n",
        "                                       bidirectional=bidirectional,\n",
        "                                       num_layers=num_layers,\n",
        "                                       dropout=dec_dropout,\n",
        "                                       out_hidden=out_hidden,\n",
        "                                       rnn_dropout=dec_rnn_dropout,\n",
        "                                       attn_type=attn_type,\n",
        "                                       pack=pack)\n",
        "\n",
        "  def forward(self, x1, x2):\n",
        "    #  [B, L1] -> [B, L1, ND*H], ([NL*ND, B, H], [NL*ND, B, H])\n",
        "    h1 = self.encoder(x1)\n",
        "    # [B, L2], ([B, L1, ND*H], ([NL*ND, B, H], [NL*ND, B, H])) -> [B, L2, E2]\n",
        "    y2 = self.decoder(x2, h1, x1)\n",
        "    return y2\n",
        "\n",
        "\n",
        "def external_attn(x1, x2, attn_dict):\n",
        "    attn_dict = {1: [1], 2: [2], 4: [4, 3]}\n",
        "    attn = []\n",
        "    B1, L1 = x1.size()\n",
        "    B2, L2 = x2.size()\n",
        "    for i in range(B1):\n",
        "      b = []\n",
        "      for j in range(L2):\n",
        "        l2 = []\n",
        "        # x2t = x2[i, j].item()\n",
        "        x2t = x2[i, j + 1].item() if j + 2 < L2 else 0 # decoder shift\n",
        "        for k in range(L1):\n",
        "          x1t = x1[i, k].item()\n",
        "          # x1t = x1[i, k + 1].item() if k + 1 < L1 else 0 # encoder shift\n",
        "          x1ts = self.attn_dict.get(x2t, [])\n",
        "          if x1t in x1ts:\n",
        "            l2.append(1.)\n",
        "          else:\n",
        "            l2.append(0.)\n",
        "        b.append(l2)\n",
        "      attn.append(b)\n",
        "    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "    attn = torch.tensor(attn).to(device)\n",
        "    attn = F.normalize(attn, p=1, dim=2)  # to [0, 1]\n",
        "    return attn\n",
        "\n",
        "\n",
        "from IPython.display import Image\n",
        "# Image(make_dot(loss).render('loss', format='png'))\n",
        "from tqdm.notebook import tqdm\n",
        "from torch import optim\n",
        "import numpy as np\n",
        "\n",
        "def plot_attention(a, x1, x2, shift=True, mask=True, \n",
        "                   suptitle='Attention', figsize=None, \n",
        "                   decoder_x1=None, decoder_x2=None, tight=False,\n",
        "                   labels=True):\n",
        "  # %matplotlib inline\n",
        "  import matplotlib.pyplot as plt\n",
        "  from mpl_toolkits.axes_grid1 import make_axes_locatable\n",
        "\n",
        "  b = a.shape[0]\n",
        "  fig, axs = plt.subplots(1, b, figsize=figsize)\n",
        "  if not isinstance(axs, np.ndarray):  # if batch size == 1\n",
        "    axs = [axs]\n",
        "  fig.suptitle(suptitle)\n",
        "  for i in range(b):\n",
        "    ax = axs[i]\n",
        "    if shift:\n",
        "      ba, bx1, bx2 = a[i,:-1,1:], x1[i,1:], x2[i,1:]\n",
        "    else:\n",
        "      ba, bx1, bx2 = a[i], x1[i], x2[i]\n",
        "    if mask:\n",
        "      mask_x1 = np.flatnonzero(bx1)\n",
        "      mask_x2 = np.flatnonzero(bx2)\n",
        "      ba, bx1, bx2 =  ba[mask_x2,:][:,mask_x1], bx1[mask_x1], bx2[mask_x2]\n",
        "    # ax.set_title(f'{i+1}', y=-0.2)\n",
        "    im = ax.imshow(ba, cmap='gray')\n",
        "    ax.set_xticks(np.arange(len(bx1)))\n",
        "    ax.set_yticks(np.arange(len(bx2)))\n",
        "    if labels:\n",
        "      ax.set_xlabel('x1')\n",
        "      ax.set_ylabel('x2')\n",
        "    ax.xaxis.tick_top()\n",
        "    ax.xaxis.set_label_position('top')\n",
        "    if decoder_x1 is not None:\n",
        "      bx1 = decoder_x1(bx1)\n",
        "    if decoder_x2 is not None:\n",
        "      bx2 = decoder_x2(bx2)\n",
        "    ax.set_xticklabels(bx1)  # rotation=90\n",
        "    ax.set_yticklabels(bx2)\n",
        "    fig.colorbar(im, ax=ax, fraction=0.05, pad=0.05)\n",
        "  if tight:\n",
        "    plt.tight_layout()\n",
        "\n",
        "# torch.backends.cudnn.enabled=False\n",
        "# torch.backends.cudnn.deterministic=True\n",
        "# torch.autograd.set_detect_anomaly(True)\n",
        "\n",
        "enc_dec_config = {\n",
        "  'enc_num_embeddings': 5,\n",
        "  'enc_embedding_dim': 2,\n",
        "  'enc_padding_idx': 0,\n",
        "  'dec_num_embeddings': 5,\n",
        "  'dec_embedding_dim': 2,\n",
        "  'dec_padding_idx': 0,\n",
        "  'rnn_type': 'RNN',\n",
        "  'hidden_size': 2,\n",
        "  'num_layers': 1,\n",
        "  'bidirectional': False,\n",
        "  'enc_rnn_dropout': 0,\n",
        "  'dec_rnn_dropout': 0,\n",
        "  'enc_dropout': 0,\n",
        "  'dec_dropout': 0,\n",
        "  'attn_type': 'soft_dist',\n",
        "  'out_hidden': 16,\n",
        "  'pack': True\n",
        "}\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "seed = 0\n",
        "torch.manual_seed(seed)\n",
        "torch.cuda.manual_seed(seed)\n",
        "model = EncoderDecoder(**enc_dec_config).to(device)\n",
        "x1 = torch.tensor([[1, 3, 2, 0], [1, 3, 4, 2]]).to(device)\n",
        "x2 = torch.tensor([[1, 3, 4, 2, 0, 0], [1, 4, 3, 4, 3, 2]]).to(device)\n",
        "opt = optim.Rprop(model.parameters(), lr=0.01)\n",
        "loss_fn = torch.nn.NLLLoss(ignore_index=0)  # 0 is <PAD>\n",
        "pbar = tqdm(range(200))\n",
        "for i in pbar:\n",
        "  opt.zero_grad()\n",
        "  t = x2[:,1:].flatten(0)\n",
        "  y2, attn = model(x1, x2)\n",
        "  p = y2[:,:-1,:].flatten(0, 1)\n",
        "  loss = loss_fn(p, t)\n",
        "  loss.backward()\n",
        "  opt.step()\n",
        "  with torch.no_grad():\n",
        "    idx = torch.nonzero(t).view(-1)\n",
        "    acc = torch.sum(torch.argmax(p, 1)[idx] == t[idx]).float()/idx.size()[0]\n",
        "    if i % 25 == 0:\n",
        "      # print(attn)\n",
        "      # print(p)\n",
        "      # print(torch.argmax(p, 1))\n",
        "      # print(t)\n",
        "      if attn is not None:\n",
        "         plot_attention(attn.detach().cpu().numpy(), \n",
        "                        x1.detach().cpu().numpy(), \n",
        "                        x2.detach().cpu().numpy(),\n",
        "                        labels=True, tight=False,\n",
        "                        shift=False, mask=False, figsize=(10, 5))\n",
        "  pbar.set_description(f'loss: {loss:.3f}, acc: {acc:.3f}')\n",
        "  if acc == 1:\n",
        "    break\n",
        "# Image(make_dot(attn).render('attn', format='png'))"
      ],
      "execution_count": 348,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "ca13eec63b404132b57eeedcee7a18c1",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=200.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkUAAAFCCAYAAAD/k6nkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3dfbRldX3n+feH4qGIgEqKTEgVUoSQ7qBJQ6eE9GI6OgTbsqUh0+n0oJGgY8LoyGqy7IyBaJMVMrZGVxhZS1a0lmF0jFhtMN2rWoshRmDZdAJWoQS7ChlLglLVKBIehIU8FHznj3OKHK5V995zOfue/bvn/VrrLM7eZ+/9+94q+PC9v7MfUlVIkiTNuoOmXYAkSVIf2BRJkiRhUyRJkgTYFEmSJAE2RZIkSYBNkSRJEmBTJGmRknwkyb+bdh2S1BWbIqnnktyU5KEkh42suyfJWSPL65NUkoMnNOabk9w8uq6q3lZVfzCJ40tSH9kUST2WZD3wT4ECzplqMZK0wtkUSf3268AtwMeBCwCSfBJ4GfCfkzyW5F3Al4bbPzxc90+G2/6vSe4czjRdn+T4fQceziy9Lck3kjyc5KoM/AzwEeCfDI/18HD7jyf5P0f2/80ku5I8mGRLkp9Y6Nhd/kFJ0gtlUyT1268Dnxq+Xpvkf6iq84FvA/+iqo6oqg8Avzjc/iXDdX+d5Fzgd4F/CRwD/Bfg03OOfzbwSuDngH8NvLaq7gTeBvz18FgvmVtUkjOB9w33ORb4FrB5oWO/gD8HSeqcTZHUU0n+R+B44DNVdRvwTeCNYxzibcD7qurOqtoL/HvglNHZIuD9VfVwVX0buBE4ZZHH/jXg6qr6SlU9CVzKYGZp/QSOLUlTYVMk9dcFwF9U1QPD5WuG6xbreODK4ddXDwMPAgHWjmzznZH3jwNHLPLYP8FgdgiAqnoM+LsJHVuSpmIiV6qoLUn+X+AXgJur6uxp16MfluRwBl85rUqyr7k4DHhJkn/E4MTrUXOXAe4F3ltVn1pCCfs73qj/zqDp2lfvi4AfBfYsYSxpLGaYuuJM0Wz6IHD+tIvQvH4ZeAY4mcHXTqcAP8PgvKBfB74L/OTI9t8Dnp2z7iPApUleDpDkxUl+dZHjfxdYl+TQA3z+aeAtSU4Z3irg3wO3VtU9izy+9EKYYeqETdEKluSVSe5IsjrJi5LsSPKKqvoi8Oi069O8LgD+76r6dlV9Z98L+DCD83neB7xn+NXYb1fV48B7gf86XPcLVfUfgT8ENif5PvDfgNctcvwbgB3Ad5I8MPfDqvpL4N8BnwXuA04EzntBP7E0hxmm5ZaqhWbJ1bLhJdSrgcOB3VX1vuH6VwO/7dSzpD4zw7ScPKdo5bsc2AY8AfybKdciSeMyw7Rs/Pps5ftRBlf9HMngty1JaokZpmVjU7TyfZTBuR+fYnB+iSS1xAzTsvHrsxUsya8DT1fVNUlWAX81vBPx7wP/EDgiyW7grVV1/TRrlaS5zDAtN0+0liRJwq/PJEmSAJsiSZIkwKZIkiQJWOFNUZKrk9yf5L9Nu5bFGt659ctJ/mZ499bfn3ZNC0lyXJIbk+wc1nzxtGtarCQbk9yVZFeSS6ZdjzSqtQxrMb9GJVmV5KtJPjftWhbScu722Yo+0TrJLwKPAf9PVb1i2vUsRpIAL6qqx5IcAtwMXFxVt0y5tANKcixwbFV9JcmRwG3AL1fVzimXNq/h1Sz/H/AaYDeDG8S9oe91a3a0lmEt5teoJO8ENgBH9f1O2a3mbt+t6JmiqvoS8OC06xhHDTw2XDxk+Op151pV91XVV4bvHwXuBNZOt6pFOQ3YVVV3V9VTwGbg3CnXJD2ntQxrMb/2SbIOeD3wsWnXshgN526vreimqFXDKdzbgfuBL1TVrdOuabGSrAdOBVqoeS1w78jybgwV6QVpOL8+BLwLeHbahYyrsdztNZuiHqqqZ6rqFGAdcFqS3k+bAyQ5gsFT03+rqr4/7XokLb8W8yvJ2cD9VXXbtGsZl7k7WTZFPVZVDwM3AhunXctChucPfBb4VFX9+bTrWaQ9wHEjy+uG6yS9QC3lF3AGcE6Sexh8jX5mkj+dbkkLazR3e82mqGeSHJPkJcP3hzM4Cfjr061qfsOTK/8EuLOqrph2PWPYBpyU5IQkhwLnAVumXJPUrBbzC6CqLq2qdVW1nkEO3FBVb5pyWfNqOHd7bUU3RUk+Dfw18A+S7E7y1mnXtAjHAjcmuYPB/7S/UFV9vzz0DOB8Br9d3T58/fNpF7WQqtoLXARcz+Akxc9U1Y7pViX9vQYzrMX8alWTudt3K/qSfEmSpMVa0TNFkiRJi2VTJEmShE2RJEkSYFMkSZIE2BRJkiQBM9IUJblw2jUsRYt1W7M0eS3+O2rNy6PFmvtsJpoioNV/aVqs25qlyWvx31FrXh4t1txbs9IUSZIkzatXN29cs2ZNrV+/fuLH/d73vscxxxwz8eMCfOtb3+rkuABPPPEEq1ev7uTYBx98cCfHffzxx/mRH/mRTo79ne98p5Pjdq2qMu0a1L2jjz661q1b18mxH3zwQY4++uiJH3fPnu4e9ffkk09y2GGHdXLso446qpPjPvrooxx55JGdHPuee+7p5Lhdm7X86ub/jEu0fv16tm/fPu0yxvKbv/mb0y5hSbpqErv0vve9b9olSAe0bt06Pv/5z0+7jLG8+93vnnYJS/JLv/RL0y5hbG9+85unXYIWwa/PJEmSsCmSJEkCbIokSZIAmyJJkiTApkiSJAmwKZIkSQJsiiRJkoCe3adImnUbN26sBx54YKx9brvttuuramNHJUnSorWeYTZFUo888MADY9/ANMmajsqRpLG0nmE2RVLP9OnRO5I0rpYzzKZI6pmWA0WSWs4wmyKpZ1oOFElqOcNsiqQeqaqmA0XSbGs9w7wkX+qZfaGy2Jck9cmkMyzJxiR3JdmV5JJ5tvuVJJVkw8i6S4f73ZXktQuN5UyR1DM2OpJaNskMS7IKuAp4DbAb2JZkS1XtnLPdkcDFwK0j604GzgNeDvwE8JdJfrqqnjnQeM4UST3jTJGklk04w04DdlXV3VX1FLAZOHc/2/0B8IfAEyPrzgU2V9WTVfW3wK7h8Q7Ipkjqkari2WefHeslSX2xxAxbk2T7yOvCkUOuBe4dWd49XPecJP8YOK6qPj+nnAX3nauzr8+SXA2cDdxfVa/oahxppXH2px/MMGlplpBhD1TVhoU3+2FJDgKuAN68lP3n6nKm6ONAL27bLbXEr8964+OYYdLYJpxhe4DjRpbXDdftcyTwCuCmJPcAvwBsGZ5svdC+P6SzpqiqvgQ82NXxpZXKpqgfzDBpaSacYduAk5KckORQBidObxkZ65GqWlNV66tqPXALcE5VbR9ud16Sw5KcAJwEfHm+wbz6TOoZGx1JLZtkhlXV3iQXAdcDq4Crq2pHksuB7VW1ZZ59dyT5DLAT2Au8Y74rz6AHTdHwhKoLAV72spdNuRppurqa/UmyEbiSQah8rKreP+fzNwMf5O+nlj9cVR+beCErzGh+rV077/mb0kzoIsOqaiuwdc66yw6w7avnLL8XeO9ix5r61WdVtamqNlTVhmOOOWba5UhT18GNz/bd5+N1wMnAG4b375jrP1TVKcOXDdEijObX0UcfPe1ypF5o+RSAqc8USXq+DkLiuft8ACTZd5+PnfPuJUlL0LdGZxydzRQl+TTw18A/SLI7yVu7GktaSTr4LWux9+r4lSR3JLk2yXH7+XymmGHS0jhTtB9V9Yauji2tVEsMiTVJto8sb6qqTWMe4z8Dn66qJ5P8b8AngDPHLWQlMcOk8fWx0RmHX59JPbOEQFnoxmcL3qujqv5uZPFjwAfGLUKSoO2vz2yKpJ7pIFCeu88Hg2boPOCNoxskObaq7hsungPcOekiJM0GmyJJE9PB5ayLuc/Hv0lyDoN7eTzIhG6ZL2n22BRJmpguAmWh+3xU1aXApRMfWNLMsSmSNBGtn6Qoaba1nmE2RVLPtBwoktRyhtkUST3TcqBIUssZZlMk9UzLgSJJLWeYTZHUMy0HiiS1nGE2RVKPtH6SoqTZ1nqG2RRJPdNyoEhSyxlmUyT1TMuBIkktZ5hNkdQzLQeKJLWcYTZFUs+0HCiS1HKG2RRJPdL6SYqSZlvrGWZTJPVMy4EiSS1nmE2R1DMtB4oktZxhvWqKHn30UW688cZplzGWn//5n592CUvy0EMPTbsEHUDLgTLL7rnnHi644IJplzGWSy+9dNolLMkNN9ww7RLG9va3v33aJYzt2muvXdJ+LWdYr5oiSW0HiiS1nGE2RVKPtH6SoqTZ1nqG2RRJPdNyoEhSyxlmUyT1TMuBIkktZ5hNkdQzLQeKJLWcYTZFUs+0HCiS1HKG2RRJPdL6SYqSZlvrGWZTJPVMy4EiSS1nmE2R1DMtB4oktZxhNkVSz7QcKJLUcoYdNO0CJD3fvu/kF/uSpD6ZdIYl2ZjkriS7klyyn8/fluRrSW5PcnOSk4fr1yf5wXD97Uk+stBYzhRJPWKjI6llk86wJKuAq4DXALuBbUm2VNXOkc2uqaqPDLc/B7gC2Dj87JtVdcpix7MpknrGpkhSyyacYacBu6rqboAkm4Fzgeeaoqr6/sj2LwKWXIBNkdQzNkWSWjbhDFsL3DuyvBs4fe5GSd4BvBM4FDhz5KMTknwV+D7wnqr6L/MN5jlFUs94TpGkli0hw9Yk2T7yunAJY15VVScCvwO8Z7j6PuBlVXUqg4bpmiRHzXccZ4qknrHRkdSyJWTYA1W14QCf7QGOG1leN1x3IJuBPx7W8STw5PD9bUm+Cfw0sP1AOztTJPXIuL9h2UBJ6pMOMmwbcFKSE5IcCpwHbBndIMlJI4uvB74xXH/M8ERtkvwkcBJw93yDOVMk9YyNjqSWTTLDqmpvkouA64FVwNVVtSPJ5cD2qtoCXJTkLOBp4CHgguHuvwhcnuRp4FngbVX14HzjddYUJVkNfAk4bDjOtVX1e12NJ60UNkX9Mfwtczuwp6rOnnY9UgsmnWFVtRXYOmfdZSPvLz7Afp8FPjvOWF3OFD0JnFlVjyU5BLg5yXVVdUuHY0rNsynqlYuBO4F5T86U9PdazrDOzimqgceGi4cMX+3+SUnLxHOK+iHJOgbnJ3xs2rVILWk5wzo9p2g49Xwb8FPAVVV1a5fjSa3rY0jMsA8B7wKOnHYhUitaz7BOrz6rqmeGt9deB5yW5BVzt0ly4b57EzzyyCNdliM1oeXfslaKJGcD91fVbQts91x+PfXUU8tUndRvLWfYslySX1UPAzfy988iGf1sU1VtqKoNL37xi5ejHKnXWg6UFeQM4Jwk9zC478mZSf507kaj+XXooYcud41SL7WcYZ01RcP7A7xk+P5wBg9z+3pX40krRReBstBTpke2+5UkleRAN1KbCVV1aVWtq6r1DO6LckNVvWnKZUlNaLkp6vKcomOBTwzPKzoI+ExVfa7D8aQVYdIhscinTJPkSAZXW3nun6Ql61ujM47OmqKqugM4tavjSytRR785LfiU6aE/AP4Q+D8mXUDLquom4KYplyE1oY+zP+PwMR9Sz3Qw9by/p0yvHd0gyT8Gjquqz0/uJ5E0i/z6TNLELCEk1iQZfcDhpqratNidkxwEXAG8edyBJWmuvjU647ApknpmCYEy3xOmYeGnTB8JvAK4KQnAjwNbkpxTVQd8mrQk7Y9NkaSJ6SBQnnvKNINm6DzgjSPjPQKs2bec5Cbgt22IJC2FTZGkiejiO/Za3FOmJekF6+N5QuOwKZJ6potAqQWeMj1n/asnXoCkmWFTJGliWg4USWo5w2yKpJ5pOVAkqeUMsymSeqblQJGkljPMpkjqkdZPUpQ021rPMJsiqWdaDhRJajnDbIqknmk5UCSp5QyzKZJ6puVAkaSWM8ymSOqZlgNFklrOMJsiqUdaP0lR0mxrPcNsiqSeaTlQJKnlDLMpknqm5UCRpJYzzKZI6pmWA0WSWs6wXjVFjzzyCNddd920yxjLW97ylmmXsCR/8Rd/Me0SdAAtB8osO/roo/m1X/u1aZcxlqeffnraJSzJ4YcfPu0SxvbhD3942iWM7fHHH1/Sfi1nWK+aImnWtX6SoqTZ1nqG2RRJPdNyoEhSyxlmUyT1TMuBIkktZ5hNkdQzLQeKJLWcYTZFUs+0HCiS1HKG2RRJPdL6SYqSZlvrGXbQtAuQ9Hz7QmWxL0nqk0lnWJKNSe5KsivJJfv5/G1Jvpbk9iQ3Jzl55LNLh/vdleS1C43lTJHUMzY6klo2yQxLsgq4CngNsBvYlmRLVe0c2eyaqvrIcPtzgCuAjcPm6Dzg5cBPAH+Z5Ker6pkDjedMkdQzzhRJatmEM+w0YFdV3V1VTwGbgXPnjPf9kcUXAfsOei6wuaqerKq/BXYNj3dAzhRJPWOjI6llE86wtcC9I8u7gdPnbpTkHcA7gUOBM0f2vWXOvmvnG8yZIqlHxv0NywZKUp8sMcPWJNk+8rpwCeNeVVUnAr8DvGep9TtTJPWMjY6kli0hwx6oqg0H+GwPcNzI8rrhugPZDPzxEvd1pkjqG2eKJLVswhm2DTgpyQlJDmVw4vSW0Q2SnDSy+HrgG8P3W4DzkhyW5ATgJODL8w3mTJHUMzY6klo2yQyrqr1JLgKuB1YBV1fVjiSXA9uragtwUZKzgKeBh4ALhvvuSPIZYCewF3jHfFeegU2R1CvO/khqWRcZVlVbga1z1l028v7iefZ9L/DexY5lUyT1jE2RpJa1nGE2RVLPtBwoktRyhnXeFA3vRrkd2FNVZ3c9ntS6lgNlpUiyGvgScBiDnLy2qn5vulVJbWg5w5Zjpuhi4E7gqGUYS2pey4GygjwJnFlVjyU5BLg5yXVVdctCO0qzruUM6/SS/CTrGFwe97Eux5FWCm/e2A818Nhw8ZDhyz9saQGtZ1jXM0UfAt4FHNnxONKK0beQmFXDr/5vA34KuKqqbp1ySVITWs6wzmaKkpwN3F9Vty2w3YX7bu39gx/8oKtypGa0/FvWSlJVz1TVKQzugntaklfM3WY0vx599NHlL1LqoZYzrMuvz84AzklyD4Pbbp+Z5E/nblRVm6pqQ1VtOPzwwzssR2pDy4GyElXVw8CNwMb9fPZcfh15pBPiErSdYZ01RVV1aVWtq6r1DG7LfUNVvamr8aSVouVAWSmSHJPkJcP3hwOvAb4+3aqkNrScYd6nSOqRPobEjDoW+MTwvKKDgM9U1eemXJPUe61n2LI0RVV1E3DTcowlta6LQEmyEbiSwbODPlZV75/z+duAdwDPAI8BF1bVzokX0oiqugM4ddp1SC1quSnq9JJ8SeOb9NTzcLbjKuB1wMnAG5KcPGeza6rqZ4cnFn8AuGLSP5ek2eDXZ5ImpoOQOA3YVVV3AyTZDJzL4MnR+8b8/sj2L8J78khaor41OuOwKZJ6poNAWQvcO7K8Gzh97kZJ3gG8EzgUOHPSRUiaDS03RX59JvXIuNPOw/BZs+9eOcPXhUsc+6qqOhH4HeA9k/y5JM2GJWZYbzhTJPXMEkLigaraMM/ne4DjRpbXDdcdyGbgj8ctQpLAmSJJE9TBb1nbgJOSnJDkUAb3DdsyukGSk0YWXw98Y2I/kKSZ4kyRpImZdEhU1d4kFwHXM7gk/+qq2pHkcmB7VW0BLkpyFvA08BBwwUSLkDQz+tbojMOmSOqRquLZZ5/t4rhbga1z1l028v7iiQ8qaeZ0lWHLxaZI6pmWf8uSpJYzzKZI6pmWA0WSWs4wmyKpZ1oOFElqOcNsiqSeaTlQJKnlDLMpknqkj5eoStJitZ5hNkVSz7QcKJLUcobNe/PGJEclOXE/63+uu5Kk2dbyjc/6xgyTll/LGXbApijJvwa+Dnw2yY4krxz5+ONdFybNqpYDpU/MMGk6Ws6w+WaKfhf4+ao6BXgL8Mkk//Pws3RemTSjWg6UnjHDpCloOcPmO6doVVXdB1BVX07yPwGfS3Ic0K+fQloh+hgSDTPDpGXWeobN1xQ9muTEqvomQFXdl+TVwH8CXt5FMUk4+OC2zv3+8R//8WmXsCS7d++edgk6gJYDpWeWNcO++93v8qEPfWjSh+3UF7/4xWmXsCSf/OQnp13C2H7jN35j2iWM7ZprrlnSfi1n2Hxfn70dOCjJyftWVNWjwEagvb9dqREtTz33jBkmTUHLGXbApqiq/qaqvgF8JsnvZOBw4Argf1+2CqUZ03Kg9IkZJk1Hyxk27yX5Q6cDxwF/BWwD/jtwRpdFSbOs5UDpKTNMWkYtZ9himqKngR8AhwOrgb+tqmc7rUqaUeOGSd8CpafMMGmZdJFhSTYmuSvJriSX7OfzdybZmeSOJF9McvzIZ88kuX342rLQWItpirYxCJRXAv8UeEOSP1vEfpKWwKZo4swwaRlNMsOSrAKuAl4HnMzgv9+T52z2VWBDVf0ccC3wgZHPflBVpwxf5yxU+2Iu9XprVW0fvr8PODfJ+YvYT9IS2OhMnBkmLaMJZ9hpwK6quhsgyWbgXGDnyHg3jmx/C/CmpQ624EzRSJiMrmvvekipEc4UTZYZJi2vCWfYWuDekeXdw3UH8lbgupHl1Um2J7klyS8vNFhbNwWSZoCNjqSWLSHD1iQZ/eVlU1VtGvcgSd4EbABeNbL6+Krak+QngRuSfK2G9y7bH5siqUec/ZHUsiVm2ANVteEAn+1hcPXoPuuG654nyVnAu4FXVdWTI/XsGf7z7iQ3AacCB2yKFnOitaRl5Ndnklo24QzbBpyU5IQkhwLnAc+7iizJqcBHgXOq6v6R9S9Nctjw/RoGt+LYyTycKZJ6xkZHUssmmWFVtTfJRcD1wCrg6qrakeRyYHtVbQE+CBwB/FkSgG8PrzT7GeCjSZ5lMAn0/qqyKZJaYlMkqWWTzrCq2gpsnbPuspH3Zx1gv78CfnacsWyKpJ6xKZLUspYzzKZI6hHPE5LUstYzzKZI6pmWA0WSWs4wmyKpZ1oOFElqOcNsiqSeaTlQJKnlDLMpknqm5UCRpJYzrLObNyY5LsmNSXYm2ZHk4q7GklaKcW961nL4tCDJqiRfTfK5adcitaD1DOtypmgv8G+r6itJjgRuS/KFhW6cJM26voXEjLsYuBM4atqFSK1oOcM6mymqqvuq6ivD948yCJb5nmwrCR/z0RdJ1gGvBz427VqklrScYctyTlGS9QwewnbrcowntaxvITHDPgS8Czhy2oVILWk5wzp/IGySI4DPAr9VVd/fz+cXJtmeZPvjjz/edTlS77X8W9ZKkeRs4P6qum2B7Z7Lr2eeeWaZqpP6reUM63SmKMkhDBqiT1XVn+9vm6raBGwCOPbYY/v1pyMtsz6GxIw6AzgnyT8HVgNHJfnTqnrT6Eaj+XX44Yf7F6eZ13qGdXn1WYA/Ae6sqiu6GkdaaVr+LWulqKpLq2pdVa0HzgNumNsQSdq/ljOsy5miM4Dzga8luX247ndr8LRbSQfQt5CQpHG0nGGdNUVVdTOQro4vrVQtB8pKVFU3ATdNuQypGS1nmHe0lnqm5UCRpJYzrPOrzyQtXld3g02yMcldSXYluWQ/n79zePf5O5J8McnxE//hJK14XWXYcrEpknpm0oGSZBVwFfA64GTgDUlOnrPZV4ENVfVzwLXAByb8Y0maETZFkiamg0A5DdhVVXdX1VPAZuDcOWPeWFX7bhR2C7Buoj+UpJnRclPkOUVSzywhJNYk2T6yvGl4/5x91gL3jizvBk6f53hvBa4btwhJgrbPKbIpknpmCYHyQFVtmMTYSd4EbABeNYnjSZo9NkWSJqKj6eQ9wHEjy+uG654nyVnAu4FXVdWTky5C0srXx6/ExmFTJPVMB4GyDTgpyQkMmqHzgDeObpDkVOCjwMaqun/SBUiaHTZFkiZm0oFSVXuTXARcD6wCrq6qHUkuB7ZX1Rbgg8ARwJ8NntDDt6vqnIkWImkm2BRJmpguAmX4eJ2tc9ZdNvL+rIkPKmkm2RRJmpiWA0WSWs4wmyKpR1o/SVHSbGs9w2yKpJ5pOVAkqeUMsymSeqblQJGkljPMpkjqmZYDRZJazjCbIqlnWg4USWo5w2yKpB5p/SRFSbOt9QzrVVO0d+9eHnzwwWmXMZaXvvSl0y5hSYY36FMPtRwos+zHfuzHePvb3z7tMsby9a9/fdolLMmJJ5447RLGduWVV067hLE9/vjjS9qv5Qw7aNoFSHq+fb9pLfYlSX0y6QxLsjHJXUl2JblkP5+/M8nOJHck+WKS40c+uyDJN4avCxYaq1czRZLa/i1LkiaZYUlWAVcBrwF2A9uSbKmqnSObfRXYUFWPJ3k78AHgf0lyNPB7wAaggNuG+z50oPGcKZJ6ZNzfsGygJPVJBxl2GrCrqu6uqqeAzcC5c8a8sar2fdd3C7Bu+P61wBeq6sFhI/QFYON8gzlTJPWMjY6klk04w9YC944s7wZOn2f7twLXzbPv2vkGsymSesamSFLLlpBha5JsH1neVFWbxj1Ikjcx+KrsVePuu49NkdQzNkWSWraEDHugqjYc4LM9wHEjy+uG654nyVnAu4FXVdWTI/u+es6+N81XiOcUST3jOUWSWjbhDNsGnJTkhCSHAucBW0Y3SHIq8FHgnKq6f+Sj64F/luSlSV4K/LPhugNypkjqERsdSS2bdIZV1d4kFzFoZlYBV1fVjiSXA9uragvwQeAI4M+G9+D7dlWdU1UPJvkDBo0VwOVVNe/NEG2KpJ6xKZLUsklnWFVtBbbOWXfZyPuz5tn3auDqxY5lUyT1jE2RpJa1nGE2RVLPtBwoktRyhtkUST3TcqBIUssZZlMk9YgnWktqWesZZlMk9UzLgSJJLWeYTZHUMy0HiiS1nGE2RVLPtBwoktRyhtkUST3TcqBIUssZZlMk9UjrJylKmm2tZ1inzz5LsjHJXUl2Jbmky7GklcJnn01fktVJvpzkb5LsSPL7065JakXLGdbZTFGSVcBVwGuA3cC2JFuqamdXY0orQd9CYkY9CZxZVY8lOQS4Ocl1VXXLtAuT+q7lDOvy67PTgF1VdTdAks3AuYBNkTSPlgNlpajBX8Jjw8VDhi//YqRFaIlIlUsAAAeGSURBVDnDuvz6bC1w78jy7uE6SfNoeep5JUmyKsntwP3AF6rq1mnXJLWg5Qzr9JyixUhyYZLtSbY/8cQT0y5Hmqpxw6RvgbKSVNUzVXUKsA44Lckr5m4zml+PPfbYDx9EmjGtZ1iXTdEe4LiR5XXDdc9TVZuqakNVbVi9enWH5UhtaDlQVqKqehi4Edi4n8+ey68jjjhi+YuTeqjlDOuyKdoGnJTkhCSHAucBWzocT1oRWg6UlSLJMUleMnx/OIMLRr4+3aqkNrScYZ2daF1Ve5NcBFwPrAKurqodXY0nrRR9C4kZdSzwieFVtAcBn6mqz025JqkJLWdYpzdvrKqtwNYux5BWmpYDZaWoqjuAU6ddh9SiljPMO1pLPdLH6WRJWqzWM8ymSOqZlgNFklrOMJsiqWdaDhRJajnDpn6fIknP18WVGws9hzDJLyb5SpK9Sf7VxH8oSTPDq88kTcykQ2KRzyH8NvBm4LcnOrikmdO3RmccNkVSj3T0m9OCzyGsqnuGnz076cElzY4+zv6Mw6ZI6pkOAmV/zyE8fdKDSBI4UyRpgpYQKGuSbB9Z3lRVmyZYkiQtmk2RpIlZQqA8UFUb5vl8Uc8hlKRJsCmSNDEdBMpzzyFk0AydB7xx0oNIErTdFHlJvtQj417Kupjwqaq9wL7nEN7J4DleO5JcnuQcgCSvTLIb+FXgo0l8TqGksXWRYcvJmSKpZ7oIif09h7CqLht5v43B12qS9IL0rdEZh02R1DMtB4oktZxhNkVSz7QcKJLUcobZFEk903KgSFLLGeaJ1lKPtH6SoqTZ1kWGvZBnNyZ5Jsntw9eWhcZypkjqGRsdSS2bZIZN4NmNP6iqUxY7nk2R1DM2RZJaNuEMW9ZnN9oUST1jU9Sm7373u1x55ZXTLmMsd91117RLWJI/+qM/mnYJYzv//POnXcLYrr322iXtN+EMe6HPblw9fAzSXuD9VfWf5tvYpkjqGZsiSS3r2fMbj6+qPUl+Erghydeq6psH2timSOoRT56W1LIlZth8z298Qc9urKo9w3/eneQm4FTggE2RV59JPePVZ5JaNuEMe+7ZjUkOZfDsxgWvIgNI8tIkhw3frwHOYORcpP1xpkjqGRsdSS2bZIZV1d4k+57duAq4et+zG4HtVbUlySuB/wi8FPgXSX6/ql4O/AyDZzk+y2AS6P1zrlr7ITZFUs/YFElq2aQzbKnPbqyqvwJ+dpyxbIqkHqkqnn32BV9VKklT0XqG2RRJPeNMkaSWtZxhNkVSz7QcKJLUcobZFEk903KgSFLLGWZTJPVMy4EiSS1nmE2R1CPee0hSy1rPMJsiqWdaDhRJajnDbIqknmk5UCSp5QyzKZJ6puVAkaSWM8ymSOqZlgNFklrOMJsiqUdaP0lR0mxrPcNsiqSeaTlQJKnlDDuoy4Mn2ZjkriS7klzS5VjSSrHvN63FvjR5SY5LcmOSnUl2JLl42jVJrWg5wzqbKUqyCrgKeA2wG9iWZEtV7exqTGkl6FtIzKi9wL+tqq8kORK4LckXzC9pYS1nWJdfn50G7KqquwGSbAbOBQwVaR4tB8pKUVX3AfcN3z+a5E5gLeaXtKCWM6zLpmgtcO/I8m7g9A7Hk5rXx+nkWZdkPXAqcOt0K5H6r/UMm/qJ1kkuBC4EOOKII6ZcjTR9LQfKSpPkCOCzwG9V1ff38/lz+XXwwVOPU6kXWs6wLv8r3gMcN7K8brjueapqE7AJ4Jhjjmn3T1KakJYDZSVJcgiDhuhTVfXn+9tmNL9Wr17tX5xE2xnWZVO0DTgpyQkMmqHzgDd2OJ60IrQcKCtFkgB/AtxZVVdMux6pJS1nWGdNUVXtTXIRcD2wCri6qnZ0NZ60UrQcKCvIGcD5wNeS3D5c97tVtXWKNUlNaDnDOv0SfBgghoi0SK2fpLhSVNXNQKZdh9Sa1jPMMwOlnmk5UCSp5QyzKZJ6puVAkaSWM8ymSOqZlgNFklrOMJsiqWdaDhRJajnDbIqkHmn9JEVJs631DDto2gVIer4unjCdZGOSu5LsSnLJfj4/LMl/GH5+6/DRFpI0ti4ybLnYFEk9M+lASbIKuAp4HXAy8IYkJ8/Z7K3AQ1X1U8D/BfzhhH8sSTPCpkjSxHQQKKcBu6rq7qp6CtgMnDtnm3OBTwzfXwv80vCuzpI0lpabIs8pknqmg5BYC9w7srwbOP1A2wzvRv8I8KPAA5MuRtLK1rdGZxw2RVK/XA+sGXOf1Um2jyxvGj6oVJKW21IyrDe/fNkUST1SVRs7OOwe4LiR5XXDdfvbZneSg4EXA3/XQS2SVrCOMmzZeE6RtPJtA05KckKSQ4HzgC1zttkCXDB8/6+AG6rlOXBJWgJniqQVbniO0EUMprVXAVdX1Y4klwPbq2oL8CfAJ5PsAh5k0DhJ0kyxKZJmQFVtBbbOWXfZyPsngF9d7rokqU/8+kySJAmbIkmSJMCmSJIkCbApkiRJAiB9uuo2yfeAb027DmmCjq+qY6ZdhLpnfmkFmrn86lVTJEmSNC1+fSZJkoRNkSRJEmBTJEmSBNgUSZIkATZFkiRJAPz/gh0qTKVEy1MAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 720x360 with 4 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ManIuY9N2YnN",
        "colab_type": "code",
        "outputId": "6e267b04-185e-4eab-96b1-9258bbed070c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 107
        }
      },
      "source": [
        "import urllib.request\n",
        "import zipfile\n",
        "import os\n",
        "\n",
        "DATA_LINK = 'http://www.manythings.org/anki/rus-eng.zip'\n",
        "DIR = './data/'\n",
        "FILE_PATH = DIR + 'rus-eng.zip'\n",
        "class AppURLopener(urllib.request.FancyURLopener):\n",
        "        version = \"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.69 Safari/537.36\"\n",
        "if not os.path.isdir(DIR):\n",
        "    os.mkdir(DIR)\n",
        "urllib._urlopener = AppURLopener()\n",
        "urllib._urlopener.retrieve(DATA_LINK, FILE_PATH)\n",
        "with zipfile.ZipFile(FILE_PATH, 'r') as zip_ref:\n",
        "    zip_ref.extractall(DIR)\n",
        "os.remove(FILE_PATH)"
      ],
      "execution_count": 276,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:12: DeprecationWarning:\n",
            "\n",
            "AppURLopener style of invoking requests is deprecated. Use newer urlopen functions/methods\n",
            "\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dW0lW8vT2xOc",
        "colab_type": "code",
        "outputId": "862a80c0-304c-40db-b75b-b08e4a795f6f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 87
        }
      },
      "source": [
        "from torch.utils.data import Dataset, DataLoader, SubsetRandomSampler, BatchSampler\n",
        "\n",
        "class Seq2SeqDataset(Dataset):\n",
        "  def __init__(self, pairs, x1_dict=None, x2_dict=None, \n",
        "               x1_tokenizer=None, x2_tokenizer=None, \n",
        "               x1_max_dict=10000, x2_max_dict=10000):\n",
        "    print(f'pairs: {len(pairs)}')\n",
        "    self.x1_tokenizer = x1_tokenizer if x1_tokenizer is not None else Seq2SeqDataset.Tokenizer()\n",
        "    self.x2_tokenizer = x2_tokenizer if x2_tokenizer is not None else Seq2SeqDataset.Tokenizer()\n",
        "    if x1_dict is None and x2_dict is None:\n",
        "      x1_dict, x2_dict = self.make_dicts(pairs)\n",
        "    x1_dict = {k: v for i, (k, v) in enumerate(x1_dict.items()) \n",
        "    if i < x1_max_dict}\n",
        "    x2_dict = {k: v for i, (k, v) in enumerate(x2_dict.items()) \n",
        "    if i < x2_max_dict}\n",
        "    self.x1_dict, self.x2_dict = x1_dict, x2_dict\n",
        "    print(f'x1 dict: {len(self.x1_dict)}, x2 dict: {len(self.x2_dict)}')\n",
        "    self.x1_w2i = {'<PAD>': 0, '<GO>': 1, '<EOS>': 2, '<UNK>': 3}\n",
        "    self.x1_i2w = {0: '<PAD>', 1: '<GO>', 2: '<EOS>', 3: '<UNK>'}\n",
        "    self.x2_w2i = {'<PAD>': 0, '<GO>': 1, '<EOS>': 2, '<UNK>': 3}\n",
        "    self.x2_i2w = {0: '<PAD>', 1: '<GO>', 2: '<EOS>', 3: '<UNK>'}\n",
        "    for i, w in enumerate(self.x1_dict):\n",
        "      self.x1_w2i[w] = i + 4\n",
        "      self.x1_i2w[i + 4] = w\n",
        "    for i, w in enumerate(self.x2_dict):\n",
        "      self.x2_w2i[w] = i + 4\n",
        "      self.x2_i2w[i + 4] = w\n",
        "    print(f'x1 idxs: {len(self.x1_w2i)}, x2 idxs: {len(self.x2_w2i)}')\n",
        "    self.pairs = []\n",
        "    x1_max_len = 0\n",
        "    x2_max_len = 0\n",
        "    for x1, x2 in pairs:\n",
        "      e1 = self.encode_x1(x1)\n",
        "      e2 = self.encode_x2(x2)\n",
        "      x1_max_len = max(len(e1), x1_max_len)\n",
        "      x2_max_len = max(len(e2), x2_max_len)\n",
        "      self.pairs.append([e1, e2])\n",
        "    print(f'x1 max len: {x1_max_len}, x2 max len: {x2_max_len}')\n",
        "\n",
        "  def __len__(self):\n",
        "    return len(self.pairs)\n",
        "\n",
        "  def __getitem__(self, index):\n",
        "    return self.pairs[index]\n",
        "\n",
        "  def encode_x1(self, s):\n",
        "    return [self.x1_w2i['<GO>']] + [self.x1_w2i.get(x, self.x1_w2i['<UNK>']) \n",
        "    for x in self.x1_tokenizer.tokenize(s)] + [self.x1_w2i['<EOS>']]\n",
        "\n",
        "  def encode_x2(self, s):\n",
        "    return [self.x2_w2i['<GO>']] + [self.x2_w2i.get(x, self.x2_w2i['<UNK>']) \n",
        "    for x in self.x2_tokenizer.tokenize(s)] + [self.x2_w2i['<EOS>']]\n",
        "\n",
        "  def decode_x1(self, v):\n",
        "    return [self.x1_i2w.get(x, '<UNK>') for x in v]\n",
        "\n",
        "  def decode_x2(self, v):\n",
        "    return [self.x2_i2w.get(x, '<UNK>') for x in v]\n",
        "\n",
        "  def make_dicts(self, pairs):\n",
        "      x1_dict, x2_dict = {}, {}\n",
        "      for x1, x2 in pairs:\n",
        "        for t in self.x1_tokenizer.tokenize(x1):\n",
        "          x1_dict[t] = x1_dict.setdefault(t, 0) + 1\n",
        "        for t in self.x2_tokenizer.tokenize(x2):\n",
        "          x2_dict[t] = x2_dict.setdefault(t, 0) + 1\n",
        "      x1_dict = dict(sorted(x1_dict.items(), key=lambda x: x[1], reverse=True))\n",
        "      x2_dict = dict(sorted(x2_dict.items(), key=lambda x: x[1], reverse=True))\n",
        "      return x1_dict, x2_dict\n",
        "\n",
        "  class Tokenizer():\n",
        "    def __init__(self):\n",
        "      pass\n",
        "\n",
        "    def tokenize(self, s):\n",
        "      return s.split() \n",
        "\n",
        "  def collate_fn(self, batch):\n",
        "    # print(batch)\n",
        "    x1s = []\n",
        "    x2s = []\n",
        "    for b in batch:\n",
        "      x1s.append(torch.tensor(b[0]))\n",
        "      x2s.append(torch.tensor(b[1]))\n",
        "    # print(x1s)\n",
        "    # print(x2s)\n",
        "    x1s = torch.nn.utils.rnn.pad_sequence(x1s, batch_first=True)\n",
        "    x2s = torch.nn.utils.rnn.pad_sequence(x2s, batch_first=True)\n",
        "    # print(x1s)\n",
        "    # print(x2s)\n",
        "    batch = [x1s, x2s]\n",
        "    # print(batch)\n",
        "    # max_x1_length = 0\n",
        "    # max_x2_length = 0\n",
        "    # print(batch)\n",
        "    # print(type(batch))\n",
        "    # for x1, x2 in batch:\n",
        "    #   max_x1_length = max(len(x1), max_x1_length)\n",
        "    #   max_x2_length = max(len(x2), max_x2_length)\n",
        "    # # print(max_x1_length, max_x2_length)\n",
        "    # x1s, x2s = [], []\n",
        "    # for x1, x2 in batch:\n",
        "    #   # if len(x1) < max_x1_length:\n",
        "    #   x1 = x1 + [self.x1_w2i['<PAD>'] for _ in range(max_x1_length - len(x1))]\n",
        "    #   # if len(x2) < max_x2_length:\n",
        "    #   x2 = x2 + [self.x2_w2i['<PAD>'] for _ in range(max_x2_length - len(x2))]  \n",
        "    #   x1s.append(x1)\n",
        "    #   x2s.append(x2)\n",
        "    # # print(x1s)\n",
        "    # # print(x2s)\n",
        "    # batch = [x1s, x2s]\n",
        "    # print(x1s)\n",
        "    # print(x2s)\n",
        "    # print(batch)\n",
        "    return batch\n",
        "\n",
        "# Get pairs as: [[x1, x2], [x1, x2], ...], where x1 and x2 - strings\n",
        "pairs = []\n",
        "with open('data/rus.txt') as f:\n",
        "  for l in f.readlines()[:100000]:\n",
        "    pairs.append(l.split('\\t')[:2])\n",
        "\n",
        "# Normalize pairs\n",
        "def normalize(s):\n",
        "  return ''.join(x.lower() for x in s if x not in '!\"#$%&\\'()*+,-./:;<=>?@[\\]^_`{|}~')\n",
        "norm_pairs = []\n",
        "for p in pairs:\n",
        "  norm_pairs.append([normalize(p[0]), normalize(p[1])])\n",
        "\n",
        "# Make dataset\n",
        "eng_rus = Seq2SeqDataset(norm_pairs, x1_max_dict=100000, x2_max_dict=100000)\n",
        "\n",
        "# Tests\n",
        "for p in eng_rus:\n",
        "  assert p[0][0] == 1\n",
        "  assert p[0][-1] == 2\n",
        "  assert p[1][0] == 1\n",
        "  assert p[1][-1] == 2\n",
        "  # print(p)\n",
        "  # print(eng_rus.decode_x1(p[0]), eng_rus.decode_x2(p[1]))\n",
        "dl = DataLoader(eng_rus, batch_size=2, shuffle=True, sampler=None,\n",
        "                batch_sampler=None, num_workers=0, collate_fn=eng_rus.collate_fn,\n",
        "                pin_memory=False, drop_last=True, timeout=0,\n",
        "                worker_init_fn=None)\n",
        "for _ in range(3):\n",
        "  for x1s, x2s in dl:\n",
        "    assert not (x1s[0][-1] == 0 and x1s[1][-1] == 0)\n",
        "    assert not (x2s[0][-1] == 0 and x2s[1][-1] == 0)\n",
        "    # print(list(map(eng_rus.decode_x1, x1s)))\n",
        "    # print(list(map(eng_rus.decode_x2, x2s)))\n",
        "\n",
        "import random\n",
        "\n",
        "def train_test_val_split(dataset, batch_size=10, drop_last=False, \n",
        "                         train_weight=0.7, test_weight=0.2, val_weight=0.1):\n",
        "  length = len(dataset)\n",
        "  idxs = list(range(length))\n",
        "  random.shuffle(idxs)\n",
        "  # print(idxs)\n",
        "  train_len = int(train_weight * length)\n",
        "  test_len = int(test_weight * length)\n",
        "  val_len = int(val_weight * length)\n",
        "  diff = length - train_len - test_len - val_len\n",
        "  if diff > 0:\n",
        "    train_len += diff\n",
        "  # print(train_len, test_len, val_len)\n",
        "  train_idxs = idxs[:train_len]\n",
        "  test_idxs = idxs[train_len:train_len+test_len]\n",
        "  val_idxs = idxs[train_len+test_len:train_len+test_len+val_len]\n",
        "  train_sampler = BatchSampler(SubsetRandomSampler(train_idxs), \n",
        "                               batch_size=batch_size, drop_last=drop_last)\n",
        "  test_sampler = BatchSampler(SubsetRandomSampler(test_idxs),\n",
        "                              batch_size=batch_size, drop_last=drop_last)\n",
        "  val_sampler = BatchSampler(SubsetRandomSampler(val_idxs), \n",
        "                             batch_size=batch_size, drop_last=drop_last)\n",
        "  train_dl = DataLoader(eng_rus, batch_sampler=train_sampler, \n",
        "                        collate_fn=dataset.collate_fn)\n",
        "  test_dl = DataLoader(eng_rus, batch_sampler=test_sampler, \n",
        "                       collate_fn=dataset.collate_fn)\n",
        "  val_dl = DataLoader(eng_rus, batch_sampler=val_sampler, \n",
        "                      collate_fn=dataset.collate_fn)\n",
        "  return train_dl, test_dl, val_dl\n",
        "\n",
        "random.seed(0)\n",
        "train_dl, test_dl, val_dl = train_test_val_split(eng_rus, batch_size=2)\n",
        "\n",
        "# for _ in range(3):\n",
        "#   for x1s, x2s in train_dl:\n",
        "#     pass\n",
        "#     # print(x1s)\n",
        "#     # print(x2s)\n",
        "#     # print(list(map(eng_rus.decode_x1, x1s)))\n",
        "#     # print(list(map(eng_rus.decode_x2, x2s)))\n",
        "#   for x1s, x2s in test_dl:\n",
        "#     pass\n",
        "#     # print(x1s)\n",
        "#     # print(x2s)\n",
        "#     # print(list(map(eng_rus.decode_x1, x1s)))\n",
        "#     # print(list(map(eng_rus.decode_x2, x2s)))\n",
        "#   for x1s, x2s in val_dl:\n",
        "#     pass\n",
        "#     # print(x1s)\n",
        "#     # print(x2s)\n",
        "#     # print(list(map(eng_rus.decode_x1, x1s)))\n",
        "#     # print(list(map(eng_rus.decode_x2, x2s)))\n",
        "# print(eng_rus.x2_w2i['я'])\n",
        "# print(eng_rus.x1_w2i['i'])\n",
        "# print(eng_rus.x1_w2i['he'])\n",
        "\n",
        "# print(eng_rus.x2_w2i)\n",
        "# print(eng_rus.x1_w2i)"
      ],
      "execution_count": 404,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "pairs: 100000\n",
            "x1 dict: 7317, x2 dict: 20872\n",
            "x1 idxs: 7321, x2 idxs: 20876\n",
            "x1 max len: 9, x2 max len: 13\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IGSR5W9BPfLt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Import PyDrive and associated libraries.\n",
        "# This only needs to be done once in a notebook.\n",
        "from pydrive.auth import GoogleAuth\n",
        "from pydrive.drive import GoogleDrive\n",
        "from google.colab import auth\n",
        "from oauth2client.client import GoogleCredentials\n",
        "\n",
        "# Authenticate and create the PyDrive client.\n",
        "# This only needs to be done once in a notebook.\n",
        "auth.authenticate_user()\n",
        "gauth = GoogleAuth()\n",
        "gauth.credentials = GoogleCredentials.get_application_default()\n",
        "drive = GoogleDrive(gauth)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2H0fgnyXljaR",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 178
        },
        "outputId": "8106aefc-ecb6-450e-bbf9-bcbdb8930286"
      },
      "source": [
        "# !pip install googletrans\n",
        "!pip install yandex-translater\n",
        "from yandex.Translater import Translater\n",
        "from tqdm.notebook import tqdm\n",
        "import json\n",
        "\n",
        "try:\n",
        "  with open('rus_eng_dict.json') as f:\n",
        "    rus_eng_dict = json.load(f)\n",
        "except Exception:\n",
        "  rus_eng_dict = {}\n",
        "  tr = Translater()\n",
        "  tr.set_key('') # Api key found on https://translate.yandex.com/developers/keys\n",
        "  tr.set_from_lang('en')\n",
        "  tr.set_to_lang('ru')\n",
        "  for x1, i1 in tqdm(list(eng_rus.x1_w2i.items())[4:14]):\n",
        "    tr.set_text(x1)\n",
        "    x2 = tr.translate()\n",
        "    rus_eng_dict.setdefault(x2, []).append(x1)\n",
        "  with open('rus_eng_dict.json', 'w') as f:\n",
        "    json.dump(rus_eng_dict, f)\n",
        "print(rus_eng_dict)\n",
        "print(len(rus_eng_dict))"
      ],
      "execution_count": 278,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: yandex-translater in /usr/local/lib/python3.6/dist-packages (6.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from yandex-translater) (2.23.0)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->yandex-translater) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->yandex-translater) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->yandex-translater) (2020.4.5.1)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->yandex-translater) (2.9)\n",
            "{'перейти': ['go'], 'я': ['i'], 'сейчас': ['now'], 'ждать': ['wait'], 'это': ['it'], 'вау': ['wow'], 'улыбка': ['smile'], 'привет': ['hi'], 'ура': ['cheers'], 'есть': ['got']}\n",
            "10\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Zo5uR4pn28Su",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 264,
          "referenced_widgets": [
            "519e9ca6deb74bb0881767954b2dd90b",
            "98be3fe6b67c41368f37afba327006c2",
            "3e79c38fbd4d4a9fad9119a9d9071bcd",
            "0c942e99434a4066b0ec1f40534e6059",
            "01f6a7c8ab2349e3909e678343cce8af",
            "5170c2ba2ed64733b02b0908345daf58",
            "d55e0902596f49399241bbfc8b35813f",
            "d9496f1fe4ae496fb51074a0985e9fa6"
          ]
        },
        "outputId": "dda4e4e7-9e65-4f17-dd7d-40b1636620c4"
      },
      "source": [
        "try:\n",
        "  with open('attn_dict.json') as f:\n",
        "    attn_dict = json.load(f)\n",
        "except Exception:\n",
        "  attn_dict = {0: [0], 1: [1], 2: [2]}\n",
        "  for x2, x1s in tqdm(rus_eng_dict.items()):\n",
        "    i2 = eng_rus.x2_w2i.get(x2, None)\n",
        "    if i2 is not None:\n",
        "      for x1 in x1s: \n",
        "        i1 = eng_rus.x1_w2i.get(x1, None)\n",
        "        if i1 is not None:\n",
        "          attn_dict.setdefault(i2, []).append(i1)\n",
        "  with open('attn_dict.json', 'w') as f:\n",
        "      json.dump(attn_dict, f)\n",
        "print(attn_dict)\n",
        "attn_dict = {int(k): v for k, v in attn_dict.items()}\n",
        "for i2, i1s in tqdm(attn_dict.items()):\n",
        "  print(eng_rus.x2_i2w[i2], [eng_rus.x1_i2w[i1] for i1 in i1s])"
      ],
      "execution_count": 279,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'0': [0], '1': [1], '2': [2], '4': [4], '47': [43], '1190': [84], '6': [6], '255': [354], '25': [51]}\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "519e9ca6deb74bb0881767954b2dd90b",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=9.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "<PAD> ['<PAD>']\n",
            "<GO> ['<GO>']\n",
            "<EOS> ['<EOS>']\n",
            "я ['i']\n",
            "сейчас ['now']\n",
            "всётаки ['wait']\n",
            "это ['it']\n",
            "привет ['hi']\n",
            "есть ['got']\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0xzc4HmtkmBU",
        "colab_type": "code",
        "outputId": "df148abb-5253-4bd2-e5bf-6ce1eab5c980",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "015ac36378d44aaebe5aa2f6f387a83a",
            "bab2bfdb6cb84a729931e0193084c5b5",
            "4afd226c31704ed7b34cd1574b8d3df4",
            "deffdd5bfbb9484ba9f42a5cddbca137",
            "df7c0db7d117416fb9c4e3b10933eb4c",
            "99648b7c36a44b9793d6b099ff2fb4de",
            "126ae19a0ea5426dabf56f6a7a0f0356",
            "ecfd340a2a8e4857bca0da7c59d9509e",
            "7d0b4fb6343d416a9cf891bf318dbaab",
            "48ecbcd3078c4c72881ee4382b0f59d3",
            "1708e394423e48d7b733f7da1a8f4bc6",
            "2fcbaf92d6a14791911cf6a02612838e",
            "c45a79677b1d469abac9785d972ba4fd",
            "a3be5489afc248f4a14dca6206b632ba",
            "2e010f9e3bb94003867b09e7b283755e",
            "1780d171a010439ba49adff481b2b0c2",
            "d61a6502d8474c918aea958da29b45ad",
            "9831a2be48034da88c3fd20bcb3ab4dc",
            "65dc6f0f37f840d7ad0d27a941ce8a10",
            "8b137f34066e46beaf99dde3845fa73c",
            "61f365be6e544858a3a8fade0fb143b8",
            "3baaa9de3e3b44109b8a65726a60e085",
            "ff3103c168c349bfb0a26b052da363ae",
            "4520da03f07340049c636d6889652f92",
            "e06ce5d183324e92a1524ae457d0d048",
            "17d17be4c1264b8badcc0b664e96e30e",
            "3189c09632924f2980250eba99a9a69b",
            "b5f41475999747aaba54ffab36d1db68",
            "c6862702472d4575b713639017423b7f",
            "3d2cb22327f3429ea91aa2fe2a91fd77",
            "1ed053867b0b4125979ef8dd056b33bc",
            "3bf1b4e9e0b54507aece7ab7fd506eb7",
            "8c9fe6addbbb4654b6624fcae7589ad7",
            "cfb0c4abc0fd4986a111d964318b4df0",
            "1b16b7b173794bbab385f4ff298c8fba",
            "48e8f8ad22dd4591aad70059920d11ad",
            "6bacb7f5c9b34dd0a70ad3d75e4adb85",
            "6f68228b808743d68bf780390b469d36",
            "0f6ac97bcbf34e4b8b08b9d7441ca5ec",
            "068f67f21dd145ec987c8f2f2f6357c9",
            "373122c5949a42d9a62953fb284af278",
            "2a70104a08394afc9a182e3babf387eb",
            "750ad57244944e2f83a9f76856031d78",
            "c819cdc4aaec4bfd9c484cb18e863c1a",
            "78eff642913f42479ebb244f86b65207",
            "a872d6a68abb46e79e395d238493603a",
            "faa1d4b6034f4476b4495bf11bf9ef71",
            "0e63da934c1c4587aa1aba0e62848305",
            "5c0a23a8a805473abc158859075cb78e",
            "55a9218b9bb44328b317b1faddbd385c",
            "dbaad36f71db41a08f52be8ab636f9a7",
            "b517d40d169341348dd34ffa7cfe2960",
            "7b4aa11e2d324c07af530263b2ccb911",
            "0dc3f5951000406fb3c5c5c665ceac99",
            "34666b0ed2904ce681b21c3e9d02b87f",
            "96681c65f9174f6a97ff5177fc260ac6",
            "82c49e4beaf14eaab296f9143fc7f340",
            "ad6452eb47a44be3849769e68e105ef1",
            "d6fd82fae8c34bcb84b42e8511edb719",
            "91beac0824194108811c44068b6fca10",
            "a662435c188e4864a26e1b2f8fa6820b",
            "4e7f073fc3ee4371a48d3b4be456b099",
            "2930419a04ea49188b6469ed1e8b530c",
            "1484181ba2ad4250a1ca10a5f09d5413",
            "93dcd822bb2447e69e21bf43affa816d",
            "17fb22c2c78d4a91a38311b3ec7bdc90",
            "4b3c524893ff44c7bebe86c65930bb31",
            "d880ec03a8df443fb33b9a7f37182829",
            "37db82650beb4c4b95205b2c60a203ab",
            "03d95c59027f49be9e92e87230e43a30",
            "0f2847dd9a884dc4a349b815565109bc",
            "d9d39ddbf03e4109aaf41da4c1c4d546",
            "0d70985d8cc94aada083b1ac1b925ddd",
            "abfcf98ad01b41ffbd9e8e5af869d0d5",
            "2d0de4e5193f4da1ab5b050f66304739",
            "f8cd7299e2ce423aa425f3906e27dbb2",
            "2574b247944f4ce69a308f0be4108096",
            "156c8294b6f84ed3b4d3e693512d3e57",
            "f64ddc92232a46bdb9c9e0e5cf25b451",
            "255c023290d940559c03cf6b761f7792",
            "9d6c86104fe6430fbe553389424e402d",
            "de8047f896c34756a5e7b97c22455ce1",
            "85acccda2f53404da4f221cbb109aebf",
            "1b5b403cc1a24404afb96c759586abcb",
            "4d805263d5334f01b7057561804514d6",
            "64d24441f4864391a1e42627a140c7ab",
            "95c078f338a24c638c764ea1fc486029",
            "a80497abd09a4448a0bba1a9f50498da",
            "8cb1f3fcbcc04ad6abbd1f039ab01db1",
            "4754d4d10689464caa72a47660ec75a9",
            "a3cdb7c5a98645b492ae02f23dd1f7f0",
            "573f248058164c78b154e8dbb83d345e",
            "5acc80d9223f453f851f55aa1ec6557c",
            "b9d924d166c44507b2cc3fc1174e90a1",
            "47bafed9151249319dfb57d31985a118",
            "e9eec820d84648c0ab0a7ee069c8f6d1",
            "d7474ea88a6f4b628753a1dcbbe1a15a",
            "4b2aec714e744b0da21ef15f754c9f4d",
            "5a3ad7f393794dd7b025453f86e5e7b6",
            "9f7a27ded9d842fa93b8fac020c87cbd",
            "cd6491cc2df64906af4115d13c2bf158",
            "d506026e72e04c6683bad9fda717f3e0",
            "dd019cb93f0d402a86c33c31b333d9ae",
            "66c1fba6371344dfbc30d5ec77ed43b7",
            "1df69c2620a3422caf56a1d8eaf17f95",
            "c13c9c3c6cf34f6ebdca372539619bca",
            "02568abcc8704e5c87d906150782e01e",
            "d1e4c6f479ca49329e3a9184b5616f6f",
            "1e6e56bc01934f538d01fbdae0888204",
            "ac0c96db065c4d0d83bbe8ca3ae279d3",
            "7440f41720f54852a6ac62baedaa5ff0",
            "929ebc32715c4755a195a53bfebcfc7f",
            "c9a88645abd5447289eb1710cfb842f8",
            "f9d5f1ea366c4fb98fe5d05e645750de",
            "042d4b6aad1a4f6d83a0f74d31e072a6",
            "72b7c97bc13842fd88fa5f7e6e3c1d4a",
            "dd3f314aa84b450fbf05b43f2cabcf6c",
            "653b98d3cfe647c88b5cb0a8aac08756",
            "39555e4075aa43b4bcc0aeb358b83428",
            "0a8b844d0b6f4335b87c7cf845f6df79",
            "9a8c19870c8845458802c45ab4dddc54",
            "e47a683c97b341d2a9126223eea05951",
            "b617eca4cc954e37aec4726ea749efa9",
            "a41518b2e4784b3385b7b8bfb9c1a0a5",
            "fa06d4da6f0740b4816cd3109f7da059",
            "96a40a9bd9ae4e9cb0c6ef701b694047",
            "4e7c18fe8a7e432681f9f6ca301ef905",
            "71811b687f0847aaafa9780298a95359",
            "fd3aa0e21e2f46e485a2ff2b74cb56db",
            "16a9d6d77e634fd29234a8874d4ab8fe",
            "2137a5531e3f44c9ad0b02bae1a89ee8",
            "fb2fe3226ca24d599b6fb35ed686b109",
            "0725e09492e24567afc215c93bb32182",
            "c9c485f438bf48d3ad512272e513ac4e",
            "d71f94f6f1864c8a979d85e082d2eb74",
            "ada8ea18a0f943b78b36340e766dab1f",
            "c5593ed3fbed4c06992bea0e95114779",
            "722d13a7be7b4198ab54c5b6fd18a68e",
            "263cb4095f44405496e0bb4868da9444",
            "9027ed42304a4baa866779377ace6374",
            "649e699b66c84628aea8d005d87ea528",
            "3a3679ca93b6484dadcc054f0683244b",
            "17d9e8ade7a84b74ae44b8e9ff7b33f6",
            "e61c35a8e41d48a8ac0327acf0b4fea7",
            "9978568b073f414db4e2add4f4e69000",
            "32be12261b0a4dca8952cc0adc7efd20",
            "f4ad4e23bb544c819495c9eb0c90bd3f",
            "a56adeb8bfd04e3a861e02ad35a31cd6",
            "36ab7eac9cbd49f78fe290acd35661e0",
            "82502c2ac5644a04a5b25228e9ed7925",
            "8d28b519d2c242afbbc4fd4f5bfdd329",
            "d4e27f51b17c4311aab7d23518c52eb9",
            "f303363789194905b1006ae323f45de0",
            "e5c5c622c2cf4fe7903ed1fb08442bb0",
            "2f652b1c68d6475eaaf3eecf95a99b84",
            "a9c48f3e4e8a4ae0a6cf7ca5503258f7",
            "1e619ab06022480c8ae02b883b2f8c4e",
            "23ee6ba51659460aaa61ba91fdced2ef",
            "c206f6e381c64d94b281b51b5e542af9",
            "c601af698a5344b7a84d7ed4e53706be",
            "b69971ddccdb440183bf5abc621328b9",
            "66ee725b7bea4466bc35fbf58cf62b4c",
            "1a32c2340ee04d40bdabf3d20f0c7bf2",
            "00cd5c0c49e34e0fb7af8e881be0b582",
            "a419d3d60aac4362b8c1dbecd489015a",
            "fcf0a524ed854f2683e868ae9632d7e6",
            "81c3c19aa8724df7bd80ceda6f936668",
            "50a2a78b647740aeb934340b07a6d59d",
            "ccccd3dddc264dc8b1a29e1fb9102c2d",
            "6b120ea53769410c84baf5a5a3b9d4d4",
            "09dc7963b2c54a9bad34bce95988dbdb",
            "7f7d5cef62484cf1b48aac85216e9152",
            "15a6c804b5774afab7b6574c84d55669",
            "b2c75d4357f74a009548f91b1271755b",
            "561b660bc3604b1e91ad0a9828773cac",
            "f7dd271f73b449c6adf722a4bf02693e",
            "9260723c67ed4b81bee3f8c7fc2b8575",
            "6de6e8d77a9d416d9733d646ca09dc11",
            "ed0f864400974c8f918706f137c24ddd",
            "72ef77085f2b4f7984961cdb71b0fb5f",
            "a33b3064035646e0b2c8d05534d9c19d",
            "bcbba3ae11c548b88c675691486a5d81",
            "844286ca26024a1695b00a0e064c9a6d",
            "b32c43fa3a9b48c6813033ea003f71a0"
          ]
        }
      },
      "source": [
        "# !pip install optuna\n",
        "import torch\n",
        "import optuna\n",
        "from tqdm.notebook import tqdm\n",
        "from pprint import pprint\n",
        "import random\n",
        "import numpy as np\n",
        "\n",
        "def objective(trial):\n",
        "  device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "  trial.set_user_attr('device', str(device))\n",
        "  opt_fn_map = {\n",
        "    'SGD': optim.SGD,\n",
        "    'Adam': optim.Adam,\n",
        "    'Adagrad': optim.Adagrad,\n",
        "    'ASGD': optim.ASGD,\n",
        "    'Adamax': optim.Adamax,\n",
        "    'SparseAdam': optim.SparseAdam,\n",
        "    'AdamW': optim.AdamW,\n",
        "    'Adadelta': optim.Adadelta,\n",
        "    'LBFGS': optim.LBFGS,\n",
        "    'RMSprop': optim.RMSprop,\n",
        "    'Rprop': optim.Rprop\n",
        "  }\n",
        "  train_config = {\n",
        "    'seed': trial.suggest_int('seed', 0, 0),\n",
        "    'n_epoches': trial.suggest_int('n_epoches', 10, 10),\n",
        "    'lr': trial.suggest_loguniform('lr', 1e-3, 1e-3),\n",
        "    # 'opt_fn': trial.suggest_categorical('opt_fn', ['SGD', 'Adam', 'Adagrad', \n",
        "    #                                           'ASGD', 'Adamax',\n",
        "    #                                           'AdamW', 'Adadelta',\n",
        "    #                                           'RMSprop', 'Rprop']),\n",
        "    'opt_fn': trial.suggest_categorical('opt_fn', ['Adam']),\n",
        "    'batch_size': trial.suggest_int('batch_size', 300, 300),\n",
        "    'weight_decay': trial.suggest_loguniform('weight_decay', 0.0, 0.0)\n",
        "  }\n",
        "  model_config = {\n",
        "    'enc_num_embeddings': len(eng_rus.x1_w2i),\n",
        "    'enc_embedding_dim': trial.suggest_int('enc_embedding_dim', 300, 300),\n",
        "    'enc_padding_idx': eng_rus.x1_w2i['<PAD>'],\n",
        "    'dec_num_embeddings': len(eng_rus.x2_w2i),\n",
        "    'dec_embedding_dim': trial.suggest_int('dec_embedding_dim', 300, 300),\n",
        "    'dec_padding_idx': eng_rus.x2_w2i['<PAD>'],\n",
        "    'rnn_type': trial.suggest_categorical('rnn_type', ['RNN']),\n",
        "    # 'rnn_type': trial.suggest_categorical('rnn_type', ['RNN']),\n",
        "    'hidden_size': trial.suggest_int('hidden_size', 300, 300),\n",
        "    'num_layers': trial.suggest_int('num_layers', 1, 1),\n",
        "    'bidirectional': trial.suggest_categorical('bidirectional', [False]),\n",
        "    # 'bidirectional': trial.suggest_categorical('bidirectional', [True]),\n",
        "    'enc_dropout': trial.suggest_uniform('enc_dropout', 0.0, 0.0),\n",
        "    'dec_dropout': trial.suggest_uniform('dec_dropout', 0.0, 0.0),\n",
        "    'enc_rnn_dropout': trial.suggest_uniform('enc_rnn_dropout', 0.0, 0.0),\n",
        "    'dec_rnn_dropout': trial.suggest_uniform('dec_rnn_dropout', 0.0, 0.0),\n",
        "    'attn_type': trial.suggest_categorical('attn_type', ['soft_dist']),\n",
        "    # 'attn_type': trial.suggest_categorical('attn_type', ['dot', 'dist', 'soft_dot', 'soft_dist', 'cos', 'soft_cos', 'none']),\n",
        "    'out_hidden': trial.suggest_int('out_hidden', 600, 600),\n",
        "    'pack': trial.suggest_categorical('pack', [True]),\n",
        "  }\n",
        "  # train\n",
        "  seed = train_config['seed']\n",
        "  opt_fn = train_config['opt_fn']\n",
        "  lr = train_config['lr']\n",
        "  n_epoches = train_config['n_epoches']\n",
        "  batch_size = train_config['batch_size']\n",
        "  weight_decay = train_config['weight_decay']\n",
        "  torch.manual_seed(seed)\n",
        "  torch.cuda.manual_seed(seed)\n",
        "  random.seed(seed)\n",
        "  np.random.seed(seed)\n",
        "  model = EncoderDecoder(**model_config).to(device)\n",
        "  if opt_fn not in ['Rprop']:\n",
        "    opt = opt_fn_map[opt_fn](model.parameters(), lr=lr, \n",
        "                             weight_decay=weight_decay)\n",
        "  else:\n",
        "    opt = opt_fn_map[opt_fn](model.parameters(), lr=lr)\n",
        "  loss_fn = torch.nn.NLLLoss(ignore_index=eng_rus.x2_w2i['<PAD>'])\n",
        "  train_dl, test_dl, val_dl = train_test_val_split(eng_rus, \n",
        "                                                   batch_size=batch_size)\n",
        "  pprint(trial.params)\n",
        "  pbar_epoch = tqdm(range(n_epoches))\n",
        "  for i in tqdm(pbar_epoch):\n",
        "    # train\n",
        "    model.train()\n",
        "    pbar_train = tqdm(train_dl)\n",
        "    accs = []\n",
        "    for x1, x2 in pbar_train:\n",
        "      opt.zero_grad()\n",
        "      x1.to(device)\n",
        "      x2.to(device)\n",
        "      t = x2[:,1:].flatten(0)\n",
        "      y2, attn = model(x1, x2)\n",
        "      p = y2[:,:-1,:].flatten(0, 1)\n",
        "      loss = loss_fn(p, t)\n",
        "      loss.backward()\n",
        "      opt.step()\n",
        "      with torch.no_grad():\n",
        "        idx = torch.nonzero(t).view(-1)\n",
        "        acc = torch.sum(torch.argmax(p, 1)[idx] == t[idx]).float()/idx.size()[0]\n",
        "        accs.append(acc)\n",
        "      pbar_train.set_description(f'epoch: {i+1}, loss: {loss:.3f}, acc: {acc:.3f}')\n",
        "    # y2d = map(eng_rus.decode_x2, torch.argmax(y2[:,:-1,:], 2).cpu().numpy())\n",
        "    # x2d = map(eng_rus.decode_x2, x2[:,1:].cpu().numpy())\n",
        "    # x1d = map(eng_rus.decode_x1, x1[:,1:].cpu().numpy())\n",
        "    # triples = list(zip(x1d, x2d, y2d))\n",
        "    # pprint(triples)\n",
        "    # validation\n",
        "    model.eval()\n",
        "    val_accs = []\n",
        "    with torch.no_grad():\n",
        "      pbar_val = tqdm(val_dl)\n",
        "      for x1, x2 in pbar_val:\n",
        "        x1.to(device)\n",
        "        x2.to(device)\n",
        "        t = x2[:,1:].flatten(0)\n",
        "        y2, attn = model(x1, x2)\n",
        "        p = y2[:,:-1,:].flatten(0, 1)\n",
        "        val_loss = loss_fn(p, t)\n",
        "        idx = torch.nonzero(t).view(-1)\n",
        "        val_acc = torch.sum(torch.argmax(p, 1)[idx] == t[idx]).float()/idx.size()[0]\n",
        "        val_accs.append(val_acc)\n",
        "      pbar_val.set_description(f'epoch: {i+1}, val_loss: {val_loss:.3f}, val_acc: {val_acc:.3f}')\n",
        "    m_acc = sum(accs)/len(accs)\n",
        "    m_val_acc = sum(val_accs)/len(val_accs)\n",
        "    pbar_epoch.set_description(f'loss: {loss:.3f}, acc: {m_acc:.3f} vloss: {val_loss:.3f}, vacc: {m_val_acc:.3f}')\n",
        "    trial.report(loss, step=i+1)\n",
        "    torch.save({'epoch': i,\n",
        "                'model_state_dict': model.state_dict(),\n",
        "                'optimizer_state_dict': opt.state_dict(),\n",
        "                'loss': loss,\n",
        "                'model_config': model_config,\n",
        "                'train_config': train_config},\n",
        "                'model.pth')\n",
        "    try:\n",
        "      uploaded = drive.CreateFile({'title': 'model.pth'})\n",
        "      uploaded.SetContentFile('model.pth')\n",
        "      uploaded.Upload()\n",
        "    except Exception as e:\n",
        "      print(e)\n",
        "  # test\n",
        "  model.eval()\n",
        "  test_accs = []\n",
        "  with torch.no_grad():\n",
        "    pbar_test = tqdm(test_dl)\n",
        "    for x1, x2 in pbar_test:\n",
        "      x1.to(device)\n",
        "      x2.to(device)\n",
        "      t = x2[:,1:].flatten(0)\n",
        "      y2, attn = model(x1, x2)\n",
        "      p = y2[:,:-1,:].flatten(0, 1)\n",
        "      test_loss = loss_fn(p, t)\n",
        "      idx = torch.nonzero(t).view(-1)\n",
        "      test_acc = torch.sum(torch.argmax(p, 1)[idx] == t[idx]).float()/idx.size()[0]\n",
        "      test_accs.append(test_acc)\n",
        "    pbar_val.set_description(f'epoch: {i+1}, test_loss: {test_loss:.3f}, test_acc: {test_acc:.3f}')\n",
        "  m_test_acc = sum(test_accs)/len(test_accs)\n",
        "  print(f'mean test_acc: {m_test_acc}')\n",
        "  if model_config['attn_type'] != 'none':\n",
        "    plot_attention(attn.clone().cpu().numpy()[:1], \n",
        "                    x1.clone().cpu().numpy()[:1], \n",
        "                    x2.clone().cpu().numpy()[:1],\n",
        "                    decoder_x1 = eng_rus.decode_x1,\n",
        "                    decoder_x2 = eng_rus.decode_x2,\n",
        "                    figsize=(10, 10),\n",
        "                    shift=True,\n",
        "                    mask=True,\n",
        "                    suptitle=model_config['attn_type'])\n",
        "  trial.set_user_attr('train_loss', loss.item())\n",
        "  trial.set_user_attr('val_loss', val_loss.item())\n",
        "  trial.set_user_attr('test_loss', test_loss.item())\n",
        "  trial.set_user_attr('train_acc', m_acc.item())\n",
        "  trial.set_user_attr('val_acc', m_val_acc.item())\n",
        "  trial.set_user_attr('test_acc', m_test_acc.item())\n",
        "  y2d = map(eng_rus.decode_x2, torch.argmax(y2[:,:-1,:], 2).cpu().numpy())\n",
        "  x2d = map(eng_rus.decode_x2, x2[:,1:].cpu().numpy())\n",
        "  x1d = map(eng_rus.decode_x1, x1[:,1:].cpu().numpy())\n",
        "  triples = list(zip(x1d, x2d, y2d))[:1]\n",
        "  pprint(triples)\n",
        "  return loss\n",
        "\n",
        "fn = 'optuna.db'\n",
        "study = optuna.create_study(study_name='16', \n",
        "                            direction='minimize', \n",
        "                            storage=f'sqlite:///{fn}', \n",
        "                            load_if_exists=True)\n",
        "study.optimize(objective, n_trials=1)\n",
        "try:\n",
        "  uploaded = drive.CreateFile({'title': fn})\n",
        "  uploaded.SetContentFile(fn)\n",
        "  uploaded.Upload()\n",
        "except Exception as e:\n",
        "  print(e)"
      ],
      "execution_count": 406,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[32m[I 2020-05-09 01:42:04,531]\u001b[0m A new study created with name: 16\u001b[0m\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "{'attn_type': 'soft_dist',\n",
            " 'batch_size': 300,\n",
            " 'bidirectional': False,\n",
            " 'dec_dropout': 0.0,\n",
            " 'dec_embedding_dim': 300,\n",
            " 'dec_rnn_dropout': 0.0,\n",
            " 'enc_dropout': 0.0,\n",
            " 'enc_embedding_dim': 300,\n",
            " 'enc_rnn_dropout': 0.0,\n",
            " 'hidden_size': 300,\n",
            " 'lr': 0.001,\n",
            " 'n_epoches': 10,\n",
            " 'num_layers': 1,\n",
            " 'opt_fn': 'Adam',\n",
            " 'out_hidden': 600,\n",
            " 'pack': True,\n",
            " 'rnn_type': 'RNN',\n",
            " 'seed': 0,\n",
            " 'weight_decay': 0.0}\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "015ac36378d44aaebe5aa2f6f387a83a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=10.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "7d0b4fb6343d416a9cf891bf318dbaab",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=10.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d61a6502d8474c918aea958da29b45ad",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=234.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e06ce5d183324e92a1524ae457d0d048",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=34.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Redirected but the response is missing a Location: header.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "8c9fe6addbbb4654b6624fcae7589ad7",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=234.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "373122c5949a42d9a62953fb284af278",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=34.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Redirected but the response is missing a Location: header.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "5c0a23a8a805473abc158859075cb78e",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=234.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "82c49e4beaf14eaab296f9143fc7f340",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=34.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Redirected but the response is missing a Location: header.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "93dcd822bb2447e69e21bf43affa816d",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=234.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0d70985d8cc94aada083b1ac1b925ddd",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=34.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Redirected but the response is missing a Location: header.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "9d6c86104fe6430fbe553389424e402d",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=234.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "8cb1f3fcbcc04ad6abbd1f039ab01db1",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=34.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Invalid client secrets file ('Error opening file', 'client_secrets.json', 'No such file or directory', 2)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d7474ea88a6f4b628753a1dcbbe1a15a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=234.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "1df69c2620a3422caf56a1d8eaf17f95",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=34.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Invalid client secrets file ('Error opening file', 'client_secrets.json', 'No such file or directory', 2)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c9a88645abd5447289eb1710cfb842f8",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=234.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "9a8c19870c8845458802c45ab4dddc54",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=34.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Invalid client secrets file ('Error opening file', 'client_secrets.json', 'No such file or directory', 2)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "fd3aa0e21e2f46e485a2ff2b74cb56db",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=234.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c5593ed3fbed4c06992bea0e95114779",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=34.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Invalid client secrets file ('Error opening file', 'client_secrets.json', 'No such file or directory', 2)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "9978568b073f414db4e2add4f4e69000",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=234.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "f303363789194905b1006ae323f45de0",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=34.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Invalid client secrets file ('Error opening file', 'client_secrets.json', 'No such file or directory', 2)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b69971ddccdb440183bf5abc621328b9",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=234.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "ccccd3dddc264dc8b1a29e1fb9102c2d",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=34.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Invalid client secrets file ('Error opening file', 'client_secrets.json', 'No such file or directory', 2)\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "9260723c67ed4b81bee3f8c7fc2b8575",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=67.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "mean test_acc: 0.700404703617096\n",
            "[(['i', 'want', 'to', 'trust', 'you', '<EOS>', '<PAD>'],\n",
            "  ['я', 'хочу', 'тебе', 'верить', '<EOS>', '<PAD>', '<PAD>', '<PAD>', '<PAD>'],\n",
            "  ['я',\n",
            "   'хочу',\n",
            "   'вам',\n",
            "   'доверять',\n",
            "   '<EOS>',\n",
            "   '<EOS>',\n",
            "   'доверять',\n",
            "   'доверять',\n",
            "   'доверять'])]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[32m[I 2020-05-09 04:42:41,462]\u001b[0m Finished trial#0 with value: 0.4556959569454193 with parameters: {'attn_type': 'soft_dist', 'batch_size': 300, 'bidirectional': False, 'dec_dropout': 0.0, 'dec_embedding_dim': 300, 'dec_rnn_dropout': 0.0, 'enc_dropout': 0.0, 'enc_embedding_dim': 300, 'enc_rnn_dropout': 0.0, 'hidden_size': 300, 'lr': 0.001, 'n_epoches': 10, 'num_layers': 1, 'opt_fn': 'Adam', 'out_hidden': 600, 'pack': True, 'rnn_type': 'RNN', 'seed': 0, 'weight_decay': 0.0}. Best is trial#0 with value: 0.4556959569454193.\u001b[0m\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Invalid client secrets file ('Error opening file', 'client_secrets.json', 'No such file or directory', 2)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAowAAAJ1CAYAAABTrqEHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3df9htZV0n/vfHo4B6zFRwGgHBIbQQlRQw04wMvXDG0b7fnMQfNTj6xSYpKyV1KkbJrhm1kcErNRlzyt9fY8pODYYzpflbz1EQBWUCkgHy1yEtcQgFPvPH3sdr83jOfh5c53n28+z1el3XvthrrXvf696bzTkf3vda+67uDgAA7MsdFj0AAAA2NwUjAABzKRgBAJhLwQgAwFwKRgAA5lIwAgAwl4IRAIC5FIwAU1X1b6vqS1V1Q1Xda0A/J1XVtTPbl1bVSftlkAALoGAESFJVd0ryqiSP6+7tSR40W/QN0d0P7O73rXL+I6uqq+qO++OcAPuTghFg4p8kOSjJpYseCMBmo2AEllJVvbCqrquqr1fV5VX1E1V1YFX956r62+njP0/33T/J5dOXfq2q3pvk3UnuM52evqGq7jPnXHeuqt+vqq9W1WVJTlhx/PNVdfL0+YlVtauq/mE6/f2qabP3z5z/hqp6xH79QAAGMPUBLJ2qekCSM5Kc0N1/W1VHJtmW5NeS/HCS45J0kj9J8uvd/RtV9cAkf5Pke7v75uk1h2/p7sPWcMp/n+So6eOumRSb+3JuknO7+81VtT3JsdP9j549/+15vwDrTcIILKNbkhyY5JiqulN3f767r0zy9CRnd/eXu/srSV6a5Gf2w/l+Oslvdfffdfc1SV49p+23knx/VR3c3Td090f3w/kB1pWCEVg63X1Fkl9K8pIkX66qd0ynlO+T5OqZpldP9w11nyTXrOh3X56V5P5JPldVO6vqCfvh/ADrSsEILKXuflt3PyrJEZlMP788yd9Ot/e473TfXru4Haf7QpLDV/S7r3H9dXc/Ncm9p2M6v6ruejvPB7ChFIzA0qmqB1TVY6rqwCT/mOTGJLcmeXuSX6+qQ6rq4CRnJXnLPrr5UpJ7VdXd13DKdyZ5cVXdo6oOS/ILc8b2jKo6pLtvTfK16e5bk3xl+s9/tobzAWwoBSOwjA5M8h+T7E7yxUzSvBcneVmSXUkuSfLpJJ+c7vsO3f25TArMq6rqa/Puks7kWsirM7lp5T1J3jyn7SlJLq2qGzK5AebU7r6xu/9Pkt9K8qHp+X54rW8WYL1Vt1kQAAD2TcIIAMBcCkaANaiqd8/8iPfs498temwA682UNAAAc0kYAQCYS8EIAMBcCkYAAOZSMAIAMJeCEQCAuRSMAADMpWAEAGAuBSMAAHMpGAEAmEvBCADAXApGAADmUjACADCXghEAgLkUjAAAzKVgBABgLgUjAABzKRgBAJhLwQgAwFwKRgAA5lIwAgAwl4IRAIC5FIwAAMylYAQAYC4FIwAAcykYAQCYS8EIAMBcCkYAAOZSMAIAMJeCEQCAuRSMAADMpWAEAGAuBSMAAHMpGAEAmEvBCADAXApGAADmUjACADCXghEAgLkUjAAAzKVgBABgLgUjAABzKRgBAJhLwQgAwFwKRgAA5lIwAgAwl4IRAIC5FIwAAMylYAQAYC4FIwAAcykYAQCYS8EIAMBcCkYAAOZSMAIAMJeCEQCAuRSMAADMpWAEAGAuBSMAAHPdcdEDAADYyk455ZTevXv3hpzrE5/4xIXdfcqGnGyGghEAYIDdu3dn165dG3Kuqjp4Q060gilpAADmkjACAAzU3YsewrqSMAIAMJeEEQBgIAkjAACjJmEEABhIwggAwKhJGAEABuhuCSMAAOMmYQQAGEjCCADAqEkYAQAGkjACADBqCkYAAOYyJQ0AMJApaQAARk3CCAAwkIQRAIBRkzACAAxgaUAAAEZPwggAMJCEEQCAUZMwAgAMJGEEAGDUJIwAAANJGAEAGDUJIwDAQBJGAABGTcIIADCAlV4AABg9BSMAAHOZkgYAGMiUNAAAo6ZgBEajqv68qr5WVX+26LEAy2XPjS/r/VgUBSMwJq9M8jOLHgTAVqNgBJZOVZ1QVZdU1UFVddequrSqju3uv0jy9UWPD1g+y54wuukFWDrdvbOqdiR5WZI7J3lLd39mwcMC2LIUjMCyOjvJziT/mOQXFzwWYIktOv3bCKakgWV1ryTbk9wtyUELHgvAliZhBJbV65P8RpL7JXl5kjMWOxxgmS17wqhgBJZOVf1skm9199uqaluSD1fVY5K8NMkPJNleVdcmeVZ3X7jIsQJsBQpGYOl095uSvGn6/JYkD58e+suFDQpYasueMLqGEQCAuSSMAAADSRgBABg1BeMWUVUfXvQYxqaqTqqqH1n0ODabqvreqvr5RY9jK9mfn5nvJeulqt5XVZdX1cXTx/kzx06vqs9NHx+vqkfNHHtCVV1UVZ+qqsuq6jmLeQeLZaUXNoXu9hfExjspyQ1JFOu39b1Jfj7Jaxc9kC1kr59ZVd2xu2++nX2dFN9L9pOqOiDJnbr7G9NdT+/uXSvaPCHJc5I8qrt3V9VDk7yrqk5Mcn2S85Kc2N3XVtWBSY6cvu4e3f3VjXovrC8J4xZRVTcsegybXVWdWVW/OH1+TlX95fT5Y6rqrVX1uqraNV1X+KUzr/t8Vb20qj5ZVZ+uqh+oqiOT/FySX57+n/aPLuI9bVL/MclR08/lldPHZ6af3VMWPbhNavYz21lVH5guXXhZVR1ZVd9etrCqXlBVL5k+/8VpYnNJVb3D9/I7VdXZVfVLM9u/VVXP29v3cprO/tlM29+pqtMWMOyFq6ofrKr/lOTyJPdfpfkLk5zZ3buTpLs/meQPkjw3kx/Gv2MmhWO6+6buvnz6uqdM/x08v6oOWY/3wcZRMLJMPpBkz1+gx2fyW3t3mu57f5Jf6+7jkzw4yY9V1YNnXru7ux+a5HVJXtDdn0/yu0nO6e7juvsDG/UmtoAXJbmyu49L8tEkxyV5SJKTk7yyqv7pIge3Sc1+ZmcmeWiS53X3an9RvyjJD3X3g5P8nO/lXr0xyc8mSVXdIcmpSa6N7+V3qKq7VtUzq+qDSf5LksuSPLi7L5pp9taZKelXTvc9MMknVnS3K8kDu/vvkuxIcnVVvb2qnj7995Du/t0kj09ylyTvr6rzq+qUPceXyUZNRy9ySnrp/qUxap9I8rCq+p4kNyX5SCaF449mUkz+dFV9MslFmfwBeMzMa/9opo8jN2rAS+BRSd7e3bd095eS/FWSExY8pq3g4939N2tod0kmf4E/I8ntnboehWkRfX1V/VCSx2Xy37fv5d59Icmzkjy7ux/V3b/X3V9f0ebp0/8ZOa67z1xLp9397CQ/keTjSV6QSRG/59g13f2bmfx5+8bp41374b2wwRSMLI3u/laSv0lyWibXd30gyY8n+f4kN2byB9lPTNOa/57bri980/Sft8S1vay/b8w8vzm3/bN49nv5L5K8JpNEcmdV+W7u3Rsy+e/+mZkpVvZi3mc9Bk9Ocl2SP6qqs6rqiDW+7rIkD1ux72FJLt2z0d2f7u5zkjw2yU/NNpxe6/jaJK9O8s4kL/7uhr+5SRhha/lAJoXh+6fPfy6TxOF7MvlL+u+r6p9kMk2ymq9ncn0OtzX7uXwgk+uUtk2vUXp0JikDtzXvu/SlJPeuqntNbxh4QvLt6dXDu/u9mVxDdvck21fpa6z+OMkpmaSIF2bf38urkxxTVQdW1fdmkoqNRne/p7ufksmsy98n+ZOq+p/Ta2PneUWSl1fVvZKkqo7LpEB/bVVtr6qTZtoel8nnnKp6XFVdkuRlSd6b5Jju/qXuvjRsOf5vlWXzgSS/luQj3f2NqvrHJB/o7k9V1UVJPpfkmiQfWkNff5rk/Kp6UpJfcL3YRHdfX1Ufmt6o8e5Mpk0/laST/Gp3f3GhA9yEVnxmN2ZSJO459q2qOjuTgua6TL6jSbItyVuq6u5JKsmru/trVeV7uUJ3f7Oq3pvka919S1X9cZJHZC/fy6p6Z5LPZDIbcdG++lxm3X19knOTnDtN/26ZOfzWqrpx+nx3d5/c3Tuq6tBM1mTvTP6n5Rnd/YWquluSX62q12fy3f5GJsVkMrkR5l9299Ub8LYWbpHp30aoZX+DACy3aRr7yST/qrv/etHjYXwe8pCH9J//+Z9vyLnuc5/7fGJ6A+eGkjACsGVV1TFJ/izJHysWWaRlD+AUjABsWd19WZJ/tuhxwLJz0wsAwECb6S7p6e9dXl5VV1TVi/bR5qenCwNcWlVvW61PCSMAwJKoqm2Z/BzXYzP5EfudVbVjmsbvaXN0Jj9v9Mju/mpV3Xu1fiWMW0hVnb7oMSwTn+f+47Pcv3ye+5fPc//xWe7dRqWLa0wYT0xyRXdf1d3fTPKOJE9a0eb/S/Kanq713d1fXq1TBePW4j/U/cvnuf/4LPcvn+f+5fPcf3yWi3dwVe2aeaz8d3JoJj8ft8e1032z7p/k/tOf+/poVZ2y2klNSQMADLSBd0nv3g8/q3PHJEcnOSnJYZms9f2g7v7avBeM3vbt2/ue97znooexqnvc4x65733vu+nv2z/wwAMXPYQ1OeSQQ3L00Udv6s/zyiuvXPQQ1uwOd7jDpv4st5qt8HlW1aKHsCZVlW3btm36z/PWW29d9BDWZPrj3Zted2+NL+j+d12Sw2e2D5vum3Vtko/tWVK3qv5XJgXkzn11qmBMcs973jPPf/7zFz2MpXHUUUcteghL48lPfvKih7BUlv130jbaAQccsOghLJUbb7xx9UasyS233LJ6o/1sE/35sjPJ0VV1v0wKxVOTPG1Fm3cleWqS/1pVB2cyRX3VvE5dwwgAsCS6++YkZ2Syrvpnk7yzuy+tqrOr6onTZhcmub6qLstkne8zp0tG7pOEEQBgiXT3BUkuWLHvrJnnneRXpo81UTACAAy0iaak14UpaQAA5pIwAgAMJGEEAGDUJIwAAAPcjmX7tiwJIwAAc0kYAQAGkjACADBqEkYAgIEkjAAAjJqEEQBgIAkjAACjJmEEABhIwggAwKhJGAEABrDSCwAAo6dgBABgLlPSAAADmZIGAGDUJIwAAANJGAEAGDUJIwDAQBJGAABGTcIIADCQhBEAgFGTMAIADGBpQAAARk/CCAAwkIQRAIBRkzACAAwkYQQAYNQkjAAAA0kYAQAYNQkjAMBAEkYAAEZNwQgAwFympAEABrA04BZVVb9RVZdX1cVVdWNVHbnoMQEAbFVLlzBW1fck+YUkR3T3jVX1mX20Oz3J6Ulyj3vcYwNHCAAsGwnj1lRJ7jyvQXef193Hd/fx27dv36BhAQBsPUuXMHb3P1TVWUmuqqqrkxy16DEBAMtNwrg1fTnJn3b3Q5JcuejBAABsZUuXMFbV9yd5QZKTFz0WAGAclj1hXLqCsbuvSPKIme1jFzgcAIAtb+kKRgCAjbbsCeOyXsMIAMB+ImEEABjASi8AAIyehBEAYCAJIwAAoyZhBAAYSMIIAMCoKRgBAJjLlDQAwECmpAEAGDUJIwDAQBJGAABGTcIIADCApQEBABg9CSMAwEASRgAARk3CCAAwkIQRAIBRkzACAAwkYQQAYNQkjAAAA0kYAQAYNQkjAMAAVnoBAGD0FIwAAMxlShoAYCBT0gAAjJqEEQBgIAkjAABbRlWdUlWXV9UVVfWivRw/raq+UlUXTx/PXq1PCSMAwECbJWGsqm1JXpPksUmuTbKzqnZ092Urmv7/3X3GWvuVMAIALI8Tk1zR3Vd19zeTvCPJk4Z2qmAEABhoz493r/cjycFVtWvmcfqKoRya5JqZ7Wun+1b6qaq6pKrOr6rDV3t/pqQBALaO3d19/MA+/jTJ27v7pqp6TpI/SPKYeS9QMAIADLDJlga8LslsYnjYdN+3dff1M5tvSPKK1TpVMCa55ppr8vznP3/Rw1gaN91006KHsDS2b9++6CEsFZ/n/nX3u9990UNYKpdccsmih8By2Jnk6Kq6XyaF4qlJnjbboKr+aXd/Ybr5xCSfXa1TBSMAwECbJWHs7pur6owkFybZluSN3X1pVZ2dZFd370jyi1X1xCQ3J/m7JKet1q+CEQBgiXT3BUkuWLHvrJnnL07y4tvTp4IRAGCgzZIwrhc/qwMAwFwSRgCAgSSMAACMmoIRAIC5TEkDAAxkShoAgFGTMAIADLDJlgZcFxJGAADmkjACAAwkYQQAYNQkjAAAA0kYAQAYNQkjAMBAEkYAAEZNwggAMJCEEQCAUZMwAgAMYKUXAABGT8IIADCQhBEAgFFTMAIAMJcpaQCAgUxJAwAwahJGAICBJIwAAIyahBEAYCAJIwAAoyZhBAAYwNKAAACMnoQRAGAgCSMAAKMmYQQAGEjCCADAqEkYAQAGkjACADBqEkYAgIEkjAAAjNqmKRir6oSquqSqDqqqu1bVpVX1oKp6ZVV9pqo+XVVPmbY9rap+Z+a1N0z/+aaq+smZ/W+tqidt/LsBAFgem2ZKurt3VtWOJC9Lcuckb0ly/yTHJXlIkoOT7Kyq9ye5NUntpZvfS/LLSd5VVXdP8iNJ/vXezldVpyc5fX+/DwBgXCwNuPHOTvLYJMcneUWSRyV5e3ff0t1fSvJXSU5Icm2Sh1TVbcbf3X+V5OiqOiTJU5P8t+6+eW8n6u7zuvv47j5+/d4OAMDWt2kSxql7Jdme5E5JDprT7n1JLk/y6ar61opjb0ryjCSnJnnmOowRAOA2lj1h3GwF4+uT/EaS+yV5eZK/TPKcqvqDJPdM8ugkZ3b3rUmetedFe65hnPr9JB9P8sXuvmyDxg0AsLQ2TcFYVT+b5Fvd/baq2pbkw0n+OMklST6VpJP8and/cV4/3f2lqvpsknet95gBABIJ44bp7jdlMp2c7r4lycOnh/4iyZmrvHb7nudVdZckRyd5+/qMFABgXDZNwbg/VNXJmdwpfU53//2ixwMAjIOEcQvp7v+Z5IhFjwMAYJksVcEIALAIy54wbrbfYQQAYJORMAIADGClFwAARk/CCAAwkIQRAIBRkzACAAwkYQQAYNQUjAAAzGVKGgBgIFPSAACMmoQRAGAAP9wNAMDoSRgBAAaSMAIAMGoSRgCAgSSMAACMmoQRAGAgCSMAAKMmYQQAGEjCCADAqEkYAQAGsNILAACjJ2EEABhIwggAwKgpGAEAmEvBCAAw0J4bX9b7sRZVdUpVXV5VV1TVi+a0+6mq6qo6frU+FYwAAEuiqrYleU2Sxyc5JslTq+qYvbS7W5LnJfnYWvpVMAIADLSJEsYTk1zR3Vd19zeTvCPJk/bS7jeTvDzJP66lUwUjAMDWcXBV7Zp5nL7i+KFJrpnZvna679uq6qFJDu/u/77Wk/pZHQCAgTbwZ3V2d/eq1xzuS1XdIcmrkpx2e14nYQQAWB7XJTl8Zvuw6b497pbk2CTvq6rPJ/nhJDtWu/FFwggAMMAmWxpwZ5Kjq+p+mRSKpyZ52p6D3f33SQ7es11V70vygu7eNa9TBSP73X3ve99FD2Fp3HDDDYsewlI56qijFj2EpfKxj63p5krW6KCDDlr0EJbGN7/5zUUPYWG6++aqOiPJhUm2JXljd19aVWcn2dXdO76bfhWMAAADbaKEMd19QZILVuw7ax9tT1pLn65hBABgLgkjAMBAmylhXA8SRgAA5pIwAgAMJGEEAGDUJIwAAANJGAEAGDUFIwAAc5mSBgAYYJMtDbguJIwAAMwlYQQAGEjCCADAqEkYAQAGkjACADBqEkYAgIEkjAAAjJqEEQBgIAkjAACjJmEEABjASi8AAIyehBEAYCAJIwAAoyZhBAAYSMIIAMCoKRgBAJjLlDQAwECmpAEAGDUJIwDAQBJGAABGTcIIADCApQEBABg9CSMAwEASRgAARk3CCAAwkIQRAIBRkzACAAwkYQQAYNQkjAAAA406Yayq76mqo/ay/8HrN6S1qar7VtWbq+rjVfWZqjp40WMCAFhG+ywYq+qnk3wuyX+rqkur6oSZw7+/3gObp6oOSvL2JL+X5OHdfWx3717kmACAcdqz0stGPBZlXsL475I8rLuPS/LMJG+uqv9neqz290Cq6pVVdXFVfbGqrps+P7uqzqyqnVV1SVW9dNr8MUnunOR3kny6ql4+08/jquojVfXJqvrDqtq+v8cKADAm865h3NbdX0iS7v54Vf14kj+rqsOT7PcSt7vPTJKqekmSG7r7t6vqcUmenOTETIrUHVX16CSHJDk0ybFJvprkPVX1k0k+mOTXk5zc3d+oqhcm+ZUkZ688X1WdnuT0/f0+AIDxWfZrGOcVjF+vqqO6+8ok6e4vVNVJSd6V5IEbMbgkj5s+Lppub09ydJJbklzY3V9Jkqp6a5JHJ7k5yTFJPlRVSXJAko/srePuPi/JedPXL/e/ZQCAAeYVjP82yR2q6pjuvixJuvvrVXVKklM3ZHSTVPE/dPfrb7Oz6v+d0/5/dPdT131kAAAjsc9rGLv7U93910neWVUvrIk7J3lVkp/foPFdmOTf7LkOsaoOrap7J/lEksdU1cFVtS3JU5P8VZKPJnlkVX3/tP1dq+r+GzRWAGCkxnzTyx4PT3J4kg8n2Znkb5M8cj0HtUd3vyfJ25J8pKo+neT8JHfr7quTvCTJ+5NcnGRXd//JdIr6tCRvr6pLMpmO/oGNGCsAwLJayw93fyvJjZnclXxQkr/p7lvXa0Dd/ZIV2+cmOXcv7d6Q5A172f+XSU5YuR8AYL0s+00va0kYd2ZSMJ6Q5EeTPLWq/nBdRwUAwKaxloTxWd29a/r8C0meVFU/s45jAgDYUkafMM4Ui7P73rw+wwEAYLNZS8IIAMA+LPoO5o2wlmsYAQAYMQkjAMBAEkYAAEZNwggAMJCEEQCAUZMwAgAMJGEEAGDUJIwAAANJGAEAGDUFIwAAc5mSBgAYwNKAAACMnoQRAGAgCSMAAKMmYQQAGEjCCADAqEkYAQAGkjACADBqEkYAgIEkjAAAjJqEEQBgACu9AAAwegpGAICB9qSM6/1Yi6o6paour6orqupFezn+c1X16aq6uKo+WFXHrNanghEAYElU1bYkr0ny+CTHJHnqXgrCt3X3g7r7uCSvSPKq1fp1DSMAwECb6BrGE5Nc0d1XJUlVvSPJk5JctqdBd//DTPu7Jll18ApGAICt4+Cq2jWzfV53nzezfWiSa2a2r03y8JWdVNVzk/xKkgOSPGa1kyoYAQC2jt3dffzQTrr7NUleU1VPS/LrSf71vPYKRgCAgTbRlPR1SQ6f2T5sum9f3pHkdat16qYXAIDlsTPJ0VV1v6o6IMmpSXbMNqiqo2c2/0WSv16tUwkjAMBAmyVh7O6bq+qMJBcm2Zbkjd19aVWdnWRXd+9IckZVnZzkW0m+mlWmoxMFIwDAUunuC5JcsGLfWTPPn3d7+1Qwst8dcMABix7C0jjiiCMWPYSl8shHPnLRQ1gqz3nOcxY9hKVy7LHHLnoIS+Ozn/3shp7P0oAAAIyehBEAYCAJIwAAoyZhBAAYSMIIAMCoSRgBAAaSMAIAMGoSRgCAgSSMAACMmoQRAGAAK70AADB6CkYAAOYyJQ0AMJApaQAARk3CCAAwkIQRAIBRkzACAAwkYQQAYNQkjAAAA0kYAQAYNQkjAMAAlgYEAGD0JIwAAANJGAEAGDUJIwDAQBJGAABGTcIIADCQhBEAgFFTMAIAMJcpaQCAgUxJAwAwahJGAIABLA0IAMDoSRgBAAaSMAIAMGoSRgCAgSSMAACMmoQRAGAgCSMAAKMmYQQAGEjCCADAqK1bwVhVR1bVjVV1cVVdVVW/Pd1/ZlXtrKpLquqlM20/V1VvrarPVtX5VXWX6bHPV9XB0+dvqarPTJ+/d9r3DVV1+fT5E6vqJVX1gvV6XwAAs/as9LIRj0VZ74Txyu4+LskjkpxWVY9LcnSSE5Mcl+RhVfXoadsHJHltd/9gkn9I8vOzHVXVg5Icu2e7u3982veuJE/v7uO6e8daB1ZVp1fVrqraNeD9AQAsvfUuGI+qqouT/K8k5yZ53PRxUZJPJvmBTArIJLmmuz80ff6WJI9a0dfLkvz7NZ73l6eJ44eq6of31qC7z+vu47v7+LW/HQCA77TsCeN63/RyZXcfN51e3pXkI0n+Q3e/frZRVR2ZZOWnMLv9I0luSPKpNZ73nO7+7ao6Ocmrpq8HAOC7sFE3vdyU5JZMCr5/U1Xbk6SqDq2qe0/b3LeqHjF9/rQkH5x5/UuSnPVdnPf6JAd8VyMGACDJ+ieMe6akD0zyP7r71VVVST4y+UduSPKMTIrJy5M8t6remOSyJK+b6edj3X3lNIlci+dW1U8muUuSF++XdwIAsA/L/rM661Ywdvfnk9x5L/vPzeR6xm+bFoI3d/cz9tL+yBV9Hrvi+Ekrtl+SSSIJAMB+4Ie7AQAGkjBugL0lhwAAbA6bomAEANjKlj1htDQgAABzSRgBAAZY9I9qbwQJIwAAc0kYAQAGkjACADBqEkYAgIEkjAAAjJqEEQBgIAkjAACjJmEEABjA7zACADB6CkYAAOYyJQ0AMJApaQAARk3CCAAwkIQRAIBRkzACAAwkYQQAYNQkjAAAA0kYAQAYNQkjAMAAlgYEAGBLqapTquryqrqiql60l+O/UlWXVdUlVfUXVXXEan0qGAEABtqTMq73YzVVtS3Ja5I8PskxSZ5aVcesaHZRkuO7+8FJzk/yitX6VTACACyPE5Nc0d1Xdfc3k7wjyZNmG3T3e7v7/0w3P5rksNU6dQ0jAMBAG3gN48FVtWtm+7zuPm9m+9Ak18xsX5vk4XP6e1aSd692UgUjAMDWsbu7j98fHVXVM5Icn+THVmurYAQAGGgT3SV9XZLDZ7YPm+67jao6OcmvJfmx7r5ptU5dwwgAsDx2Jjm6qu5XVQckOTXJjtkGVfVDSV6f5Ind/eW1dKpgBABYEt19c5IzklyY5LNJ3tndl1bV2VX1xGmzVybZnuQPq+riqtqxj+6+zZQ0AMBAm2hKOt19QZILVuw7a+b5ybe3TwkjAABzSRgBAAYYw9KACsapW2+9ddFDWBpXX331ooewNLZt2yWcAMoAAAa+SURBVLboISyVd7971Z8a43Z42tOetughLJUHPOABix7C0jjnnHMWPYSlo2AEABho2RNG1zACADCXhBEAYCAJIwAAoyZhBAAYSMIIAMCoSRgBAAaSMAIAMGoSRgCAAcaw0ouEEQCAuSSMAAADSRgBABg1BSMAAHOZkgYAGMiUNAAAoyZhBAAYSMIIAMCoSRgBAAaSMAIAMGoSRgCAASwNCADA6EkYAQAGkjACADBqEkYAgIEkjAAAjJqEEQBgIAkjAACjJmEEABhIwggAwKgpGAEAmMuUNADAAJYGBABg9CSMAAADSRgBABg1CSMAwEASRgAARk3CCAAwkIQRAIBRW1jBWFXvq6rLq+ri6eP8mWOnV9Xnpo+PV9WjZo49oaouqqpPVdVlVfWcxbwDAICJPb/FuN6PRdnQKemqOiDJnbr7G9NdT+/uXSvaPCHJc5I8qrt3V9VDk7yrqk5Mcn2S85Kc2N3XVtWBSY6cvu4e3f3VjXovAABjsSEJY1X9YFX9pySXJ7n/Ks1fmOTM7t6dJN39ySR/kOS5Se6WSZF7/fTYTd19+fR1T6mqz1TV86vqkPV4HwAAK21UurjIhHHdCsaqumtVPbOqPpjkvyS5LMmDu/uimWZvnZmSfuV03wOTfGJFd7uSPLC7/y7JjiRXV9Xbq+rpVXWHJOnu303y+CR3SfL+qjq/qk7Zc3wv4zu9qnZV1a69HQcAYGI9p6S/kOSSJM/u7s/to813TEmvprufXVUPSnJykhckeWyS06bHrknym1X1skyKxzdmUmw+cS/9nJfJ9HaqarlvbQIA1pW7pL97T05yXZI/qqqzquqINb7usiQPW7HvYUku3bPR3Z/u7nMyKRZ/arbh9FrH1yZ5dZJ3Jnnxdzd8AACSdUwYu/s9Sd5TVfdK8owkf1JVuzNJHD8/56WvSPLyqjqlu6+vquMySRAfXlXbkxzf3e+btj0uydVJUlWPS/LbSb6Y5A1Jntfd39z/7wwA4LaWPWFc97uku/v6JOcmOXea/t0yc/itVXXj9Pnu7j65u3dU1aFJPjydKv56kmd09xeq6m5JfrWqXp/kxiTfyHQ6OpMbYf5ld1+93u8JAGBMNvRndbr74zPPT5rT7nVJXreX/V9P8s/38ZqVN8oAAGyIZU8YrfQCAMBcCkYAAOba0ClpAIBlZEoaAIBRkzACAAyw6GX7NoKEEQCAuSSMAAADSRgBABg1CSMAwEASRgAARk3CCAAwkIQRAIBRkzACAAwkYQQAYNQkjAAAA1jpBQCA0ZMwAgAMJGEEAGDUFIwAAMxlShoAYCBT0gAAjJqEEQBgIAkjAACjJmEEABhIwggAwJZRVadU1eVVdUVVvWgvxx9dVZ+sqpur6slr6VPCCAAwwGZaGrCqtiV5TZLHJrk2yc6q2tHdl800+99JTkvygrX2q2AEAFgeJya5oruvSpKqekeSJyX5dsHY3Z+fHrt1rZ0qGAEABtrAhPHgqto1s31ed583s31okmtmtq9N8vChJ1UwAgBsHbu7+/iNPqmCEQBgoM1yDWOS65IcPrN92HTfIO6SBgBYHjuTHF1V96uqA5KcmmTH0E4VjAAAA+25U3q9H2sYx81JzkhyYZLPJnlnd19aVWdX1ROTpKpOqKprk/yrJK+vqktX69eUNADAEunuC5JcsGLfWTPPd2YyVb1mCkYAgIE20TWM68KUNAAAcykYAQCYq5Y9Ql2LqvpKkqsXPQ4AYL84orsP2aiTHXDAAf193/d9G3Kua6655hN+h3FBNvJLBQCw1SgYAQAGWvYZW9cwAgAwl4QRAGAgCSMAAKMmYQQAGEjCCADAqEkYAQAGkjACADBqEkYAgAG6W8IIAMC4SRgBAAaSMAIAMGoSRgCAgSSMAACMmoIRAIC5TEkDAAxkShoAgFGTMAIADCRhBABg1CSMAAADWBoQAIDRkzACAAwkYQQAYNQkjAAAA0kYAQAYNQkjAMBAEkYAAEZNwggAMJCEEQCAUZMwAgAMYKUXAABGT8EIAMBcpqQBAAYyJQ0AwKhJGAEABpIwAgAwahJGAICBJIwAAIyahBEAYCAJIwAAoyZhBAAYwNKAAACMnoQRAGAgCSMAAKMmYQQAGEjCCADAqEkYAQAGkjACADBqCkYAAOYyJQ0AMJApaQAARk3CCAAwgKUBAQAYPQkjAMBAEkYAAEZNwggAMJCEEQCAUZMwAgAMJGEEAGDUJIwAAANJGAEAGDUJIwDAMBcmOXiDzrV7g85zG7XsESoAAMOYkgYAYC4FIwAAcykYAQCYS8EIAMBcCkYAAOb6v2lID+ODR8TTAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 720x720 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q3PWXX209ZHw",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 961
        },
        "outputId": "fd8250d1-282f-4137-b8d2-1e72033e6b06"
      },
      "source": [
        "from pprint import pprint\n",
        "pprint(study.best_params)\n",
        "pprint(study.best_value)\n",
        "pprint(study.best_trial)\n",
        "pprint(study.direction)\n",
        "%load_ext google.colab.data_table\n",
        "study.trials_dataframe()"
      ],
      "execution_count": 317,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'attn_type': 'soft_dist',\n",
            " 'batch_size': 100,\n",
            " 'bidirectional': True,\n",
            " 'dec_dropout': 0.0,\n",
            " 'dec_embedding_dim': 300,\n",
            " 'dec_rnn_dropout': 0.0,\n",
            " 'enc_dropout': 0.0,\n",
            " 'enc_embedding_dim': 300,\n",
            " 'enc_rnn_dropout': 0.0,\n",
            " 'hidden_size': 300,\n",
            " 'lr': 0.001,\n",
            " 'n_epoches': 5,\n",
            " 'num_layers': 1,\n",
            " 'opt_fn': 'Adam',\n",
            " 'out_hidden': 600,\n",
            " 'pack': False,\n",
            " 'rnn_type': 'RNN',\n",
            " 'seed': 0,\n",
            " 'weight_decay': 0.0}\n",
            "0.13512815535068512\n",
            "FrozenTrial(number=7, value=0.13512815535068512, datetime_start=datetime.datetime(2020, 5, 9, 0, 16, 11, 937425), datetime_complete=datetime.datetime(2020, 5, 9, 0, 18, 43, 106384), params={'attn_type': 'soft_dist', 'batch_size': 100, 'bidirectional': True, 'dec_dropout': 0.0, 'dec_embedding_dim': 300, 'dec_rnn_dropout': 0.0, 'enc_dropout': 0.0, 'enc_embedding_dim': 300, 'enc_rnn_dropout': 0.0, 'hidden_size': 300, 'lr': 0.001, 'n_epoches': 5, 'num_layers': 1, 'opt_fn': 'Adam', 'out_hidden': 600, 'pack': False, 'rnn_type': 'RNN', 'seed': 0, 'weight_decay': 0.0}, distributions={'attn_type': CategoricalDistribution(choices=('soft_dist', 'none')), 'batch_size': IntUniformDistribution(high=100, low=100, step=1), 'bidirectional': CategoricalDistribution(choices=(False, True)), 'dec_dropout': UniformDistribution(high=0.0, low=0.0), 'dec_embedding_dim': IntUniformDistribution(high=300, low=300, step=1), 'dec_rnn_dropout': UniformDistribution(high=0.0, low=0.0), 'enc_dropout': UniformDistribution(high=0.0, low=0.0), 'enc_embedding_dim': IntUniformDistribution(high=300, low=300, step=1), 'enc_rnn_dropout': UniformDistribution(high=0.0, low=0.0), 'hidden_size': IntUniformDistribution(high=300, low=300, step=1), 'lr': LogUniformDistribution(high=0.001, low=0.001), 'n_epoches': IntUniformDistribution(high=5, low=5, step=1), 'num_layers': IntUniformDistribution(high=1, low=1, step=1), 'opt_fn': CategoricalDistribution(choices=('Adam',)), 'out_hidden': IntUniformDistribution(high=600, low=600, step=1), 'pack': CategoricalDistribution(choices=(False, True)), 'rnn_type': CategoricalDistribution(choices=('RNN', 'GRU', 'LSTM')), 'seed': IntUniformDistribution(high=0, low=0, step=1), 'weight_decay': LogUniformDistribution(high=0.0, low=0.0)}, user_attrs={'device': 'cpu', 'train_acc': 0.9747191071510315, 'train_loss': 0.13512815535068512, 'val_acc': 0.8252149224281311, 'val_loss': 1.6250296831130981}, system_attrs={}, intermediate_values={1: 3.1183221340179443, 2: 2.057677745819092, 3: 1.0529454946517944, 4: 0.5272083878517151, 5: 0.13512815535068512}, trial_id=126, state=TrialState.COMPLETE)\n",
            "<StudyDirection.MINIMIZE: 1>\n",
            "The google.colab.data_table extension is already loaded. To reload it, use:\n",
            "  %reload_ext google.colab.data_table\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>number</th>\n",
              "      <th>value</th>\n",
              "      <th>datetime_start</th>\n",
              "      <th>datetime_complete</th>\n",
              "      <th>params_attn_type</th>\n",
              "      <th>params_batch_size</th>\n",
              "      <th>params_bidirectional</th>\n",
              "      <th>params_dec_dropout</th>\n",
              "      <th>params_dec_embedding_dim</th>\n",
              "      <th>params_dec_rnn_dropout</th>\n",
              "      <th>params_enc_dropout</th>\n",
              "      <th>params_enc_embedding_dim</th>\n",
              "      <th>params_enc_rnn_dropout</th>\n",
              "      <th>params_hidden_size</th>\n",
              "      <th>params_lr</th>\n",
              "      <th>params_n_epoches</th>\n",
              "      <th>params_num_layers</th>\n",
              "      <th>params_opt_fn</th>\n",
              "      <th>params_out_hidden</th>\n",
              "      <th>params_pack</th>\n",
              "      <th>params_rnn_type</th>\n",
              "      <th>params_seed</th>\n",
              "      <th>params_weight_decay</th>\n",
              "      <th>user_attrs_device</th>\n",
              "      <th>user_attrs_train_acc</th>\n",
              "      <th>user_attrs_train_loss</th>\n",
              "      <th>user_attrs_val_acc</th>\n",
              "      <th>user_attrs_val_loss</th>\n",
              "      <th>state</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>0.224853</td>\n",
              "      <td>2020-05-08 23:57:13.972534</td>\n",
              "      <td>2020-05-09 00:00:27.987888</td>\n",
              "      <td>soft_dist</td>\n",
              "      <td>100</td>\n",
              "      <td>True</td>\n",
              "      <td>0.0</td>\n",
              "      <td>300</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>300</td>\n",
              "      <td>0.0</td>\n",
              "      <td>300</td>\n",
              "      <td>0.001</td>\n",
              "      <td>5</td>\n",
              "      <td>1</td>\n",
              "      <td>Adam</td>\n",
              "      <td>600</td>\n",
              "      <td>True</td>\n",
              "      <td>GRU</td>\n",
              "      <td>0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>cpu</td>\n",
              "      <td>0.948718</td>\n",
              "      <td>0.224853</td>\n",
              "      <td>0.810056</td>\n",
              "      <td>1.702161</td>\n",
              "      <td>COMPLETE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>0.496093</td>\n",
              "      <td>2020-05-09 00:00:28.019622</td>\n",
              "      <td>2020-05-09 00:02:48.862425</td>\n",
              "      <td>none</td>\n",
              "      <td>100</td>\n",
              "      <td>True</td>\n",
              "      <td>0.0</td>\n",
              "      <td>300</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>300</td>\n",
              "      <td>0.0</td>\n",
              "      <td>300</td>\n",
              "      <td>0.001</td>\n",
              "      <td>5</td>\n",
              "      <td>1</td>\n",
              "      <td>Adam</td>\n",
              "      <td>600</td>\n",
              "      <td>True</td>\n",
              "      <td>RNN</td>\n",
              "      <td>0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>cpu</td>\n",
              "      <td>0.901130</td>\n",
              "      <td>0.496093</td>\n",
              "      <td>0.802941</td>\n",
              "      <td>1.955279</td>\n",
              "      <td>COMPLETE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>1.640806</td>\n",
              "      <td>2020-05-09 00:02:48.893567</td>\n",
              "      <td>2020-05-09 00:05:24.323928</td>\n",
              "      <td>none</td>\n",
              "      <td>100</td>\n",
              "      <td>False</td>\n",
              "      <td>0.0</td>\n",
              "      <td>300</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>300</td>\n",
              "      <td>0.0</td>\n",
              "      <td>300</td>\n",
              "      <td>0.001</td>\n",
              "      <td>5</td>\n",
              "      <td>1</td>\n",
              "      <td>Adam</td>\n",
              "      <td>600</td>\n",
              "      <td>False</td>\n",
              "      <td>GRU</td>\n",
              "      <td>0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>cpu</td>\n",
              "      <td>0.618384</td>\n",
              "      <td>1.640806</td>\n",
              "      <td>0.577904</td>\n",
              "      <td>2.727423</td>\n",
              "      <td>COMPLETE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>0.586174</td>\n",
              "      <td>2020-05-09 00:05:24.355761</td>\n",
              "      <td>2020-05-09 00:08:51.982904</td>\n",
              "      <td>none</td>\n",
              "      <td>100</td>\n",
              "      <td>True</td>\n",
              "      <td>0.0</td>\n",
              "      <td>300</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>300</td>\n",
              "      <td>0.0</td>\n",
              "      <td>300</td>\n",
              "      <td>0.001</td>\n",
              "      <td>5</td>\n",
              "      <td>1</td>\n",
              "      <td>Adam</td>\n",
              "      <td>600</td>\n",
              "      <td>True</td>\n",
              "      <td>LSTM</td>\n",
              "      <td>0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>cpu</td>\n",
              "      <td>0.873846</td>\n",
              "      <td>0.586174</td>\n",
              "      <td>0.780899</td>\n",
              "      <td>1.809779</td>\n",
              "      <td>COMPLETE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>1.023411</td>\n",
              "      <td>2020-05-09 00:08:52.019142</td>\n",
              "      <td>2020-05-09 00:11:12.052000</td>\n",
              "      <td>soft_dist</td>\n",
              "      <td>100</td>\n",
              "      <td>False</td>\n",
              "      <td>0.0</td>\n",
              "      <td>300</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>300</td>\n",
              "      <td>0.0</td>\n",
              "      <td>300</td>\n",
              "      <td>0.001</td>\n",
              "      <td>5</td>\n",
              "      <td>1</td>\n",
              "      <td>Adam</td>\n",
              "      <td>600</td>\n",
              "      <td>True</td>\n",
              "      <td>RNN</td>\n",
              "      <td>0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>cpu</td>\n",
              "      <td>0.728045</td>\n",
              "      <td>1.023411</td>\n",
              "      <td>0.584527</td>\n",
              "      <td>2.836457</td>\n",
              "      <td>COMPLETE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>5</td>\n",
              "      <td>1.122952</td>\n",
              "      <td>2020-05-09 00:11:12.088577</td>\n",
              "      <td>2020-05-09 00:13:51.048209</td>\n",
              "      <td>soft_dist</td>\n",
              "      <td>100</td>\n",
              "      <td>False</td>\n",
              "      <td>0.0</td>\n",
              "      <td>300</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>300</td>\n",
              "      <td>0.0</td>\n",
              "      <td>300</td>\n",
              "      <td>0.001</td>\n",
              "      <td>5</td>\n",
              "      <td>1</td>\n",
              "      <td>Adam</td>\n",
              "      <td>600</td>\n",
              "      <td>True</td>\n",
              "      <td>GRU</td>\n",
              "      <td>0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>cpu</td>\n",
              "      <td>0.725490</td>\n",
              "      <td>1.122952</td>\n",
              "      <td>0.609827</td>\n",
              "      <td>2.727381</td>\n",
              "      <td>COMPLETE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>6</td>\n",
              "      <td>0.496093</td>\n",
              "      <td>2020-05-09 00:13:51.081768</td>\n",
              "      <td>2020-05-09 00:16:11.896839</td>\n",
              "      <td>none</td>\n",
              "      <td>100</td>\n",
              "      <td>True</td>\n",
              "      <td>0.0</td>\n",
              "      <td>300</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>300</td>\n",
              "      <td>0.0</td>\n",
              "      <td>300</td>\n",
              "      <td>0.001</td>\n",
              "      <td>5</td>\n",
              "      <td>1</td>\n",
              "      <td>Adam</td>\n",
              "      <td>600</td>\n",
              "      <td>True</td>\n",
              "      <td>RNN</td>\n",
              "      <td>0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>cpu</td>\n",
              "      <td>0.901130</td>\n",
              "      <td>0.496093</td>\n",
              "      <td>0.802941</td>\n",
              "      <td>1.955279</td>\n",
              "      <td>COMPLETE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>7</td>\n",
              "      <td>0.135128</td>\n",
              "      <td>2020-05-09 00:16:11.937425</td>\n",
              "      <td>2020-05-09 00:18:43.106384</td>\n",
              "      <td>soft_dist</td>\n",
              "      <td>100</td>\n",
              "      <td>True</td>\n",
              "      <td>0.0</td>\n",
              "      <td>300</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>300</td>\n",
              "      <td>0.0</td>\n",
              "      <td>300</td>\n",
              "      <td>0.001</td>\n",
              "      <td>5</td>\n",
              "      <td>1</td>\n",
              "      <td>Adam</td>\n",
              "      <td>600</td>\n",
              "      <td>False</td>\n",
              "      <td>RNN</td>\n",
              "      <td>0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>cpu</td>\n",
              "      <td>0.974719</td>\n",
              "      <td>0.135128</td>\n",
              "      <td>0.825215</td>\n",
              "      <td>1.625030</td>\n",
              "      <td>COMPLETE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>8</td>\n",
              "      <td>0.605857</td>\n",
              "      <td>2020-05-09 00:18:43.144302</td>\n",
              "      <td>2020-05-09 00:22:00.473383</td>\n",
              "      <td>none</td>\n",
              "      <td>100</td>\n",
              "      <td>True</td>\n",
              "      <td>0.0</td>\n",
              "      <td>300</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>300</td>\n",
              "      <td>0.0</td>\n",
              "      <td>300</td>\n",
              "      <td>0.001</td>\n",
              "      <td>5</td>\n",
              "      <td>1</td>\n",
              "      <td>Adam</td>\n",
              "      <td>600</td>\n",
              "      <td>False</td>\n",
              "      <td>GRU</td>\n",
              "      <td>0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>cpu</td>\n",
              "      <td>0.876833</td>\n",
              "      <td>0.605857</td>\n",
              "      <td>0.820513</td>\n",
              "      <td>1.700572</td>\n",
              "      <td>COMPLETE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>9</td>\n",
              "      <td>0.334463</td>\n",
              "      <td>2020-05-09 00:22:00.510652</td>\n",
              "      <td>2020-05-09 00:25:50.224702</td>\n",
              "      <td>soft_dist</td>\n",
              "      <td>100</td>\n",
              "      <td>True</td>\n",
              "      <td>0.0</td>\n",
              "      <td>300</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>300</td>\n",
              "      <td>0.0</td>\n",
              "      <td>300</td>\n",
              "      <td>0.001</td>\n",
              "      <td>5</td>\n",
              "      <td>1</td>\n",
              "      <td>Adam</td>\n",
              "      <td>600</td>\n",
              "      <td>False</td>\n",
              "      <td>LSTM</td>\n",
              "      <td>0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>cpu</td>\n",
              "      <td>0.935574</td>\n",
              "      <td>0.334463</td>\n",
              "      <td>0.811268</td>\n",
              "      <td>1.913971</td>\n",
              "      <td>COMPLETE</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   number     value  ... user_attrs_val_loss     state\n",
              "0       0  0.224853  ...            1.702161  COMPLETE\n",
              "1       1  0.496093  ...            1.955279  COMPLETE\n",
              "2       2  1.640806  ...            2.727423  COMPLETE\n",
              "3       3  0.586174  ...            1.809779  COMPLETE\n",
              "4       4  1.023411  ...            2.836457  COMPLETE\n",
              "5       5  1.122952  ...            2.727381  COMPLETE\n",
              "6       6  0.496093  ...            1.955279  COMPLETE\n",
              "7       7  0.135128  ...            1.625030  COMPLETE\n",
              "8       8  0.605857  ...            1.700572  COMPLETE\n",
              "9       9  0.334463  ...            1.913971  COMPLETE\n",
              "\n",
              "[10 rows x 29 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 317
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IjcNmFIB-Nm2",
        "colab_type": "code",
        "outputId": "26b9736a-c8dd-4d4b-a508-35c751a1d3b0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 542
        }
      },
      "source": [
        "# optuna.visualization.plot_contour(study, params=['out_hidden', 'attn_type'])\n",
        "# optuna.visualization.plot_optimization_history(study)\n",
        "# optuna.visualization.plot_slice(study)\n",
        "# optuna.visualization.plot_parallel_coordinate(study)  # BUG\n",
        "optuna.visualization.plot_intermediate_values(study)"
      ],
      "execution_count": 319,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html>\n",
              "<head><meta charset=\"utf-8\" /></head>\n",
              "<body>\n",
              "    <div>\n",
              "            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>\n",
              "                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
              "        <script src=\"https://cdn.plot.ly/plotly-latest.min.js\"></script>    \n",
              "            <div id=\"eef1c5bb-f1b4-4d14-b86a-ba05cdc731fe\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>\n",
              "            <script type=\"text/javascript\">\n",
              "                \n",
              "                    window.PLOTLYENV=window.PLOTLYENV || {};\n",
              "                    \n",
              "                if (document.getElementById(\"eef1c5bb-f1b4-4d14-b86a-ba05cdc731fe\")) {\n",
              "                    Plotly.newPlot(\n",
              "                        'eef1c5bb-f1b4-4d14-b86a-ba05cdc731fe',\n",
              "                        [{\"marker\": {\"maxdisplayed\": 10}, \"mode\": \"lines+markers\", \"name\": \"Trial0\", \"type\": \"scatter\", \"x\": [1, 2, 3, 4, 5], \"y\": [3.411008358001709, 2.2167744636535645, 1.4129889011383057, 0.6963837146759033, 0.22485308349132538]}, {\"marker\": {\"maxdisplayed\": 10}, \"mode\": \"lines+markers\", \"name\": \"Trial1\", \"type\": \"scatter\", \"x\": [1, 2, 3, 4, 5], \"y\": [3.3765158653259277, 2.568922519683838, 1.6102677583694458, 1.0795210599899292, 0.49609336256980896]}, {\"marker\": {\"maxdisplayed\": 10}, \"mode\": \"lines+markers\", \"name\": \"Trial2\", \"type\": \"scatter\", \"x\": [1, 2, 3, 4, 5], \"y\": [4.232237339019775, 3.3544094562530518, 2.751138210296631, 1.8131709098815918, 1.640805721282959]}, {\"marker\": {\"maxdisplayed\": 10}, \"mode\": \"lines+markers\", \"name\": \"Trial3\", \"type\": \"scatter\", \"x\": [1, 2, 3, 4, 5], \"y\": [3.6278884410858154, 2.905818462371826, 2.004249334335327, 1.1413625478744507, 0.5861742496490479]}, {\"marker\": {\"maxdisplayed\": 10}, \"mode\": \"lines+markers\", \"name\": \"Trial4\", \"type\": \"scatter\", \"x\": [1, 2, 3, 4, 5], \"y\": [3.8354756832122803, 2.948737382888794, 1.8906593322753906, 1.48545503616333, 1.0234112739562988]}, {\"marker\": {\"maxdisplayed\": 10}, \"mode\": \"lines+markers\", \"name\": \"Trial5\", \"type\": \"scatter\", \"x\": [1, 2, 3, 4, 5], \"y\": [3.9591429233551025, 3.2066218852996826, 2.048017978668213, 1.4399868249893188, 1.1229523420333862]}, {\"marker\": {\"maxdisplayed\": 10}, \"mode\": \"lines+markers\", \"name\": \"Trial6\", \"type\": \"scatter\", \"x\": [1, 2, 3, 4, 5], \"y\": [3.3765158653259277, 2.568922519683838, 1.6102677583694458, 1.0795210599899292, 0.49609336256980896]}, {\"marker\": {\"maxdisplayed\": 10}, \"mode\": \"lines+markers\", \"name\": \"Trial7\", \"type\": \"scatter\", \"x\": [1, 2, 3, 4, 5], \"y\": [3.1183221340179443, 2.057677745819092, 1.0529454946517944, 0.5272083878517151, 0.13512815535068512]}, {\"marker\": {\"maxdisplayed\": 10}, \"mode\": \"lines+markers\", \"name\": \"Trial8\", \"type\": \"scatter\", \"x\": [1, 2, 3, 4, 5], \"y\": [3.5932199954986572, 2.5197157859802246, 1.8401373624801636, 1.067029356956482, 0.6058573722839355]}, {\"marker\": {\"maxdisplayed\": 10}, \"mode\": \"lines+markers\", \"name\": \"Trial9\", \"type\": \"scatter\", \"x\": [1, 2, 3, 4, 5], \"y\": [3.509397029876709, 2.8164827823638916, 1.7662299871444702, 0.9596610069274902, 0.33446332812309265]}],\n",
              "                        {\"showlegend\": false, \"template\": {\"data\": {\"bar\": [{\"error_x\": {\"color\": \"#2a3f5f\"}, \"error_y\": {\"color\": \"#2a3f5f\"}, \"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"bar\"}], \"barpolar\": [{\"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"baxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"choropleth\"}], \"contour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"contour\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"contourcarpet\"}], \"heatmap\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmap\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmapgl\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"histogram\"}], \"histogram2d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2d\"}], \"histogram2dcontour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2dcontour\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"parcoords\"}], \"pie\": [{\"automargin\": true, \"type\": \"pie\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter\"}], \"scatter3d\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattermapbox\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolar\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolargl\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"#EBF0F8\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"#C8D4E3\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowcolor\": \"#2a3f5f\", \"arrowhead\": 0, \"arrowwidth\": 1}, \"coloraxis\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"colorscale\": {\"diverging\": [[0, \"#8e0152\"], [0.1, \"#c51b7d\"], [0.2, \"#de77ae\"], [0.3, \"#f1b6da\"], [0.4, \"#fde0ef\"], [0.5, \"#f7f7f7\"], [0.6, \"#e6f5d0\"], [0.7, \"#b8e186\"], [0.8, \"#7fbc41\"], [0.9, \"#4d9221\"], [1, \"#276419\"]], \"sequential\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"sequentialminus\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]]}, \"colorway\": [\"#636efa\", \"#EF553B\", \"#00cc96\", \"#ab63fa\", \"#FFA15A\", \"#19d3f3\", \"#FF6692\", \"#B6E880\", \"#FF97FF\", \"#FECB52\"], \"font\": {\"color\": \"#2a3f5f\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"#E5ECF6\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hoverlabel\": {\"align\": \"left\"}, \"hovermode\": \"closest\", \"mapbox\": {\"style\": \"light\"}, \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"#E5ECF6\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"line\": {\"color\": \"#2a3f5f\"}}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"title\": {\"x\": 0.05}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"title\": {\"standoff\": 15}, \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"title\": {\"standoff\": 15}, \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}}}, \"title\": {\"text\": \"Intermediate Values Plot\"}, \"xaxis\": {\"title\": {\"text\": \"Step\"}}, \"yaxis\": {\"title\": {\"text\": \"Intermediate Value\"}}},\n",
              "                        {\"responsive\": true}\n",
              "                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('eef1c5bb-f1b4-4d14-b86a-ba05cdc731fe');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })\n",
              "                };\n",
              "                \n",
              "            </script>\n",
              "        </div>\n",
              "</body>\n",
              "</html>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gqoErSQduyIj",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 87
        },
        "outputId": "2c125381-5407-4862-9d44-666f7a3ddf4a"
      },
      "source": [
        "import torch\n",
        "\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "checkpoint = torch.load('model.pth')\n",
        "\n",
        "model = EncoderDecoder(**checkpoint['model_config']).to(device)\n",
        "model.load_state_dict(checkpoint['model_state_dict'])\n",
        "\n",
        "model.eval()\n",
        "with torch.no_grad():\n",
        "  x1 = eng_rus.encode_x1(normalize('i am tom'))\n",
        "  print(eng_rus.decode_x1(x1))\n",
        "  x1 = torch.tensor([x1]).to(device)\n",
        "  x2 = torch.tensor([[eng_rus.x2_w2i['<GO>']]]).to(device)\n",
        "  cnt = 0\n",
        "  while x2.item() != eng_rus.x2_w2i['<EOS>'] and cnt < 10:\n",
        "    # print(x1)\n",
        "    # print(x2)\n",
        "    y2, attn = model(x1, x2)\n",
        "    # print(y2)\n",
        "    p = torch.argmax(y2, 2)\n",
        "    x2 = p\n",
        "    print(eng_rus.decode_x2([p.item()]))\n",
        "    cnt += 1\n",
        "\n",
        "# model = TheModelClass(*args, **kwargs)\n",
        "# optimizer = TheOptimizerClass(*args, **kwargs)\n",
        "# model.load_state_dict(checkpoint['model_state_dict'])\n",
        "# optimizer.load_state_dict(checkpoint['optimizer_state_dict'])\n",
        "# epoch = checkpoint['epoch']\n",
        "# loss = checkpoint['loss']\n",
        "# model.eval()\n",
        "# # - or -\n",
        "# model.train()"
      ],
      "execution_count": 402,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['<GO>', 'i', 'am', 'tom', '<EOS>']\n",
            "['я']\n",
            "['позвонил']\n",
            "['<EOS>']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YsWSxCnrBCw3",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 381
        },
        "outputId": "5ab0d9e6-e6ef-455d-c63a-81ae2b09d470"
      },
      "source": [
        "import sacrebleu\n",
        "\n",
        "translated = []\n",
        "target = []\n",
        "\n",
        "for source, target_ in test:\n",
        "  translated.append(translate(model, source, device))\n",
        "  target.append(target_)\n",
        "\n",
        "bleu = sacrebleu.corpus_bleu(translated, [target])\n",
        "print(bleu.score)"
      ],
      "execution_count": 397,
      "outputs": [
        {
          "output_type": "error",
          "ename": "ModuleNotFoundError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-397-9d811e57c135>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0msacrebleu\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mtranslated\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mtarget\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'sacrebleu'",
            "",
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0;32m\nNOTE: If your import is failing due to a missing package, you can\nmanually install dependencies using either !pip or !apt.\n\nTo view examples of installing some common dependencies, click the\n\"Open Examples\" button below.\n\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0SDR3bmCSmjf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install torchviz"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xeh5fNJbSlhX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from graphviz import Digraph\n",
        "import torch\n",
        "from torch.autograd import Variable\n",
        "\n",
        "\n",
        "from torchviz import make_dot\n",
        "make_dot(x).view()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oduDznnmegxZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "torch.manual_seed(0)\n",
        "\n",
        "# input is of size N x C = 3 x 5\n",
        "x = torch.randn(2, 2, requires_grad=True)\n",
        "y = torch.randn(2, 2, requires_grad=True)\n",
        "# each element in target has to have 0 <= value < C\n",
        "t = torch.tensor([0, 0])\n",
        "print(x)\n",
        "print(nn.Softmax(dim=1)(x))\n",
        "print(nn.Softmax(dim=1)(y))\n",
        "print(nn.LogSoftmax(dim=1)(x))\n",
        "print(t)\n",
        "# l = -t*log(x) = -1*log(x)\n",
        "print(nn.NLLLoss(reduction='sum')(nn.LogSoftmax(dim=1)(x), t))\n",
        "print(nn.CrossEntropyLoss(reduction='sum')(x, t))\n",
        "# l(divergence x from t) = -t*log(x) - -t*log(t) = -t(log(x)-log(t)) = -t(log(x/t))\n",
        "print(nn.KLDivLoss(reduction='none')(nn.LogSoftmax(dim=1)(x),\n",
        "                                     nn.Softmax(dim=1)(y)))\n",
        "print(nn.KLDivLoss(reduction='sum')(nn.LogSoftmax(dim=1)(x),\n",
        "                                     nn.Softmax(dim=1)(y)))\n",
        "print(nn.KLDivLoss(reduction='mean')(nn.LogSoftmax(dim=1)(x),\n",
        "                                     nn.Softmax(dim=1)(y)))\n",
        "print(nn.KLDivLoss(reduction='batchmean')(nn.LogSoftmax(dim=1)(x),\n",
        "                                     nn.Softmax(dim=1)(y)))\n",
        "# l = abs(x - t)\n",
        "print(nn.L1Loss(reduction='none')(nn.Softmax(dim=1)(x), nn.Softmax(dim=1)(y)))\n",
        "# l = (x - t)**2\n",
        "print(nn.MSELoss(reduction='none')(nn.Softmax(dim=1)(x), nn.Softmax(dim=1)(y)))"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}