Fine Tuned LLamav2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "machine_shape": "hm",
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "1b0c327b00924e058279cf90eaa1ebd4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_1f1172ed28ab41849ed8292e059ffba1",
              "IPY_MODEL_5de581ff708a45ce98ca2b8b5562bd8f",
              "IPY_MODEL_e7d7c11ace8348e2913c8219f9b2a6ca"
            ],
            "layout": "IPY_MODEL_d9457ba8e4ce420686ae5f465c8de207"
          }
        },
        "1f1172ed28ab41849ed8292e059ffba1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ec6c3941856242afa4bceb119b8a78c3",
            "placeholder": "​",
            "style": "IPY_MODEL_2af9f921dded412e87c6df02491d7ee3",
            "value": "Downloading data files: 100%"
          }
        },
        "5de581ff708a45ce98ca2b8b5562bd8f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_23dfde8232234d68a4a18a0e745d2b34",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_2a9918506a77453991e1636a8c8f2c82",
            "value": 1
          }
        },
        "e7d7c11ace8348e2913c8219f9b2a6ca": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b353f89801884853a9d18f165da97fbc",
            "placeholder": "​",
            "style": "IPY_MODEL_7acff9adbdda494f9e3bb19863dc1169",
            "value": " 1/1 [00:00&lt;00:00, 90.61it/s]"
          }
        },
        "d9457ba8e4ce420686ae5f465c8de207": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ec6c3941856242afa4bceb119b8a78c3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2af9f921dded412e87c6df02491d7ee3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "23dfde8232234d68a4a18a0e745d2b34": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2a9918506a77453991e1636a8c8f2c82": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b353f89801884853a9d18f165da97fbc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7acff9adbdda494f9e3bb19863dc1169": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c0d95379fe774439b9d5f036fd915a1e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_9303ed1daf00437c8f098ae0c3b2445b",
              "IPY_MODEL_cc57a9b51aee439f82bf03f8c88cffbc",
              "IPY_MODEL_132ef319341b492684fa1880200d7ef8"
            ],
            "layout": "IPY_MODEL_9a58b5d8e9654a1d97ad5624a5ff3d26"
          }
        },
        "9303ed1daf00437c8f098ae0c3b2445b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fb7d8b739438467fbaa8e65ab1bf6480",
            "placeholder": "​",
            "style": "IPY_MODEL_e7dcd50c58b14653b4c5a3524bf5541b",
            "value": "Extracting data files: 100%"
          }
        },
        "cc57a9b51aee439f82bf03f8c88cffbc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cbd56d5f0985453f8591694599d05806",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_4ec9d4edcadf4c6fa98ac90a58df107c",
            "value": 1
          }
        },
        "132ef319341b492684fa1880200d7ef8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6ff40f96d8c54422908e5d4ae3b59ef1",
            "placeholder": "​",
            "style": "IPY_MODEL_13f2748395cb4592a053f232c624c466",
            "value": " 1/1 [00:00&lt;00:00, 58.76it/s]"
          }
        },
        "9a58b5d8e9654a1d97ad5624a5ff3d26": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fb7d8b739438467fbaa8e65ab1bf6480": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e7dcd50c58b14653b4c5a3524bf5541b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "cbd56d5f0985453f8591694599d05806": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4ec9d4edcadf4c6fa98ac90a58df107c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "6ff40f96d8c54422908e5d4ae3b59ef1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "13f2748395cb4592a053f232c624c466": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c9affde07dca416cb77b66a12b3f5f6b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_fc1b1ff7cb5b4134bfe61fe49276f0d6",
              "IPY_MODEL_ad097d03ba4e482796e72731a8b543c7",
              "IPY_MODEL_500a1ab0269f43a4a3530effaa20eafa"
            ],
            "layout": "IPY_MODEL_2e6fbf2f5d34457a9313e890e12aa4fe"
          }
        },
        "fc1b1ff7cb5b4134bfe61fe49276f0d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_82c633d1d13e4197b67ec528b00cb0f4",
            "placeholder": "​",
            "style": "IPY_MODEL_65546e5682bc4aa2ba014d9f3cd1cf45",
            "value": "Generating train split: "
          }
        },
        "ad097d03ba4e482796e72731a8b543c7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b1bfa3bef66f43548e070e9dede9c8df",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_4008324a0d9948d99d327ca24a5104bb",
            "value": 1
          }
        },
        "500a1ab0269f43a4a3530effaa20eafa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_62c4ba98ee714372bcf59d84f4ff44d3",
            "placeholder": "​",
            "style": "IPY_MODEL_733753f36ef64567a21ccf2fa00979e6",
            "value": " 796/0 [00:00&lt;00:00, 2640.71 examples/s]"
          }
        },
        "2e6fbf2f5d34457a9313e890e12aa4fe": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "82c633d1d13e4197b67ec528b00cb0f4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "65546e5682bc4aa2ba014d9f3cd1cf45": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b1bfa3bef66f43548e070e9dede9c8df": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "20px"
          }
        },
        "4008324a0d9948d99d327ca24a5104bb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "62c4ba98ee714372bcf59d84f4ff44d3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "733753f36ef64567a21ccf2fa00979e6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5fbec24b8a6d433c86ed828b1decee16": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_dff443f9edb242afa5e7147bc5d0d3cb",
              "IPY_MODEL_7e485d1d8e7548ac9e6ce50fa3c7ee0e",
              "IPY_MODEL_d228042d99a14d3280f6b8202b8be3fa"
            ],
            "layout": "IPY_MODEL_bdf94f4e12a041529d8cd90fe8b35e2b"
          }
        },
        "dff443f9edb242afa5e7147bc5d0d3cb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_eaf2fa3e79ca44d4b6ef725fde5fd630",
            "placeholder": "​",
            "style": "IPY_MODEL_223f5ee8fe7242728b3658a9958e1870",
            "value": "Loading checkpoint shards: 100%"
          }
        },
        "7e485d1d8e7548ac9e6ce50fa3c7ee0e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1d6ed8f2688b45b996c3de45330f8d84",
            "max": 14,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_4aee6aa741ec4881af795c921ea59461",
            "value": 14
          }
        },
        "d228042d99a14d3280f6b8202b8be3fa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_016b2a48d2c14d82bfc087328964fcc0",
            "placeholder": "​",
            "style": "IPY_MODEL_76b56151f2c449cb8393dd4a282de4ea",
            "value": " 14/14 [01:23&lt;00:00,  5.09s/it]"
          }
        },
        "bdf94f4e12a041529d8cd90fe8b35e2b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "eaf2fa3e79ca44d4b6ef725fde5fd630": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "223f5ee8fe7242728b3658a9958e1870": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1d6ed8f2688b45b996c3de45330f8d84": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4aee6aa741ec4881af795c921ea59461": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "016b2a48d2c14d82bfc087328964fcc0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "76b56151f2c449cb8393dd4a282de4ea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "02c5802b93484806983cc69a79a042c3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_c04ae58d651743c8bc50050d0e1fa565",
              "IPY_MODEL_6abeef6405904fcc8f8f492d1f75f776",
              "IPY_MODEL_0123c5c251c948248ed88f14951c74d7"
            ],
            "layout": "IPY_MODEL_646f9524cae1462f8c1a075a654bbe9a"
          }
        },
        "c04ae58d651743c8bc50050d0e1fa565": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_86e525ce807b4afc9ec33ca47598a239",
            "placeholder": "​",
            "style": "IPY_MODEL_151fc41391b24f89ace9d5f7c2dda938",
            "value": "Map: 100%"
          }
        },
        "6abeef6405904fcc8f8f492d1f75f776": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5a0a62834ac9426f9d9f0836e1961741",
            "max": 796,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_3ec50f3584734ce6a13743f95dad1523",
            "value": 796
          }
        },
        "0123c5c251c948248ed88f14951c74d7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_dda57fd9fb574f4eb07c167fdaeb7c56",
            "placeholder": "​",
            "style": "IPY_MODEL_06440cb51b4445d0a8a595895d758ba7",
            "value": " 796/796 [00:00&lt;00:00, 3164.43 examples/s]"
          }
        },
        "646f9524cae1462f8c1a075a654bbe9a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "86e525ce807b4afc9ec33ca47598a239": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "151fc41391b24f89ace9d5f7c2dda938": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5a0a62834ac9426f9d9f0836e1961741": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3ec50f3584734ce6a13743f95dad1523": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "dda57fd9fb574f4eb07c167fdaeb7c56": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "06440cb51b4445d0a8a595895d758ba7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9bf10ab2d16d4b4c920a6e7ce1fb7309": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_325418c4a8594e568c14a6055ce9cda9",
              "IPY_MODEL_c0faa5acaa684081b317c19aa8242dc1",
              "IPY_MODEL_6f99853f5e834f8eb99d274a50bb8e6e"
            ],
            "layout": "IPY_MODEL_32386d08705b49469d528cf6699aec68"
          }
        },
        "325418c4a8594e568c14a6055ce9cda9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c6cd08532f7b47d28f917d5c42b4ce87",
            "placeholder": "​",
            "style": "IPY_MODEL_815015a7d3e2424e8b4343260dcc81bc",
            "value": "Map: 100%"
          }
        },
        "c0faa5acaa684081b317c19aa8242dc1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d363bf05282043f4b02f20f01c31cbcf",
            "max": 796,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_8eabf48f3d3643ad87dc7c843eedcc30",
            "value": 796
          }
        },
        "6f99853f5e834f8eb99d274a50bb8e6e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e4554ebad1cc4d5f90a738086350e991",
            "placeholder": "​",
            "style": "IPY_MODEL_4c90bee51934431f94e664f8a88836d7",
            "value": " 796/796 [00:05&lt;00:00, 142.18 examples/s]"
          }
        },
        "32386d08705b49469d528cf6699aec68": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c6cd08532f7b47d28f917d5c42b4ce87": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "815015a7d3e2424e8b4343260dcc81bc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d363bf05282043f4b02f20f01c31cbcf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8eabf48f3d3643ad87dc7c843eedcc30": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "e4554ebad1cc4d5f90a738086350e991": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4c90bee51934431f94e664f8a88836d7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "19250960968645ae944b5d926a3cce37": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b5f2884ca58e4ac0a6bd6acc0b756034",
              "IPY_MODEL_b759687a8dc445869215c1595d68c7e3",
              "IPY_MODEL_dfc4602675f946528a1601c9397cecff"
            ],
            "layout": "IPY_MODEL_cfc7ee23b87a48adbadec3150be3427a"
          }
        },
        "b5f2884ca58e4ac0a6bd6acc0b756034": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_61fd2b8872bd45ee9850ab731bd083b0",
            "placeholder": "​",
            "style": "IPY_MODEL_999593bab1704c54bf6c18b9b4c6774c",
            "value": "Filter: 100%"
          }
        },
        "b759687a8dc445869215c1595d68c7e3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8ef98168b10046c3a81ac6ebdebbaa42",
            "max": 796,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_0c8451cc5e0844689002ac179fd42819",
            "value": 796
          }
        },
        "dfc4602675f946528a1601c9397cecff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e81751c422af490d9e5094d947fe4bac",
            "placeholder": "​",
            "style": "IPY_MODEL_b7979df9a42a4578ab20abc0ebccadf0",
            "value": " 796/796 [00:01&lt;00:00, 621.79 examples/s]"
          }
        },
        "cfc7ee23b87a48adbadec3150be3427a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "61fd2b8872bd45ee9850ab731bd083b0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "999593bab1704c54bf6c18b9b4c6774c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8ef98168b10046c3a81ac6ebdebbaa42": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0c8451cc5e0844689002ac179fd42819": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "e81751c422af490d9e5094d947fe4bac": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b7979df9a42a4578ab20abc0ebccadf0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "aa904ca6d9574f048e3521943d3ec478": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_ca360b3361624630a080bf2c548ecaec",
              "IPY_MODEL_2608a665b1684a81979a3ebce713ae98",
              "IPY_MODEL_42f9477a3a4c4a3ca6845a613b878bb2"
            ],
            "layout": "IPY_MODEL_a9d98bbd85a34fed9c8c228bde08d6dd"
          }
        },
        "ca360b3361624630a080bf2c548ecaec": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_97410e6200f447fd8d525d79420f08f1",
            "placeholder": "​",
            "style": "IPY_MODEL_ac31b8fedca14baf915ebfd9c7278c2c",
            "value": "Loading checkpoint shards: 100%"
          }
        },
        "2608a665b1684a81979a3ebce713ae98": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_097cd288bb674328899799cb47c4a534",
            "max": 14,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_f49e8e866fd541f9b061464d4e628dc7",
            "value": 14
          }
        },
        "42f9477a3a4c4a3ca6845a613b878bb2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_abc3a6650e9b433b91a1a74ea76949e6",
            "placeholder": "​",
            "style": "IPY_MODEL_5730521e3daa47c3ac1b94bb8c841a5e",
            "value": " 14/14 [00:14&lt;00:00,  1.17it/s]"
          }
        },
        "a9d98bbd85a34fed9c8c228bde08d6dd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "97410e6200f447fd8d525d79420f08f1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ac31b8fedca14baf915ebfd9c7278c2c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "097cd288bb674328899799cb47c4a534": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f49e8e866fd541f9b061464d4e628dc7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "abc3a6650e9b433b91a1a74ea76949e6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5730521e3daa47c3ac1b94bb8c841a5e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a0b393129f944a8391d9b6e579aea447": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_3e0c3235642d4012a1af1089b79dedb5",
              "IPY_MODEL_19eee33dcb2845dfa631e7a27e315b5a",
              "IPY_MODEL_09a968fae2c64ea0b61396af29077a2a"
            ],
            "layout": "IPY_MODEL_587d5d8eebb94a35a3de6bda2167fba0"
          }
        },
        "3e0c3235642d4012a1af1089b79dedb5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4a4f20cf6113485eae718684ab46007e",
            "placeholder": "​",
            "style": "IPY_MODEL_6dea2bf72db2486d9800049eb4c6a054",
            "value": "Loading checkpoint shards: 100%"
          }
        },
        "19eee33dcb2845dfa631e7a27e315b5a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_60b4abf673284b9db7cd1a27326593d8",
            "max": 2,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e7a6338df52347f090f17bc8400069e6",
            "value": 2
          }
        },
        "09a968fae2c64ea0b61396af29077a2a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_957d5f71596342f387da7ccb2efc623d",
            "placeholder": "​",
            "style": "IPY_MODEL_f8f6bc6d1a34423695aff499c670833f",
            "value": " 2/2 [01:03&lt;00:00, 27.97s/it]"
          }
        },
        "587d5d8eebb94a35a3de6bda2167fba0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4a4f20cf6113485eae718684ab46007e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6dea2bf72db2486d9800049eb4c6a054": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "60b4abf673284b9db7cd1a27326593d8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e7a6338df52347f090f17bc8400069e6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "957d5f71596342f387da7ccb2efc623d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f8f6bc6d1a34423695aff499c670833f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GLXwJqbjtPho"
      },
      "outputs": [],
      "source": [
        "!pip install -q accelerate==0.21.0 --progress-bar off\n",
        "!pip install -q peft==0.4.0 --progress-bar off\n",
        "!pip install -q bitsandbytes==0.40.2 --progress-bar off\n",
        "!pip install -q transformers==4.31.0 --progress-bar off\n",
        "!pip install -q trl==0.4.7 --progress-bar off"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "from random import randrange\n",
        "from functools import partial\n",
        "import torch\n",
        "from datasets import load_dataset\n",
        "from transformers import (AutoModelForCausalLM,\n",
        "                          AutoTokenizer,\n",
        "                          BitsAndBytesConfig,\n",
        "                          HfArgumentParser,\n",
        "                          Trainer,\n",
        "                          TrainingArguments,\n",
        "                          DataCollatorForLanguageModeling,\n",
        "                          EarlyStoppingCallback,\n",
        "                          pipeline,\n",
        "                          logging,\n",
        "                          set_seed)\n",
        "\n",
        "import bitsandbytes as bnb\n",
        "from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training, PeftModel, AutoPeftModelForCausalLM\n",
        "from trl import SFTTrainer\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "id": "nAMzy_0FtaUZ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "684b7420-06aa-48b0-a2ab-c286e28235ab"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Hugging Face Hub Login\n",
        "\n",
        "Meta's family of Llama 2 models is gated. You will require approval to access it using the Hugging Face Hub.\n",
        "\n",
        "Below are the steps to request permission for the Llama-2-7B model:\n",
        "1. Get approval from Hugging Face (https://huggingface.co/meta-llama/Llama-2-7b-hf).\n",
        "2. Get approval from Meta (https://ai.meta.com/resources/models-and-libraries/llama-downloads/).\n",
        "3. Create a WRITE access token on Hugging Face (https://huggingface.co/settings/tokens).\n",
        "4. Execute `!huggingface-cli login` in Google Colab Notebook, enter the token, and enter \"Y.\"\n",
        "\n",
        "Note: Make sure your email address on your Hugging Face account is the same as the one you enter on Meta's website for approval.\n",
        "\n",
        "If you don't want to perform the above steps, use a cloned version of Llama-2-7B, such as https://huggingface.co/daryl149/llama-2-7b-chat-hf. Additionally, you'll have to set `use_auth_token` to `False` while loading the model and its tokenizer."
      ],
      "metadata": {
        "id": "N2jeE5EyvB8e"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def create_bnb_config(load_in_4bit, bnb_4bit_use_double_quant, bnb_4bit_quant_type, bnb_4bit_compute_dtype):\n",
        "    \"\"\"\n",
        "    Configures model quantization method using bitsandbytes to speed up training and inference\n",
        "\n",
        "    :param load_in_4bit: Load model in 4-bit precision mode\n",
        "    :param bnb_4bit_use_double_quant: Nested quantization for 4-bit model\n",
        "    :param bnb_4bit_quant_type: Quantization data type for 4-bit model\n",
        "    :param bnb_4bit_compute_dtype: Computation data type for 4-bit model\n",
        "    \"\"\"\n",
        "\n",
        "    bnb_config = BitsAndBytesConfig(\n",
        "        load_in_4bit = load_in_4bit,\n",
        "        bnb_4bit_use_double_quant = bnb_4bit_use_double_quant,\n",
        "        bnb_4bit_quant_type = bnb_4bit_quant_type,\n",
        "        bnb_4bit_compute_dtype = bnb_4bit_compute_dtype,\n",
        "    )\n",
        "\n",
        "    return bnb_config"
      ],
      "metadata": {
        "id": "H2o8PO-T0JWx"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def load_model(model_name, bnb_config):\n",
        "    \"\"\"\n",
        "    Loads model and model tokenizer\n",
        "\n",
        "    :param model_name: Hugging Face model name\n",
        "    :param bnb_config: Bitsandbytes configuration\n",
        "    \"\"\"\n",
        "\n",
        "    # Get number of GPU device and set maximum memory\n",
        "    n_gpus = torch.cuda.device_count()\n",
        "    max_memory = f'{40960}MB'\n",
        "\n",
        "    # Load model\n",
        "    model = AutoModelForCausalLM.from_pretrained(\n",
        "        model_name,\n",
        "        quantization_config = bnb_config,\n",
        "        device_map = \"auto\", # dispatch the model efficiently on the available resources\n",
        "        max_memory = {i: max_memory for i in range(n_gpus)},\n",
        "    )\n",
        "\n",
        "    # Load model tokenizer with the user authentication token\n",
        "    tokenizer = AutoTokenizer.from_pretrained(model_name, use_auth_token = True)\n",
        "\n",
        "    # Set padding token as EOS token\n",
        "    tokenizer.pad_token = tokenizer.eos_token\n",
        "\n",
        "    return model, tokenizer"
      ],
      "metadata": {
        "id": "gIqPR8O-s7WE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "################################################################################\n",
        "# transformers parameters\n",
        "################################################################################\n",
        "\n",
        "# The pre-trained model from the Hugging Face Hub to load and fine-tune\n",
        "model_name = \"TinyPixel/Llama-2-7B-bf16-sharded\"\n",
        "\n",
        "################################################################################\n",
        "# bitsandbytes parameters\n",
        "################################################################################\n",
        "\n",
        "# Activate 4-bit precision base model loading\n",
        "load_in_4bit = True\n",
        "\n",
        "# Activate nested quantization for 4-bit base models (double quantization)\n",
        "bnb_4bit_use_double_quant = True\n",
        "\n",
        "# Quantization type (fp4 or nf4)\n",
        "bnb_4bit_quant_type = \"nf4\"\n",
        "\n",
        "# Compute data type for 4-bit base models\n",
        "bnb_4bit_compute_dtype = torch.bfloat16"
      ],
      "metadata": {
        "id": "_BxEZPWQtK6K"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "\n",
        "import pandas as pd\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "# Read the original dataset\n",
        "df = pd.read_csv('/content/drive/MyDrive/alldata_1_for_kaggle.csv', encoding='ISO-8859-1')\n",
        "df.head()\n",
        "\n"
      ],
      "metadata": {
        "id": "qLK_o6Id0ZZ5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "outputId": "989b0f43-f638-4649-dcd8-7f42a28caada"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   Unnamed: 0               0  \\\n",
              "0           0  Thyroid_Cancer   \n",
              "1           1  Thyroid_Cancer   \n",
              "2           2  Thyroid_Cancer   \n",
              "3           3  Thyroid_Cancer   \n",
              "4           4  Thyroid_Cancer   \n",
              "\n",
              "                                                   a  \n",
              "0  Thyroid surgery in  children in a single insti...  \n",
              "1  \" The adopted strategy was the same as that us...  \n",
              "2  coronary arterybypass grafting thrombosis ï¬b...  \n",
              "3   Solitary plasmacytoma SP of the skull is an u...  \n",
              "4   This study aimed to investigate serum matrix ...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-5608dc86-25fb-4bb6-9968-9cffcffa01b8\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>0</th>\n",
              "      <th>a</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>Thyroid_Cancer</td>\n",
              "      <td>Thyroid surgery in  children in a single insti...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>Thyroid_Cancer</td>\n",
              "      <td>\" The adopted strategy was the same as that us...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>Thyroid_Cancer</td>\n",
              "      <td>coronary arterybypass grafting thrombosis ï¬b...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>Thyroid_Cancer</td>\n",
              "      <td>Solitary plasmacytoma SP of the skull is an u...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>Thyroid_Cancer</td>\n",
              "      <td>This study aimed to investigate serum matrix ...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-5608dc86-25fb-4bb6-9968-9cffcffa01b8')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-5608dc86-25fb-4bb6-9968-9cffcffa01b8 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-5608dc86-25fb-4bb6-9968-9cffcffa01b8');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-de82094b-3289-4336-a20f-2f467a75af72\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-de82094b-3289-4336-a20f-2f467a75af72')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "    background-color: #E8F0FE;\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: #1967D2;\n",
              "    height: 32px;\n",
              "    padding: 0 0 0 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: #E2EBFA;\n",
              "    box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: #174EA6;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "    background-color: #3B4455;\n",
              "    fill: #D2E3FC;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart:hover {\n",
              "    background-color: #434B5C;\n",
              "    box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "    filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "    fill: #FFFFFF;\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const charts = await google.colab.kernel.invokeFunction(\n",
              "          'suggestCharts', [key], {});\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-de82094b-3289-4336-a20f-2f467a75af72 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df= df.drop (columns=['Unnamed: 0'])\n",
        "df.columns = ['label', 'text']\n",
        "df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "5Z4n0qUrolRQ",
        "outputId": "c7af46ca-a120-424a-a4d7-4b7bd660e6d8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "            label                                               text\n",
              "0  Thyroid_Cancer  Thyroid surgery in  children in a single insti...\n",
              "1  Thyroid_Cancer  \" The adopted strategy was the same as that us...\n",
              "2  Thyroid_Cancer  coronary arterybypass grafting thrombosis ï¬b...\n",
              "3  Thyroid_Cancer   Solitary plasmacytoma SP of the skull is an u...\n",
              "4  Thyroid_Cancer   This study aimed to investigate serum matrix ..."
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-a9dba096-59f6-42e8-8780-6d6802939c2b\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>label</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Thyroid_Cancer</td>\n",
              "      <td>Thyroid surgery in  children in a single insti...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Thyroid_Cancer</td>\n",
              "      <td>\" The adopted strategy was the same as that us...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Thyroid_Cancer</td>\n",
              "      <td>coronary arterybypass grafting thrombosis ï¬b...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Thyroid_Cancer</td>\n",
              "      <td>Solitary plasmacytoma SP of the skull is an u...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Thyroid_Cancer</td>\n",
              "      <td>This study aimed to investigate serum matrix ...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-a9dba096-59f6-42e8-8780-6d6802939c2b')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-a9dba096-59f6-42e8-8780-6d6802939c2b button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-a9dba096-59f6-42e8-8780-6d6802939c2b');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-fc846831-916f-4606-87ad-cbca99911c19\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-fc846831-916f-4606-87ad-cbca99911c19')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "    background-color: #E8F0FE;\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: #1967D2;\n",
              "    height: 32px;\n",
              "    padding: 0 0 0 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: #E2EBFA;\n",
              "    box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: #174EA6;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "    background-color: #3B4455;\n",
              "    fill: #D2E3FC;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart:hover {\n",
              "    background-color: #434B5C;\n",
              "    box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "    filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "    fill: #FFFFFF;\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const charts = await google.colab.kernel.invokeFunction(\n",
              "          'suggestCharts', [key], {});\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-fc846831-916f-4606-87ad-cbca99911c19 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "len(df)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JMSRMP2vpjsp",
        "outputId": "620c51a0-2cff-440b-82da-159434e4f745"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "7570"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.drop_duplicates(subset=\"text\", keep=\"first\", inplace=True)\n",
        "len(df)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7Mt982Impsh_",
        "outputId": "e13e621a-4815-4e6b-e93d-bc15ff6c9887"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "996"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_df, test_df = train_test_split(df, test_size=0.2, random_state=42)  # 80% training, 20% testing\n",
        "\n",
        "# Save the splits into separate CSV files\n",
        "train_df.to_csv('/content/drive/MyDrive/train_data_new.csv', index=False)\n",
        "test_df.to_csv('/content/drive/MyDrive/test_data_new.csv', index=False)"
      ],
      "metadata": {
        "id": "lNE2jf4_pgCd"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "train_df= pd.read_csv('/content/drive/MyDrive/train_data_new.csv',encoding='ISO-8859-1')\n",
        "test_df= pd.read_csv ('/content/drive/MyDrive/test_data_new.csv', encoding='ISO-8859-1')"
      ],
      "metadata": {
        "id": "PG6T2KKXlp81"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "0EPo3YvImjSF",
        "outputId": "abe0f965-1d05-467a-9828-3abe07ecbd66"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "          label                                               text\n",
              "0   Lung_Cancer  \" Despite previous investigations it remains u...\n",
              "1  Colon_Cancer  \"  the influence of anastomotic leakage al on ...\n",
              "2   Lung_Cancer  \"About half (49%) of UNCeqRMETA mutations had ...\n",
              "3  Colon_Cancer  \"circadian clocks have important physiological...\n",
              "4   Lung_Cancer  \"Thymidylate synthase (TS) gene expression in ..."
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-345a6b32-57a2-494a-a4b0-c724ecfb46ff\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>label</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Lung_Cancer</td>\n",
              "      <td>\" Despite previous investigations it remains u...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Colon_Cancer</td>\n",
              "      <td>\"  the influence of anastomotic leakage al on ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Lung_Cancer</td>\n",
              "      <td>\"About half (49%) of UNCeqRMETA mutations had ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Colon_Cancer</td>\n",
              "      <td>\"circadian clocks have important physiological...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Lung_Cancer</td>\n",
              "      <td>\"Thymidylate synthase (TS) gene expression in ...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-345a6b32-57a2-494a-a4b0-c724ecfb46ff')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-345a6b32-57a2-494a-a4b0-c724ecfb46ff button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-345a6b32-57a2-494a-a4b0-c724ecfb46ff');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-7b63c144-f552-41e2-a0b8-117530187016\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-7b63c144-f552-41e2-a0b8-117530187016')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "    background-color: #E8F0FE;\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: #1967D2;\n",
              "    height: 32px;\n",
              "    padding: 0 0 0 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: #E2EBFA;\n",
              "    box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: #174EA6;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "    background-color: #3B4455;\n",
              "    fill: #D2E3FC;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart:hover {\n",
              "    background-color: #434B5C;\n",
              "    box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "    filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "    fill: #FFFFFF;\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const charts = await google.colab.kernel.invokeFunction(\n",
              "          'suggestCharts', [key], {});\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-7b63c144-f552-41e2-a0b8-117530187016 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "len(train_df), len(test_df)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rlejCiRup9AE",
        "outputId": "28d754bf-a0c6-419f-a9b0-630d0ad5e36f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(796, 200)"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_texts = train_df['text'].tolist()\n",
        "train_labels = train_df['label'].tolist()\n",
        "#idx2label = {'Colon_Cancer': 0, 'Lung_Cancer': 1, 'Thyroid_Cancer': 2}\n",
        "dataset_data = []\n",
        "\n",
        "# Assuming you have train_texts and train_labels defined somewhere\n",
        "# in your code\n",
        "\n",
        "for text, label in zip(train_texts, train_labels):\n",
        "   # label = list(idx2label.keys())[list(idx2label.values()).index(label_idx)]\n",
        "\n",
        "    entry = {\n",
        "        \"instruction\": \"Categorize the input article into one of the three categories\\nColon_Cancer\\nLung_Cancer\\nThyroid_Cancer\\n\\n.\",\n",
        "        \"input\": text,\n",
        "        \"output\": label\n",
        "    }\n",
        "    dataset_data.append(entry)\n",
        "\n",
        "# Now, dataset_data contains entries with class names instead of encoded labels\n",
        "\n",
        "# Import the necessary library\n",
        "import csv\n",
        "\n",
        "# Define the CSV file name\n",
        "csv_file = 'dataset.csv'\n",
        "\n",
        "# Write the dataset_data to a CSV file\n",
        "with open(csv_file, 'w', newline='', encoding='utf-8') as file:\n",
        "    fieldnames = ['instruction', 'input', 'output']\n",
        "    writer = csv.DictWriter(file, fieldnames=fieldnames)\n",
        "    writer.writeheader()\n",
        "    for entry in dataset_data:\n",
        "        writer.writerow(entry)\n",
        "\n",
        "print(f\"CSV file '{csv_file}' has been created.\")\n",
        "\n",
        "#test_texts = test_df['text'].tolist()\n",
        "#test_labels = test_df['label'].tolist()"
      ],
      "metadata": {
        "id": "XbjWmCt4qYir",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "55876979-64a3-4bcc-da74-3631e0d68a42"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "CSV file 'dataset.csv' has been created.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n"
      ],
      "metadata": {
        "id": "sBgBWxHey7Fn"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "dataset_name='/content/dataset.csv'\n",
        "data= pd.read_csv('/content/dataset.csv',encoding='ISO-8859-1')\n"
      ],
      "metadata": {
        "id": "PCXGazE4zAJh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "m-rKWldlzI44",
        "outputId": "9ab623ef-b33f-4a18-fb7a-527018a39274"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                         instruction  \\\n",
              "0  Categorize the input article into one of the t...   \n",
              "1  Categorize the input article into one of the t...   \n",
              "2  Categorize the input article into one of the t...   \n",
              "3  Categorize the input article into one of the t...   \n",
              "4  Categorize the input article into one of the t...   \n",
              "\n",
              "                                               input        output  \n",
              "0  \" Despite previous investigations it remains u...   Lung_Cancer  \n",
              "1  \"  the influence of anastomotic leakage al on ...  Colon_Cancer  \n",
              "2  \"About half (49%) of UNCeqRMETA mutations had ...   Lung_Cancer  \n",
              "3  \"circadian clocks have important physiological...  Colon_Cancer  \n",
              "4  \"Thymidylate synthase (TS) gene expression in ...   Lung_Cancer  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-6fb9786b-75dc-472e-b5c6-bf20dc39aaef\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>instruction</th>\n",
              "      <th>input</th>\n",
              "      <th>output</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Categorize the input article into one of the t...</td>\n",
              "      <td>\" Despite previous investigations it remains u...</td>\n",
              "      <td>Lung_Cancer</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Categorize the input article into one of the t...</td>\n",
              "      <td>\"  the influence of anastomotic leakage al on ...</td>\n",
              "      <td>Colon_Cancer</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Categorize the input article into one of the t...</td>\n",
              "      <td>\"About half (49%) of UNCeqRMETA mutations had ...</td>\n",
              "      <td>Lung_Cancer</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Categorize the input article into one of the t...</td>\n",
              "      <td>\"circadian clocks have important physiological...</td>\n",
              "      <td>Colon_Cancer</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Categorize the input article into one of the t...</td>\n",
              "      <td>\"Thymidylate synthase (TS) gene expression in ...</td>\n",
              "      <td>Lung_Cancer</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-6fb9786b-75dc-472e-b5c6-bf20dc39aaef')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-6fb9786b-75dc-472e-b5c6-bf20dc39aaef button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-6fb9786b-75dc-472e-b5c6-bf20dc39aaef');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-1a93342d-3f8e-4987-bcc7-7f4374950486\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-1a93342d-3f8e-4987-bcc7-7f4374950486')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "    background-color: #E8F0FE;\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: #1967D2;\n",
              "    height: 32px;\n",
              "    padding: 0 0 0 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: #E2EBFA;\n",
              "    box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: #174EA6;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "    background-color: #3B4455;\n",
              "    fill: #D2E3FC;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart:hover {\n",
              "    background-color: #434B5C;\n",
              "    box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "    filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "    fill: #FFFFFF;\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const charts = await google.colab.kernel.invokeFunction(\n",
              "          'suggestCharts', [key], {});\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-1a93342d-3f8e-4987-bcc7-7f4374950486 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Load dataset\n",
        "dataset = load_dataset(\"csv\", data_files = dataset_name, split = \"train\")"
      ],
      "metadata": {
        "id": "Ftmfi4M9RJAp",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 113,
          "referenced_widgets": [
            "1b0c327b00924e058279cf90eaa1ebd4",
            "1f1172ed28ab41849ed8292e059ffba1",
            "5de581ff708a45ce98ca2b8b5562bd8f",
            "e7d7c11ace8348e2913c8219f9b2a6ca",
            "d9457ba8e4ce420686ae5f465c8de207",
            "ec6c3941856242afa4bceb119b8a78c3",
            "2af9f921dded412e87c6df02491d7ee3",
            "23dfde8232234d68a4a18a0e745d2b34",
            "2a9918506a77453991e1636a8c8f2c82",
            "b353f89801884853a9d18f165da97fbc",
            "7acff9adbdda494f9e3bb19863dc1169",
            "c0d95379fe774439b9d5f036fd915a1e",
            "9303ed1daf00437c8f098ae0c3b2445b",
            "cc57a9b51aee439f82bf03f8c88cffbc",
            "132ef319341b492684fa1880200d7ef8",
            "9a58b5d8e9654a1d97ad5624a5ff3d26",
            "fb7d8b739438467fbaa8e65ab1bf6480",
            "e7dcd50c58b14653b4c5a3524bf5541b",
            "cbd56d5f0985453f8591694599d05806",
            "4ec9d4edcadf4c6fa98ac90a58df107c",
            "6ff40f96d8c54422908e5d4ae3b59ef1",
            "13f2748395cb4592a053f232c624c466",
            "c9affde07dca416cb77b66a12b3f5f6b",
            "fc1b1ff7cb5b4134bfe61fe49276f0d6",
            "ad097d03ba4e482796e72731a8b543c7",
            "500a1ab0269f43a4a3530effaa20eafa",
            "2e6fbf2f5d34457a9313e890e12aa4fe",
            "82c633d1d13e4197b67ec528b00cb0f4",
            "65546e5682bc4aa2ba014d9f3cd1cf45",
            "b1bfa3bef66f43548e070e9dede9c8df",
            "4008324a0d9948d99d327ca24a5104bb",
            "62c4ba98ee714372bcf59d84f4ff44d3",
            "733753f36ef64567a21ccf2fa00979e6"
          ]
        },
        "outputId": "e79f194a-1a29-41b4-de23-80a671a69457"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading data files:   0%|          | 0/1 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "1b0c327b00924e058279cf90eaa1ebd4"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Extracting data files:   0%|          | 0/1 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "c0d95379fe774439b9d5f036fd915a1e"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating train split: 0 examples [00:00, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "c9affde07dca416cb77b66a12b3f5f6b"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(f'Number of prompts: {len(dataset)}')\n",
        "print(f'Column names are: {dataset.column_names}')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tLoaZvdTmIZT",
        "outputId": "5c122a0e-8587-4541-ef92-c26775d75405"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Number of prompts: 796\n",
            "Column names are: ['instruction', 'input', 'output']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def create_prompt_formats(sample):\n",
        "    \"\"\"\n",
        "    Creates a formatted prompt template for a prompt in the instruction dataset\n",
        "\n",
        "    :param sample: Prompt or sample from the instruction dataset\n",
        "    \"\"\"\n",
        "\n",
        "    # Initialize static strings for the prompt template\n",
        "    INTRO_BLURB = \"Below is an instruction that describes a task. Write a response that appropriately completes the request.\"\n",
        "    INSTRUCTION_KEY = \"### Instruction:\"\n",
        "    INPUT_KEY = \"Input:\"\n",
        "    RESPONSE_KEY = \"### Response:\"\n",
        "    END_KEY = \"### End\"\n",
        "\n",
        "    # Combine a prompt with the static strings\n",
        "    blurb = f\"{INTRO_BLURB}\"\n",
        "    instruction = f\"{INSTRUCTION_KEY}\\n{sample['instruction']}\"\n",
        "    input_context = f\"{INPUT_KEY}\\n{sample['input']}\" if sample[\"input\"] else None\n",
        "    response = f\"{RESPONSE_KEY}\\n{sample['output']}\"\n",
        "    end = f\"{END_KEY}\"\n",
        "\n",
        "    # Create a list of prompt template elements\n",
        "    parts = [part for part in [blurb, instruction, input_context, response, end] if part]\n",
        "\n",
        "    # Join prompt template elements into a single string to create the prompt template\n",
        "    formatted_prompt = \"\\n\\n\".join(parts)\n",
        "\n",
        "    # Store the formatted prompt template in a new key \"text\"\n",
        "    sample[\"text\"] = formatted_prompt\n",
        "\n",
        "    return sample"
      ],
      "metadata": {
        "id": "KgLM5FR0mSP2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "create_prompt_formats(dataset[randrange(len(dataset))])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iL1qmnMER4nH",
        "outputId": "ddb9b3b6-3fef-41a1-d513-5df7f4c58af1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'instruction': 'Categorize the input article into one of the three categories\\nColon_Cancer\\nLung_Cancer\\nThyroid_Cancer\\n\\n.',\n",
              " 'input': '\"The lower replication rate of adipose meQTLs in whole-blood samples6 might be explained by the heterogeneity of different cell types in whole blood and by their more liberal P-value threshold (8.6Â\\x9710?4) which led to the identification of a large number of weak cis-meQTLs. Compared with cis-regulation trans-eQTL regulation is typically considered to be more complex has smaller effect sizes and is more difficult to be replicated even in the same tissue. However in our study the lung trans-meQTLs are highly reproducible in TCGA lung breast and kidney tissues. Notably this similarity allows mapping meQTLs with substantially improved power by borrowing strength across tissues51. meQTL SNPs are strongly associated with multiple epigenetic marks. Chromatin regulators play a role in maintaining genomic integrity and anization52. We found that meQTL SNPs were strongly enriched for DNase hypersensitive sites and sequences bound by CTCF or modified histones. SNPs could affect these epigenetic marks by several mechanisms such as by affecting the core recognition sequences (exemplified for rs2816057 on chromosome 1 for CTCF) causing loss or gain of a CpG within a binding region which when methylated could affect binding27 or altering the binding sequence for interacting factors53. CTCF could cause changes in epigenetic marks through its multiple key roles including genome anization through mediating intra- and inter-chromosomal contacts5455 the regulation of transcription by binding between enhancers and promoters5456 and the regulation of splicing which may impact tissue specificity during tissue development39. These changes can impact regulation of distant genes and not the genes proximal to the SNPs that would be typically investigated in eQTL studies. This may be one reason for the previously observed lack of correlation between eQTLs and meQTLs347. Future large studies integrating SNP profiles the DNA methylome and transcriptome data through tissue developmental stages will hopefully shed light on this possibility. There may be a myriad of other DNA-binding factors whose binding is directly or indirectly affected by SNPs. For example among the histone marks the strongest enrichment of meQTLs in our study was for H3K4me3 in both SAEC and hAEC cell types. As H3K4me3 is the chromatin mark primarily associated with regulatory elements at promoters and enhancers this suggests a strong influence of meQTLs on regulating gene activity. Unfortunately transcription factor binding data in SAEC or hAEC are not available so we could not test whether SNPs in their core sequence could affect the deposition of epigenetic marks e.g. by recruiting DNA methyltransferases57. It will be important to obtain ChIP data from relevant primary cells for numerous DNA-binding regulatory factors to further elucidate the mechanisms whereby meQTLs and other SNP-affected epigenetic marks arise. In  we show here that genetic variation has a profound impact on the DNA methylome with implications for cancer risk tissue specificity and chromatin structure and anization. The meQTL data (Supplementary Data) attached to this manuscript provides an important resource for studying genetic-DNA methylation interactions in lung tissue. Methods Sample collection We assayed 244 fresh frozen paired tumor and non-involved lung tissue samples from Stage I to IIIA non-small cell lung cancer (NSCLC) cases from the Environment And Genetics in Lung cancer Etiology (EAGLE) study18. EAGLE includes 2100 incident lung cancer cases and 2120 population controls enrolled in 2002Â\\x932005 within 216 municipalities of the Lombardy region of Italy. Cases were newly diagnosed primary cancers of lung trachea and bronchus verified by tissue pathology (67.0%) cytology (28.0%) or review of clinical records (5.0%). They were 35?79 years of age at diagnosis and were recruited from 13 hospitals which cover over 80% of the lung cancer cases from the study area. The study was approved by local and NCI Institutional Review Boards and all participants signed an informed consent form. Lung tissue samples were snap-frozen in liquid nitrogen within 20 minutes of surgical resection. Surgeons and pathologists were together in the surgery room at the time of resection and sample collection to ensure correct sampling of tissue from the tumor the area adjacent to the tumor and an additional area distant from the tumor (1Â\\x935 cm). The precise site of tissue sampling was indicated on a lung drawing and the pathologists classified the samples as tumor adjacent lung tissue and distant non-involved lung tissue. For the current study we used lung tissue sampled from an area distant from the tumor to reduce the potential effects of field cancerization. DNA methylation profiling and data quality control Fresh frozen lung tissue samples remained frozen while approximately 30 mg was subsampled for DNA extraction into pre-chilled 2.0 ml microcentrifuge tubes. Lysates for DNA extraction were generated by incubating 30 mg of tissue in 1 ml of 0.2 mg/ml Proteinase K (Ambion) in DNA Lysis Buffer (10 mM Tris-Cl (pH 8.0) 0.1 M EDTA (pH 8.0) and 0.5% (w/v) SDS) for 24 hrs at 56Â°C with shaking at 850 rpm in Thermomixer R (Eppendorf). DNA was extracted from the generated lysate using the QIAamp DNA Blood Maxi Kit (Qiagen) according to the manufacturerÂ\\x99s protocol. Bisulfite treatment and Illumina Infinium HumanMethylation450 BeadChip assays were performed by the Southern California Genotyping Consortium at the University of California Los Angeles (UCLA) following IlluminaÂ\\x99s protocols. This assay generates DNA methylation data for 485512 cytosine targets (482421 CpG and 3091 CpH) and 65 SNP probes for the purpose of data quality control. Raw methylated and unmethylated intensities were background corrected and dye-bias equalized to correct for technical variation in signal between arrays. For background correction we applied a normal-exponential convolution using the intensity of the Infinium I probes in the channel opposite their design to measure non-specific signal58. \"',\n",
              " 'output': 'Lung_Cancer',\n",
              " 'text': 'Below is an instruction that describes a task. Write a response that appropriately completes the request.\\n\\n### Instruction:\\nCategorize the input article into one of the three categories\\nColon_Cancer\\nLung_Cancer\\nThyroid_Cancer\\n\\n.\\n\\nInput:\\n\"The lower replication rate of adipose meQTLs in whole-blood samples6 might be explained by the heterogeneity of different cell types in whole blood and by their more liberal P-value threshold (8.6Â\\x9710?4) which led to the identification of a large number of weak cis-meQTLs. Compared with cis-regulation trans-eQTL regulation is typically considered to be more complex has smaller effect sizes and is more difficult to be replicated even in the same tissue. However in our study the lung trans-meQTLs are highly reproducible in TCGA lung breast and kidney tissues. Notably this similarity allows mapping meQTLs with substantially improved power by borrowing strength across tissues51. meQTL SNPs are strongly associated with multiple epigenetic marks. Chromatin regulators play a role in maintaining genomic integrity and anization52. We found that meQTL SNPs were strongly enriched for DNase hypersensitive sites and sequences bound by CTCF or modified histones. SNPs could affect these epigenetic marks by several mechanisms such as by affecting the core recognition sequences (exemplified for rs2816057 on chromosome 1 for CTCF) causing loss or gain of a CpG within a binding region which when methylated could affect binding27 or altering the binding sequence for interacting factors53. CTCF could cause changes in epigenetic marks through its multiple key roles including genome anization through mediating intra- and inter-chromosomal contacts5455 the regulation of transcription by binding between enhancers and promoters5456 and the regulation of splicing which may impact tissue specificity during tissue development39. These changes can impact regulation of distant genes and not the genes proximal to the SNPs that would be typically investigated in eQTL studies. This may be one reason for the previously observed lack of correlation between eQTLs and meQTLs347. Future large studies integrating SNP profiles the DNA methylome and transcriptome data through tissue developmental stages will hopefully shed light on this possibility. There may be a myriad of other DNA-binding factors whose binding is directly or indirectly affected by SNPs. For example among the histone marks the strongest enrichment of meQTLs in our study was for H3K4me3 in both SAEC and hAEC cell types. As H3K4me3 is the chromatin mark primarily associated with regulatory elements at promoters and enhancers this suggests a strong influence of meQTLs on regulating gene activity. Unfortunately transcription factor binding data in SAEC or hAEC are not available so we could not test whether SNPs in their core sequence could affect the deposition of epigenetic marks e.g. by recruiting DNA methyltransferases57. It will be important to obtain ChIP data from relevant primary cells for numerous DNA-binding regulatory factors to further elucidate the mechanisms whereby meQTLs and other SNP-affected epigenetic marks arise. In  we show here that genetic variation has a profound impact on the DNA methylome with implications for cancer risk tissue specificity and chromatin structure and anization. The meQTL data (Supplementary Data) attached to this manuscript provides an important resource for studying genetic-DNA methylation interactions in lung tissue. Methods Sample collection We assayed 244 fresh frozen paired tumor and non-involved lung tissue samples from Stage I to IIIA non-small cell lung cancer (NSCLC) cases from the Environment And Genetics in Lung cancer Etiology (EAGLE) study18. EAGLE includes 2100 incident lung cancer cases and 2120 population controls enrolled in 2002Â\\x932005 within 216 municipalities of the Lombardy region of Italy. Cases were newly diagnosed primary cancers of lung trachea and bronchus verified by tissue pathology (67.0%) cytology (28.0%) or review of clinical records (5.0%). They were 35?79 years of age at diagnosis and were recruited from 13 hospitals which cover over 80% of the lung cancer cases from the study area. The study was approved by local and NCI Institutional Review Boards and all participants signed an informed consent form. Lung tissue samples were snap-frozen in liquid nitrogen within 20 minutes of surgical resection. Surgeons and pathologists were together in the surgery room at the time of resection and sample collection to ensure correct sampling of tissue from the tumor the area adjacent to the tumor and an additional area distant from the tumor (1Â\\x935 cm). The precise site of tissue sampling was indicated on a lung drawing and the pathologists classified the samples as tumor adjacent lung tissue and distant non-involved lung tissue. For the current study we used lung tissue sampled from an area distant from the tumor to reduce the potential effects of field cancerization. DNA methylation profiling and data quality control Fresh frozen lung tissue samples remained frozen while approximately 30 mg was subsampled for DNA extraction into pre-chilled 2.0 ml microcentrifuge tubes. Lysates for DNA extraction were generated by incubating 30 mg of tissue in 1 ml of 0.2 mg/ml Proteinase K (Ambion) in DNA Lysis Buffer (10 mM Tris-Cl (pH 8.0) 0.1 M EDTA (pH 8.0) and 0.5% (w/v) SDS) for 24 hrs at 56Â°C with shaking at 850 rpm in Thermomixer R (Eppendorf). DNA was extracted from the generated lysate using the QIAamp DNA Blood Maxi Kit (Qiagen) according to the manufacturerÂ\\x99s protocol. Bisulfite treatment and Illumina Infinium HumanMethylation450 BeadChip assays were performed by the Southern California Genotyping Consortium at the University of California Los Angeles (UCLA) following IlluminaÂ\\x99s protocols. This assay generates DNA methylation data for 485512 cytosine targets (482421 CpG and 3091 CpH) and 65 SNP probes for the purpose of data quality control. Raw methylated and unmethylated intensities were background corrected and dye-bias equalized to correct for technical variation in signal between arrays. For background correction we applied a normal-exponential convolution using the intensity of the Infinium I probes in the channel opposite their design to measure non-specific signal58. \"\\n\\n### Response:\\nLung_Cancer\\n\\n### End'}"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def get_max_length(model):\n",
        "    \"\"\"\n",
        "    Extracts maximum token length from the model configuration\n",
        "\n",
        "    :param model: Hugging Face model\n",
        "    \"\"\"\n",
        "\n",
        "    # Pull model configuration\n",
        "    conf = model.config\n",
        "    # Initialize a \"max_length\" variable to store maximum sequence length as null\n",
        "    max_length = None\n",
        "    # Find maximum sequence length in the model configuration and save it in \"max_length\" if found\n",
        "    for length_setting in [\"n_positions\", \"max_position_embeddings\", \"seq_length\"]:\n",
        "        max_length = getattr(model.config, length_setting, None)\n",
        "        if max_length:\n",
        "            print(f\"Found max lenth: {max_length}\")\n",
        "            break\n",
        "    # Set \"max_length\" to 1024 (default value) if maximum sequence length is not found in the model configuration\n",
        "    if not max_length:\n",
        "        max_length = 1024\n",
        "        print(f\"Using default max length: {max_length}\")\n",
        "    return max_length"
      ],
      "metadata": {
        "id": "q6bAo-8nmtFn"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def preprocess_batch(batch, tokenizer, max_length):\n",
        "    \"\"\"\n",
        "    Tokenizes dataset batch\n",
        "\n",
        "    :param batch: Dataset batch\n",
        "    :param tokenizer: Model tokenizer\n",
        "    :param max_length: Maximum number of tokens to emit from the tokenizer\n",
        "    \"\"\"\n",
        "\n",
        "    return tokenizer(\n",
        "        batch[\"text\"],\n",
        "        max_length = max_length,\n",
        "        truncation = True,\n",
        "    )"
      ],
      "metadata": {
        "id": "C7vwFmJhmt6t"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def preprocess_dataset(tokenizer: AutoTokenizer, max_length: int, seed, dataset: str):\n",
        "    \"\"\"\n",
        "    Tokenizes dataset for fine-tuning\n",
        "\n",
        "    :param tokenizer (AutoTokenizer): Model tokenizer\n",
        "    :param max_length (int): Maximum number of tokens to emit from the tokenizer\n",
        "    :param seed: Random seed for reproducibility\n",
        "    :param dataset (str): Instruction dataset\n",
        "    \"\"\"\n",
        "\n",
        "    # Add prompt to each sample\n",
        "    print(\"Preprocessing dataset...\")\n",
        "    dataset = dataset.map(create_prompt_formats)\n",
        "\n",
        "    # Apply preprocessing to each batch of the dataset & and remove \"instruction\", \"input\", \"output\", and \"text\" fields\n",
        "    _preprocessing_function = partial(preprocess_batch, max_length = max_length, tokenizer = tokenizer)\n",
        "    dataset = dataset.map(\n",
        "        _preprocessing_function,\n",
        "        batched = True,\n",
        "        remove_columns = [\"instruction\", \"input\", \"output\", \"text\"],\n",
        "    )\n",
        "\n",
        "    # Filter out samples that have \"input_ids\" exceeding \"max_length\"\n",
        "    dataset = dataset.filter(lambda sample: len(sample[\"input_ids\"]) < max_length)\n",
        "\n",
        "    # Shuffle dataset\n",
        "    dataset = dataset.shuffle(seed = seed)\n",
        "\n",
        "    return dataset"
      ],
      "metadata": {
        "id": "S9UkOnqgmvlZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!huggingface-cli login"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "efee10ce-c028-4a19-d14d-d99e8c78ee47",
        "id": "lequWYyA2vQf"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "    _|    _|  _|    _|    _|_|_|    _|_|_|  _|_|_|  _|      _|    _|_|_|      _|_|_|_|    _|_|      _|_|_|  _|_|_|_|\n",
            "    _|    _|  _|    _|  _|        _|          _|    _|_|    _|  _|            _|        _|    _|  _|        _|\n",
            "    _|_|_|_|  _|    _|  _|  _|_|  _|  _|_|    _|    _|  _|  _|  _|  _|_|      _|_|_|    _|_|_|_|  _|        _|_|_|\n",
            "    _|    _|  _|    _|  _|    _|  _|    _|    _|    _|    _|_|  _|    _|      _|        _|    _|  _|        _|\n",
            "    _|    _|    _|_|      _|_|_|    _|_|_|  _|_|_|  _|      _|    _|_|_|      _|        _|    _|    _|_|_|  _|_|_|_|\n",
            "    \n",
            "    A token is already saved on your machine. Run `huggingface-cli whoami` to get more information or `huggingface-cli logout` if you want to log out.\n",
            "    Setting a new token will erase the existing one.\n",
            "    To login, `huggingface_hub` requires a token generated from https://huggingface.co/settings/tokens .\n",
            "Token: \n",
            "Add token as git credential? (Y/n) Y\n",
            "Token is valid (permission: read).\n",
            "\u001b[1m\u001b[31mCannot authenticate through git-credential as no helper is defined on your machine.\n",
            "You might have to re-authenticate when pushing to the Hugging Face Hub.\n",
            "Run the following command in your terminal in case you want to set the 'store' credential helper as default.\n",
            "\n",
            "git config --global credential.helper store\n",
            "\n",
            "Read https://git-scm.com/book/en/v2/Git-Tools-Credential-Storage for more details.\u001b[0m\n",
            "Token has not been saved to git credential helper.\n",
            "Your token has been saved to /root/.cache/huggingface/token\n",
            "Login successful\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Load model from Hugging Face Hub with model name and bitsandbytes configuration\n",
        "\n",
        "bnb_config = create_bnb_config(load_in_4bit, bnb_4bit_use_double_quant, bnb_4bit_quant_type, bnb_4bit_compute_dtype)\n",
        "\n",
        "model, tokenizer = load_model(model_name, bnb_config)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 104,
          "referenced_widgets": [
            "5fbec24b8a6d433c86ed828b1decee16",
            "dff443f9edb242afa5e7147bc5d0d3cb",
            "7e485d1d8e7548ac9e6ce50fa3c7ee0e",
            "d228042d99a14d3280f6b8202b8be3fa",
            "bdf94f4e12a041529d8cd90fe8b35e2b",
            "eaf2fa3e79ca44d4b6ef725fde5fd630",
            "223f5ee8fe7242728b3658a9958e1870",
            "1d6ed8f2688b45b996c3de45330f8d84",
            "4aee6aa741ec4881af795c921ea59461",
            "016b2a48d2c14d82bfc087328964fcc0",
            "76b56151f2c449cb8393dd4a282de4ea"
          ]
        },
        "id": "Pb6Q94ZttDEB",
        "outputId": "dc07645d-5e10-428f-e54c-14d0af271faa"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Loading checkpoint shards:   0%|          | 0/14 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "5fbec24b8a6d433c86ed828b1decee16"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py:1714: FutureWarning: The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers.\n",
            "  warnings.warn(\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Random seed\n",
        "seed = 33\n",
        "\n",
        "max_length = get_max_length(model)\n",
        "preprocessed_dataset = preprocess_dataset(tokenizer, max_length, seed, dataset)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 148,
          "referenced_widgets": [
            "02c5802b93484806983cc69a79a042c3",
            "c04ae58d651743c8bc50050d0e1fa565",
            "6abeef6405904fcc8f8f492d1f75f776",
            "0123c5c251c948248ed88f14951c74d7",
            "646f9524cae1462f8c1a075a654bbe9a",
            "86e525ce807b4afc9ec33ca47598a239",
            "151fc41391b24f89ace9d5f7c2dda938",
            "5a0a62834ac9426f9d9f0836e1961741",
            "3ec50f3584734ce6a13743f95dad1523",
            "dda57fd9fb574f4eb07c167fdaeb7c56",
            "06440cb51b4445d0a8a595895d758ba7",
            "9bf10ab2d16d4b4c920a6e7ce1fb7309",
            "325418c4a8594e568c14a6055ce9cda9",
            "c0faa5acaa684081b317c19aa8242dc1",
            "6f99853f5e834f8eb99d274a50bb8e6e",
            "32386d08705b49469d528cf6699aec68",
            "c6cd08532f7b47d28f917d5c42b4ce87",
            "815015a7d3e2424e8b4343260dcc81bc",
            "d363bf05282043f4b02f20f01c31cbcf",
            "8eabf48f3d3643ad87dc7c843eedcc30",
            "e4554ebad1cc4d5f90a738086350e991",
            "4c90bee51934431f94e664f8a88836d7",
            "19250960968645ae944b5d926a3cce37",
            "b5f2884ca58e4ac0a6bd6acc0b756034",
            "b759687a8dc445869215c1595d68c7e3",
            "dfc4602675f946528a1601c9397cecff",
            "cfc7ee23b87a48adbadec3150be3427a",
            "61fd2b8872bd45ee9850ab731bd083b0",
            "999593bab1704c54bf6c18b9b4c6774c",
            "8ef98168b10046c3a81ac6ebdebbaa42",
            "0c8451cc5e0844689002ac179fd42819",
            "e81751c422af490d9e5094d947fe4bac",
            "b7979df9a42a4578ab20abc0ebccadf0"
          ]
        },
        "id": "50XY9r6ssu-x",
        "outputId": "a3d880b8-1371-43e7-fb93-bf108d47fba5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Found max lenth: 2048\n",
            "Preprocessing dataset...\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map:   0%|          | 0/796 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "02c5802b93484806983cc69a79a042c3"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map:   0%|          | 0/796 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "9bf10ab2d16d4b4c920a6e7ce1fb7309"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Filter:   0%|          | 0/796 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "19250960968645ae944b5d926a3cce37"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(preprocessed_dataset)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "C-aACsuus0ps",
        "outputId": "36b084a6-8ebc-4221-ca98-1a5f38486d1c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Dataset({\n",
            "    features: ['input_ids', 'attention_mask'],\n",
            "    num_rows: 111\n",
            "})\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(preprocessed_dataset[0])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6wN2y7yAIfN1",
        "outputId": "fddd0d9e-239b-4f25-daab-a056f8682a46"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'input_ids': [1, 13866, 338, 385, 15278, 393, 16612, 263, 3414, 29889, 14350, 263, 2933, 393, 7128, 2486, 1614, 2167, 278, 2009, 29889, 13, 13, 2277, 29937, 2799, 4080, 29901, 13, 29907, 20440, 675, 278, 1881, 4274, 964, 697, 310, 278, 2211, 13997, 13, 1625, 265, 29918, 6028, 2265, 13, 29931, 686, 29918, 6028, 2265, 13, 1349, 29891, 1007, 29918, 6028, 2265, 13, 13, 29889, 13, 13, 4290, 29901, 13, 29908, 23036, 26229, 310, 278, 7744, 573, 323, 3038, 2933, 9251, 756, 1063, 28585, 304, 19224, 2820, 2462, 29871, 29896, 29900, 30212, 30344, 29896, 29946, 1400, 325, 5753, 3381, 411, 2087, 29899, 29902, 29875, 29899, 19903, 470, 2087, 29899, 19903, 372, 471, 10579, 15668, 304, 1284, 393, 3517, 325, 5753, 3381, 1258, 451, 4556, 738, 4328, 297, 18530, 4603, 1090, 1438, 5855, 518, 29896, 29906, 29962, 518, 29906, 29896, 1822, 2398, 445, 3117, 9432, 29879, 393, 372, 4893, 777, 931, 363, 278, 325, 5753, 457, 29899, 19910, 1133, 3677, 2101, 29899, 14940, 323, 9101, 304, 18414, 5987, 297, 8002, 3694, 297, 278, 301, 3085, 1584, 472, 278, 3171, 310, 278, 5198, 1540, 2933, 29889, 6549, 565, 278, 301, 3085, 892, 23968, 515, 22829, 14914, 286, 625, 29871, 29896, 29896, 3841, 1156, 21622, 272, 297, 542, 2785, 263, 2821, 29899, 7582, 4328, 297, 278, 18530, 4603, 471, 1476, 746, 17420, 325, 5753, 262, 630, 322, 1661, 29899, 29894, 5753, 262, 630, 286, 625, 29889, 910, 13284, 304, 2761, 21622, 272, 18530, 4603, 471, 19949, 363, 472, 3203, 263, 4098, 322, 3117, 5520, 1156, 325, 5753, 3381, 29889, 9788, 591, 1476, 393, 565, 286, 625, 892, 18066, 287, 472, 278, 19224, 310, 278, 7601, 7307, 29947, 323, 3038, 2933, 694, 901, 1135, 29871, 29946, 3841, 892, 3734, 304, 22222, 263, 8359, 4328, 1546, 325, 5753, 262, 630, 322, 443, 29894, 5753, 262, 630, 286, 625, 29889, 323, 9424, 4208, 1438, 2582, 22366, 393, 410, 11461, 433, 20009, 325, 5753, 3381, 411, 2087, 29899, 19903, 470, 2087, 29899, 29902, 29875, 29899, 19903, 297, 3153, 508, 12566, 2750, 9232, 273, 4125, 1539, 579, 25101, 304, 278, 301, 3085, 29889, 8797, 10230, 310, 1749, 660, 29899, 9026, 29934, 1158, 304, 28557, 7418, 21677, 21248, 310, 1539, 579, 2454, 1701, 455, 1090, 263, 766, 2042, 20710, 1883, 4338, 17845, 393, 21622, 272, 2761, 1033, 367, 17809, 8859, 491, 1749, 716, 1158, 29889, 6549, 1539, 579, 2454, 1701, 455, 1033, 451, 367, 29115, 411, 3058, 1017, 2745, 2462, 29871, 29945, 30212, 30344, 29953, 1156, 18766, 1550, 263, 20376, 297, 21622, 272, 18530, 4603, 1122, 2307, 367, 8900, 491, 2462, 29871, 29946, 29889, 5853, 4100, 368, 408, 372, 1033, 367, 28705, 393, 20376, 310, 21622, 272, 18530, 4603, 1795, 451, 9432, 12212, 21622, 272, 14321, 591, 723, 763, 304, 22884, 393, 8943, 2582, 892, 7625, 746, 21622, 272, 1539, 579, 25101, 471, 19030, 1090, 278, 1021, 5855, 773, 278, 1023, 1422, 1158, 11763, 29889, 11637, 278, 1556, 4100, 4328, 1546, 278, 1023, 1223, 1036, 12185, 297, 278, 2114, 393, 1550, 7604, 1663, 27988, 871, 1136, 11376, 3694, 310, 1539, 579, 2454, 1701, 455, 17126, 310, 2159, 278, 660, 29899, 9026, 29934, 2948, 4893, 964, 3633, 451, 871, 3694, 310, 1701, 455, 541, 884, 3694, 310, 9101, 639, 1539, 579, 2454, 8569, 4550, 12021, 292, 263, 901, 4323, 23378, 8954, 310, 21622, 272, 3038, 14321, 29889, 910, 4328, 1122, 367, 310, 3153, 29527, 749, 297, 18845, 988, 278, 3677, 277, 398, 272, 2779, 1090, 22522, 338, 758, 24130, 10835, 429, 814, 287, 472, 263, 5683, 7408, 1156, 1539, 579, 25101, 3265, 1135, 3646, 292, 278, 1539, 579, 2454, 21622, 272, 3038, 3528, 29889, 512, 24899, 936, 6055, 25300, 936, 28744, 310, 7601, 21622, 943, 756, 1063, 28585, 304, 7910, 278, 12045, 310, 21622, 272, 1539, 579, 25101, 1951, 372, 508, 1121, 297, 278, 6507, 310, 21622, 272, 9101, 964, 278, 18342, 362, 518, 29906, 29953, 29962, 30212, 30344, 29961, 29906, 29929, 1822, 739, 756, 884, 1063, 1476, 393, 25300, 708, 1122, 27391, 1539, 579, 25101, 26377, 368, 491, 9013, 3277, 14321, 310, 758, 29899, 342, 370, 3726, 20710, 456, 300, 579, 25101, 1549, 278, 20376, 310, 9418, 29899, 574, 29875, 6352, 293, 13879, 1549, 278, 7910, 310, 5492, 14321, 13879, 408, 1532, 408, 491, 278, 21445, 310, 5198, 12609, 14889, 23881, 8581, 491, 278, 16083, 8792, 470, 23603, 3372, 1230, 22884, 322, 14919, 21549, 518, 29906, 29953, 1822, 24428, 304, 286, 326, 293, 278, 6434, 310, 325, 5753, 3381, 21699, 411, 25300, 1711, 20974, 1539, 579, 25101, 286, 625, 892, 325, 5753, 262, 630, 322, 21622, 272, 18066, 287, 373, 278, 1021, 2462, 322, 301, 3085, 892, 23968, 29871, 29896, 29896, 3841, 2678, 29889, 23829, 11687, 591, 1476, 263, 2821, 4328, 297, 9232, 273, 4125, 18530, 4603, 1546, 325, 5753, 262, 630, 322, 1661, 29899, 29894, 5753, 262, 630, 286, 625, 29889, 512, 6124, 2087, 29899, 29902, 29875, 29899, 19903, 325, 5753, 262, 630, 286, 625, 10018, 16710, 13047, 2750, 1539, 579, 25101, 9401, 304, 2087, 29899, 19903, 29889, 4525, 2582, 4368, 393, 297, 263, 24899, 936, 4444, 325, 5753, 3381, 411, 263, 4608, 1316, 408, 2087, 29899, 29902, 29875, 29899, 19903, 21984, 411, 25300, 936, 28744, 310, 278, 7601, 21622, 272, 1122, 23228, 368, 10032, 278, 4188, 22342, 310, 1539, 579, 2454, 9677, 8581, 491, 278, 25300, 708, 29889, 9788, 3517, 11898, 505, 4318, 393, 278, 2087, 29899, 29902, 29875, 29899, 19903, 29899, 19910, 1133, 2761, 310, 7773, 269, 29889, 29883, 29889, 350, 29896, 29953, 29889, 29943, 29896, 29900, 29887, 29886, 21622, 943, 338, 14278, 373, 7307, 29947, 323, 9101, 322, 10762, 29940, 29973, 16684, 758, 24130, 10835, 297, 263, 1513, 13460, 373, 278, 21622, 272, 9101, 518, 29896, 29941, 29962, 1213, 13, 13, 2277, 29937, 13291, 29901, 13, 29931, 686, 29918, 6028, 2265, 13, 13, 2277, 29937, 2796], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def create_peft_config(r, lora_alpha, target_modules, lora_dropout, bias, task_type):\n",
        "    \"\"\"\n",
        "    Creates Parameter-Efficient Fine-Tuning configuration for the model\n",
        "\n",
        "    :param r: LoRA attention dimension\n",
        "    :param lora_alpha: Alpha parameter for LoRA scaling\n",
        "    :param modules: Names of the modules to apply LoRA to\n",
        "    :param lora_dropout: Dropout Probability for LoRA layers\n",
        "    :param bias: Specifies if the bias parameters should be trained\n",
        "    \"\"\"\n",
        "    config = LoraConfig(\n",
        "        r = r,\n",
        "        lora_alpha = lora_alpha,\n",
        "        target_modules = target_modules,\n",
        "        lora_dropout = lora_dropout,\n",
        "        bias = bias,\n",
        "        task_type = task_type,\n",
        "    )\n",
        "\n",
        "    return config"
      ],
      "metadata": {
        "id": "913uHanlnYef"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def find_all_linear_names(model):\n",
        "    \"\"\"\n",
        "    Find modules to apply LoRA to.\n",
        "\n",
        "    :param model: PEFT model\n",
        "    \"\"\"\n",
        "\n",
        "    cls = bnb.nn.Linear4bit\n",
        "    lora_module_names = set()\n",
        "    for name, module in model.named_modules():\n",
        "        if isinstance(module, cls):\n",
        "            names = name.split('.')\n",
        "            lora_module_names.add(names[0] if len(names) == 1 else names[-1])\n",
        "\n",
        "    if 'lm_head' in lora_module_names:\n",
        "        lora_module_names.remove('lm_head')\n",
        "    print(f\"LoRA module names: {list(lora_module_names)}\")\n",
        "    return list(lora_module_names)"
      ],
      "metadata": {
        "id": "dg8pIUgMm_bX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def print_trainable_parameters(model, use_4bit = False):\n",
        "    \"\"\"\n",
        "    Prints the number of trainable parameters in the model.\n",
        "\n",
        "    :param model: PEFT model\n",
        "    \"\"\"\n",
        "\n",
        "    trainable_params = 0\n",
        "    all_param = 0\n",
        "\n",
        "    for _, param in model.named_parameters():\n",
        "        num_params = param.numel()\n",
        "        if num_params == 0 and hasattr(param, \"ds_numel\"):\n",
        "            num_params = param.ds_numel\n",
        "        all_param += num_params\n",
        "        if param.requires_grad:\n",
        "            trainable_params += num_params\n",
        "\n",
        "    if use_4bit:\n",
        "        trainable_params /= 2\n",
        "\n",
        "    print(\n",
        "        f\"All Parameters: {all_param:,d} || Trainable Parameters: {trainable_params:,d} || Trainable Parameters %: {100 * trainable_params / all_param}\"\n",
        "    )"
      ],
      "metadata": {
        "id": "F6iEs6pVnCac"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip3 install langchain openai\n",
        "!pip install openai==0.27.7 wandb\n",
        "!pip3 install langchain openai"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "X-vX5C4CSUNf",
        "outputId": "3a3255d4-3d6d-4e61-e83c-99d2d8a37716"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: langchain in /usr/local/lib/python3.10/dist-packages (0.0.271)\n",
            "Requirement already satisfied: openai in /usr/local/lib/python3.10/dist-packages (0.27.7)\n",
            "Requirement already satisfied: PyYAML>=5.3 in /usr/local/lib/python3.10/dist-packages (from langchain) (6.0.1)\n",
            "Requirement already satisfied: SQLAlchemy<3,>=1.4 in /usr/local/lib/python3.10/dist-packages (from langchain) (2.0.20)\n",
            "Requirement already satisfied: aiohttp<4.0.0,>=3.8.3 in /usr/local/lib/python3.10/dist-packages (from langchain) (3.8.5)\n",
            "Requirement already satisfied: async-timeout<5.0.0,>=4.0.0 in /usr/local/lib/python3.10/dist-packages (from langchain) (4.0.3)\n",
            "Requirement already satisfied: dataclasses-json<0.6.0,>=0.5.7 in /usr/local/lib/python3.10/dist-packages (from langchain) (0.5.14)\n",
            "Requirement already satisfied: langsmith<0.1.0,>=0.0.21 in /usr/local/lib/python3.10/dist-packages (from langchain) (0.0.26)\n",
            "Requirement already satisfied: numexpr<3.0.0,>=2.8.4 in /usr/local/lib/python3.10/dist-packages (from langchain) (2.8.5)\n",
            "Requirement already satisfied: numpy<2,>=1 in /usr/local/lib/python3.10/dist-packages (from langchain) (1.23.5)\n",
            "Requirement already satisfied: pydantic<3,>=1 in /usr/local/lib/python3.10/dist-packages (from langchain) (2.2.0)\n",
            "Requirement already satisfied: requests<3,>=2 in /usr/local/lib/python3.10/dist-packages (from langchain) (2.31.0)\n",
            "Requirement already satisfied: tenacity<9.0.0,>=8.1.0 in /usr/local/lib/python3.10/dist-packages (from langchain) (8.2.3)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from openai) (4.66.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (23.1.0)\n",
            "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (3.2.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (6.0.4)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.9.2)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.4.0)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.3.1)\n",
            "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in /usr/local/lib/python3.10/dist-packages (from dataclasses-json<0.6.0,>=0.5.7->langchain) (3.20.1)\n",
            "Requirement already satisfied: typing-inspect<1,>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from dataclasses-json<0.6.0,>=0.5.7->langchain) (0.9.0)\n",
            "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1->langchain) (0.5.0)\n",
            "Requirement already satisfied: pydantic-core==2.6.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1->langchain) (2.6.0)\n",
            "Requirement already satisfied: typing-extensions>=4.6.1 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1->langchain) (4.7.1)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain) (2.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain) (2023.7.22)\n",
            "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.10/dist-packages (from SQLAlchemy<3,>=1.4->langchain) (2.0.2)\n",
            "Requirement already satisfied: packaging>=17.0 in /usr/local/lib/python3.10/dist-packages (from marshmallow<4.0.0,>=3.18.0->dataclasses-json<0.6.0,>=0.5.7->langchain) (23.1)\n",
            "Requirement already satisfied: mypy-extensions>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from typing-inspect<1,>=0.4.0->dataclasses-json<0.6.0,>=0.5.7->langchain) (1.0.0)\n",
            "Requirement already satisfied: openai==0.27.7 in /usr/local/lib/python3.10/dist-packages (0.27.7)\n",
            "Requirement already satisfied: wandb in /usr/local/lib/python3.10/dist-packages (0.15.8)\n",
            "Requirement already satisfied: requests>=2.20 in /usr/local/lib/python3.10/dist-packages (from openai==0.27.7) (2.31.0)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from openai==0.27.7) (4.66.1)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from openai==0.27.7) (3.8.5)\n",
            "Requirement already satisfied: Click!=8.0.0,>=7.1 in /usr/local/lib/python3.10/dist-packages (from wandb) (8.1.7)\n",
            "Requirement already satisfied: GitPython!=3.1.29,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from wandb) (3.1.32)\n",
            "Requirement already satisfied: psutil>=5.0.0 in /usr/local/lib/python3.10/dist-packages (from wandb) (5.9.5)\n",
            "Requirement already satisfied: sentry-sdk>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from wandb) (1.29.2)\n",
            "Requirement already satisfied: docker-pycreds>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from wandb) (0.4.0)\n",
            "Requirement already satisfied: PyYAML in /usr/local/lib/python3.10/dist-packages (from wandb) (6.0.1)\n",
            "Requirement already satisfied: pathtools in /usr/local/lib/python3.10/dist-packages (from wandb) (0.1.2)\n",
            "Requirement already satisfied: setproctitle in /usr/local/lib/python3.10/dist-packages (from wandb) (1.3.2)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from wandb) (67.7.2)\n",
            "Requirement already satisfied: appdirs>=1.4.3 in /usr/local/lib/python3.10/dist-packages (from wandb) (1.4.4)\n",
            "Requirement already satisfied: protobuf!=4.21.0,<5,>=3.19.0 in /usr/local/lib/python3.10/dist-packages (from wandb) (3.20.3)\n",
            "Requirement already satisfied: six>=1.4.0 in /usr/local/lib/python3.10/dist-packages (from docker-pycreds>=0.4.0->wandb) (1.16.0)\n",
            "Requirement already satisfied: gitdb<5,>=4.0.1 in /usr/local/lib/python3.10/dist-packages (from GitPython!=3.1.29,>=1.0.0->wandb) (4.0.10)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai==0.27.7) (3.2.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai==0.27.7) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai==0.27.7) (2.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai==0.27.7) (2023.7.22)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai==0.27.7) (23.1.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai==0.27.7) (6.0.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai==0.27.7) (4.0.3)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai==0.27.7) (1.9.2)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai==0.27.7) (1.4.0)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai==0.27.7) (1.3.1)\n",
            "Requirement already satisfied: smmap<6,>=3.0.1 in /usr/local/lib/python3.10/dist-packages (from gitdb<5,>=4.0.1->GitPython!=3.1.29,>=1.0.0->wandb) (5.0.0)\n",
            "Requirement already satisfied: langchain in /usr/local/lib/python3.10/dist-packages (0.0.271)\n",
            "Requirement already satisfied: openai in /usr/local/lib/python3.10/dist-packages (0.27.7)\n",
            "Requirement already satisfied: PyYAML>=5.3 in /usr/local/lib/python3.10/dist-packages (from langchain) (6.0.1)\n",
            "Requirement already satisfied: SQLAlchemy<3,>=1.4 in /usr/local/lib/python3.10/dist-packages (from langchain) (2.0.20)\n",
            "Requirement already satisfied: aiohttp<4.0.0,>=3.8.3 in /usr/local/lib/python3.10/dist-packages (from langchain) (3.8.5)\n",
            "Requirement already satisfied: async-timeout<5.0.0,>=4.0.0 in /usr/local/lib/python3.10/dist-packages (from langchain) (4.0.3)\n",
            "Requirement already satisfied: dataclasses-json<0.6.0,>=0.5.7 in /usr/local/lib/python3.10/dist-packages (from langchain) (0.5.14)\n",
            "Requirement already satisfied: langsmith<0.1.0,>=0.0.21 in /usr/local/lib/python3.10/dist-packages (from langchain) (0.0.26)\n",
            "Requirement already satisfied: numexpr<3.0.0,>=2.8.4 in /usr/local/lib/python3.10/dist-packages (from langchain) (2.8.5)\n",
            "Requirement already satisfied: numpy<2,>=1 in /usr/local/lib/python3.10/dist-packages (from langchain) (1.23.5)\n",
            "Requirement already satisfied: pydantic<3,>=1 in /usr/local/lib/python3.10/dist-packages (from langchain) (2.2.0)\n",
            "Requirement already satisfied: requests<3,>=2 in /usr/local/lib/python3.10/dist-packages (from langchain) (2.31.0)\n",
            "Requirement already satisfied: tenacity<9.0.0,>=8.1.0 in /usr/local/lib/python3.10/dist-packages (from langchain) (8.2.3)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from openai) (4.66.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (23.1.0)\n",
            "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (3.2.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (6.0.4)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.9.2)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.4.0)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.3.1)\n",
            "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in /usr/local/lib/python3.10/dist-packages (from dataclasses-json<0.6.0,>=0.5.7->langchain) (3.20.1)\n",
            "Requirement already satisfied: typing-inspect<1,>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from dataclasses-json<0.6.0,>=0.5.7->langchain) (0.9.0)\n",
            "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1->langchain) (0.5.0)\n",
            "Requirement already satisfied: pydantic-core==2.6.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1->langchain) (2.6.0)\n",
            "Requirement already satisfied: typing-extensions>=4.6.1 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1->langchain) (4.7.1)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain) (2.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain) (2023.7.22)\n",
            "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.10/dist-packages (from SQLAlchemy<3,>=1.4->langchain) (2.0.2)\n",
            "Requirement already satisfied: packaging>=17.0 in /usr/local/lib/python3.10/dist-packages (from marshmallow<4.0.0,>=3.18.0->dataclasses-json<0.6.0,>=0.5.7->langchain) (23.1)\n",
            "Requirement already satisfied: mypy-extensions>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from typing-inspect<1,>=0.4.0->dataclasses-json<0.6.0,>=0.5.7->langchain) (1.0.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%env OPENAI_API_KEY=sk-"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0CeKUFCiSrcb",
        "outputId": "06a878a3-43ca-4349-fa62-27adfb15d5b7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "env: OPENAI_API_KEY=sk-\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "KUykGNtfS08D"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def fine_tune(model,\n",
        "          tokenizer,\n",
        "          dataset,\n",
        "          lora_r,\n",
        "          lora_alpha,\n",
        "          lora_dropout,\n",
        "          bias,\n",
        "          task_type,\n",
        "          per_device_train_batch_size,\n",
        "          gradient_accumulation_steps,\n",
        "          warmup_steps,\n",
        "          max_steps,\n",
        "          learning_rate,\n",
        "          fp16,\n",
        "          logging_steps,\n",
        "          output_dir,\n",
        "          optim):\n",
        "    \"\"\"\n",
        "    Prepares and fine-tune the pre-trained model.\n",
        "\n",
        "    :param model: Pre-trained Hugging Face model\n",
        "    :param tokenizer: Model tokenizer\n",
        "    :param dataset: Preprocessed training dataset\n",
        "    \"\"\"\n",
        "\n",
        "    # Enable gradient checkpointing to reduce memory usage during fine-tuning\n",
        "    model.gradient_checkpointing_enable()\n",
        "\n",
        "    # Prepare the model for training\n",
        "    model = prepare_model_for_kbit_training(model)\n",
        "\n",
        "    # Get LoRA module names\n",
        "    target_modules = find_all_linear_names(model)\n",
        "\n",
        "    # Create PEFT configuration for these modules and wrap the model to PEFT\n",
        "    peft_config = create_peft_config(lora_r, lora_alpha, target_modules, lora_dropout, bias, task_type)\n",
        "    model = get_peft_model(model, peft_config)\n",
        "\n",
        "    # Print information about the percentage of trainable parameters\n",
        "    print_trainable_parameters(model)\n",
        "\n",
        "    # Training parameters\n",
        "    trainer = Trainer(\n",
        "        model = model,\n",
        "        train_dataset = dataset,\n",
        "        args = TrainingArguments(\n",
        "            per_device_train_batch_size = per_device_train_batch_size,\n",
        "            gradient_accumulation_steps = gradient_accumulation_steps,\n",
        "            warmup_steps = warmup_steps,\n",
        "            max_steps = max_steps,\n",
        "            learning_rate = learning_rate,\n",
        "            fp16 = fp16,\n",
        "            logging_steps = logging_steps,\n",
        "            output_dir = output_dir,\n",
        "            optim = optim,\n",
        "        ),\n",
        "        data_collator = DataCollatorForLanguageModeling(tokenizer, mlm = False)\n",
        "    )\n",
        "\n",
        "    model.config.use_cache = False\n",
        "\n",
        "    do_train = True\n",
        "\n",
        "    # Launch training and log metrics\n",
        "    print(\"Training...\")\n",
        "\n",
        "    if do_train:\n",
        "        train_result = trainer.train()\n",
        "        metrics = train_result.metrics\n",
        "        trainer.log_metrics(\"train\", metrics)\n",
        "        trainer.save_metrics(\"train\", metrics)\n",
        "        trainer.save_state()\n",
        "        print(metrics)\n",
        "\n",
        "    # Save model\n",
        "    print(\"Saving last checkpoint of the model...\")\n",
        "    os.makedirs(output_dir, exist_ok = True)\n",
        "    trainer.model.save_pretrained(output_dir)\n",
        "\n",
        "    # Free memory for merging weights\n",
        "    del model\n",
        "    del trainer\n",
        "    torch.cuda.empty_cache()"
      ],
      "metadata": {
        "id": "tCbnhnxtnhvh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "################################################################################\n",
        "# QLoRA parameters\n",
        "################################################################################\n",
        "\n",
        "# LoRA attention dimension\n",
        "lora_r = 16\n",
        "\n",
        "# Alpha parameter for LoRA scaling\n",
        "lora_alpha = 64\n",
        "\n",
        "# Dropout probability for LoRA layers\n",
        "lora_dropout = 0.1\n",
        "\n",
        "# Bias\n",
        "bias = \"none\"\n",
        "\n",
        "# Task type\n",
        "task_type = \"CAUSAL_LM\""
      ],
      "metadata": {
        "id": "FxC2aY8eUzH7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "################################################################################\n",
        "# TrainingArguments parameters\n",
        "################################################################################\n",
        "\n",
        "# Output directory where the model predictions and checkpoints will be stored\n",
        "output_dir = \"/content/drive/MyDrive/result_new_2\"\n",
        "\n",
        "# Batch size per GPU for training\n",
        "per_device_train_batch_size = 1\n",
        "\n",
        "# Number of update steps to accumulate the gradients for\n",
        "gradient_accumulation_steps = 4\n",
        "\n",
        "# Initial learning rate (AdamW optimizer)\n",
        "learning_rate = 2e-4\n",
        "\n",
        "# Optimizer to use\n",
        "optim = \"paged_adamw_32bit\"\n",
        "\n",
        "# Number of training steps (overrides num_train_epochs)\n",
        "max_steps = 20\n",
        "\n",
        "# Linear warmup steps from 0 to learning_rate\n",
        "warmup_steps = 2\n",
        "\n",
        "# Enable fp16/bf16 training (set bf16 to True with an A100)\n",
        "fp16 = True\n",
        "\n",
        "# Log every X updates steps\n",
        "logging_steps = 1"
      ],
      "metadata": {
        "id": "nnfOKEvipXLG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "fine_tune(model,\n",
        "      tokenizer,\n",
        "      preprocessed_dataset,\n",
        "      lora_r,\n",
        "      lora_alpha,\n",
        "      lora_dropout,\n",
        "      bias,\n",
        "      task_type,\n",
        "      per_device_train_batch_size,\n",
        "      gradient_accumulation_steps,\n",
        "      warmup_steps,\n",
        "      max_steps,\n",
        "      learning_rate,\n",
        "      fp16,\n",
        "      logging_steps,\n",
        "      output_dir,\n",
        "      optim)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "rRJZ6L7qpPIM",
        "outputId": "76fbd163-420f-479f-ba19-29dec5a3c3e2"
      },
      "execution_count": null,
      "outputs": [
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "LoRA module names: ['gate_proj', 'v_proj', 'down_proj', 'q_proj', 'up_proj', 'k_proj', 'o_proj']\n",
            "All Parameters: 3,540,389,888 || Trainable Parameters: 39,976,960 || Trainable Parameters %: 1.1291682911958425\n",
            "Training...\n"
          ]
        },
        {
          "data": {
            "application/javascript": [
              "\n",
              "        window._wandbApiKey = new Promise((resolve, reject) => {\n",
              "            function loadScript(url) {\n",
              "            return new Promise(function(resolve, reject) {\n",
              "                let newScript = document.createElement(\"script\");\n",
              "                newScript.onerror = reject;\n",
              "                newScript.onload = resolve;\n",
              "                document.body.appendChild(newScript);\n",
              "                newScript.src = url;\n",
              "            });\n",
              "            }\n",
              "            loadScript(\"https://cdn.jsdelivr.net/npm/postmate/build/postmate.min.js\").then(() => {\n",
              "            const iframe = document.createElement('iframe')\n",
              "            iframe.style.cssText = \"width:0;height:0;border:none\"\n",
              "            document.body.appendChild(iframe)\n",
              "            const handshake = new Postmate({\n",
              "                container: iframe,\n",
              "                url: 'https://wandb.ai/authorize'\n",
              "            });\n",
              "            const timeout = setTimeout(() => reject(\"Couldn't auto authenticate\"), 5000)\n",
              "            handshake.then(function(child) {\n",
              "                child.on('authorize', data => {\n",
              "                    clearTimeout(timeout)\n",
              "                    resolve(data)\n",
              "                });\n",
              "            });\n",
              "            })\n",
              "        });\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Logging into wandb.ai. (Learn how to deploy a W&B server locally: https://wandb.me/wandb-server)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: You can find your API key in your browser here: https://wandb.ai/authorize\n",
            "wandb: Paste an API key from your profile and hit enter, or press ctrl+c to quit:"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " ··········\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "Tracking run with wandb version 0.15.8"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "Run data is saved locally in <code>/content/wandb/run-20230823_175047-ccai77og</code>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "Syncing run <strong><a href='https://wandb.ai/samidataset/huggingface/runs/ccai77og' target=\"_blank\">ethereal-snowball-4</a></strong> to <a href='https://wandb.ai/samidataset/huggingface' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              " View project at <a href='https://wandb.ai/samidataset/huggingface' target=\"_blank\">https://wandb.ai/samidataset/huggingface</a>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              " View run at <a href='https://wandb.ai/samidataset/huggingface/runs/ccai77og' target=\"_blank\">https://wandb.ai/samidataset/huggingface/runs/ccai77og</a>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "You're using a LlamaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='20' max='20' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [20/20 10:44, Epoch 0/1]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Step</th>\n",
              "      <th>Training Loss</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>2.087700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>1.934600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>1.692300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>1.735100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>1.502000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>1.502900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>1.514100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>1.925100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>1.889200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>1.544900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>1.507800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>1.560800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>1.588300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>1.491400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>1.379400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>1.768200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>1.576800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>1.666500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>2.327000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>1.617200</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "***** train metrics *****\n",
            "  epoch                    =       0.72\n",
            "  total_flos               =  2281077GF\n",
            "  train_loss               =     1.6906\n",
            "  train_runtime            = 0:15:57.62\n",
            "  train_samples_per_second =      0.084\n",
            "  train_steps_per_second   =      0.021\n",
            "{'train_runtime': 957.621, 'train_samples_per_second': 0.084, 'train_steps_per_second': 0.021, 'total_flos': 2449288063918080.0, 'train_loss': 1.69057697057724, 'epoch': 0.72}\n",
            "Saving last checkpoint of the model...\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Load fine-tuned weights\n",
        "model = AutoPeftModelForCausalLM.from_pretrained(output_dir, device_map = \"auto\", torch_dtype = torch.bfloat16)\n",
        "# Merge the LoRA layers with the base model\n",
        "model = model.merge_and_unload()\n",
        "\n",
        "# Save fine-tuned model at a new location\n",
        "output_merged_dir = \"/content/drive/MyDrive/merged_result/news_classification_llama2_7b/final_merged_checkpoint\"\n",
        "os.makedirs(output_merged_dir, exist_ok = True)\n",
        "model.save_pretrained(output_merged_dir, safe_serialization = True)\n",
        "\n",
        "# Save tokenizer for easy inference\n",
        "tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
        "tokenizer.save_pretrained(output_merged_dir)"
      ],
      "metadata": {
        "id": "kXMHbUeq0gRn",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 101,
          "referenced_widgets": [
            "aa904ca6d9574f048e3521943d3ec478",
            "ca360b3361624630a080bf2c548ecaec",
            "2608a665b1684a81979a3ebce713ae98",
            "42f9477a3a4c4a3ca6845a613b878bb2",
            "a9d98bbd85a34fed9c8c228bde08d6dd",
            "97410e6200f447fd8d525d79420f08f1",
            "ac31b8fedca14baf915ebfd9c7278c2c",
            "097cd288bb674328899799cb47c4a534",
            "f49e8e866fd541f9b061464d4e628dc7",
            "abc3a6650e9b433b91a1a74ea76949e6",
            "5730521e3daa47c3ac1b94bb8c841a5e"
          ]
        },
        "outputId": "92b124df-10c1-4cb9-f45f-160c693fad40"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Loading checkpoint shards:   0%|          | 0/14 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "aa904ca6d9574f048e3521943d3ec478"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "('/content/drive/MyDrive/merged_result/news_classification_llama2_7b/final_merged_checkpoint/tokenizer_config.json',\n",
              " '/content/drive/MyDrive/merged_result/news_classification_llama2_7b/final_merged_checkpoint/special_tokens_map.json',\n",
              " '/content/drive/MyDrive/merged_result/news_classification_llama2_7b/final_merged_checkpoint/tokenizer.json')"
            ]
          },
          "metadata": {},
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model"
      ],
      "metadata": {
        "id": "DALr5u8V4drE",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "33194829-1d8a-4159-c16e-f1a9cd2414a5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "LlamaForCausalLM(\n",
              "  (model): LlamaModel(\n",
              "    (embed_tokens): Embedding(32000, 4096, padding_idx=0)\n",
              "    (layers): ModuleList(\n",
              "      (0-31): 32 x LlamaDecoderLayer(\n",
              "        (self_attn): LlamaAttention(\n",
              "          (q_proj): Linear4bit(in_features=4096, out_features=4096, bias=False)\n",
              "          (k_proj): Linear4bit(in_features=4096, out_features=4096, bias=False)\n",
              "          (v_proj): Linear4bit(in_features=4096, out_features=4096, bias=False)\n",
              "          (o_proj): Linear4bit(in_features=4096, out_features=4096, bias=False)\n",
              "          (rotary_emb): LlamaRotaryEmbedding()\n",
              "        )\n",
              "        (mlp): LlamaMLP(\n",
              "          (gate_proj): Linear4bit(in_features=4096, out_features=11008, bias=False)\n",
              "          (up_proj): Linear4bit(in_features=4096, out_features=11008, bias=False)\n",
              "          (down_proj): Linear4bit(in_features=11008, out_features=4096, bias=False)\n",
              "          (act_fn): SiLUActivation()\n",
              "        )\n",
              "        (input_layernorm): LlamaRMSNorm()\n",
              "        (post_attention_layernorm): LlamaRMSNorm()\n",
              "      )\n",
              "    )\n",
              "    (norm): LlamaRMSNorm()\n",
              "  )\n",
              "  (lm_head): Linear(in_features=4096, out_features=32000, bias=False)\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "MB1DRaVA7EGQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer"
      ],
      "metadata": {
        "id": "KOsIH8F14fFj",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "27029532-2685-4d08-ba11-24a361380e82"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "LlamaTokenizerFast(name_or_path='TinyPixel/Llama-2-7B-bf16-sharded', vocab_size=32000, model_max_length=1000000000000000019884624838656, is_fast=True, padding_side='left', truncation_side='right', special_tokens={'bos_token': AddedToken(\"<s>\", rstrip=False, lstrip=False, single_word=False, normalized=True), 'eos_token': AddedToken(\"</s>\", rstrip=False, lstrip=False, single_word=False, normalized=True), 'unk_token': AddedToken(\"<unk>\", rstrip=False, lstrip=False, single_word=False, normalized=True)}, clean_up_tokenization_spaces=False)"
            ]
          },
          "metadata": {},
          "execution_count": 42
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import AutoModelForSequenceClassification, AutoTokenizer\n",
        "\n",
        "output_merged_dir = \"/content/drive/MyDrive/merged_result/news_classification_llama2_7b/final_merged_checkpoint\"\n",
        "model = AutoModelForSequenceClassification.from_pretrained(output_merged_dir)\n",
        "tokenizer = AutoTokenizer.from_pretrained(output_merged_dir)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 104,
          "referenced_widgets": [
            "a0b393129f944a8391d9b6e579aea447",
            "3e0c3235642d4012a1af1089b79dedb5",
            "19eee33dcb2845dfa631e7a27e315b5a",
            "09a968fae2c64ea0b61396af29077a2a",
            "587d5d8eebb94a35a3de6bda2167fba0",
            "4a4f20cf6113485eae718684ab46007e",
            "6dea2bf72db2486d9800049eb4c6a054",
            "60b4abf673284b9db7cd1a27326593d8",
            "e7a6338df52347f090f17bc8400069e6",
            "957d5f71596342f387da7ccb2efc623d",
            "f8f6bc6d1a34423695aff499c670833f"
          ]
        },
        "id": "If8e0ReKLEj5",
        "outputId": "cdd97e08-c9d3-4731-c154-d01037662857"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "a0b393129f944a8391d9b6e579aea447"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of LlamaForSequenceClassification were not initialized from the model checkpoint at /content/drive/MyDrive/merged_result/news_classification_llama2_7b/final_merged_checkpoint and are newly initialized: ['score.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test_texts=test_df['text'][1]\n",
        "print(test_texts)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dhNtHLz7LhfM",
        "outputId": "85746e74-6fa1-4414-ebe7-d997188e3e1f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\"confidence. If you choose not to take part in this survey it will not affect the care you receive from the NHS in any way. Please do not write your name and address anywhere on the questionnaire as this information is not required. No information you give in this questionnaire will be shared in a way that allows you to be identified. How to complete the survey and how long it will take. The questionnaire is short and will take 5Â10?min to complete. Please try to answer every question. Please return your questionnaire even if you have not answered every question. If English is not your first language or if you if you have difficulty understanding the questions then please ask a relative or carer to help you complete the questionnaire. Questions or help? If you have any questions please contact your local lung clinical nurse specialist team. Please select one answer to each question by placing a in the appropriate box. There is space at the end of the survey for you to write any comments. This work is published under the standard license to publish agreement. After 12 months the work will become freely available and the license terms will switch to a Creative Commons Attribution-NonCommercial-Share Alike 3.0 Unported License. Abdel-Rahman M Stockton D Rachet B Hakulinen T Coleman MP 2009 What if cancer survival in Britain were the same as in Europe: how many deaths are avoidable Br J Cancer 101 (Suppl 2 S115 S124 19956155 Aveling EL Martin G Jimnez GarcÂ­a S Martin L Herbert G Armstrong N Dixon-Woods M Woolhouse I 2012 Reciprocal peer review for quality improvement: an ethnographic case study of the Improving Lung Cancer Outcomes Project BMJ Qual Saf 21 1034 1041 Beckett P Woolhouse I Stanley R Peake MD 2012 Exploring variations in lung cancer care across the UK-the Âstory so far' for the National Lung Cancer Audit Clin Med 12 14 18 22372213 Department of Health2012National Cancer Patients' Experience Survey Programme 2012/13. England. Health And Social Care Information Centre2012National Lung Cancer Audit Report. Institute for Healthcare Improvement2003The Breakthrough Series: IHI's Collaborative Model for Achieving Breakthrough Improvement. Boston. Khakwani A Rich AL Powell HA Tata LJ Stanley RA Baldwin DR Duffy JP Hubbard RB 2013 Lung cancer survival in England: trends in non-small-cell lung cancer survival over the duration of the National Lung Cancer Audit Br J Cancer 109 (8 2058 2065 24052044 Kwon S Florence M Grigas P Horton M Horvath K Johnson M Jurkovich G Klamp W Peterson K Quigley T Raum W Rogers T Thirlby R Farrokhi E Flum D 2012 Creating a learning healthcare system in surgery: Washington State's Surgical Care and Outcomes Assessment Program (SCOAP) at 5 years Surgery 151 146 152 22129638 National Institute for Health and Care Excellence 2011 The Diagnosis And Treatment Of Lung Cancer (Update Of Nice Clinical Guideline 24) Clinical guidelines CG121 London UK Pronovost P Needham D Berenholtz S Sinopoli D Chu H Cosgrove S Sexton B Hyzy R Welsh R Roth G Bander J Kepros J Goeschel C 2006 An intervention to decrease catheter-related bloodstream infections in the ICU N Engl J Med 355 2725 2732 17192537 Roberts CM Stone RA Buckingham RJ Pursey NA Lowe D Potter JM 2012 A randomized trial of peer review: the UK National Chronic Obstructive Pulmonary Disease Resources and Outcomes Project: three-year evaluation J Eval Clin Pract 18 (3 599 605 21332611 Walters S Maringe C Coleman MP Peake MD Butler J Young N BergstrÂ¶m S Hanna L Jakobsen E KÂ¶lbeck K SundstrÂ¸m S Engholm G Gavin A Gjerstorff ML Hatcher J Johannesen TB Linklater KM McGahan CE Steward J Tracey E Turner D Richards MA Rachet B ICBP Module 1 Working Group 2013 Lung cancer survival and stage at diagnosis in Australia Canada Denmark Norway Sweden and the UK: a population-based study 2004-2007 Thorax 68 551 564 23399908 Wise J 2010 Health atlas shows large variations in care in England BMJ 341 c6809 c6809  Study timelines.  Consort diagram disposal of eligible trusts including screening randomisation and follow-up.  Run chart showing the waiting times from the multidisciplinary team meeting to the first treatment for 10 consecutive small-cell lung cancer patients following the implementation of the quality improvement plan at one trust in the intervention group.  Mean change in national lung cancer audit metrics from baseline (2009) to 2011. P=0.055 active treatmentÂintervention vs controls. Intervention n=31 trusts control n=47 trusts and non-intervention (control and non-participants combined) n=66 trusts. Abbreviations: CNS clinical nurse specialist; MDT multidisciplinary team; SCLC small-cell lung cancer.  Total patient questionnaire scores by the multidisciplinary team in the intervention group at baseline (pre) and at the end of the study (post). A low score indicates better experience. Each symbol represents the mean score for each trust in the intervention group. The maximum possible score for the questionnaire is 11.  Quality improvement plan themes Quality improvement plan theme Number of plans Multidisciplinary team effectiveness 31 Diagnostic pathways 13 Treatment pathways 9 Access to clinical nurse specialists 8 Clinical trial recruitment 4 Patient experience 2  Baseline (2009) national lung cancer audit indicators   Control ( n =47) Intervention ( n =31) Excluded ( n =67) P -value   Mean (%) s.e.m. Mean (%) s.e.m. Mean (%) s.e.m. Control vs intervention vs non-participant control vs intervention Case ascertainment 158.1 38.6 122.0 7.2 107.4 3.6 0.220 0.455 Discussed at the MDT meeting 95.2 0.7 93.7 1.7 90.9 1.9 0.155 0.370 Histological confirmation rate 75.7 1.2 76.4 1.8 78.4 1.6 0.409 0.739 Active treatment 59.5 1.2 55.9 2.2 59.5 1.5 0.305 0.131 Surgery (all cases) 13.4 0.6 13.0 0.8 14.2 0.7 0.469 0.648 SCLC (chemo) 65.1 2.2 66.5 3.9 63.3 2.7 0.746 0.733 Seen by CNS 70.3 3.8 76.6 3.2 58.3 4.2 0.007 0.243 CNS present diagnosis 44.0 3.8 49.4 5.4 38.7 3.8 0.237 0.403 Abbreviations: CNS=clinical nurse specialist; MDT=mulitdisciplinary team; SCLC=small-cell lung cancer. Data are shown as mean and s.e. proportion of patients. BMC Cancer BMC Cancer BMC Cancer 1471-2407 BioMed Central 24386906 3893473 1471-2407-14-3 10.1186/1471-2407-14-3 Study Protocol Study protocol of a randomized controlled trial comparing Mindfulness-Based Stress Reduction with treatment as usual in reducing psychological distress in patients with lung cancer and their partners: the MILON study Schellekens Melanie PJ 1 Melanie.Schellekensradboudumc.nl van den Hurk Desiree GM 2 Desiree.vandenHurkradboudumc.nl Prins Judith B 3 Judith.Prinsradboudumc.nl Molema Johan 2 Johan.Molemaradboudumc.nl Donders A Rogier T 4 Rogier.Dondersradboudumc.nl Woertman Willem H 4 Willem.Woertmanradboudumc.nl van der Drift Miep A 2 Miep.vanderDriftradboudumc.nl Speckens Anne EM 1 Anne.Speckensradboudumc.nl 1Department of Psychiatry Radboud University Nijmegen Medical Centre Nijmegen The Netherlands 2Department of Pulmonary Diseases Radboud University Nijmegen Medical Centre Nijmegen The Netherlands 3Department of Medical Psychology Radboud University Nijmegen Medical Centre Nijmegen The Netherlands 4Department of Epidemiology Biostatistics and Health Technology Assessment Radboud University Nijmegen Medical Centre Nijmegen The Netherlands 2014 3 1 2014 14 3 3 28 6 2013 19 12 2013 Copyright  2014 Schellekens et al.; licensee BioMed Central Ltd. 2014 Schellekens et al.; licensee BioMed Central Ltd. This is an open access  distributed under the terms of the Creative Commons Attribution License (http://creativecommons./licenses/by/2.0) which permits unrestricted use distribution and reproduction in any medium provided the original work is properly cited. Background Lung cancer is the leading cause of cancer death worldwide and characterized by a poor prognosis. It has a major impact on the psychological wellbeing of patients and their partners. Recently it has been shown that Mindfulness-Based Stress Reduction (MBSR) is effective in reducing anxiety and depressive symptoms in cancer patients. The generalization of these results is limited since most participants were female patients with breast cancer. Moreover only one study examined the effectiveness of MBSR in partners of cancer patients. Therefore in the present trial we study the effectiveness of MBSR versus treatment as usual (TAU) in patients with lung cancer and their partners. Methods/Design A parallel group randomized controlled trial is conducted to compare MBSR with TAU. Lung cancer patients who have received or are still under treatment and their partners are recruited. Assessments will take place at baseline post intervention and at three-month follow-up. The primary outcome is psychological distress (i.e. anxiety and depressive symptoms). Secondary outcomes are quality of life (only for patients) caregiver appraisal (only for partners) relationship quality and spirituality. In addition cost-effectiveness ratio (only in patients) and several process variables are assessed. Discussion This trial will provide information about the clinical and cost-effectiveness of MBSR compared to TAU in patients with lung cancer and their partners. Trial registration ClinicalTrials.gov NCT01494883. Mindfulness-based stress reduction Lung cancer patients Partners Psychological distress Randomized controlled trial Background With an estimated 1.4 million deaths per year lung cancer is the leading cause of death by cancer worldwide. Even with the best available treatment five-year survival is merely 16% and about 60 to 70% of patients die within the first year after diagnosis [1]. This poor prognosis is often caused by a late diagnosis as the presentation usually occurs when the lung cancer is advanced. Patients may develop burdensome symptoms like pain dyspnoea fatigue and cough and they may undergo radical treatment including surgery chemo- and radiotherapy. Not surprisingly lung cancer has a major impact on the psychological wellbeing of patients and their family. Akechi and colleagues [2] showed that 19% of patients with advanced lung cancer meets the criteria of psychiatric disorders especially depressive and adjustment disorders. Of patients who had been successfully treated for lung cancer 15% met the criteria for a minor or major depressive disorder [3]. The prevalence rate of depressive and anxiety symptoms among lung cancer patients ranges from 20 to 47% [4-7]. Compared to patients with other cancer diagnoses lung cancer patients report the highest rates of distress (43 to 58%) [89] resulting in a lower quality of life [10]. Family friends and especially partners of patients with lung cancer also have to deal with its psychological impact [11-14]. Partners not only provide emotional and practical support they also have to cope with their own concerns including the uncertainty regarding the course of the illness and the fear of losing their partner [15]. More than 50% of partners of lung cancer patients report negative emotional effects of caregiving [16]. Around 40% of partners of patients with advanced lung cancer report high levels of distress [17]. The relationship between patient and partner can also be affected by the cancer. It has been shown that some partners report a lower quality of their relationship after the diagnosis of lung cancer [18]. Though numerous studies examined the psychological distress of lung cancer patients and their partners [2-22] not much research is done on how to alleviate distress in these groups [23]. In addition the available studies on managing the psychosocial care needs of cancer patients and their families have focused on care at the very end of life (e.g. [24-26]). Recently studies have demonstrated that palliative care initiated early in treatment improves the quality of life and depressive symptoms of lung cancer patients [1027]. This stresses the importance of integrating psychosocial care for lung cancer patients and their partners early in the treatment rather than instigating it once life-prolonging therapies fail. In the past ten years MBSR has become a promising psychosocial intervention for cancer patients. Mindfulness is defined as intentionally paying attention to moment-by-moment experiences in a non-judgmental way [28]. MBSR is an 8-week group-based training consisting of meditation practices such as the bodyscan gentle yoga sitting and walking meditation. By repeatedly bringing attention back to the current experience participants gradually learn to disengage from dysfunctional thoughts and directly experience the emotions and bodily sensations of the present moment. MBSR aims to provide participants with the ability to step back from ruminating about the past or worrying about the future and simply allow experiences to unfold [2829]. A recent meta-analysis [30] of 13 nonrandomized studies and 9 randomized controlled trials (RCT) concluded there is positive evidence for the use of mindfulness-based interventions in reducing psychological distress in cancer patients. Among the RCTÂs a reduction in symptom severity was found for both anxiety and depression corresponding to moderate pooled controlled effect sizes (HedgesÂs g = 0.37 and HedgesÂs g = 0.44 respectively) [30]. Though mindfulness-based interventions seem to be effective the authors note that across studies the majority of participants were women (85%) and diagnosed with breast cancer (77%). Compared to breast cancer patients patients with lung cancer are more often male older and have a poorer prognosis. Furthermore of these 22 studies only one study included the partners of the patients showing that partners also benefit from the MBSR training [31]. This is quite surprising since partners of cancer patients also report high levels of distress [32]. Aims The aim of the Mindfulness for Lung Oncology Nijmegen (MILON) study is to examine the effectiveness of MBSR compared to TAU in reducing psychological distress in patients with lung cancer and their partners. We hypothesize that patients in the MBSR group will report a lower level of psychological distress (i.e. anxiety and depressive symptoms) higher levels of quality of life quality of relationship and spirituality than those in the TAU group. Medical and societal costs will be lower in the MBSR versus TAU group. We expect partners in the MBSR group to report a lower level of psychological distress and higher levels of caregiver appraisal relationship quality and spirituality than their counterparts in the TAU group. With regard to the working mechanisms of the MBSR programme we will examine changes in mindfulness skills self-compassion rumination intrusion avoidance and adherence to MBSR. Methods/Design Study design The design of the ÂMILONÂ study is a parallel group randomized controlled trial with an embedded process study. Participants are randomized between MBSR and TAU. The study protocol has been approved by our ethical review board (CMO Arnhem-Nijmegen) and registered under number 2011Â519. Participants and procedure Patients and partners are recruited at the outpatient \"\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Add the [PAD] token as a special token for padding\n",
        "tokenizer.add_special_tokens({'pad_token': '[PAD]'})\n",
        "\n",
        "input_text = test_texts\n",
        "inputs = tokenizer(input_text, return_tensors=\"pt\", padding=True, truncation=True)"
      ],
      "metadata": {
        "id": "PBN-YF9ULeV7",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "40d5ddac-0f7a-475f-a027-5f3195aef334"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Asking to truncate to max_length but no maximum length is provided and the model has no predefined maximum length. Default to no truncation.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "with torch.no_grad():\n",
        "    logits = model(**inputs).logits\n",
        "\n",
        "predicted_class_idx = logits.argmax().item()\n",
        "#predicted_class = idx2label[predicted_class_idx]\n",
        "print(\"Predicted class:\", predicted_class_idx)\n",
        "\n"
      ],
      "metadata": {
        "id": "8CQFEYtZMUzc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "test_df['label'][1]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hN6wFS2YM8iv",
        "outputId": "92a8fba7-1f77-46f8-d445-40817fc89b18"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "2"
            ]
          },
          "metadata": {},
          "execution_count": 63
        }
      ]
    }
  ]
}