submit_to_milton.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![analyse_all_the_things.png](attachment:analyse_all_the_things.png)\n",
    "# RUN ALL THE THINGS \n",
    "### on Milton\n",
    "\n",
    "\n",
    "This is a test interface between Jupyter Notebook and Milton. \n",
    "Requires: \n",
    "\n",
    "- ~~A working environment in your home area~~\n",
    "\n",
    "- A little knowledge\n",
    "\n",
    "- A litte luck\n",
    "\n",
    "### Work in progress. Still to do:\n",
    " - Check files and folders exist or don't depending on settings\n",
    "     - this often causes jobs to fail\n",
    " - Implement a file chooser? \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import sys\n",
    "from read_roi import read_roi_zip, read_roi_file #Need a kernel with this installed "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# First define your workflow settings\n",
    "\n",
    "\n",
    "Microscopy address:\n",
    "- /stornext/Img/data/prkfs1/m/Microscopy/\n",
    "\n",
    " GP Transfer\n",
    "- /stornext/General/scratch/GP_Transfer/\n",
    "\n",
    " Your Home Area\n",
    "- /home/users/allstaff/[your_user_name]/\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = {\n",
    "    \"input\":\"/stornext/Img/data/prkfs1/m/Microscopy/LLS/LLSZ/Cindy/20221109/RON3_CMXRos_SiRActin_R1-01.czi\",\n",
    "    \"output\":\"/vast/scratch/users/whitehead/\",\n",
    "    \"roi_file\":\"/stornext/Img/data/prkfs1/m/Microscopy/LLS/LLSZ/Cindy/20221109/RON3_CMXRos_SiRActin_R1-01_RoiSet.zip\",\n",
    "    \"workflow_path\":None,\n",
    "    \"processing\":\"crop\",\n",
    "    \"deconvolution\": \"cuda_gpu\",                           \n",
    "    \"deconvolution_num_iter\":20,\n",
    "    \"deconvolution_psf\":[\"/stornext/Img/data/prkfs1/m/Microscopy/LLS/LLSZ/Cindy/psfs/488.tif\",\n",
    "                         \"/stornext/Img/data/prkfs1/m/Microscopy/LLS/LLSZ/Cindy/psfs/561.tif\",\n",
    "                         \"/stornext/Img/data/prkfs1/m/Microscopy/LLS/LLSZ/Cindy/psfs/640.tif\",\n",
    "                         \"/stornext/Img/data/prkfs1/m/Microscopy/LLS/LLSZ/Cindy/psfs/640.tif\"],\n",
    "    \"time_range\":None, #Change to None to do all\n",
    "    \"channel_range\":None, #ditto    \n",
    "    \"output_file_type\":\"h5\"\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sanity checks \n",
    "##### If no errors, then should be good to continue. Otherwise go back and check above"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "if not os.path.exists(config[\"input\"]):\n",
    "    print (\"ERROR: Input file path is incorrect\")\n",
    "\n",
    "if not os.path.exists(config[\"output\"]):\n",
    "    print(\"ERROR: Output file path is incorrect\")\n",
    "    \n",
    "if \"roi_file\" in config and config[\"roi_file\"] is not None:\n",
    "    if not os.path.exists(config[\"roi_file\"]):\n",
    "        print(\"ERROR: ROI File path is incorrect\")\n",
    "    \n",
    "if \"workflow_path\" in config and config[\"workflow_path\"] is not None:\n",
    "    if not os.path.exists(config[\"workflow_path\"]):\n",
    "        print(\"ERROR: Workflow file path is incorrect\")\n",
    "        \n",
    "if \"deconvolution_psf\" in config:\n",
    "    for idx,psf in enumerate(config[\"deconvolution_psf\"]):\n",
    "        if not os.path.exists(psf):\n",
    "            print(\"ERROR: PSF file path %i is incorrect\" % idx)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "# Next Decide on your Milton Settings "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "milton_config = {\n",
    "    'job-name':'jupyter_submission_test',\n",
    "    #how long does one \"job\" take\n",
    "    #Note that job will fail if it goes over this time\n",
    "    \"time\":\"4:00:00\",  #hours:minutes:secounds\n",
    "    #logFile output\n",
    "    \"logfiles_path\":\"/vast/scratch/users/whitehead/outputs/\", #make sure this exists\n",
    "    #cpus per task\n",
    "    \"cpus-per-task\":12,\n",
    "    #GB memory required\n",
    "    \"mem\":128,\n",
    "    #email \n",
    "    \"email_address\":\"whitehead@wehi.edu.au\",\n",
    "    \"use_gpu_nodes\":True,\n",
    "    #This is a shared and tested environment - but you can change if you have a local one\n",
    "    #\"conda_env_name\":\"/stornext/Img/data/prkfs1/m/Microscopy/BAC_Conda_envs/napari_milton/\",\n",
    "    \"conda_env_name\":\"napari_dev\",\n",
    "}\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sanity Checks 2 \n",
    "#### Again address any errors above"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.exists(milton_config[\"logfiles_path\"]):\n",
    "    print(\"ERROR: Logfiles path does not exist\")\n",
    "\n",
    "if not os.path.exists(milton_config[\"conda_env_name\"]):\n",
    "    print(\"ERROR: Can't find Conda Environment\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### If you're using roi_file, this is handy:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_rois(config_settings):\n",
    "    if \"roi_file\" in config_settings.keys() and \"roi_file\" != None:    \n",
    "        roi_file_path = config_settings[\"roi_file\"]\n",
    "        if os.path.exists(roi_file_path):\n",
    "            print(\"Found roi file\")\n",
    "            try:\n",
    "                rois = read_roi_zip(roi_file_path)\n",
    "            except: \n",
    "                rois = read_roi_file(roi_file_path)\n",
    "            nrois = len(list(rois.keys()))\n",
    "            print(\"There are %i rois\" % nrois)\n",
    "            return nrois\n",
    "        else:\n",
    "            print(\"Roi file defined but not found\")\n",
    "            return False\n",
    "    else:\n",
    "        print(\"No roi file found\")\n",
    "        return False    \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### This function makes a config file to send to napari_lattice --config "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def write_config_file(config_settings,output_file_location):\n",
    "    with open(output_file_location,'w') as f:\n",
    "        for key,val in config_settings.items():    \n",
    "            if val is not None:\n",
    "                if type(val) is str:\n",
    "                    print('%s: \"%s\"' % (key,val),file=f)\n",
    "\n",
    "                if type(val) is int:\n",
    "                    print('%s: %i' % (key,val),file=f)\n",
    "\n",
    "                if type(val) is list:\n",
    "                    print(\"%s:\" % key,file=f)\n",
    "                    for x in val:\n",
    "                        if type(x) is int:\n",
    "                            print(\" - %i\" % x,file=f)\n",
    "                        else:\n",
    "                            print(' - \"%s\"' % x,file=f)\n",
    "            \n",
    "    print(\"Config found written to %s\" % output_file_location)\n",
    "                            "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### This one prepares the milton submission script"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def write_submission_sh(milton_config_settings,\n",
    "                        config_yml_location,\n",
    "                        parallelisation_keyword,\n",
    "                        n_jobs,\n",
    "                        output_file_location):\n",
    "    \n",
    "    with open(output_file_location,'w') as f:\n",
    "\n",
    "        print(\"#!/bin/bash\",file=f)\n",
    "        print(\"#SBATCH --job-name=%s\" % milton_config_settings['job-name'],file=f)\n",
    "        print(\"#SBATCH --time=%s\" % milton_config_settings['time'],file=f)\n",
    "        print(\"#SBATCH --ntasks=1\",file=f)\n",
    "        print(\"#SBATCH --output=%s%s_%%A_%%a.out\" % \n",
    "              (milton_config_settings['logfiles_path'],milton_config_settings['job-name']),file=f)\n",
    "        print(\"#SBATCH --cpus-per-task=%i\" % milton_config_settings['cpus-per-task'],file=f)\n",
    "        print(\"#SBATCH --mem=%iG\" % milton_config_settings['mem'],file=f)\n",
    "        print(\"#SBATCH --mail-type=END\",file=f)\n",
    "        print(\"#SBATCH --mail-use=%s\" % milton_config_settings['email_address'],file=f)\n",
    "        if(milton_config_settings['use_gpu_nodes']):\n",
    "            print(\"#SBATCH --partition=gpuq\",file=f)\n",
    "            print(\"#SBATCH --gres=gpu:A30:1\",file=f)\n",
    "        print(\"#SBATCH --array=0-%i\" % (n_jobs-1),file=f)\n",
    "\n",
    "\n",
    "        print(\"\",file=f)\n",
    "        print(\"source /stornext/System/data/apps/anaconda3/anaconda3-2019.03/etc/profile.d/conda.sh\",file=f)\n",
    "        print(\"conda activate %s\" % milton_config_settings['conda_env_name'],file=f)\n",
    "        print(\"\",file=f)\n",
    "\n",
    "        print(\"export TMPDIR=/vast/scratch/users/$USER/tmp\",file=f)\n",
    "        print(\"mkdir -p $TMPDIR\",file=f)\n",
    "\n",
    "        print(\"\",file=f)\n",
    "\n",
    "        #print(\"xvfb-run -s '-terminate' -d napari_lattice --config %s --%s $SLURM_ARRAY_TASK_ID\" % \n",
    "         #     (config_yml_location,parallelisation_keyword), file=f)\n",
    "        \n",
    "        print(\"lls-pipeline --config %s --%s $SLURM_ARRAY_TASK_ID\" % \n",
    "              (config_yml_location,parallelisation_keyword), file=f)\n",
    "        \n",
    "\n",
    "#write_submission_sh(milton_config,'/path/to/config.yml','roi_number',4,'asdf/test.sh')    \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Where are you going to put the settings files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "config_file = 'test/configtest.yml'\n",
    "submission_file = 'test/submission_test.sh'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Make the config file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "write_config_file(config,config_file)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Currently parallelising over roi_number - but can be changed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_rois = get_rois(config)\n",
    "#n_rois = 1\n",
    "\n",
    "write_submission_sh(milton_config,\n",
    "                    config_file,\n",
    "                    'roi_number',\n",
    "                    n_rois,\n",
    "                    submission_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SUBMIT TO MILTON!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!sbatch $submission_file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!squeue -u whitehead"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Things that might cause failures\n",
    "\n",
    " - forgot Xvfb import \n",
    " - Image files / roi files / workflow files don't exists where you've specified\n",
    " - Output files already exist\n",
    " - Something I've overlooked\n",
    " - Something I haven't overlooked, but neglected to fix\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (on_demand)",
   "language": "python",
   "name": "on_demand"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}