Scripts for traning models on idun

YOLO-detect-buoys/Job.slurm

@@ -0,0 +1,50 @@
+#!/bin/bash
+#SBATCH --partition=GPUQ
+#SBATCH --account=ie-idi
+#SBATCH --time=999:99:99
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=4
+#SBATCH --gres=gpu:a100:4
+#SBATCH --constraint="gpu40g|gpu80g|gpu32g"
+#SBATCH --job-name="vortex-img-process"
+#SBATCH --output=vortex_img_process_log.out
+#SBATCH --mem=32G
+
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/cluster/apps/eb/software/Python/3.10.4-GCCcore-11.3.0/lib/
+
+set -e
+
+module purge
+module --ignore_cache load foss/2022a
+module --ignore_cache load Python/3.10.4-GCCcore-11.3.0
+
+pip cache purge
+
+# makes sure that the pip is up to date
+python3 -m pip install --upgrade pip
+
+# Create a temporary virtual environment
+VENV_DIR=$(mktemp -d -t env-repaint-XXXXXXXXXX)
+python3 -m venv $VENV_DIR
+source $VENV_DIR/bin/activate
+
+pip install --upgrade pip 
+
+# install the required packages
+pip install -r requirements.txt
+#pip install pyyaml # used to read the configuration file
+#pip install blobfile # install blobfile to download the dataset
+#pip install kagglehub # install kagglehub to download the dataset
+pip install --force-reinstall torch -U 
+pip install torchvision torchaudio 
+#pip install diffusers transformers accelerate --user
+
+# Mixing expandable_segments:True with max_split_size doesn't make sense because the expandable segment is the size of RAM and so it could never be split with max_split_size.
+# export PYTORCH_CUDA_ALLOC_CONF="expandable_segments:True,max_split_size_mb:128"
+export PYTORCH_CUDA_ALLOC_CONF="expandable_segments:True"
+
+python3 train.py
+
+# Deactivate and remove the virtual environment
+deactivate
+rm -rf $VENV_DIR

YOLO-detect-buoys/__main__.py

@@ -2,26 +2,39 @@
 The main entry point for YOLO detecter buoys.
 """
 
-from os import getcwd, path
+from os import getcwd, getenv, path
 
+from dotenv import load_dotenv
+from huggingface_hub import HfApi, Repository
 from ultralytics import YOLO
 from utils import get_data, get_device, process_video
 
 if "__main__" == __name__:
 
+    load_dotenv()
+
     device = get_device()
+    print("device", device)
     model = YOLO("yolov8n.pt")
 
     file_path = path.abspath(getcwd())
     dataset = get_data()
 
     result = model.train(
-        data=dataset.location + "\\data.yaml", epochs=50, imgsz=640, device=device
+        data=dataset.location + "/data.yaml",
+        epochs=100,
+        imgsz=640,
+        device=device,
+        batch=4,
+        cache=False,
     )
 
     model.val()
 
-    process_video("https://youtu.be/4WGpIOwkLA4?feature=shared", model)
+    path = model.export(format="onnx")  # export to onnx
+
+    print("Model exported to: " + path)
+    print(path)
 
 # References:
 # https://docs.ultralytics.com/quickstart/#install-ultralytics

YOLO-detect-buoys/requirements.txt

@@ -0,0 +1,10 @@
+huggingface_hub==0.23.3
+numpy==1.26.4
+opencv_contrib_python==4.9.0.80
+opencv_python==4.9.0.80
+pafy==0.5.5
+python-dotenv==1.0.1
+roboflow==1.1.24
+torch==2.2.1
+protobuf==4.24.0
+ultralytics==8.0.196

unet_roboflow_training/Job.slurm

@@ -0,0 +1,50 @@
+#!/bin/bash
+#SBATCH --partition=GPUQ
+#SBATCH --account=ie-idi
+#SBATCH --time=999:99:99
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=4
+#SBATCH --gres=gpu:a100:4
+#SBATCH --constraint="gpu40g|gpu80g|gpu32g"
+#SBATCH --job-name="vortex-img-process"
+#SBATCH --output=vortex_img_process_log.out
+#SBATCH --mem=32G
+
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/cluster/apps/eb/software/Python/3.10.4-GCCcore-11.3.0/lib/
+
+set -e
+
+module purge
+module --ignore_cache load foss/2022a
+module --ignore_cache load Python/3.10.4-GCCcore-11.3.0
+
+pip cache purge
+
+# makes sure that the pip is up to date
+python3 -m pip install --upgrade pip
+
+# Create a temporary virtual environment
+VENV_DIR=$(mktemp -d -t env-repaint-XXXXXXXXXX)
+python3 -m venv $VENV_DIR
+source $VENV_DIR/bin/activate
+
+pip install --upgrade pip 
+
+# install the required packages
+pip install -r requirements.txt
+#pip install pyyaml # used to read the configuration file
+#pip install blobfile # install blobfile to download the dataset
+#pip install kagglehub # install kagglehub to download the dataset
+pip install --force-reinstall torch -U 
+pip install torchvision torchaudio 
+#pip install diffusers transformers accelerate --user
+
+# Mixing expandable_segments:True with max_split_size doesn't make sense because the expandable segment is the size of RAM and so it could never be split with max_split_size.
+# export PYTORCH_CUDA_ALLOC_CONF="expandable_segments:True,max_split_size_mb:128"
+export PYTORCH_CUDA_ALLOC_CONF="expandable_segments:True"
+
+python3 train.py
+
+# Deactivate and remove the virtual environment
+deactivate
+rm -rf $VENV_DIR