Setting up GitHub actions for model validation (#1)

Combined preprocessing, training, deployment and validation steps in a unique Snakemake workflow executed on CINECA Leonardo via GitHub actions.

Author: landerlini

.github/workflows/train.yaml

@@ -0,0 +1,33 @@
+name: Train models
+
+on:
+  pull_request:
+    branches:
+      - main
+
+concurrency:
+  group: pull_request-${{ github.head_ref }}
+  cancel-in-progress: true
+
+
+jobs:
+    pid-2016MagUp-Sim10b:
+        uses: LamarrSim/release-models/.github/workflows/release.yaml@main
+        permissions:
+          contents: write
+          pull-requests: write
+        with:
+            name: pp-2016-MU-Sim10b-${{ github.head_ref }}
+            snakemake_dir: pidgan
+            target: cache_container validate_all
+            model_prefix: pid
+            additional_config: |
+                storage_folder: pid-2016MagUp-Sim10b
+
+            additional_profile: |
+                jobs: 1
+
+            generated_model_path: /tmp/compiled_model.c
+        secrets:
+            SECRETS_YAML: ${{ secrets.STORAGE_SECRETS }}
+

pidgan/config/config.yaml

@@ -6,8 +6,8 @@ apptainer_cached_image: /tmp/lb-pidsim-train.sif
 sample: 2016MU
 storage_folder: tests
 temp_dir: /tmp
-training_time_limit_seconds: 60 #0
-entries_for_validation: 10000 #00
+training_time_limit_seconds: 600
+entries_for_validation: 1000000
 
 nbconvert_args:
     - --allow-errors
@@ -19,7 +19,6 @@ training_data:
     - training-data/2016MU-sim10b/LamarrTraining-j109.10.root
     - training-data/2016MU-sim10b/LamarrTraining-j109.100.root
     - training-data/2016MU-sim10b/LamarrTraining-j109.101.root
-other:
 
     - training-data/2016MU-sim10b/LamarrTraining-j109.102.root
     - training-data/2016MU-sim10b/LamarrTraining-j109.103.root

pidgan/workflow/Snakefile

@@ -9,9 +9,13 @@ PARTICLES = ['muon', 'pion', 'kaon', 'proton']
 ITEMS = ['model.keras', 'tX.pkl', 'tY.pkl']
 
 
-rule build_container:
+rule validate_all:
+    input: smkstore(expand("validate-{particle}.html", particle=PARTICLES), 's3reports')
+
+
+rule build_apptainer_image:
   input:
-    definition_file="workflow/envs/pidgan.def"
+    definition_file=ancient("workflow/envs/pidgan.def")
 
   output:
     sif=smkstore("lhcb-pidsim.sif", 's3images')
@@ -38,7 +42,7 @@ rule build_container:
 
 
 rule cache_container:
-  input: ancient(smkstore("lhcb-pidsim.sif", 's3images'))
+  input: smkstore("lhcb-pidsim.sif", 's3images')
   output: config['apptainer_cached_image']
   shell: "cp {input} {output}"
 
@@ -59,7 +63,7 @@ rule preprocessing:
         report=report(
             smkstore("{model}-{particle}-preprocessing.html", 's3reports'),
             category="Preprocessing",
-            labels=dict(model="{model}", particle="{particle}")
+            labels=dict(model="{model}", particle="{particle}", type='html')
         )
 
     log: smkstore("{model}-{particle}-preprocessing.html", 's3reports')
@@ -120,16 +124,16 @@ rule train:
         history_csv=report(
             smkstore("history/generator-{model}-{particle}.csv", 's3reports'),
             category="Training",
-            labels=dict(model="{model}", particle="{particle}")
+            labels=dict(model="{model}", particle="{particle}", type='csv')
         ),
 
         output_validation_set=smkstore("ready2validate/{model}-{particle}.npz", 's3reports'),
         output_model=smkstore("{model}/{particle}/model.keras", "s3models"),
 
         report=report(
             smkstore("{model}-{particle}-train.html", 's3reports'),
-            category="Preprocessing",
-            labels=dict(model="{model}", particle="{particle}")
+            category="Training",
+            labels=dict(model="{model}", particle="{particle}", type='html')
         )
 
     log: smkstore("{model}-{particle}-train.html", 's3reports')
@@ -138,8 +142,8 @@ rule train:
 
 
     resources:
-        cpu=8,
-        mem_mb=32000,
+        cpu=4,
+        mem_mb=15000,
         gpu=1
 
     params:
@@ -193,12 +197,12 @@ rule deploy:
 
     output:
         hook=touch("/tmp/deploy.touch"),
-        generated_c_file="/tmp/compiled_pid_model.c",
+        generated_c_file="/tmp/compiled_model.c",
         generated_library=smkstore("pid_compiled_model/generated.so", 's3models'),
         report=report(
             smkstore("deploy.html", 's3reports'),
             category="Deploy",
-            labels=dict(model="ALL", particle="ALL")
+            labels=dict(model="ALL", particle="ALL", type='html')
         )
 
     params:
@@ -243,8 +247,8 @@ rule validate:
 
         report=report(
             smkstore("validate-{particle}.html", 's3reports'),
-            category="Preprocessing",
-            labels=dict(model="ALL", particle="{particle}")
+            category="Validation",
+            labels=dict(model="ALL", particle="{particle}", type="html")
         )
 
     log: smkstore("validate-{particle}.html", 's3reports')
@@ -259,15 +263,15 @@ rule validate:
         max_entries=config['entries_for_validation'],
 
     resources:
-        cpu=8,
-        mem_mb=32000,
+        cpu=4,
+        mem_mb=15000,
         gpu=0
 
     shell:
         """
         INPUT_FILES='{params.data_files}' \
         PARTICLE='{wildcards.particle}' \
-        GENERATED_LIBRARY='{input.generated_library}' \
+        SHARED_OBJECT='{input.generated_library}' \
         MAX_ENTRIES='{params.max_entries}' \
         SAMPLE='{params.sample}' \
         jupyter nbconvert --to html --execute \
@@ -278,7 +282,5 @@ rule validate:
         {input.notebook} --output {log}
         """
 
-rule validate_all:
-    input: smkstore(expand("validate-{particle}.html", particle=PARTICLES), 's3reports')