TPV: reject docker for all tools

reorganize tpv local and make it use template if none defined

Author: martenson

files/galaxy-qa1.galaxy.cloud.e-infra.cz/tpv_rules_local.yml

@@ -44,6 +44,6 @@ tools:
     mem: 16
 
   .*/fastqc/.*:
-    cores: 1 
+    cores: 1
     context:
       walltime: 2

files/galaxy-umsa.grid.cesnet.cz/tpv_rules_local.yml

@@ -1,3 +1,2 @@
-# prevent tpv lint complaints
 global:
   default_inherits: default

files/galaxy-qa2.galaxy.cloud.e-infra.cz/tpv_rules_local.yml renamed to files/galaxy/config/tpv_rules_local.yml

@@ -1,2 +1,26 @@
 global:
   default_inherits: default
+destinations:
+  tpv_pulsar_rosettafold:
+    inherits: tpv_pulsar
+    runner: pulsar_tpv_runner
+    params:
+      singularity_enabled: false
+      submit_native_specification: "-l select=1:ncpus={int(cores)}:mem={int(mem)}gb:scratch_local={int(scratch)}gb:ngpus={int(gpus)}:gpu_mem={int(gpu_mem)}gb -l walltime={int(walltime)}:00:00 -q [email protected] -N pulsar_cz_j{job.id}__{tool.id if '/' not in tool.id else tool.id.split('/')[-2]+'_v'+tool.id.split('/')[-1]}__{user.username if user and hasattr(user, 'username') else 'anonymous'}"
+    scheduling:
+      require:
+        - rosettafold
+  tpv_pulsar_rosettafold_sing:
+    inherits: tpv_pulsar_rosettafold
+    container_resolvers:
+    -   type: explicit_singularity
+    -   cache_directory: /cvmfs/singularity.metacentrum.cz/RoseTTAFold2/
+        type: cached_mulled_singularity
+    params:
+      singularity_enabled: true
+      singularity_run_extra_arguments: '--nv --env SCRATCHDIR="$SCRATCHDIR" --env SCRATCH=$SCRATCHDIR'
+      singularity_volumes: '$job_directory:ro,$tool_directory:ro,$job_directory/outputs:rw,$working_directory:rw,$SCRATCHDIR,/scratch.ssd/galaxyeu/permanent/rosettafold_data/bfd:/opt/RoseTTAFold2/bfd,/scratch.ssd/galaxyeu/permanent/rosettafold_data/pdb100_2021Mar03:/opt/RoseTTAFold2/pdb100_2021Mar03,/scratch.ssd/galaxyeu/permanent/rosettafold_data/UniRef30_2020_06:/opt/RoseTTAFold2/UniRef30_2020_06,/scratch.ssd/galaxyeu/permanent/rosettafold_data/weights:/opt/RoseTTAFold2/network/weights'
+      singularity_default_container_id: "/cvmfs/singularity.metacentrum.cz/RoseTTAFold2/rosettafold_image.sif"
+    scheduling:
+      require:
+        - singularity

files/usegalaxy.cz/tpv_rules_local.yml

@@ -17,7 +17,7 @@ pulsar:
 galaxy_config_files:
   - src: files/galaxy/themes.yml
     dest: "{{ galaxy_config.galaxy.themes_config_file }}"
-  - src: "files/{{ inventory_hostname }}/tpv_rules_local.yml"
+  - src: "{{ lookup('first_found', ['files/'+inventory_hostname+'/tpv_rules_local.yml', 'files/galaxy/config/tpv_rules_local.yml']) }}"
     dest: "{{ tpv_mutable_dir }}/tpv_rules_{{ inventory_hostname }}.yml"
   - src: "{{ lookup('first_found', ['files/'+inventory_hostname+'/config/tool_conf.xml', 'files/galaxy/config/tool_conf.xml']) }}"
     dest: "{{ galaxy_config_dir }}/tool_conf.xml"

host_vars/galaxy-qa1.galaxy.cloud.e-infra.cz/vars.yml

@@ -16,7 +16,7 @@ pulsar:
 galaxy_config_files:
   - src: files/galaxy/themes.yml
     dest: "{{ galaxy_config.galaxy.themes_config_file }}"
-  - src: "files/{{ inventory_hostname }}/tpv_rules_local.yml"
+  - src: "{{ lookup('first_found', ['files/'+inventory_hostname+'/tpv_rules_local.yml', 'files/galaxy/config/tpv_rules_local.yml']) }}"
     dest: "{{ tpv_mutable_dir }}/tpv_rules_{{ inventory_hostname }}.yml"
   - src: files/{{ inventory_hostname }}/file_sources_conf.yml
     dest: "{{ galaxy_config_dir }}/file_sources_conf.yml"

host_vars/galaxy-qa2.galaxy.cloud.e-infra.cz/vars.yml

@@ -31,7 +31,7 @@ pulsar:
 galaxy_config_files:
   - src: files/galaxy/themes.yml
     dest: "{{ galaxy_config.galaxy.themes_config_file }}"
-  - src: "files/{{ inventory_hostname }}/tpv_rules_local.yml"
+  - src: "{{ lookup('first_found', ['files/'+inventory_hostname+'/tpv_rules_local.yml', 'files/galaxy/config/tpv_rules_local.yml']) }}"
     dest: "{{ tpv_mutable_dir }}/tpv_rules_{{ inventory_hostname }}.yml"
   - src: "{{ lookup('first_found', ['files/'+inventory_hostname+'/config/tool_conf.xml', 'files/galaxy/config/tool_conf.xml']) }}"
     dest: "{{ galaxy_config_dir }}/tool_conf.xml"

host_vars/galaxy-umsa.grid.cesnet.cz/vars.yml

@@ -26,7 +26,7 @@ pulsar:
 galaxy_config_files:
   - src: files/galaxy/themes.yml
     dest: "{{ galaxy_config.galaxy.themes_config_file }}"
-  - src: "files/{{ inventory_hostname }}/tpv_rules_local.yml"
+  - src: "{{ lookup('first_found', ['files/'+inventory_hostname+'/tpv_rules_local.yml', 'files/galaxy/config/tpv_rules_local.yml']) }}"
     dest: "{{ tpv_mutable_dir }}/tpv_rules_{{ inventory_hostname }}.yml"
   - src: "{{ lookup('first_found', ['files/'+inventory_hostname+'/config/tool_conf.xml', 'files/galaxy/config/tool_conf.xml']) }}"
     dest: "{{ galaxy_config_dir }}/tool_conf.xml"

host_vars/usegalaxy.cz/vars.yml

@@ -18,6 +18,7 @@ tools:
         - pulsar
       reject:
         - offline
+        - docker
     rules: []
     rank: |
       helpers.weighted_random_sampling(candidate_destinations)
@@ -43,15 +44,15 @@ tools:
     gpus: 1
     context:
       walltime: 24
-      scratch: 25 
+      scratch: 25
       gpu_mem: 16
-    scheduling:   
+    scheduling:
       require:
         - rosettafold
 
   rosettafold2:
     inherits: testing_rosettafold2
-    scheduling:   
+    scheduling:
       require:
         - singularity
 
@@ -234,27 +235,3 @@ destinations:
     scheduling:
       require:
         - training
-# usegalaxy.cz specific:
-  tpv_pulsar_rosettafold:
-    inherits: tpv_pulsar
-    runner: pulsar_tpv_runner
-    params:
-      singularity_enabled: false
-      submit_native_specification: "-l select=1:ncpus={int(cores)}:mem={int(mem)}gb:scratch_local={int(scratch)}gb:ngpus={int(gpus)}:gpu_mem={int(gpu_mem)}gb -l walltime={int(walltime)}:00:00 -q [email protected] -N pulsar_cz_j{job.id}__{tool.id if '/' not in tool.id else tool.id.split('/')[-2]+'_v'+tool.id.split('/')[-1]}__{user.username if user and hasattr(user, 'username') else 'anonymous'}"
-    scheduling:
-      require:
-        - rosettafold
-  tpv_pulsar_rosettafold_sing:
-    inherits: tpv_pulsar_rosettafold
-    container_resolvers:
-    -   type: explicit_singularity
-    -   cache_directory: /cvmfs/singularity.metacentrum.cz/RoseTTAFold2/
-        type: cached_mulled_singularity
-    params:
-      singularity_enabled: true
-      singularity_run_extra_arguments: '--nv --env SCRATCHDIR="$SCRATCHDIR" --env SCRATCH=$SCRATCHDIR'
-      singularity_volumes: '$job_directory:ro,$tool_directory:ro,$job_directory/outputs:rw,$working_directory:rw,$SCRATCHDIR,/scratch.ssd/galaxyeu/permanent/rosettafold_data/bfd:/opt/RoseTTAFold2/bfd,/scratch.ssd/galaxyeu/permanent/rosettafold_data/pdb100_2021Mar03:/opt/RoseTTAFold2/pdb100_2021Mar03,/scratch.ssd/galaxyeu/permanent/rosettafold_data/UniRef30_2020_06:/opt/RoseTTAFold2/UniRef30_2020_06,/scratch.ssd/galaxyeu/permanent/rosettafold_data/weights:/opt/RoseTTAFold2/network/weights'
-      singularity_default_container_id: "/cvmfs/singularity.metacentrum.cz/RoseTTAFold2/rosettafold_image.sif"
-    scheduling:
-      require:
-        - singularity