Skip to content

Commit cf6a54a

Browse files
martensonmartenson
committed
seperate tpv rules into proper local files
and clean up
1 parent 85124f5 commit cf6a54a

File tree

9 files changed

+86
-111
lines changed

9 files changed

+86
-111
lines changed

‎files/galaxy-qa1.galaxy.cloud.e-infra.cz/tpv_rules_local.yml‎

Lines changed: 0 additions & 5 deletions
This file was deleted.

‎files/galaxy-umsa.grid.cesnet.cz/tpv_rules_local.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,6 @@ tools:
4444
mem: 16
4545

4646
.*/fastqc/.*:
47-
cores: 1
47+
cores: 1
4848
context:
4949
walltime: 2

files/galaxy-qa2.galaxy.cloud.e-infra.cz/tpv_rules_local.yml renamed to files/galaxy/config/tpv_rules_local.yml

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,2 @@
1-
# prevent tpv lint complaints
21
global:
32
default_inherits: default

‎files/usegalaxy.cz/tpv_rules_local.yml‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,82 @@
11
global:
22
default_inherits: default
3+
destinations:
4+
tpv_pulsar_rosettafold:
5+
inherits: tpv_pulsar
6+
runner: pulsar_tpv_runner
7+
params:
8+
singularity_enabled: false
9+
submit_native_specification: "-l select=1:ncpus={int(cores)}:mem={int(mem)}gb:scratch_local={int(scratch)}gb:ngpus={int(gpus)}:gpu_mem={int(gpu_mem)}gb -l walltime={int(walltime)}:00:00 -q galaxy_gpu@pbs-m1.metacentrum.cz -N pulsar_cz_j{job.id}__{tool.id if '/' not in tool.id else tool.id.split('/')[-2]+'_v'+tool.id.split('/')[-1]}__{user.username if user and hasattr(user, 'username') else 'anonymous'}"
10+
scheduling:
11+
require:
12+
- rosettafold
13+
tpv_pulsar_rosettafold_sing:
14+
inherits: tpv_pulsar_rosettafold
15+
container_resolvers:
16+
- type: explicit_singularity
17+
- cache_directory: /cvmfs/singularity.metacentrum.cz/RoseTTAFold2/
18+
type: cached_mulled_singularity
19+
params:
20+
singularity_enabled: true
21+
singularity_run_extra_arguments: '--nv --env SCRATCHDIR="$SCRATCHDIR" --env SCRATCH=$SCRATCHDIR'
22+
singularity_volumes: '$job_directory:ro,$tool_directory:ro,$job_directory/outputs:rw,$working_directory:rw,$SCRATCHDIR,/scratch.ssd/galaxyeu/permanent/rosettafold_data/bfd:/opt/RoseTTAFold2/bfd,/scratch.ssd/galaxyeu/permanent/rosettafold_data/pdb100_2021Mar03:/opt/RoseTTAFold2/pdb100_2021Mar03,/scratch.ssd/galaxyeu/permanent/rosettafold_data/UniRef30_2020_06:/opt/RoseTTAFold2/UniRef30_2020_06,/scratch.ssd/galaxyeu/permanent/rosettafold_data/weights:/opt/RoseTTAFold2/network/weights'
23+
singularity_default_container_id: "/cvmfs/singularity.metacentrum.cz/RoseTTAFold2/rosettafold_image.sif"
24+
scheduling:
25+
require:
26+
- singularity
27+
tpv_pulsar_alphafold:
28+
inherits: tpv_pulsar
29+
max_accepted_gpus: 1
30+
max_gpus: 1
31+
runner: pulsar_tpv_runner
32+
params:
33+
singularity_run_extra_arguments: '--nv'
34+
singularity_volumes: '$job_directory:ro,$tool_directory:ro,$job_directory/outputs:rw,$working_directory:rw,$SCRATCHDIR,$ALPHAFOLD_DB:/data/2.3:ro'
35+
submit_native_specification: "-l select=1:ncpus={int(cores)}:mem={int(mem)}gb:scratch_local={int(scratch)}gb:ngpus={int(gpus)}:gpu_mem={int(gpu_mem)}gb -l walltime={int(walltime)}:00:00 -q {{ pulsar.pbs_gpu_queue }} -N {{ pulsar.nfs_prefix }}_j{job.id}__{tool.id if '/' not in tool.id else tool.id.split('/')[-2]+'_v'+tool.id.split('/')[-1]}__{user.username if user and hasattr(user, 'username') else 'anonymous'}"
36+
scheduling:
37+
require:
38+
- alphafold
39+
tpv_pulsar_nasty_java:
40+
inherits: tpv_pulsar
41+
runner: pulsar_tpv_runner
42+
params:
43+
singularity_run_extra_arguments: '--env _JAVA_OPTIONS="-Xmx{int(mem)}g -Djava.io.tmpdir=$SCRATCHDIR"'
44+
scheduling:
45+
require:
46+
- nasty-java
47+
tools:
48+
testing_rosettafold2:
49+
cores: 8
50+
mem: 64
51+
gpus: 1
52+
context:
53+
walltime: 24
54+
scratch: 25
55+
gpu_mem: 16
56+
scheduling:
57+
require:
58+
- rosettafold
59+
rosettafold2:
60+
inherits: testing_rosettafold2
61+
scheduling:
62+
require:
63+
- singularity
64+
toolshed.g2.bx.psu.edu/repos/galaxy-australia/alphafold2/alphafold/.*:
65+
cores: 8
66+
mem: 120
67+
gpus: 1
68+
context:
69+
walltime: 24
70+
scratch: 100
71+
gpu_mem: 16
72+
env:
73+
MPLCONFIGDIR: "$SCRATCHDIR"
74+
ALPHAFOLD_DB: "/scratch.ssd/galaxyeu/permanent/alphafold.db"
75+
ALPHAFOLD_USE_GPU: True
76+
scheduling:
77+
require:
78+
- alphafold
79+
toolshed.g2.bx.psu.edu/repos/iuc/goenrichment/goenrichment/2.0.1/.*:
80+
scheduling:
81+
require:
82+
- nasty-java

‎host_vars/galaxy-qa1.galaxy.cloud.e-infra.cz/vars.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ pulsar:
2121
galaxy_config_files:
2222
- src: files/galaxy/themes.yml
2323
dest: "{{ galaxy_config.galaxy.themes_config_file }}"
24-
- src: "files/{{ inventory_hostname }}/tpv_rules_local.yml"
24+
- src: "{{ lookup('first_found', ['files/'+inventory_hostname+'/tpv_rules_local.yml', 'files/galaxy/config/tpv_rules_local.yml']) }}"
2525
dest: "{{ tpv_mutable_dir }}/tpv_rules_{{ inventory_hostname }}.yml"
2626
- src: "{{ lookup('first_found', ['files/'+inventory_hostname+'/config/tool_conf.xml', 'files/galaxy/config/tool_conf.xml']) }}"
2727
dest: "{{ galaxy_config_dir }}/tool_conf.xml"

‎host_vars/galaxy-qa2.galaxy.cloud.e-infra.cz/vars.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ pulsar:
1717
galaxy_config_files:
1818
- src: files/galaxy/themes.yml
1919
dest: "{{ galaxy_config.galaxy.themes_config_file }}"
20-
- src: "files/{{ inventory_hostname }}/tpv_rules_local.yml"
20+
- src: "{{ lookup('first_found', ['files/'+inventory_hostname+'/tpv_rules_local.yml', 'files/galaxy/config/tpv_rules_local.yml']) }}"
2121
dest: "{{ tpv_mutable_dir }}/tpv_rules_{{ inventory_hostname }}.yml"
2222
- src: files/{{ inventory_hostname }}/file_sources_conf.yml
2323
dest: "{{ galaxy_config_dir }}/file_sources_conf.yml"

‎host_vars/galaxy-umsa.grid.cesnet.cz/vars.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ pulsar:
3232
galaxy_config_files:
3333
- src: files/galaxy/themes.yml
3434
dest: "{{ galaxy_config.galaxy.themes_config_file }}"
35-
- src: "files/{{ inventory_hostname }}/tpv_rules_local.yml"
35+
- src: "{{ lookup('first_found', ['files/'+inventory_hostname+'/tpv_rules_local.yml', 'files/galaxy/config/tpv_rules_local.yml']) }}"
3636
dest: "{{ tpv_mutable_dir }}/tpv_rules_{{ inventory_hostname }}.yml"
3737
- src: "{{ lookup('first_found', ['files/'+inventory_hostname+'/config/tool_conf.xml', 'files/galaxy/config/tool_conf.xml']) }}"
3838
dest: "{{ galaxy_config_dir }}/tool_conf.xml"

‎host_vars/usegalaxy.cz/vars.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ pulsar:
2727
galaxy_config_files:
2828
- src: files/galaxy/themes.yml
2929
dest: "{{ galaxy_config.galaxy.themes_config_file }}"
30-
- src: "files/{{ inventory_hostname }}/tpv_rules_local.yml"
30+
- src: "{{ lookup('first_found', ['files/'+inventory_hostname+'/tpv_rules_local.yml', 'files/galaxy/config/tpv_rules_local.yml']) }}"
3131
dest: "{{ tpv_mutable_dir }}/tpv_rules_{{ inventory_hostname }}.yml"
3232
- src: "{{ lookup('first_found', ['files/'+inventory_hostname+'/config/tool_conf.xml', 'files/galaxy/config/tool_conf.xml']) }}"
3333
dest: "{{ galaxy_config_dir }}/tool_conf.xml"

‎templates/galaxy/config/tpv_rules_meta.yml.j2‎

Lines changed: 1 addition & 100 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
global:
22
default_inherits: default
3-
43
tools:
54
default:
65
cores: 1
@@ -20,43 +19,23 @@ tools:
2019
- pulsar
2120
reject:
2221
- offline
22+
- docker
2323
rules: []
2424
rank: |
2525
helpers.weighted_random_sampling(candidate_destinations)
26-
2726
local_running_tools:
2827
abstract: True
2928
scheduling:
3029
require:
3130
- local
3231
reject:
3332
- pulsar
34-
3533
local_sing_running_tools:
3634
inherits: local_running_tools
3735
abstract: True
3836
scheduling:
3937
require:
4038
- singularity
41-
42-
testing_rosettafold2:
43-
cores: 8
44-
mem: 64
45-
gpus: 1
46-
context:
47-
walltime: 24
48-
scratch: 25
49-
gpu_mem: 16
50-
scheduling:
51-
require:
52-
- rosettafold
53-
54-
rosettafold2:
55-
inherits: testing_rosettafold2
56-
scheduling:
57-
require:
58-
- singularity
59-
6039
.*testing.*:
6140
cores: 1
6241
mem: 1
@@ -70,73 +49,40 @@ tools:
7049
# last line in block must evaluate to a value - which determines whether the TPV if conditional matches or not
7150
not user or (user.email not in admin_users and user.email not in ['galaxy@cesnet.cz','galaxy@usegalaxy.cz'])
7251
fail: Unauthorized. Only admins can execute this tool.
73-
7452
- id: resource_params_defined
7553
if: |
7654
param_dict = job.get_param_values(app)
7755
param_dict.get('__job_resource', {}).get('__job_resource__select') == 'yes'
7856
cores: int(job.get_param_values(app)['__job_resource']['cores'])
7957
context:
8058
walltime: "{int(job.get_param_values(app)['__job_resource']['time'])}"
81-
82-
.*/alphafold/.*:
83-
cores: 8
84-
mem: 120
85-
gpus: 1
86-
context:
87-
walltime: 24
88-
scratch: 100
89-
gpu_mem: 16
90-
env:
91-
MPLCONFIGDIR: "$SCRATCHDIR"
92-
ALPHAFOLD_DB: "/scratch.ssd/galaxyeu/permanent/alphafold.db"
93-
ALPHAFOLD_USE_GPU: True
94-
scheduling:
95-
require:
96-
- alphafold
97-
98-
.*/goenrichment/.*:
99-
scheduling:
100-
require:
101-
- nasty-java
102-
10359
.*/bedtools_annotatebed/.*:
10460
context:
10561
walltime: 48
106-
10762
.*/megahit/.*:
10863
cores: 16
10964
mem: 256
11065
context:
11166
walltime: 96
11267
scratch: 100
113-
11468
.*/hapog/.*:
11569
cores: 8
11670
context:
11771
walltime: 48
118-
11972
__SET_.*:
12073
inherits: local_running_tools
121-
12274
__IMPORT_HISTORY__:
12375
inherits: local_running_tools
124-
12576
__EXPORT_HISTORY__:
12677
inherits: local_running_tools
127-
12878
"Show .*":
12979
inherits: local_running_tools
130-
13180
"Remove .*":
13281
inherits: local_running_tools
133-
13482
cat1:
13583
inherits: local_running_tools
136-
13784
.*/data_manager_.*:
13885
inherits: local_sing_running_tools
139-
14086
roles:
14187
training.*:
14288
max_cores: 4
@@ -145,7 +91,6 @@ roles:
14591
require:
14692
- pulsar
14793
- training
148-
14994
destinations:
15095
tpv_local:
15196
runner: local_runner
@@ -218,53 +163,9 @@ destinations:
218163
scheduling:
219164
require:
220165
- pulsar
221-
tpv_pulsar_nasty_java:
222-
inherits: tpv_pulsar
223-
runner: pulsar_tpv_runner
224-
params:
225-
singularity_run_extra_arguments: '--env _JAVA_OPTIONS="-Xmx{int(mem)}g -Djava.io.tmpdir=$SCRATCHDIR"'
226-
scheduling:
227-
require:
228-
- nasty-java
229-
tpv_pulsar_alphafold:
230-
inherits: tpv_pulsar
231-
max_accepted_gpus: 1
232-
max_gpus: 1
233-
runner: pulsar_tpv_runner
234-
params:
235-
singularity_run_extra_arguments: '--nv'
236-
singularity_volumes: '$job_directory:ro,$tool_directory:ro,$job_directory/outputs:rw,$working_directory:rw,$SCRATCHDIR,$ALPHAFOLD_DB:/data/2.3:ro'
237-
submit_native_specification: "-l select=1:ncpus={int(cores)}:mem={int(mem)}gb:scratch_local={int(scratch)}gb:ngpus={int(gpus)}:gpu_mem={int(gpu_mem)}gb -l walltime={int(walltime)}:00:00 -q {{ pulsar.pbs_gpu_queue }} -N {{ pulsar.nfs_prefix }}_j{job.id}__{tool.id if '/' not in tool.id else tool.id.split('/')[-2]+'_v'+tool.id.split('/')[-1]}__{user.username if user and hasattr(user, 'username') else 'anonymous'}"
238-
scheduling:
239-
require:
240-
- alphafold
241166
tpv_pulsar_training:
242167
inherits: tpv_pulsar
243168
runner: pulsar_tpv_runner
244169
scheduling:
245170
require:
246171
- training
247-
# usegalaxy.cz specific:
248-
tpv_pulsar_rosettafold:
249-
inherits: tpv_pulsar
250-
runner: pulsar_tpv_runner
251-
params:
252-
singularity_enabled: false
253-
submit_native_specification: "-l select=1:ncpus={int(cores)}:mem={int(mem)}gb:scratch_local={int(scratch)}gb:ngpus={int(gpus)}:gpu_mem={int(gpu_mem)}gb -l walltime={int(walltime)}:00:00 -q galaxy_gpu@pbs-m1.metacentrum.cz -N pulsar_cz_j{job.id}__{tool.id if '/' not in tool.id else tool.id.split('/')[-2]+'_v'+tool.id.split('/')[-1]}__{user.username if user and hasattr(user, 'username') else 'anonymous'}"
254-
scheduling:
255-
require:
256-
- rosettafold
257-
tpv_pulsar_rosettafold_sing:
258-
inherits: tpv_pulsar_rosettafold
259-
container_resolvers:
260-
- type: explicit_singularity
261-
- cache_directory: /cvmfs/singularity.metacentrum.cz/RoseTTAFold2/
262-
type: cached_mulled_singularity
263-
params:
264-
singularity_enabled: true
265-
singularity_run_extra_arguments: '--nv --env SCRATCHDIR="$SCRATCHDIR" --env SCRATCH=$SCRATCHDIR'
266-
singularity_volumes: '$job_directory:ro,$tool_directory:ro,$job_directory/outputs:rw,$working_directory:rw,$SCRATCHDIR,/scratch.ssd/galaxyeu/permanent/rosettafold_data/bfd:/opt/RoseTTAFold2/bfd,/scratch.ssd/galaxyeu/permanent/rosettafold_data/pdb100_2021Mar03:/opt/RoseTTAFold2/pdb100_2021Mar03,/scratch.ssd/galaxyeu/permanent/rosettafold_data/UniRef30_2020_06:/opt/RoseTTAFold2/UniRef30_2020_06,/scratch.ssd/galaxyeu/permanent/rosettafold_data/weights:/opt/RoseTTAFold2/network/weights'
267-
singularity_default_container_id: "/cvmfs/singularity.metacentrum.cz/RoseTTAFold2/rosettafold_image.sif"
268-
scheduling:
269-
require:
270-
- singularity

0 commit comments

Comments
 (0)