From a3017b27f5f070e705ad136acd653b2bc3df1efa Mon Sep 17 00:00:00 2001 From: Carsten Ehbrecht Date: Tue, 15 Sep 2020 23:27:31 +0200 Subject: [PATCH] * allow unlimited maxprocesses * updated docs for drmaa_native_specification configuration * added drmaa_native_specification --- docs/configuration.rst | 12 ++++++++---- pywps/app/Process.py | 2 +- pywps/configuration.py | 2 ++ pywps/processing/scheduler.py | 3 +++ 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index c6be181b3..b845e2ce4 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -95,7 +95,7 @@ configuration file `_. the URL of the WPS service endpoint :language: - a comma-separated list of ISO 639-1 language and ISO 3166-1 alpha2 country + a comma-separated list of ISO 639-1 language and ISO 3166-1 alpha2 country code of the service (e.g. ``en-CA``, ``fr-CA``, ``en-US``) @@ -110,14 +110,14 @@ configuration file `_. of cores in the processor of the hosting machine. As well, speed and response time of hard drives impact ultimate processing performance. A reasonable number of parallel running processes is not higher than the - number of processor cores. + number of processor cores. -1 for no limit. :maxrequestsize: maximal request size. 0 for no limit :maxprocesses: maximal number of requests being stored in queue, waiting till they can be - processed (see ``parallelprocesses`` configuration option). + processed (see ``parallelprocesses`` configuration option). -1 for no limit. :workdir: a directory to store all temporary files (which should be always deleted, @@ -170,6 +170,11 @@ configuration file `_. the `scheduler` backend and is by default set automatically: `os.path.dirname(os.path.realpath(sys.argv[0]))` +:drmaa_native_specification: + option to set the DRMAA native specification, for example to limit number of + CPUs and memory usage. Example: `--cpus-per-task=1 --mem=1024`. + See DRMAA docs for details: https://github.com/natefoo/slurm-drmaa + [logging] --------- @@ -280,4 +285,3 @@ Sample file prefix=appname/coolapp/ public=true encrypt=false - diff --git a/pywps/app/Process.py b/pywps/app/Process.py index 3be1ebfb4..dd604d15c 100644 --- a/pywps/app/Process.py +++ b/pywps/app/Process.py @@ -208,7 +208,7 @@ def _execute_process(self, async_, wps_request, wps_response): # try to store for later usage else: maxprocesses = int(config.get_config_value('server', 'maxprocesses')) - if stored >= maxprocesses: + if stored >= maxprocesses and maxprocesses != -1: raise ServerBusy('Maximum number of processes in queue reached. Please try later.') LOGGER.debug("Store process in job queue, uuid={}".format(self.uuid)) dblog.store_process(self.uuid, wps_request) diff --git a/pywps/configuration.py b/pywps/configuration.py index f0c3b4ab3..ff7ade193 100755 --- a/pywps/configuration.py +++ b/pywps/configuration.py @@ -100,6 +100,8 @@ def load_configuration(cfgfiles=None): CONFIG.add_section('processing') CONFIG.set('processing', 'mode', 'default') CONFIG.set('processing', 'path', os.path.dirname(os.path.realpath(sys.argv[0]))) + # https://github.com/natefoo/slurm-drmaa + CONFIG.set('processing', 'drmaa_native_specification', '') CONFIG.add_section('logging') CONFIG.set('logging', 'file', '') diff --git a/pywps/processing/scheduler.py b/pywps/processing/scheduler.py index 9bfd355b5..0e4b2d2fd 100644 --- a/pywps/processing/scheduler.py +++ b/pywps/processing/scheduler.py @@ -53,6 +53,9 @@ def run_job(self): jt.args = ['-c', cfg_file, dump_filename] else: jt.args = [dump_filename] + drmaa_native_specification = config.get_config_value('processing', 'drmaa_native_specification') + if drmaa_native_specification: + jt.nativeSpecification = drmaa_native_specification jt.joinFiles = True jt.outputPath = ":{}".format(os.path.join(self.job.workdir, "job-output.txt")) # run job