update scheduler with drmaa config

docs/configuration.rst

@@ -95,7 +95,7 @@ configuration file <https://docs.pycsw.org/en/latest/configuration.html>`_.
     the URL of the WPS service endpoint
 
 :language:
-    a comma-separated list of ISO 639-1 language and ISO 3166-1 alpha2 country 
+    a comma-separated list of ISO 639-1 language and ISO 3166-1 alpha2 country
     code of the service
     (e.g. ``en-CA``, ``fr-CA``, ``en-US``)
 
@@ -110,14 +110,14 @@ configuration file <https://docs.pycsw.org/en/latest/configuration.html>`_.
     of cores  in the processor of the hosting machine. As well, speed and
     response time of hard drives impact ultimate processing performance. A
     reasonable number of parallel running processes is not higher than the
-    number of processor cores.
+    number of processor cores. -1 for no limit.
 
 :maxrequestsize:
     maximal request size. 0 for no limit
 
 :maxprocesses:
     maximal number of requests being stored in queue, waiting till they can be
-    processed (see ``parallelprocesses`` configuration option).
+    processed (see ``parallelprocesses`` configuration option). -1 for no limit.
 
 :workdir:
     a directory to store all temporary files (which should be always deleted,
@@ -170,6 +170,11 @@ configuration file <https://docs.pycsw.org/en/latest/configuration.html>`_.
     the `scheduler` backend and is by default set automatically:
     `os.path.dirname(os.path.realpath(sys.argv[0]))`
 
+:drmaa_native_specification:
+    option to set the DRMAA native specification, for example to limit number of
+    CPUs and memory usage. Example: `--cpus-per-task=1 --mem=1024`.
+    See DRMAA docs for details: https://github.com/natefoo/slurm-drmaa
+
 [logging]
 ---------
 
@@ -280,4 +285,3 @@ Sample file
   prefix=appname/coolapp/
   public=true
   encrypt=false
-

pywps/app/Process.py

@@ -208,7 +208,7 @@ def _execute_process(self, async_, wps_request, wps_response):
             # try to store for later usage
             else:
                 maxprocesses = int(config.get_config_value('server', 'maxprocesses'))
-                if stored >= maxprocesses:
+                if stored >= maxprocesses and maxprocesses != -1:
                     raise ServerBusy('Maximum number of processes in queue reached. Please try later.')
                 LOGGER.debug("Store process in job queue, uuid={}".format(self.uuid))
                 dblog.store_process(self.uuid, wps_request)

pywps/configuration.py

@@ -100,6 +100,8 @@ def load_configuration(cfgfiles=None):
     CONFIG.add_section('processing')
     CONFIG.set('processing', 'mode', 'default')
     CONFIG.set('processing', 'path', os.path.dirname(os.path.realpath(sys.argv[0])))
+    # https://github.com/natefoo/slurm-drmaa
+    CONFIG.set('processing', 'drmaa_native_specification', '')
 
     CONFIG.add_section('logging')
     CONFIG.set('logging', 'file', '')

pywps/processing/scheduler.py

@@ -53,6 +53,9 @@ def run_job(self):
                 jt.args = ['-c', cfg_file, dump_filename]
             else:
                 jt.args = [dump_filename]
+            drmaa_native_specification = config.get_config_value('processing', 'drmaa_native_specification')
+            if drmaa_native_specification:
+                jt.nativeSpecification = drmaa_native_specification
             jt.joinFiles = True
             jt.outputPath = ":{}".format(os.path.join(self.job.workdir, "job-output.txt"))
             # run job