psutil: allow accessing psutil object methods (#6733)

* The `_is_process_running` check was requesting the `psutil.Process`
  object via the `cylc psutil` command.
* It would appear that these objects can get large enough to fill up the
  buffer. This means that `subprocess.Process.communicate` or `.wait`
  will hang indefinitely unless something reads from stdout to clear out
  the buffers.
* This commit changes the `is_process_running` check to request only the
  one field it actually requires, `psutil.Process.cmdline` to reduce the
  data volume to a level where pipe polling is not required.

Author: oliver-sanders

changes.d/6733.fix.md

@@ -0,0 +1 @@
+Fix an issue where the message "Cannot tell if the workflow is running" error could appear erroneously.

cylc/flow/scripts/psutil.py

@@ -46,25 +46,47 @@ def get_option_parser():
     return OptionParser(__doc__)
 
 
+def _call_psutil_interface(interface, *args):
+    """Call a psutil interface with the provided arguments.
+
+    Args:
+        interface:
+            The psutil method we want to call, e.g. "cpu_percent".
+
+            In the case of objects, this may include attributes, e.g.
+            "Process.cmdline".
+        args:
+            The arguments to provide to the psutil method.
+
+    Returns:
+        The result of the psutil method call.
+
+    """
+    result = psutil
+    is_first = True
+    for fcn in interface.split('.'):
+        try:
+            method = getattr(result, fcn)
+        except AttributeError as exc:
+            # error obtaining interfaces from psutil e.g:
+            # * requesting a method which does not exist
+            print(exc, file=sys.stderr)
+            sys.exit(2)
+
+        if is_first:
+            is_first = False
+        else:
+            args = ()
+
+        result = method(*args)
+    return result
+
+
 def _psutil(metrics_json):
     metrics = parse_dirty_json(metrics_json)
 
     try:
-        methods = [
-            getattr(psutil, key[0])
-            for key in metrics
-        ]
-    except AttributeError as exc:
-        # error obtaining interfaces from psutil e.g:
-        # * requesting a method which does not exist
-        print(exc, file=sys.stderr)
-        sys.exit(2)
-
-    try:
-        ret = [
-            method(*key[1:])
-            for key, method in zip(metrics, methods)
-        ]
+        ret = [_call_psutil_interface(*key) for key in metrics]
     except Exception as exc:
         # error extracting metrics from psutil e.g:
         # * requesting information on a resource which does not exist

cylc/flow/workflow_files.py

@@ -411,7 +411,7 @@ def _is_process_running(
     from cylc.flow.terminal import parse_dirty_json
 
     # See if the process is still running or not.
-    metric = f'[["Process", {pid}]]'
+    metric = f'[["Process.cmdline", {pid}]]'
     if is_remote_host(host):
         cmd = ['psutil']
         cmd = construct_cylc_server_ssh_cmd(cmd, host)
@@ -459,7 +459,7 @@ def _is_process_running(
             f'\n{command}'
         )
 
-    return cli_format(process['cmdline']) == command
+    return cli_format(process) == command
 
 
 def detect_old_contact_file(

tests/integration/test_workflow_files.py

@@ -291,9 +291,7 @@ def communicate(self, *args, **kwargs):
     # respond with something Cylc should be able to make sense of
     stdout = dedent('''
         % simulated stdout pollution %
-        [{
-            "cmdline": ["expected", "command"]
-        }]
+        [["expected", "command"]]
     ''')
 
     caplog.set_level(logging.WARN, logger=CYLC_LOG)

tests/unit/test_psutil.py

@@ -16,6 +16,9 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import json
+import os
+
+from psutil import Process
 
 from cylc.flow.scripts.psutil import _psutil
 
@@ -38,3 +41,11 @@ def test_psutil_basic():
         assert key in mem
         # all of which should be integers
         assert isinstance(mem[key], int)
+
+
+def test_psutil_object():
+    """It should call object methods."""
+    # obtain the commandline for this test process
+    ret = _psutil(f'[["Process.cmdline", {os.getpid()}]]')
+
+    assert ret[0] == Process().cmdline()