tests/f: fix remote tests & improve portability

cylc/flow/task_remote_cmd.py

@@ -105,6 +105,8 @@ def remote_init(install_target: str, rund: str, *dirs_to_symlink: str) -> None:
             print(f'Error occurred when symlinking.'
                   f' {target} contains an invalid environment variable.')
             return
+        if cylc.flow.flags.verbosity > 1:
+            print(f'$ ln -s "{target}" "{path}"')
         make_symlink_dir(path, target)
     srvd = os.path.join(rund, WorkflowFiles.Service.DIRNAME)
     os.makedirs(srvd, exist_ok=True)

tests/functional/cylc-cat-log/09-cat-running.t

@@ -19,6 +19,7 @@
 export REQUIRE_PLATFORM='loc:remote comms:tcp'
 . "$(dirname "$0")/test_header"
 set_test_number 2
+export REFTEST_OPTS="--abort-if-any-task-fails"
 reftest
 purge
 exit

tests/functional/cylc-cat-log/09-cat-running/flow.cylc

@@ -9,15 +9,29 @@
 
 [scheduling]
     [[graph]]
-        R1 = local-task:echo & remote-task:echo => cat-log
+        R1 = """
+            # start "local-task" and "remote-task", then run cat-log
+            local-task:echo & remote-task:echo => cat-log
+
+            # add a downstream task to ensure "local-task" and "remote-task"
+            # succeed (for the reference test)
+            local-task & remote-task => fin
+        """
 
 [runtime]
     [[ECHO]]
         script = """
             cylc__job__wait_cylc_message_started
             echo rubbish
             echo garbage >&2
-            cylc message 'echo done'
+            cylc message -- 'echo done'
+
+            # wait up to PT1M for the cat-log task to succeed
+            # (the workflow will shut down if cat-log fails)
+            sleep 60
+
+            # fail if the task was not orphaned by this point
+            false
         """
         [[[outputs]]]
             echo = "echo done"
@@ -26,15 +40,20 @@
     [[remote-task]]
         platform = {{ environ['CYLC_TEST_PLATFORM'] }}
         inherit = ECHO
+
     [[cat-log]]
         script = """
-            cylc__job__wait_cylc_message_started
             for TASK in '1/local-task' '1/remote-task'; do
                 cylc cat-log --debug -f o \
                     "${CYLC_WORKFLOW_ID}//${TASK}" \
                     | grep 'rubbish'
                 cylc cat-log --debug -f e \
                     "${CYLC_WORKFLOW_ID}//${TASK}" \
                     | grep 'garbage'
+                # orphan the sleep command
+                cylc set "${CYLC_WORKFLOW_ID}//${TASK}"
             done
         """
+
+      [[fin]]
+          run mode = skip

tests/functional/cylc-cat-log/09-cat-running/reference.log

@@ -3,3 +3,4 @@ Final point: 1
 1/local-task -triggered off []
 1/remote-task -triggered off []
 1/cat-log -triggered off ['1/local-task', '1/remote-task']
+1/fin -triggered off ['1/local-task', '1/remote-task']

tests/functional/cylc-combination-scripts/10-CYLC_WORKFLOW_SRC_DIR.t

@@ -36,7 +36,7 @@ __HERE__
 
 run_ok "${TEST_NAME_BASE}-view" cylc view -p .
 
-named_grep_ok "src-path-in-view-p" "TEST: $PWD" \
+named_grep_ok "${TEST_NAME_BASE}-src-path-in-view-p" "TEST: $PWD" \
     "${TEST_NAME_BASE}-view.stdout"
 
 # It starts playing:
@@ -47,7 +47,7 @@ run_ok "${TEST_NAME_BASE}-vip" \
         --workflow-name "${WORKFLOW_NAME}"
 
 # It can get CYLC_WORKFLOW_SRC_DIR
-named_grep_ok "src-path-available" \
+named_grep_ok "${TEST_NAME_BASE}-src-path-available" \
     "TEST: $PWD" "${RUN_DIR}/${WORKFLOW_NAME}/log/config/flow-processed.cylc"
 
 # It can be updated with Cylc VR
@@ -57,7 +57,7 @@ run_ok "${TEST_NAME_BASE}-vr" \
 poll_grep "meta" "${RUN_DIR}/${WORKFLOW_NAME}/log/config/flow-processed.cylc"
 
 # It can get CYLC_WORKFLOW_SRC_DIR
-named_grep_ok "src-path-available" \
+named_grep_ok "${TEST_NAME_BASE}-src-path-available" \
     "TEST: $PWD" "${RUN_DIR}/${WORKFLOW_NAME}/log/config/flow-processed.cylc"
 
 cylc stop "${WORKFLOW_NAME}"

tests/functional/cylc-lint/00.lint.t

@@ -30,15 +30,15 @@ rm etc/global.cylc
 
 TEST_NAME="${TEST_NAME_BASE}.vanilla"
 run_fail "${TEST_NAME}" cylc lint .
-named_grep_ok "check-for-error-code" "S004" "${TEST_NAME}.stdout"
+named_grep_ok "${TEST_NAME_BASE}-check-for-error-code" "S004" "${TEST_NAME}.stdout"
 
 TEST_NAME="${TEST_NAME_BASE}.pick-a-ruleset"
 run_fail "${TEST_NAME}" cylc lint . -r 728
-named_grep_ok "check-for-error-code" "U998" "${TEST_NAME}.stdout"
+named_grep_ok "${TEST_NAME_BASE}-check-for-error-code" "U998" "${TEST_NAME}.stdout"
 
 TEST_NAME="${TEST_NAME_BASE}.inplace"
 run_fail "${TEST_NAME}" cylc lint . -i
-named_grep_ok "check-for-error-code-in-file" "U998" flow.cylc
+named_grep_ok "${TEST_NAME_BASE}-check-for-error-code-in-file" "U998" flow.cylc
 
 rm flow.cylc
 
@@ -49,7 +49,7 @@ __HERE__
 
 TEST_NAME="${TEST_NAME_BASE}.pick-a-ruleset-728"
 run_fail "${TEST_NAME}" cylc lint . -r 728
-named_grep_ok "do-not-upgrade-check-if-compat-mode" "Lint after renaming" "${TEST_NAME}.stderr"
+named_grep_ok "${TEST_NAME_BASE}-do-not-upgrade-check-if-compat-mode" "Lint after renaming" "${TEST_NAME}.stderr"
 
 TEST_NAME="${TEST_NAME_BASE}.pick-a-ruleset-728-exit-zero"
 run_ok "${TEST_NAME}" cylc lint . -r 728 --exit-zero
@@ -69,15 +69,15 @@ __HERE__
 
 TEST_NAME="${TEST_NAME_BASE}.zero-issues"
 run_ok "${TEST_NAME}" cylc lint .
-named_grep_ok "message on no errors" "found no issues" "${TEST_NAME}.stdout"
+named_grep_ok "${TEST_NAME_BASE}-message-on-no-errors" "found no issues" "${TEST_NAME}.stdout"
 
 # It returns an error message if you attempt to lint a non-existant location
 TEST_NAME="it-fails-if-not-target"
-run_fail ${TEST_NAME} cylc lint "a-$(uuidgen)"
+run_fail "${TEST_NAME}" cylc lint "a-$(uuidgen)"
 grep_ok "Workflow ID not found" "${TEST_NAME}.stderr"
 
 # It returns a reference in reference mode
-TEST_NAME="it-returns-a-reference"
+TEST_NAME="${TEST_NAME_BASE}-it-returns-a-reference"
 run_ok "${TEST_NAME}" cylc lint --list-codes
 named_grep_ok "${TEST_NAME}-contains-style-codes" "^S001:" "${TEST_NAME}.stdout"
 TEST_NAME="it-returns-a-reference-style"

tests/functional/cylc-lint/01.lint-toml.t

@@ -48,12 +48,12 @@ __HERE__
 
 
 # Control tests
-TEST_NAME="it lints without toml file"
+TEST_NAME='it-lints-without-toml-file'
 run_fail "${TEST_NAME}" cylc lint
 TESTOUT="${TEST_NAME}.stdout"
-named_grep_ok "it returns error code" "S004" "${TESTOUT}"
-named_grep_ok "it returns error from subdirectory" "niwa.cylc" "${TESTOUT}"
-named_grep_ok "it returns a 728 upgrade code" "^\[U" "${TESTOUT}"
+named_grep_ok "${TEST_NAME_BASE}-returns-error-code" "S004" "${TESTOUT}"
+named_grep_ok "${TEST_NAME_BASE}-returns-error-from-subdirectory" "niwa.cylc" "${TESTOUT}"
+named_grep_ok "${TEST_NAME_BASE}-returns-728-upgrade-code" "^\[U" "${TESTOUT}"
 
 
 # Add a pyproject.toml file
@@ -75,7 +75,7 @@ exclude = [
 __HERE__
 
 # Test that results are different:
-TEST_NAME="it_lints_with_toml_file"
+TEST_NAME='it-lints-with-toml-file'
 run_fail "${TEST_NAME}" cylc lint
 TESTOUT="${TEST_NAME}.stdout"
 grep_fail "S004" "${TESTOUT}"
@@ -93,13 +93,13 @@ cat > flow.cylc <<__HERE__
         """
 __HERE__
 
-TEST_NAME="it_fails_if_max-line-length_set"
+TEST_NAME="${TEST_NAME_BASE}-it-fails-if-max-line-length-set"
 run_fail "${TEST_NAME}" cylc lint
 named_grep_ok "${TEST_NAME}-line-too-long-message" \
     "\[${LINE_LEN_NO}\] flow.cylc:2: line > 4 characters." \
     "${TEST_NAME}.stdout"
 
-TEST_NAME="it_does_not_fail_if_max-line-length_set_but_ignored"
+TEST_NAME='it-does-not-fail-if-max-line-length-set-but-ignored'
 cat > pyproject.toml <<__HERE__
 [tool.cylc.lint]
 # Check against these rules

tests/functional/cylc-poll/04-poll-multi-hosts/flow.cylc

@@ -1,20 +1,29 @@
 #!Jinja2
+
 [scheduler]
     UTC mode = True
     [[events]]
         expected task failures = 1/local-fail-1, 1/local-fail-2, 1/remote-fail-1
+
 [scheduling]
     [[graph]]
         R1 = """
             POLLABLE:start-all => poller
             POLLABLE:succeed-any  # (make member success optional)
         """
+
 [runtime]
+    # turn off error trapping to disable "cylc message" calls
+    # (this prevents the "succeeded" / "failed" messages getting through)
     [[POLLABLE]]
         init-script = cylc__job__disable_fail_signals ERR EXIT
+
+    # these tasks will fail (but not send the failed message or write it to the
+    # job.status file)
     [[FAIL]]
         inherit = POLLABLE
         script = """
+            cylc__job__wait_cylc_message_started
             echo 'I am failing...' >&2
             exit 1
         """
@@ -23,9 +32,13 @@
     [[remote-fail-1]]
         inherit = FAIL
         platform = {{ environ['CYLC_TEST_PLATFORM'] }}
+
+    # these tasks will succeed (but not send the succeeded message or write it
+    # to the job.status file)
     [[SUCCESS]]
         inherit = POLLABLE
         script = """
+            cylc__job__wait_cylc_message_started
             echo 'I am OK.'
             {
                 echo 'CYLC_JOB_EXIT=SUCCEEDED'
@@ -38,8 +51,18 @@
     [[remote-success-1, remote-success-2]]
         inherit = SUCCESS
         platform = {{ environ['CYLC_TEST_PLATFORM'] }}
+
+    # this task will poll the active jobs and hopefully detect their
+    # success / failure
     [[poller]]
         script = """
+            # give a bit of grace time for the job to leave the job runner queue
+            sleep 5
+
+            # poll the job (it should change to a final state)
             cylc poll "${CYLC_WORKFLOW_ID}//*/POLLABLE"
+
+            # shut down the scheduler (there should not be any active tasks
+            # left to delay shutdown)
             cylc stop "${CYLC_WORKFLOW_ID}"
         """

tests/functional/events/11-cycle-task-event-job-logs-retrieve.t

@@ -27,12 +27,12 @@ create_test_global_config '' "
 [platforms]
     [[_retrieve]]
         $(cylc config -i "[platforms][$CYLC_TEST_PLATFORM]")
+    [[_retrieve]]
         retrieve job logs = True
-        install target = $CYLC_TEST_PLATFORM
     [[_no_retrieve]]
         $(cylc config -i "[platforms][$CYLC_TEST_PLATFORM]")
+    [[_no_retrieve]]
         retrieve job logs = False
-        install target = $CYLC_TEST_PLATFORM
 "
 
 run_ok "${TEST_NAME_BASE}-validate" \

tests/functional/events/17-task-event-job-logs-retrieve-command.t

@@ -49,6 +49,8 @@ workflow_run_ok "${TEST_NAME_BASE}-run" \
 
 WORKFLOW_LOG_D="${RUN_DIR}/${WORKFLOW_NAME}/log"
 sed 's/^.* -v //' "${WORKFLOW_LOG_D}/scheduler/my-rsync.log" >'my-rsync.log.edited'
+sed -i -E 's/--max-size=[^ ]* //' 'my-rsync.log.edited'  # strip "retrieve job logs max size" arg
+sort -u 'my-rsync.log.edited'  # strip out duplicates (can result from PBS log file spooling)
 
 OPT_HEAD='--include=/1 --include=/1/t1'
 OPT_TAIL='--exclude=/**'

tests/functional/intelligent-host-selection/00-mixedhost.t

@@ -56,7 +56,7 @@ workflow_run_ok "${TEST_NAME_BASE}-run" \
 # Run a bunch of tests on the workflow logs to ensure that warning messages
 # produced by Intelligent Host Selection Logic have happened.
 
-named_grep_ok "unreachable host warning" \
+named_grep_ok "${TEST_NAME_BASE}-unreachable-host-warning" \
     'unreachable_host has been added to the list of unreachable hosts' \
     "${WORKFLOW_RUN_DIR}/log/scheduler/log"
 

tests/functional/intelligent-host-selection/02-badhosts.t

@@ -67,16 +67,16 @@ workflow_run_ok "${TEST_NAME_BASE}-run" \
 LOGFILE="${WORKFLOW_RUN_DIR}/log/scheduler/log"
 
 # Check that badhosttask has submit failed, but not good or mixed
-named_grep_ok "badhost task submit failed" \
+named_grep_ok "${TEST_NAME_BASE}-badhost-task-submit-failed" \
     "1/badhosttask.* submit-failed" "${LOGFILE}"
-named_grep_ok "goodhost suceeded" \
+named_grep_ok "${TEST_NAME_BASE}-goodhost-suceeded" \
     "1/mixedhosttask.* succeeded" "${LOGFILE}"
-named_grep_ok "mixedhost task suceeded" \
+named_grep_ok "${TEST_NAME_BASE}-mixedhost-task-suceeded" \
     "1/goodhosttask.* succeeded" "${LOGFILE}"
 
 # Check that when a task fail badhosts associated with that task's platform
 # are removed from the badhosts set.
-named_grep_ok "remove task platform bad hosts after submit-fail" \
+named_grep_ok "${TEST_NAME_BASE}-remove-task-platform-bad-hosts-after-submit-fail" \
     "initialisation did not complete (no hosts were reachable)" \
     "${LOGFILE}"
 

tests/functional/intelligent-host-selection/03-polling.t

@@ -75,11 +75,12 @@ LOGFILE="${WORKFLOW_RUN_DIR}/log/scheduler/log"
 # Check that when a task fail badhosts associated with that task's platform
 # are removed from the badhosts set.
 named_grep_ok \
-    "job poll fails" \
+    "${TEST_NAME_BASE}-job-poll-fails" \
     "unreachable_host has been added to the list of unreachable hosts" \
     "${LOGFILE}"
 
-named_grep_ok "job poll retries & succeeds" \
+named_grep_ok \
+    "${TEST_NAME_BASE}-job-poll-retries-and-succeeds" \
     "\[jobs-poll out\] \[TASK JOB SUMMARY\].*1/mixedhosttask/01" \
     "${LOGFILE}"