From f4c0a85a54873aeaf4ef246b88df9363160f5a36 Mon Sep 17 00:00:00 2001 From: Stephane Thiell Date: Sun, 3 Aug 2025 16:28:02 -0700 Subject: [PATCH 1/2] tests: use mock hostname command remotely As `hostname` is used by rcopy in tree mode via tar, mock it remotely so we can simulate multiple target nodes in tests. --- .github/workflows/nosetests.yml | 8 +++++++- MANIFEST.in | 1 + tests/bin/hostname | 4 ++++ 3 files changed, 12 insertions(+), 1 deletion(-) create mode 100755 tests/bin/hostname diff --git a/.github/workflows/nosetests.yml b/.github/workflows/nosetests.yml index c0913bef..e4ace9d6 100644 --- a/.github/workflows/nosetests.yml +++ b/.github/workflows/nosetests.yml @@ -35,11 +35,17 @@ jobs: run: | sed -i "1iexport CLUSTERSHELL_GW_PYTHON_EXECUTABLE=$(which python)" ~/.bashrc sed -i "2iexport PYTHONPATH=\$PYTHONPATH:$(realpath $pythonLocation/lib/python*/site-packages)" ~/.bashrc - head ~/.bashrc + head -2 ~/.bashrc - name: Install pdsh to test WorkerPdsh run: | sudo apt-get -y install pdsh sudo sh -c 'echo ssh > /etc/pdsh/rcmd_default' + - name: Add tests/bin to remote PATH + run: | + sed -i "1iexport PATH=$PWD/tests/bin:\$PATH" ~/.bashrc + head -1 ~/.bashrc + ssh 127.0.0.2 which hostname + ssh 127.0.0.2 hostname - name: Post to Slack (start) continue-on-error: true id: slack-start diff --git a/MANIFEST.in b/MANIFEST.in index a12265ed..17635815 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -32,3 +32,4 @@ include doc/examples/*.py include doc/examples/defaults.conf-rsh include doc/epydoc/*.conf include tests/*.py +include tests/bin/* diff --git a/tests/bin/hostname b/tests/bin/hostname new file mode 100755 index 00000000..38cc30e0 --- /dev/null +++ b/tests/bin/hostname @@ -0,0 +1,4 @@ +#!/bin/bash +# This is useful for simulating multiple nodes on localhost IPs, +# such as 127.0.0.2, during testing scenarios. +echo $SSH_CONNECTION | { IFS=' ' read -r _ _ IP _; echo "$IP"; } From 730a18debfbbee61a65182fc7b110c4fbb2d7801 Mon Sep 17 00:00:00 2001 From: Stephane Thiell Date: Sun, 3 Aug 2025 14:40:01 -0700 Subject: [PATCH 2/2] tests: improve rcopy tree tests Add test for rcopy with file as the source and with multiple targets. Test the change from #545. --- tests/TreeWorkerTest.py | 131 +++++++++++++++++++++++++++++++--------- 1 file changed, 102 insertions(+), 29 deletions(-) diff --git a/tests/TreeWorkerTest.py b/tests/TreeWorkerTest.py index 77410a84..236192ab 100644 --- a/tests/TreeWorkerTest.py +++ b/tests/TreeWorkerTest.py @@ -28,8 +28,9 @@ NODE_HEAD = HOSTNAME NODE_GATEWAY = 'localhost' NODE_DISTANT = '127.0.0.2' -NODE_DIRECT = '127.0.0.3' -NODE_FOREIGN = '127.0.0.4' +NODE_DISTANT2 = '127.0.0.[2-3]' +NODE_DIRECT = '127.0.0.4' +NODE_FOREIGN = '127.0.0.5' class TEventHandlerBase(object): @@ -105,7 +106,7 @@ class TreeWorkerTest(unittest.TestCase): """ TreeWorkerTest: test TreeWorker - NODE_HEAD -> NODE_GATEWAY -> NODE_DISTANT + NODE_HEAD -> NODE_GATEWAY -> NODE_DISTANT2 -> NODE_DIRECT [defined in topology] -> NODE_FOREIGN [not defined in topology] @@ -120,7 +121,7 @@ def setUp(self): # set task topology graph = TopologyGraph() graph.add_route(NodeSet(HOSTNAME), NodeSet(NODE_GATEWAY)) - graph.add_route(NodeSet(NODE_GATEWAY), NodeSet(NODE_DISTANT)) + graph.add_route(NodeSet(NODE_GATEWAY), NodeSet(NODE_DISTANT2)) graph.add_route(NodeSet(HOSTNAME), NodeSet(NODE_DIRECT)) # NODE_FOREIGN is not included self.task.topology = graph.to_tree(HOSTNAME) @@ -254,12 +255,13 @@ def _tree_run_write(self, target, separate_thread=False): if separate_thread: task_wait() task_cleanup() + target_cnt = len(NodeSet(target)) self.assertEqual(teh.ev_start_cnt, 1) - self.assertEqual(teh.ev_pickup_cnt, 1) - self.assertEqual(teh.ev_read_cnt, 1) - self.assertEqual(teh.ev_written_cnt, 1) - self.assertEqual(teh.ev_written_sz, len('Lorem Ipsum')) - self.assertEqual(teh.ev_hup_cnt, 1) + self.assertEqual(teh.ev_pickup_cnt, target_cnt) + self.assertEqual(teh.ev_read_cnt, target_cnt) + self.assertEqual(teh.ev_written_cnt, target_cnt) + self.assertEqual(teh.ev_written_sz, target_cnt * len('Lorem Ipsum')) + self.assertEqual(teh.ev_hup_cnt, target_cnt) self.assertEqual(teh.ev_timedout_cnt, 0) self.assertEqual(teh.ev_close_cnt, 1) self.assertEqual(teh.last_read, b'Lorem Ipsum') @@ -268,6 +270,10 @@ def test_tree_run_write_distant(self): """test tree run with write(), distant target""" self._tree_run_write(NODE_DISTANT) + def test_tree_run_write_distant2(self): + """test tree run with write(), distant 2 targets""" + self._tree_run_write(NODE_DISTANT2) + def test_tree_run_write_direct(self): """test tree run with write(), direct target, in topology""" self._tree_run_write(NODE_DIRECT) @@ -284,6 +290,10 @@ def test_tree_run_write_distant_mt(self): """test tree run with write(), distant target, separate thread""" self._tree_run_write(NODE_DISTANT, separate_thread=True) + def test_tree_run_write_distant2_mt(self): + """test tree run with write(), distant 2 targets, separate thread""" + self._tree_run_write(NODE_DISTANT2, separate_thread=True) + def test_tree_run_write_direct_mt(self): """test tree run with write(), direct target, in topology, separate thread""" self._tree_run_write(NODE_DIRECT, separate_thread=True) @@ -303,11 +313,12 @@ def _tree_copy_file(self, target): try: worker = self.task.copy(srcf.name, dest, nodes=target, handler=teh) self.task.run() + target_cnt = len(NodeSet(target)) self.assertEqual(teh.ev_start_cnt, 1) - self.assertEqual(teh.ev_pickup_cnt, 1) + self.assertEqual(teh.ev_pickup_cnt, target_cnt) self.assertEqual(teh.ev_read_cnt, 0) #self.assertEqual(teh.ev_written_cnt, 0) # FIXME - self.assertEqual(teh.ev_hup_cnt, 1) + self.assertEqual(teh.ev_hup_cnt, target_cnt) self.assertEqual(teh.ev_timedout_cnt, 0) self.assertEqual(teh.ev_close_cnt, 1) with open(dest, 'r') as destf: @@ -319,6 +330,10 @@ def test_tree_copy_file_distant(self): """test tree copy: file, distant target""" self._tree_copy_file(NODE_DISTANT) + def test_tree_copy_file_distant2(self): + """test tree copy: file, distant 2 targets""" + self._tree_copy_file(NODE_DISTANT2) + def test_tree_copy_file_direct(self): """test tree copy: file, direct target, in topology""" self._tree_copy_file(NODE_DIRECT) @@ -346,11 +361,12 @@ def _tree_copy_dir(self, target): worker = self.task.copy(srcdir.name, destdir.name + '/', nodes=target, handler=teh) self.task.run() + target_cnt = len(NodeSet(target)) self.assertEqual(teh.ev_start_cnt, 1) - self.assertEqual(teh.ev_pickup_cnt, 1) + self.assertEqual(teh.ev_pickup_cnt, target_cnt) self.assertEqual(teh.ev_read_cnt, 0) #self.assertEqual(teh.ev_written_cnt, 0) # FIXME - self.assertEqual(teh.ev_hup_cnt, 1) + self.assertEqual(teh.ev_hup_cnt, target_cnt) self.assertEqual(teh.ev_timedout_cnt, 0) self.assertEqual(teh.ev_close_cnt, 1) @@ -370,6 +386,11 @@ def test_tree_copy_dir_distant(self): """test tree copy: directory, distant target""" self._tree_copy_dir(NODE_DISTANT) + def test_tree_copy_dir_distant2(self): + """test tree copy: directory, distant 2 targets""" + self._tree_copy_dir(NODE_DISTANT2) + + def test_tree_copy_dir_direct(self): """test tree copy: directory, direct target, in topology""" self._tree_copy_dir(NODE_DIRECT) @@ -382,37 +403,37 @@ def test_tree_copy_dir_gateway(self): """test tree copy: directory, gateway is target""" self._tree_copy_dir(NODE_GATEWAY) - def _tree_rcopy_dir(self, target, dirsuffix=None): + def _tree_rcopy_dir(self, target): teh = TEventHandler() + b1 = b'Lorem Ipsum Unum' * 100 + b2 = b'Lorem Ipsum Duo' * 100 + srcdir = make_temp_dir() destdir = make_temp_dir() - file1 = make_temp_file(b'Lorem Ipsum Unum', suffix=".txt", - dir=srcdir.name) - file2 = make_temp_file(b'Lorem Ipsum Duo', suffix=".txt", - dir=srcdir.name) + file1 = make_temp_file(b1, suffix=".txt", dir=srcdir.name) + file2 = make_temp_file(b2, suffix=".txt", dir=srcdir.name) try: worker = self.task.rcopy(srcdir.name, destdir.name, nodes=target, handler=teh) self.task.run() + target_cnt = len(NodeSet(target)) self.assertEqual(teh.ev_start_cnt, 1) - self.assertEqual(teh.ev_pickup_cnt, 1) + self.assertEqual(teh.ev_pickup_cnt, target_cnt) self.assertEqual(teh.ev_read_cnt, 0) #self.assertEqual(teh.ev_written_cnt, 0) # FIXME - self.assertEqual(teh.ev_hup_cnt, 1) + self.assertEqual(teh.ev_hup_cnt, target_cnt) self.assertEqual(teh.ev_timedout_cnt, 0) self.assertEqual(teh.ev_close_cnt, 1) # rcopy successful? - if not dirsuffix: - dirsuffix = target - rcopy_dest = join(destdir.name, - basename(srcdir.name) + '.' + dirsuffix) - with open(join(rcopy_dest, basename(file1.name)), 'rb') as rfile1: - self.assertEqual(rfile1.read(), b'Lorem Ipsum Unum') - with open(join(rcopy_dest, basename(file2.name)), 'rb') as rfile2: - self.assertEqual(rfile2.read(), b'Lorem Ipsum Duo') + for tgt in NodeSet(target): + rcopy_dest = join(destdir.name, basename(srcdir.name) + '.' + tgt) + with open(join(rcopy_dest, basename(file1.name)), 'rb') as rfile1: + self.assertEqual(rfile1.read(), b1) + with open(join(rcopy_dest, basename(file2.name)), 'rb') as rfile2: + self.assertEqual(rfile2.read(), b2) finally: file1.close() file2.close() @@ -423,7 +444,11 @@ def test_tree_rcopy_dir_distant(self): """test tree rcopy: directory, distant target""" # In distant tree mode, the returned result will include the # hostname of the distant host, not target name - self._tree_rcopy_dir(NODE_DISTANT, dirsuffix=HOSTNAME) + self._tree_rcopy_dir(NODE_DISTANT) + + def test_tree_rcopy_dir_distant2(self): + """test tree rcopy: directory, distant 2 targets""" + self._tree_rcopy_dir(NODE_DISTANT2) def test_tree_rcopy_dir_direct(self): """test tree rcopy: directory, direct target, in topology""" @@ -437,6 +462,54 @@ def test_tree_rcopy_dir_gateway(self): """test tree rcopy: directory, gateway is target""" self._tree_rcopy_dir(NODE_GATEWAY) + def _tree_rcopy_file(self, target): + teh = TEventHandler() + + # The file needs to be large enough to test GH#545 + b1 = b'Lorem Ipsum' * 1100000 + + srcdir = make_temp_dir() + destdir = make_temp_dir() + srcfile = make_temp_file(b1, suffix=".txt", dir=srcdir.name) + + try: + worker = self.task.rcopy(srcfile.name, destdir.name, nodes=target, handler=teh) + self.task.run() + target_cnt = len(NodeSet(target)) + self.assertEqual(teh.ev_start_cnt, 1) + self.assertEqual(teh.ev_pickup_cnt, target_cnt) + self.assertEqual(teh.ev_read_cnt, 0) + #self.assertEqual(teh.ev_written_cnt, 0) # FIXME + self.assertEqual(teh.ev_hup_cnt, target_cnt) + self.assertEqual(teh.ev_timedout_cnt, 0) + self.assertEqual(teh.ev_close_cnt, 1) + + # rcopy successful? + for tgt in NodeSet(target): + rcopy_dest = join(destdir.name, basename(srcfile.name) + '.' + tgt) + with open(rcopy_dest, 'rb') as tfile: + self.assertEqual(tfile.read(), b1) + finally: + srcfile.close() + srcdir.cleanup() + destdir.cleanup() + + def test_tree_rcopy_file_distant(self): + """test tree rcopy: file, distant target""" + self._tree_rcopy_file(NODE_DISTANT) + + def test_tree_rcopy_file_distant2(self): + """test tree rcopy: file, distant 2 targets""" + self._tree_rcopy_file(NODE_DISTANT2) + + def test_tree_rcopy_file_direct(self): + """test tree rcopy: file, direct target, in topology""" + self._tree_rcopy_file(NODE_DIRECT) + + def test_tree_rcopy_file_foreign(self): + """test tree rcopy: file, direct target, not in topology""" + self._tree_rcopy_file(NODE_FOREIGN) + def test_tree_worker_missing_arguments(self): """test TreeWorker with missing arguments""" teh = TEventHandler()