Add examples of (1) common setup per class and serial test execution (2) replication testing

build.sh

@@ -94,10 +94,10 @@ if [ ! -z "${ASAN_BUILD}" ]; then
 else
     # TEST_PATTERN can be used to run specific tests or test patterns.
     if [[ -n "$TEST_PATTERN" ]]; then
-        python3 -m pytest --cache-clear -v "$SCRIPT_DIR/tests/" -k $TEST_PATTERN
+        python3 -m pytest --cache-clear -v "$SCRIPT_DIR/tests/" -k $TEST_PATTERN --order-scope=class
     else
         echo "TEST_PATTERN is not set. Running all integration tests."
-        python3 -m pytest --cache-clear -v "$SCRIPT_DIR/tests/"
+        python3 -m pytest --cache-clear -v "$SCRIPT_DIR/tests/" --order-scope=class
     fi
 fi
 

requirements.txt

@@ -1,3 +1,4 @@
 valkey
 pytest==6
 black
+pytest-order

src/valkey_test_case.py

@@ -176,7 +176,7 @@ def children_pids(self):
 
     def wait_for_replicas(self, num_of_replicas):
         wait_for_equal(
-            lambda: self.client.info()["connected_slaves"],
+            lambda: self.client.info(section="replication")["connected_slaves"],
             num_of_replicas,
             timeout=MAX_REPLICA_WAIT_TIME,
         )
@@ -323,7 +323,7 @@ def num_replicas_online(self, client=None):
         if client is None:
             client = self.client
         count = 0
-        for k, v in client.info().items():
+        for k, v in client.info(section="replication").items():
             if re.match("^slave[0-9]", k) and v["state"] == "online":
                 count += 1
         return count
@@ -345,8 +345,8 @@ def is_primary_link_up(self, client=None):
             client = self.client
         """Returns True if role is slave and master_link_status is up"""
         if (
-            client.info()["role"] == "slave"
-            and client.info()["master_link_status"] == "up"
+            client.info(section="replication")["role"] == "slave"
+            and client.info(section="replication")["master_link_status"] == "up"
         ):
             return True
         return False
@@ -479,7 +479,13 @@ def get_valkey_handle(self):
 
     # Expose bind_ip parameter to caller to have more flexible
     def create_server(
-        self, testdir, bind_ip=None, port=None, server_path=server_path, args=""
+        self,
+        testdir,
+        bind_ip=None,
+        port=None,
+        server_path=server_path,
+        args="",
+        skip_teardown=False,
     ):
         if not bind_ip:
             bind_ip = self.get_bind_ip()
@@ -495,7 +501,8 @@ def create_server(
             cwd=testdir,
             server_path=server_path,
         )
-        self.server_list.append(valkey_server)
+        if not skip_teardown:
+            self.server_list.append(valkey_server)
         valkey_server.args.update(args)
         valkey_cli = valkey_server.start()
         return valkey_server, valkey_cli
@@ -540,21 +547,33 @@ def exit(self, remove_rdb=True, remove_nodes_conf=True):
 
 
 class ReplicationTestCase(ValkeyTestCase):
-    num_replicas = 1
-
-    def setup_replication(self, num_replicas=num_replicas):
+    num_replicas = 0
+    skip_teardown = False
+    replicas = []
+    # Primary server
+    server = None
+
+    def setup_replication(
+        self, num_replicas=1, primary_server=None, skip_teardown=False
+    ):
+        self.num_replicas = num_replicas
+        self.replicas = []
+        if primary_server is not None:
+            self.server = primary_server
+        self.skip_teardown = skip_teardown
         self.create_replicas(num_replicas)
         self.start_replicas()
-        self.wait_for_all_replicas_online(self.num_replicas)
         self.wait_for_replicas(self.num_replicas)
         self.wait_for_primary_link_up_all_replicas()
         self.wait_for_all_replicas_online(self.num_replicas)
         for i in range(len(self.replicas)):
             self.waitForReplicaToSyncUp(self.replicas[i])
+        return self.replicas
 
     def teardown(self):
-        ValkeyTestCase.teardown(self)
-        self.destroy_replicas()
+        if not self.skip_teardown:
+            self.destroy_replicas()
+            ValkeyTestCase.teardown(self)
 
     def _create_replica(self, primaryhost, primaryport, server_path):
         return ValkeyReplica(
@@ -575,9 +594,6 @@ def create_replicas(
         connection_type="tcp",
         server_path=None,
     ):
-
-        self.destroy_replicas()
-
         default_primaryhost = None
         default_port = None
         if connection_type == "tcp":
@@ -599,8 +615,6 @@ def create_replicas(
         if not primaryport:
             primaryport = default_port
 
-        self.num_replicas = num_replicas
-        self.replicas = []
         for _ in range(self.num_replicas):
             replica = self._create_replica(primaryhost, primaryport, server_path)
             replica.set_startup_args(self.args)
@@ -635,9 +649,9 @@ def wait_for_value_propagate_to_replicas(self, key, value, db=0):
             )
 
     def waitForReplicaOffsetToSyncUp(self, primary, replica):
-        pinfo = primary.info()["master_repl_offset"]
+        pinfo = primary.info(section="replication")["master_repl_offset"]
         wait_for_equal(
-            lambda: replica.client.info()["slave_repl_offset"],
+            lambda: replica.client.info(section="replication")["slave_repl_offset"],
             pinfo.get_primary_repl_offset(),
             timeout=TEST_MAX_WAIT_TIME_SECONDS,
         )

tests/test_reuse_replication_setup_per_class.py

@@ -0,0 +1,70 @@
+from conftest import resource_port_tracker
+from valkey_test_case import ReplicationTestCase
+import pytest
+import os
+
+
+class TestExampleReuseReplicationSetupPerClass(ReplicationTestCase):
+    """
+    Every test will use the same replication setup and the servers will be torn down in the final test.
+    This also adds ordering of the tests in a serial manner.
+    """
+
+    common_server = None
+    common_client = None
+    common_replicas = []
+
+    def setup_server_and_client(self):
+        server_path = f"{os.path.dirname(os.path.realpath(__file__))}/.build/binaries/{os.environ['SERVER_VERSION']}/valkey-server"
+        # This is just to avoid a delay on startup for setting up replication.
+        additional_startup_args = {
+            "repl-diskless-sync": "yes",
+            "repl-diskless-sync-delay": "0",
+        }
+        (
+            primary_server,
+            TestExampleReuseReplicationSetupPerClass.common_client,
+        ) = self.create_server(
+            testdir=self.testdir,
+            server_path=server_path,
+            args=additional_startup_args,
+            skip_teardown=True,
+        )
+        TestExampleReuseReplicationSetupPerClass.common_server = primary_server
+        TestExampleReuseReplicationSetupPerClass.common_replicas = (
+            self.setup_replication(
+                num_replicas=1, skip_teardown=True, primary_server=primary_server
+            )
+        )
+
+    @pytest.mark.order(1)
+    def test_replication_basic1(self):
+        # Set up the server and client just one time.
+        self.setup_server_and_client()
+        server = TestExampleReuseReplicationSetupPerClass.common_server
+        replicas = TestExampleReuseReplicationSetupPerClass.common_replicas
+        client = TestExampleReuseReplicationSetupPerClass.common_client
+        server = TestExampleReuseReplicationSetupPerClass.common_server
+        replicas = TestExampleReuseReplicationSetupPerClass.common_replicas
+        client.execute_command("SET K V")
+        self.waitForReplicaToSyncUp(replicas[0])
+        assert replicas[0].client.execute_command("GET K") == b"V"
+
+    @pytest.mark.order(2)
+    def test_replication_basic2(self):
+        server = TestExampleReuseReplicationSetupPerClass.common_server
+        replicas = TestExampleReuseReplicationSetupPerClass.common_replicas
+        client = TestExampleReuseReplicationSetupPerClass.common_client
+        assert replicas[0].client.execute_command("GET K") == b"V"
+        client.execute_command("SET K VV")
+        self.waitForReplicaToSyncUp(replicas[0])
+        assert replicas[0].client.execute_command("GET K") == b"VV"
+
+    @pytest.mark.order(3)
+    def test_replication_basic3(self):
+        server = TestExampleReuseReplicationSetupPerClass.common_server
+        replicas = TestExampleReuseReplicationSetupPerClass.common_replicas
+        replicas[0].client.execute_command("CONFIG SET repl-timeout 5") == b"OK"
+        assert replicas[0].client.execute_command("GET K") == b"VV"
+        server.exit()
+        replicas[0].exit()

tests/test_reuse_setup_per_class.py

@@ -0,0 +1,45 @@
+from conftest import resource_port_tracker
+from valkey_test_case import ValkeyTestCase
+import pytest
+import sys
+import os
+
+
+class TestExampleReuseSetupPerClass(ValkeyTestCase):
+    """
+    Every test will use the same server and client instance and the server will be torn down in the final test.
+    This also adds ordering of the tests in a serial manner.
+    """
+
+    common_server = None
+    common_client = None
+
+    def setup_server_and_client(self):
+        server_path = f"{os.path.dirname(os.path.realpath(__file__))}/.build/binaries/{os.environ['SERVER_VERSION']}/valkey-server"
+        additional_startup_args = ""
+        (
+            TestExampleReuseSetupPerClass.common_server,
+            TestExampleReuseSetupPerClass.common_client,
+        ) = self.create_server(
+            testdir=self.testdir,
+            server_path=server_path,
+            args=additional_startup_args,
+            skip_teardown=True,
+        )
+
+    @pytest.mark.order(1)
+    def test_basic1(self):
+        # Set up the server and client just one time.
+        self.setup_server_and_client()
+        TestExampleReuseSetupPerClass.common_client.execute_command("PING")
+
+    @pytest.mark.order(2)
+    def test_basic2(self):
+        TestExampleReuseSetupPerClass.common_client.execute_command("PING")
+        c = TestExampleReuseSetupPerClass.common_server.get_new_client()
+        c.execute_command("PING")
+
+    @pytest.mark.order(3)
+    def test_basic3(self):
+        TestExampleReuseSetupPerClass.common_client.execute_command("SET K V")
+        TestExampleReuseSetupPerClass.common_server.exit()

tests/test_specific_replication_setup_per_class.py

@@ -0,0 +1,31 @@
+from conftest import resource_port_tracker
+from valkey_test_case import ReplicationTestCase
+import pytest
+import sys
+import os
+
+
+class TestExampleReplication(ReplicationTestCase):
+    @pytest.fixture(autouse=True)
+    def setup_test(self, setup):
+        # This is just to avoid a delay on startup for setting up replication.
+        additional_startup_args = {
+            "repl-diskless-sync": "yes",
+            "repl-diskless-sync-delay": "0",
+        }
+        server_path = f"{os.path.dirname(os.path.realpath(__file__))}/.build/binaries/{os.environ['SERVER_VERSION']}/valkey-server"
+        self.server, self.client = self.create_server(
+            testdir=self.testdir, server_path=server_path, args=additional_startup_args
+        )
+        self.setup_replication(num_replicas=1)
+
+    def test_replication1(self):
+        self.replicas[0].client.execute_command("CONFIG SET repl-timeout 5") == b"OK"
+        self.client.execute_command("SET K V")
+        self.waitForReplicaToSyncUp(self.replicas[0])
+        assert self.replicas[0].client.execute_command("GET K") == b"V"
+
+    def test_replication2(self):
+        self.client.execute_command("SET K VV")
+        self.waitForReplicaToSyncUp(self.replicas[0])
+        assert self.replicas[0].client.execute_command("GET K") == b"VV"

tests/test_example_setup_per_class.py → tests/test_specific_setup_per_class.py

@@ -15,7 +15,7 @@ def setup_test(self, setup):
         )
 
 
-class TestExamplePerClassSetup(ExampleTestCaseBase):
+class TestExampleSpecificSetupPerClass(ExampleTestCaseBase):
     """
     Every test will use the same server startup from the ExampleTestCaseBase.
     """

tests/test_example_setup_per_test.py → tests/test_specific_setup_per_test.py

@@ -5,7 +5,7 @@
 import os
 
 
-class TestExamplePerTestSetup(ValkeyTestCase):
+class TestExampleSpecificSetupPerTest(ValkeyTestCase):
     """
     Every test in this class will have a custom setup that is defined individually.
     """