Added check_core_file_exists and test_probe_glusterd (#456)

* Added check_core_file_exists

Added the function check_core_file_exists.

Fixes: #359

Signed-off-by: Ayush Ujjwal <[email protected]>

* replaced run and modified the output

Signed-off-by: Ayush Ujjwal <[email protected]>

* Rebased and removved the conflicts

Signed-off-by: Ayush Ujjwal <[email protected]>

* fixed lint issues

Signed-off-by: Ayush Ujjwal <[email protected]>

* added try except to prevent failure even when ls cannot access the file

Signed-off-by: Ayush Ujjwal <[email protected]>

* added a test case

Signed-off-by: Ayush Ujjwal <[email protected]>

* indentation modified

Signed-off-by: Ayush Ujjwal <[email protected]>

* changed the nature of test case

Signed-off-by: Ayush Ujjwal <[email protected]>

* multiple values in ret[msg] handled; modified the boolean values

Signed-off-by: Ayush Ujjwal <[email protected]>

Author: aujjwal-redhat

.pylintrc

@@ -163,7 +163,8 @@ disable=print-statement,
         F811,
         F841,
         C901,
-        R0915
+        R0915,
+        W0102
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option

common/ops/support_ops/io_ops.py

@@ -3,6 +3,7 @@
 holds API for running all the IO commands.
 """
 import os
+import re
 from common.ops.abstract_ops import AbstractOps
 
 
@@ -126,6 +127,90 @@ def create_deep_dirs_with_files(self, path: str, dir_start_no: int,
                f"--num-of-files {no_files} {path}")
         return self.execute_command_async(cmd, node)
 
+    def check_core_file_exists(self, nodes: list, testrun_timestamp,
+                               paths=['/', '/var/log/core',
+                                      '/tmp', '/var/crash', '~/']):
+        '''
+        Listing directories and files in "/", /var/log/core, /tmp,
+        "/var/crash", "~/" directory for checking if the core file
+        created or not
+
+        Args:
+
+            nodes(list):
+                List of nodes need to pass from test method
+            testrun_timestamp:
+                This time stamp need to pass from test method
+                test case running started time, time format is EPOCH
+                time format, use below command for getting timestamp
+                of test case 'date +%s'
+            paths(list):
+                By default core file will be verified in "/","/tmp",
+                "/var/log/core", "/var/crash", "~/"
+        Return:
+         bool : True if core file was created
+                else False
+        If test case need to verify core file in specific path,
+        need to pass path from test method
+        '''
+        count = 0
+        cmd_list = []
+        for path in paths:
+            cmd = ' '.join(['cd', path, '&&', 'ls', 'core*'])
+            cmd_list.append(cmd)
+
+        # Checks for core file in "/", "/var/log/core", "/tmp" "/var/crash",
+        # "~/" directory
+        for node in nodes:
+            cmd = 'grep -r "time of crash" /var/log/glusterfs/'
+            try:
+                ret = self.execute_abstract_op_node(cmd, node)
+                logfiles = " ".join(ret['msg'])
+                if ret['error_code'] == 0:
+                    self.logger.error(" Seems like there was a crash,"
+                                      " kindly check the logfiles, "
+                                      "even if you don't see a core file")
+                for logfile in logfiles.strip('\n').split('\n'):
+                    self.logger.error(f"Core was found in "
+                                      f"{logfile.split(':')[0]}")
+            except Exception as error:
+                self.logger.info(f"Error: {error}")
+
+            for cmd in cmd_list:
+                try:
+                    ret = self.execute_abstract_op_node(cmd, node)
+                    out = " ".join(ret['msg'])
+                    self.logger.info("storing all files and directory "
+                                     "names into list")
+                    dir_list = re.split(r'\s+', out)
+
+                    # checking for core file created or not in "/"
+                    # "/var/log/core", "/tmp" directory
+                    self.logger.info("checking core file created or not")
+                    for file1 in dir_list:
+                        if re.search(r'\bcore\.[\S]+\b', file1):
+                            file_path_list = re.split(r'[\s]+', cmd)
+                            file_path = file_path_list[1] + '/' + file1
+                            time_cmd = 'stat ' + '-c ' + '%X ' + file_path
+                            ret = self.execute_abstract_op_node(time_cmd, node)
+                            file_timestamp = ret['msg'][0].rstrip('\n')
+                            file_timestamp = file_timestamp.strip()
+                            if file_timestamp > testrun_timestamp:
+                                count += 1
+                                self.logger.error(f"New core file was created "
+                                                  f"and found at {file1}")
+                            else:
+                                self.logger.info("Old core file Found")
+                except Exception as error:
+                    self.logger.info(f"Error: {error}")
+        # return the status of core file
+        if count >= 1:
+            self.logger.error("Core file created glusterd crashed")
+            return True
+        else:
+            self.logger.info("No core files found ")
+            return False
+
     def collect_mounts_arequal(self, mounts: dict, path=''):
         """
         Collects arequal from all the mounts

tests/functional/glusterd/test_probe_glusterd.py

@@ -0,0 +1,75 @@
+"""
+Copyright (C) 2020 Red Hat, Inc. <http://www.redhat.com>
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+any later version.
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+Description:
+    Test Cases in this module related to peer probe invalid ip,
+    non existing ip, non existing host.
+"""
+# disruptive;
+
+from tests.nd_parent_test import NdParentTest
+
+
+class TestCase(NdParentTest):
+
+    def run_test(self, redant):
+        '''
+        Test script to verify peer probe non existing ip,
+        non_exsting_host and invalid-ip, peer probe has to
+        be fail for invalid-ip, non-existing-ip and
+        non existing host, verify Glusterd services up and
+        running or not after invalid peer probe,
+        and core file should not get created
+        under "/", /var/log/core and /tmp  directory
+        '''
+        ret = redant.execute_abstract_op_node('date +%s', self.server_list[0])
+        test_timestamp = ret['msg'][0].strip()
+        # Assigning non existing ip to variable
+        non_exist_ip = '256.256.256.256'
+
+        # Assigning invalid ip to variable
+        invalid_ip = '10.11.a'
+
+        # Assigning non existing host to variable
+        non_exist_host = 'abc.lab.eng.blr.redhat.com'
+
+        # Peer probe checks for non existing host
+        try:
+            ret = redant.peer_probe(non_exist_host, self.server_list[0])
+        except Exception as error:
+            redant.logger.info("Peer probe failed for non-existing server")
+            redant.logger.info(f"Error: {error}")
+
+        try:
+            ret = redant.peer_probe(invalid_ip, self.server_list[0])
+        except Exception as error:
+            redant.logger.info("Peer probe failed for invalid IP")
+            redant.logger.info(f"Error: {error}")
+
+        try:
+            ret = redant.peer_probe(non_exist_ip, self.server_list[0])
+        except Exception as error:
+            redant.logger.info("Peer probe failed for non-existing IP")
+            redant.logger.info(f"Error: {error}")
+
+        # Checks Glusterd services running or not after peer probe
+        # to invalid host and non existing host
+
+        redant.is_glusterd_running(self.server_list[0])
+
+        # Chekcing core file created or not in "/", "/tmp" and
+        # "/var/log/core" directory
+        ret = redant.check_core_file_exists(self.server_list, test_timestamp)
+        if ret:
+            raise Exception('Core file found')