merge DIOPI-TEST

DIOPI-TEST/python/conformance/conformance_test.py

@@ -66,11 +66,12 @@ def allclose(cfg: dict, tensor1: np.ndarray, tensor2: np.ndarray, sum_to_compare
     passed = np.allclose(tensor1, tensor2, rtol, atol, True)
     if record:
         save_precision(cfg, tensor1, tensor2, passed, var_name)
-    if not passed and logger.level == 10:
+    if not passed:
         sum1 = tensor1.sum()
         sum2 = tensor2.sum()
         mask = np.isclose(tensor1, tensor2, rtol, atol, True)
         max_diff = np.abs(tensor1 - tensor2).max()
+        logger.info(f"Max of diff is {max_diff}.")
         logger.debug(f"Sum of {var_name} is {sum1}, Sum of {var_name}_ref is {sum2}, Max of diff is {max_diff}. \
                      \n" + f"{var_name} is {tensor1},\n{var_name}_ref is {tensor2},\nMask is {mask}\n")
     return passed
@@ -198,7 +199,7 @@ def test_normal(mean, std, size=None):
         out_numpy = out_numpy.flatten()
         p_value = stats.kstest(out_numpy, 'norm', args=(mean, std))[1]
         # pytorch use 0.0001, but stats.kstest use 0.05 as threshold
-        assert p_value > 0.001, "failed to execute normal"
+        assert p_value > 0.0005, "failed to execute normal"
 
     def test_normal_(input, mean, std, shape=None):
         from scipy import stats
@@ -208,13 +209,45 @@ def test_normal_(input, mean, std, shape=None):
         p_value = stats.kstest(out_numpy, 'norm', args=(mean, std))[1]
         assert p_value > 0.05, "failed to execute normal_"
 
+    def test_multinomial(input, num_samples, replacement):
+        out = F.multinomial(input, num_samples, replacement)
+        out_numpy = out.numpy()
+        has_duplicates = False
+        if len(out.size()) == 2:
+            has_duplicates = len(out_numpy[0]) != len(set(out_numpy[0]))
+        else:
+            has_duplicates = len(out_numpy) != len(set(out_numpy))
+        if not replacement:
+            assert has_duplicates is False, "failed to execute multinomial"
+        out_numpy = out_numpy.flatten()
+        assert len(out_numpy) % num_samples == 0, "failed to execute multinomial"
+
+
+def config_to_format_string(data, indent=0):
+    yaml_str = ""
+    if isinstance(data, dict):
+        for key, value in data.items():
+            if value is None or value == [] or value == {} or value == "":
+                continue
+            yaml_str += "\n" + " " * indent + f"{key}: "
+            if key not in ["shape", "value"]:
+                yaml_str += config_to_format_string(value, indent + 2)
+            else:
+                yaml_str += config_to_format_string(str(value), indent + 2)
+    elif isinstance(data, (list, tuple)):
+        for item in data:
+            yaml_str += "\n" + " " * indent + "- " + config_to_format_string(item, indent + 2)
+    else:
+        yaml_str += f"{data}"
+    return yaml_str
+
 
 class ConformanceTest(object):
     r'''
     Run all functions by using input, then compare_with_gen_output with saved output
     '''
     @staticmethod
-    def run(func_name, model_name, filter_dtype_str_list):
+    def run(func_name, model_name, filter_dtype_str_list, debug_level):
 
         _cur_dir = os.path.dirname(os.path.abspath(__file__))
         inputs_dir_path = os.path.join(_cur_dir, "../data/" + model_name + "/inputs")
@@ -263,8 +296,18 @@ def run(func_name, model_name, filter_dtype_str_list):
                     sum_to_compare = True if 'sorted' in kwargs and ~kwargs['sorted'] else False
                     passed = compare_with_gen_output(output, data['cfg'], output_reference, sum_to_compare) \
                         if need_output else True
-                    logger.info(f"Run diopi_functions.{cfg_func_name} succeed") \
-                        if passed else logger.error(f"Run diopi_functions.{cfg_func_name} failed", tag=test_tag, info=tensor_info)
+                    if passed:
+                        logger.info(f"Run diopi_functions.{cfg_func_name} succeed")
+                    else:
+                        logger.error(f"Run diopi_functions.{cfg_func_name} failed", tag=test_tag, info=tensor_info)
+                        if debug_level > 0:
+                            logger.error("failed config:\n%s", config_to_format_string(data['cfg']))
+                            if debug_level > 1:
+                                logger.error("failed arguments:")
+                                for key, arg in kwargs.items():
+                                    logger.error(f"{key}: {arg}")
+                                logger.error(f"output_reference:\n{output_reference}")
+                                logger.error(f"output:\n{output}")
                 except FunctionNotImplementedError as e:
                     logger.error(f"NotImplemented: {e}")
                     continue
@@ -299,10 +342,21 @@ def run(func_name, model_name, filter_dtype_str_list):
 
                     try:
                         grad_input = eval(f"F.{cfg_func_name}_backward(**kwargs, **backward_para)")
-                        # import pdb;pdb.set_trace()
                         passed = compare_with_gen_output(grad_input, data['cfg'], backward_out_reference)
-                        logger.info(f"Run diopi_functions.{cfg_func_name}_backward succeed") \
-                            if passed else logger.error(f"Run diopi_functions.{cfg_func_name}_backward failed", tag=test_tag, info=tensor_info)
+                        if passed:
+                            logger.info(f"Run diopi_functions.{cfg_func_name}_backward succeed")
+                        else:
+                            logger.error(f"Run diopi_functions.{cfg_func_name}_backward failed", tag=test_tag, info=tensor_info)
+                            if debug_level > 0:
+                                logger.error("failed config:\n%s", config_to_format_string(data['cfg']))
+                                if debug_level > 1:
+                                    logger.error("failed arguments:")
+                                    for key, arg in kwargs.items():
+                                        logger.error(f"{key}: {arg}")
+                                    for key, arg in backward_para.items():
+                                        logger.error(f"{key}: {arg}")
+                                    logger.error(f"grad_reference:\n{backward_out_reference}")
+                                    logger.error(f"grad:\n{grad_input}")
                         write_precision(data["cfg"], cfg_func_name + '_bp', passed)
                     except FunctionNotImplementedError as e:
                         logger.error(f"NotImplemented: {e}")

DIOPI-TEST/python/conformance/device_config_helper.py

@@ -0,0 +1,124 @@
+import copy
+from .config import _must_be_the_type, _must_exist, expand_cfg_by_name
+
+
+class Skip:
+    def __init__(self, value):
+        self.value = value
+
+
+def _must_be_the_list_or_tuple_of_type(cfg_path: str, cfg_dict: dict, required_type, cfg_keys: list) -> None:
+    if isinstance(required_type, (list, tuple)):
+        types_str = ""
+        for i in required_type:
+            types_str += i.__name__
+            types_str += ' or '
+        types_str = types_str[:-4]
+    else:
+        types_str = required_type.__name__
+
+    err = f"key %s should be the list or tuple of {types_str} in {cfg_path}"
+    for key in cfg_keys:
+        if key in cfg_dict.keys():
+            assert isinstance(cfg_dict[key], (list, tuple)), err % key
+            for v in cfg_dict[key]:
+                assert isinstance(v, required_type), err % key
+
+
+def check_configs_format(cfgs_dict: dict):
+    for case_k, case_v in cfgs_dict.items():
+        domain = f"device_configs.{case_k}"
+        _must_be_the_type(domain, case_v, list, ["dtype"])
+        if "dtype" in case_v.keys():
+            _must_be_the_list_or_tuple_of_type(domain, case_v, Skip, ["dtype"])
+
+        _must_exist(domain, case_v, ['name'])
+        _must_be_the_type(domain, case_v, list, ['name'])
+
+        if "tensor_para" in case_v.keys():
+            _must_be_the_type(domain, case_v, dict, ['tensor_para'])
+            _must_exist(domain + ".tensor_para", case_v["tensor_para"], ["args"])
+            _must_be_the_type(domain + ".tensor_para", case_v["tensor_para"],
+                              (list, tuple), ['args'])
+            domain_tmp = domain + ".tensor_para.args"
+            for arg in case_v["tensor_para"]['args']:
+                _must_exist(domain_tmp, arg, ["ins"])
+                _must_be_the_list_or_tuple_of_type(domain_tmp, arg, Skip, ['shape', 'value', 'dtype'])
+
+        if "para" in case_v.keys():
+            _must_be_the_type(domain, case_v, dict, ['para'])
+            dict_obj = case_v["para"]
+            _must_be_the_list_or_tuple_of_type(domain + ".para", dict_obj, Skip,
+                                               [i for i in dict_obj.keys()])
+
+
+def expand_tensor_paras_args_by_ins(cfgs_dict):
+    '''
+    [
+        {
+            "ins": ['x1', 'x2'],
+            "shape": [(2, 3, 16), (4, 32, 7, 7)],
+        },
+    ]
+    ====>
+    {
+        'x1':{
+            "ins": ['x1'],
+            "shape": [(2, 3, 16), (4, 32, 7, 7)],
+        },
+        'x2':{
+            "ins": ['x2'],
+            "shape": [(2, 3, 16), (4, 32, 7, 7)],
+        },
+    }
+    '''
+    for cfg_name in cfgs_dict:
+        tensor_para_args = cfgs_dict[cfg_name]["tensor_para"]["args"]
+        tmp_tensor_para_args = {}
+        for arg in tensor_para_args:
+            assert isinstance(arg["ins"], (list, tuple))
+            for in_name in arg["ins"]:
+                tmp_tensor_para_args[in_name] = copy.deepcopy(arg)
+                tmp_tensor_para_args[in_name]["ins"] = [in_name]
+        cfgs_dict[cfg_name]["tensor_para"]["args"] = tmp_tensor_para_args
+
+
+def format_cfg(cases):
+    for case_k, case_v in cases.items():
+        # set [] for defalut para, tensor_para, para
+        if "tensor_para" not in case_v.keys():
+            case_v["tensor_para"] = {}
+        if "args" not in case_v["tensor_para"].keys():
+            case_v["tensor_para"]["args"] = []
+        if "para" not in case_v.keys():
+            case_v["para"] = {}
+
+
+def extract_value_from_skip(cfgs_dict):
+    for case_k, case_v in cfgs_dict.items():
+        if "dtype" in case_v.keys():
+            case_v["dtype"] = [x.value for x in case_v["dtype"]]
+        for para_k, para_v in case_v["para"].items():
+            case_v["para"][para_k] = [x.value for x in para_v]
+        for arg_k, arg_v in case_v["tensor_para"]["args"].items():
+            if "shape" in arg_v:
+                arg_v["shape"] = [x.value for x in arg_v["shape"]]
+            if "value" in arg_v:
+                arg_v["value"] = [x.value for x in arg_v["value"]]
+            if "dtype" in arg_v:
+                arg_v["dtype"] = [x.value for x in arg_v["dtype"]]
+
+
+class DeviceConfig(object):
+    r"""
+    Process device config file
+    """
+
+    @staticmethod
+    def process_configs(cfgs_dict: dict):
+        check_configs_format(cfgs_dict)
+        cfgs_dict = expand_cfg_by_name(cfgs_dict, 'name')
+        format_cfg(cfgs_dict)
+        expand_tensor_paras_args_by_ins(cfgs_dict)
+        extract_value_from_skip(cfgs_dict)
+        return cfgs_dict