From b41710140432f9ed0afd54ce01ee1b8cfb093351 Mon Sep 17 00:00:00 2001 From: niushengxiao Date: Tue, 15 Jul 2025 13:47:23 +0800 Subject: [PATCH] feat: extend the num head terms for fp8 calibration --- .../common/offline_fp8_quant_mem_manager.py | 11 +- lightllm/server/api_cli.py | 1 - ...st_kv_cache_calib_per_head_qwen3_235b.json | 1968 ++++++++++++----- ..._kv_cache_calib_per_tensor_qwen3_235b.json | 276 +-- 4 files changed, 1506 insertions(+), 750 deletions(-) diff --git a/lightllm/common/offline_fp8_quant_mem_manager.py b/lightllm/common/offline_fp8_quant_mem_manager.py index 686371231..5cc0b12d0 100755 --- a/lightllm/common/offline_fp8_quant_mem_manager.py +++ b/lightllm/common/offline_fp8_quant_mem_manager.py @@ -25,7 +25,6 @@ def __init__( self.qmax = torch.finfo(torch.float8_e4m3fn).max self.qmin = torch.finfo(torch.float8_e4m3fn).min - self.layer_num = layer_num self.total_head_num = head_num * dist.get_world_size() if dist.is_initialized() else head_num self.count = 0 self.scales = None @@ -45,7 +44,13 @@ def __init__( self.scales_list = cfg["scales"] self.scales = torch.tensor(self.scales_list, dtype=torch.float32, device="cuda").view(cfg["scales_shape"]) if not get_env_start_args().enable_fa3: - self.scales = torch.repeat_interleave(self.scales, self.head_num, dim=-1) + self.scales = torch.repeat_interleave(self.scales, head_num, dim=-1) + elif cfg["num_head"] > self.total_head_num: + factor = cfg["num_head"] // self.total_head_num + self.scales = self.scales[..., ::factor].contiguous() + elif cfg["num_head"] < self.total_head_num: + factor = self.total_head_num // cfg["num_head"] + self.scales = torch.repeat_interleave(self.scales, factor, dim=-1).contiguous() if get_env_start_args().enable_fa3 and dist.is_initialized() and dist.get_world_size() > 1: half_head = self.total_head_num // 2 start_head = dist.get_rank() * head_num @@ -77,7 +82,7 @@ def _load_and_check_config(self): raise ValueError( f"num_layers {cfg['num_layers']} in config " f"not match current layer_num {self.layer_num}" ) - if cfg["num_head"] != self.total_head_num: + if cfg["num_head"] % self.total_head_num != 0 and self.total_head_num % cfg["num_head"] != 0: raise ValueError( f"num_head {cfg['num_head']} in config " f"not match current model head num {self.total_head_num}" ) diff --git a/lightllm/server/api_cli.py b/lightllm/server/api_cli.py index c0ccd7a1c..3763d5862 100644 --- a/lightllm/server/api_cli.py +++ b/lightllm/server/api_cli.py @@ -175,7 +175,6 @@ def make_argument_parser() -> argparse.ArgumentParser: export_fp8kv_calibration record and export kv cache quant calibration results to a json file. It can be used for llama and qwen model. Calibration need to disable cudagraph and use fa3 or flashinfer backend. - Tp size must no more than head num when calibration. ppl_int8kv mode use int8 to store kv cache, and use ppl fast kernel; ppl_fp16 mode use ppl fast fp16 decode attention kernel; you need to read source code to make sure the supported detail mode for all models""", diff --git a/test/advanced_config/fp8_calibration_per_head/test_kv_cache_calib_per_head_qwen3_235b.json b/test/advanced_config/fp8_calibration_per_head/test_kv_cache_calib_per_head_qwen3_235b.json index 870ee695a..50a343f00 100644 --- a/test/advanced_config/fp8_calibration_per_head/test_kv_cache_calib_per_head_qwen3_235b.json +++ b/test/advanced_config/fp8_calibration_per_head/test_kv_cache_calib_per_head_qwen3_235b.json @@ -5,951 +5,1703 @@ "qmin": -448.0, "qmax": 448.0, "num_layers": 94, - "num_head": 4, + "num_head": 8, "scales_shape": [ 94, - 8 + 16 ], "scales": [ [ - 0.0334821455180645, + 0.0336216539144516, + 0.0336216539144516, 0.0493861623108387, - 0.0385044664144516, + 0.0493861623108387, + 0.0382254496216774, + 0.0382254496216774, + 0.0552455373108387, 0.0552455373108387, 9.264265099773183e-05, + 9.264265099773183e-05, + 0.00022888185048941523, 0.00022888185048941523, - 0.00015912737580947578, + 0.00016566686099395156, + 0.00016566686099395156, + 0.00015476772387046367, 0.00015476772387046367 ], [ - 0.037109375, - 0.0333426371216774, - 0.0333426371216774, + 0.0373883955180645, + 0.0373883955180645, + 0.0336216539144516, + 0.0336216539144516, + 0.0343191996216774, + 0.0343191996216774, + 0.0398995541036129, 0.0398995541036129, - 0.00028773717349395156, + 0.0002899169921875, + 0.0002899169921875, + 0.00011607579654082656, + 0.00011607579654082656, 0.00010844640200957656, - 0.00010517665941733867, - 0.00013024467625655234 + 0.00010844640200957656, + 0.00013078963092993945, + 0.00013078963092993945 ], [ - 0.0719866082072258, - 0.0809151828289032, + 0.0711495578289032, + 0.0711495578289032, + 0.0775669664144516, + 0.0775669664144516, 0.0731026828289032, - 0.068359375, - 0.0002833775361068547, - 0.00018964495393447578, - 0.00017547608877066523, - 0.00014168876805342734 + 0.0731026828289032, + 0.0736607164144516, + 0.0736607164144516, + 0.0003073556290473789, + 0.0003073556290473789, + 0.00018310548330191523, + 0.00018310548330191523, + 0.0001787458168109879, + 0.0001787458168109879, + 0.00015476772387046367, + 0.00015476772387046367 ], [ - 0.0613839328289032, - 0.0703125, - 0.0591517873108387, - 0.0521763414144516, - 0.00028773717349395156, - 0.00042506627505645156, - 0.0002615792618598789, - 0.0004381452454254031 + 0.063058041036129, + 0.063058041036129, + 0.0700334832072258, + 0.0700334832072258, + 0.0616629496216774, + 0.0616629496216774, + 0.0544084832072258, + 0.0544084832072258, + 0.0002735682937782258, + 0.0002735682937782258, + 0.00047956197522580624, + 0.00047956197522580624, + 0.00027247838443145156, + 0.00027247838443145156, + 0.00044904439710080624, + 0.00044904439710080624 ], [ 0.0725446492433548, - 0.0697544664144516, + 0.0725446492433548, 0.0694754496216774, - 0.0482700914144516, - 0.0003291539032943547, + 0.0694754496216774, + 0.0694754496216774, + 0.0694754496216774, + 0.0510602705180645, + 0.0510602705180645, + 0.00029427665867842734, + 0.00029427665867842734, + 0.0002899169921875, + 0.0002899169921875, + 0.0003836495743598789, + 0.0003836495743598789, 0.00026702880859375, - 0.00040108818211592734, - 0.0003117152664344758 + 0.00026702880859375 ], [ 0.1484375, - 0.1311383992433548, - 0.1037946492433548, - 0.1233258992433548, - 0.00038582939305342734, - 0.00033569338847883046, - 0.00031825475161895156, - 0.0006583078065887094 + 0.1484375, + 0.1328125, + 0.1328125, + 0.1065848246216774, + 0.1065848246216774, + 0.125, + 0.125, + 0.0003749302704818547, + 0.0003749302704818547, + 0.0003160749329254031, + 0.0003160749329254031, + 0.0003008161438629031, + 0.0003008161438629031, + 0.0006626674439758062, + 0.0006626674439758062 ], [ - 0.037109375, + 0.0373883955180645, + 0.0373883955180645, + 0.0491071455180645, 0.0491071455180645, 0.0421316996216774, - 0.0390625, - 0.0005558559205383062, - 0.0006495884736068547, - 0.0003444126923568547, - 0.0003422328445594758 + 0.0421316996216774, + 0.0398995541036129, + 0.0398995541036129, + 0.000579833984375, + 0.000579833984375, + 0.0007062639924697578, + 0.0007062639924697578, + 0.00030299596255645156, + 0.00030299596255645156, + 0.00037057060399092734, + 0.00037057060399092734 ], [ - 0.0345982164144516, - 0.0538504496216774, - 0.0463169664144516, - 0.0569196455180645, - 0.00038582939305342734, - 0.00030953544774092734, - 0.0007193429628387094, - 0.00047738212742842734 + 0.0327845998108387, + 0.0327845998108387, + 0.0558035746216774, + 0.0558035746216774, + 0.0496651791036129, + 0.0496651791036129, + 0.0566406287252903, + 0.0566406287252903, + 0.00042724612285383046, + 0.00042724612285383046, + 0.00031825475161895156, + 0.00031825475161895156, + 0.0007585798157379031, + 0.0007585798157379031, + 0.00042724612285383046, + 0.00042724612285383046 ], [ - 0.1082589328289032, - 0.1010044664144516, - 0.0831473246216774, - 0.1093750074505806, - 0.0008326939423568547, + 0.1104910746216774, + 0.1104910746216774, + 0.0987723246216774, + 0.0987723246216774, + 0.0803571492433548, + 0.0803571492433548, + 0.1104910746216774, + 0.1104910746216774, + 0.0008065360016189516, + 0.0008065360016189516, + 0.0005122593720443547, + 0.0005122593720443547, + 0.0005841936799697578, + 0.0005841936799697578, 0.0004686628235504031, - 0.0005711147096008062, - 0.00046212333836592734 + 0.0004686628235504031 ], [ + 0.0555245578289032, 0.0555245578289032, 0.0412946455180645, + 0.0412946455180645, 0.0449218787252903, - 0.02929687686264515, - 0.0007672991487197578, - 0.0009765625, - 0.000518798828125, - 0.0004359654267318547 + 0.0449218787252903, + 0.02859933115541935, + 0.02859933115541935, + 0.0008021763642318547, + 0.0008021763642318547, + 0.0011160714784637094, + 0.0011160714784637094, + 0.0005035400390625, + 0.0005035400390625, + 0.00044904439710080624, + 0.00044904439710080624 ], [ - 0.0853794664144516, - 0.0870535746216774, - 0.1037946492433548, - 0.086495541036129, - 0.00033351354068145156, - 0.0004141671524848789, - 0.0004207066376693547, - 0.0003160749329254031 + 0.0831473246216774, + 0.0831473246216774, + 0.0881696492433548, + 0.0881696492433548, + 0.1049107164144516, + 0.1049107164144516, + 0.0859375074505806, + 0.0859375074505806, + 0.0003683907852973789, + 0.0003683907852973789, + 0.0003422328445594758, + 0.0003422328445594758, + 0.00048828125, + 0.00048828125, + 0.0003291539032943547, + 0.0003291539032943547 ], [ 0.0482700914144516, + 0.0482700914144516, + 0.0396205373108387, 0.0396205373108387, 0.0429687537252903, + 0.0429687537252903, 0.0407366082072258, - 0.0003465925110504031, - 0.0004381452454254031, - 0.0004207066376693547, - 0.0004512242157943547 + 0.0407366082072258, + 0.0003771100891754031, + 0.0003771100891754031, + 0.00042724612285383046, + 0.00042724612285383046, + 0.00045776370097883046, + 0.00045776370097883046, + 0.0004817417939193547, + 0.0004817417939193547 ], [ - 0.03027343936264515, - 0.0357142873108387, + 0.0318080373108387, + 0.0318080373108387, + 0.0354352705180645, + 0.0354352705180645, + 0.0368303582072258, 0.0368303582072258, 0.0538504496216774, - 0.0005427769501693547, - 0.0003531319962348789, - 0.00030517578125, - 0.00033569338847883046 + 0.0538504496216774, + 0.0005384173127822578, + 0.0005384173127822578, + 0.00033351354068145156, + 0.00033351354068145156, + 0.0002964564773719758, + 0.0002964564773719758, + 0.0003378732071723789, + 0.0003378732071723789 ], [ - 0.0583147332072258, - 0.0585937537252903, + 0.0591517873108387, + 0.0591517873108387, + 0.0574776828289032, + 0.0574776828289032, 0.0493861623108387, - 0.0454799123108387, - 0.00038146975566633046, - 0.0004359654267318547, - 0.00047956197522580624, - 0.00040108818211592734 + 0.0493861623108387, + 0.0457589291036129, + 0.0457589291036129, + 0.0004446847306098789, + 0.0004446847306098789, + 0.00045776370097883046, + 0.00045776370097883046, + 0.0004686628235504031, + 0.0004686628235504031, + 0.000579833984375, + 0.000579833984375 ], [ - 0.0675223246216774, - 0.0753348246216774, - 0.0652901828289032, - 0.0594308078289032, - 0.0002833775361068547, - 0.00031825475161895156, - 0.0003422328445594758, - 0.0004991804016754031 + 0.068917416036129, + 0.068917416036129, + 0.07421875, + 0.07421875, + 0.0686383992433548, + 0.0686383992433548, + 0.0605468787252903, + 0.0605468787252903, + 0.0003008161438629031, + 0.0003008161438629031, + 0.00035531181492842734, + 0.00035531181492842734, + 0.0003771100891754031, + 0.0003771100891754031, + 0.0005296979798004031, + 0.0005296979798004031 ], [ + 0.0368303582072258, 0.0368303582072258, 0.0440848246216774, + 0.0440848246216774, + 0.0471540205180645, 0.0471540205180645, 0.0505022332072258, - 0.0006059919251129031, - 0.00033569338847883046, - 0.00043160576024092734, - 0.0004446847306098789 + 0.0505022332072258, + 0.00044686454930342734, + 0.00044686454930342734, + 0.0003313337219879031, + 0.0003313337219879031, + 0.00041852681897580624, + 0.00041852681897580624, + 0.00044686454930342734, + 0.00044686454930342734 ], [ - 0.0594308078289032, + 0.0599888414144516, + 0.0599888414144516, 0.0482700914144516, - 0.0426897332072258, - 0.0449218787252903, + 0.0482700914144516, + 0.0446428582072258, + 0.0446428582072258, + 0.0468750037252903, + 0.0468750037252903, 0.0003989083634223789, - 0.0006495884736068547, - 0.0006277902284637094, - 0.00030299596255645156 + 0.0003989083634223789, + 0.0008632115204818547, + 0.0008632115204818547, + 0.0007324219332076609, + 0.0007324219332076609, + 0.00035531181492842734, + 0.00035531181492842734 ], [ - 0.0524553582072258, - 0.0714285746216774, - 0.0758928582072258, - 0.0725446492433548, - 0.00039454869693145156, - 0.0004752023087348789, - 0.0004141671524848789, - 0.0003465925110504031 + 0.0558035746216774, + 0.0558035746216774, + 0.0719866082072258, + 0.0719866082072258, + 0.078125, + 0.078125, + 0.0736607164144516, + 0.0736607164144516, + 0.00040108818211592734, + 0.00040108818211592734, + 0.0005275181611068547, + 0.0005275181611068547, + 0.0003901890595443547, + 0.0003901890595443547, + 0.0004817417939193547, + 0.0004817417939193547 ], [ - 0.0691964328289032, + 0.0680803582072258, + 0.0680803582072258, 0.0758928582072258, - 0.0516183041036129, + 0.0758928582072258, + 0.0507812537252903, + 0.0507812537252903, 0.0638950914144516, - 0.0005275181611068547, - 0.00045776370097883046, - 0.00034877232974395156, - 0.0003574916336219758 + 0.0638950914144516, + 0.0005144391907379031, + 0.0005144391907379031, + 0.0004664830048568547, + 0.0004664830048568547, + 0.0003422328445594758, + 0.0003422328445594758, + 0.0003683907852973789, + 0.0003683907852973789 ], [ 0.0446428582072258, - 0.0330636166036129, - 0.0404575914144516, - 0.0412946455180645, - 0.0003683907852973789, - 0.00044250491191633046, - 0.0003749302704818547, - 0.0005580357392318547 + 0.0446428582072258, + 0.0329241082072258, + 0.0329241082072258, + 0.0415736623108387, + 0.0415736623108387, + 0.0407366082072258, + 0.0407366082072258, + 0.00036621096660383046, + 0.00036621096660383046, + 0.0004054478486068547, + 0.0004054478486068547, + 0.0003727504226844758, + 0.0003727504226844758, + 0.0005841936799697578, + 0.0005841936799697578 ], [ + 0.0499441996216774, 0.0499441996216774, 0.0387834832072258, - 0.0426897332072258, - 0.0488281287252903, - 0.00044686454930342734, - 0.0005253383424133062, - 0.00038146975566633046, - 0.0003749302704818547 + 0.0387834832072258, + 0.0452008955180645, + 0.0452008955180645, + 0.0496651791036129, + 0.0496651791036129, + 0.0005166190094314516, + 0.0005166190094314516, + 0.0006016322877258062, + 0.0006016322877258062, + 0.0003618513001129031, + 0.0003618513001129031, + 0.0003683907852973789, + 0.0003683907852973789 ], [ + 0.0446428582072258, 0.0446428582072258, 0.0390625, - 0.0449218787252903, - 0.0396205373108387, - 0.00038146975566633046, - 0.0003923688782379031, - 0.00040980748599395156, - 0.0004076276673004031 + 0.0390625, + 0.0440848246216774, + 0.0440848246216774, + 0.0376674123108387, + 0.0376674123108387, + 0.0003531319962348789, + 0.0003531319962348789, + 0.0004207066376693547, + 0.0004207066376693547, + 0.00040108818211592734, + 0.00040108818211592734, + 0.00045776370097883046, + 0.00045776370097883046 ], [ - 0.0315290205180645, + 0.0318080373108387, + 0.0318080373108387, 0.0415736623108387, + 0.0415736623108387, + 0.0479910746216774, 0.0479910746216774, 0.0446428582072258, - 0.0003727504226844758, - 0.0004076276673004031, - 0.00046430318616330624, - 0.00032479423680342734 + 0.0446428582072258, + 0.0004294259415473789, + 0.0004294259415473789, + 0.00042506627505645156, + 0.00042506627505645156, + 0.0004512242157943547, + 0.0004512242157943547, + 0.00034877232974395156, + 0.00034877232974395156 ], [ - 0.0404575914144516, - 0.0516183041036129, - 0.0318080373108387, + 0.0421316996216774, + 0.0421316996216774, + 0.0518973246216774, + 0.0518973246216774, + 0.0334821455180645, + 0.0334821455180645, 0.0499441996216774, - 0.0004512242157943547, - 0.00033569338847883046, - 0.0005449567688629031, - 0.0003749302704818547 + 0.0499441996216774, + 0.00047084264224395156, + 0.00047084264224395156, + 0.0003465925110504031, + 0.0003465925110504031, + 0.0004817417939193547, + 0.0004817417939193547, + 0.0004141671524848789, + 0.0004141671524848789 ], [ 0.0463169664144516, - 0.0339006707072258, + 0.0463169664144516, + 0.0333426371216774, + 0.0333426371216774, + 0.0446428582072258, 0.0446428582072258, 0.0424107164144516, - 0.0006147112580947578, - 0.0003836495743598789, - 0.0004207066376693547, - 0.0005296979798004031 + 0.0424107164144516, + 0.0006103515625, + 0.0006103515625, + 0.0003618513001129031, + 0.0003618513001129031, + 0.00042724612285383046, + 0.00042724612285383046, + 0.0005275181611068547, + 0.0005275181611068547 ], [ - 0.0797991082072258, - 0.0987723246216774, - 0.0616629496216774, - 0.065011166036129, - 0.00046212333836592734, - 0.0004599435196723789, - 0.00039672854472883046, - 0.0006277902284637094 + 0.0792410746216774, + 0.0792410746216774, + 0.1004464328289032, + 0.1004464328289032, + 0.0602678582072258, + 0.0602678582072258, + 0.0655691996216774, + 0.0655691996216774, + 0.0004664830048568547, + 0.0004664830048568547, + 0.0004991804016754031, + 0.0004991804016754031, + 0.00047302249004133046, + 0.00047302249004133046, + 0.0006365095032379031, + 0.0006365095032379031 ], [ - 0.0354352705180645, + 0.0368303582072258, + 0.0368303582072258, 0.0418526791036129, - 0.0463169664144516, - 0.03125, + 0.0418526791036129, + 0.0460379496216774, + 0.0460379496216774, + 0.0291573666036129, + 0.0291573666036129, + 0.0006147112580947578, + 0.0006147112580947578, 0.0006103515625, - 0.0005711147096008062, - 0.0004817417939193547, - 0.0013950893189758062 + 0.0006103515625, + 0.0005275181611068547, + 0.0005275181611068547, + 0.0011335101444274187, + 0.0011335101444274187 ], [ 0.0326450914144516, - 0.0315290205180645, + 0.0326450914144516, + 0.0309709832072258, + 0.0309709832072258, + 0.0485491082072258, 0.0485491082072258, 0.0563616082072258, - 0.0006713867769576609, - 0.0005231585237197578, - 0.0006888253847137094, - 0.0004664830048568547 + 0.0563616082072258, + 0.0007890973938629031, + 0.0007890973938629031, + 0.0005841936799697578, + 0.0005841936799697578, + 0.0007106236298568547, + 0.0007106236298568547, + 0.00043160576024092734, + 0.00043160576024092734 ], [ - 0.02943638525903225, - 0.0429687537252903, 0.02929687686264515, - 0.03125, - 0.0005275181611068547, - 0.0005078997346572578, - 0.00044686454930342734, - 0.00043160576024092734 + 0.02929687686264515, + 0.0429687537252903, + 0.0429687537252903, + 0.0306919664144516, + 0.0306919664144516, + 0.0316685289144516, + 0.0316685289144516, + 0.0005035400390625, + 0.0005035400390625, + 0.0005623953766189516, + 0.0005623953766189516, + 0.00047084264224395156, + 0.00047084264224395156, + 0.00042724612285383046, + 0.00042724612285383046 ], [ 0.0549665205180645, + 0.0549665205180645, + 0.0457589291036129, 0.0457589291036129, 0.0438058041036129, - 0.0290178582072258, - 0.0013078962219879031, - 0.0013253348879516125, - 0.0007542201783508062, - 0.0005841936799697578 + 0.0438058041036129, + 0.0288783498108387, + 0.0288783498108387, + 0.0013776507694274187, + 0.0013776507694274187, + 0.0012904576724395156, + 0.0012904576724395156, + 0.0007455008453689516, + 0.0007455008453689516, + 0.000579833984375, + 0.000579833984375 ], [ 0.0959821492433548, - 0.08203125, - 0.1116071492433548, - 0.1037946492433548, + 0.0959821492433548, + 0.0837053582072258, + 0.0837053582072258, + 0.1121651828289032, + 0.1121651828289032, + 0.1010044664144516, + 0.1010044664144516, + 0.0005754743469879031, + 0.0005754743469879031, + 0.0005318777984939516, + 0.0005318777984939516, 0.0006277902284637094, - 0.0005296979798004031, - 0.0006365095032379031, - 0.0005057199159637094 + 0.0006277902284637094, + 0.0005667550722137094, + 0.0005667550722137094 ], [ + 0.0491071455180645, 0.0491071455180645, 0.0440848246216774, + 0.0440848246216774, + 0.0443638414144516, 0.0443638414144516, 0.0432477705180645, - 0.00044032506411895156, - 0.0003880092117469758, - 0.0005057199159637094, - 0.0005427769501693547 + 0.0432477705180645, + 0.0004926409455947578, + 0.0004926409455947578, + 0.00043160576024092734, + 0.00043160576024092734, + 0.0005318777984939516, + 0.0005318777984939516, + 0.0005209787050262094, + 0.0005209787050262094 ], [ - 0.0329241082072258, - 0.0393415205180645, - 0.0373883955180645, + 0.0311104916036129, + 0.0311104916036129, + 0.0404575914144516, + 0.0404575914144516, + 0.0376674123108387, + 0.0376674123108387, 0.0563616082072258, - 0.0005885533173568547, - 0.0004076276673004031, - 0.0004752023087348789, - 0.0008980887942016125 - ], - [ - 0.0577566996216774, - 0.0588727705180645, - 0.0524553582072258, - 0.0460379496216774, + 0.0563616082072258, + 0.0006147112580947578, + 0.0006147112580947578, + 0.0004294259415473789, + 0.0004294259415473789, 0.0004970005829818547, - 0.0006844656891189516, - 0.0005711147096008062, - 0.0011160714784637094 + 0.0004970005829818547, + 0.0008414132753387094, + 0.0008414132753387094 ], [ - 0.0619419664144516, - 0.0809151828289032, - 0.066964291036129, - 0.0638950914144516, - 0.000579833984375, - 0.0004970005829818547, - 0.0004686628235504031, - 0.0004926409455947578 + 0.0580357164144516, + 0.0580357164144516, + 0.0558035746216774, + 0.0558035746216774, + 0.0521763414144516, + 0.0521763414144516, + 0.0465959832072258, + 0.0465959832072258, + 0.0004839216126129031, + 0.0004839216126129031, + 0.0007455008453689516, + 0.0007455008453689516, + 0.0005667550722137094, + 0.0005667550722137094, + 0.0009504046174697578, + 0.0009504046174697578 + ], + [ + 0.0602678582072258, + 0.0602678582072258, + 0.08203125, + 0.08203125, + 0.068359375, + 0.068359375, + 0.0636160746216774, + 0.0636160746216774, + 0.00046430318616330624, + 0.00046430318616330624, + 0.0005929129547439516, + 0.0005929129547439516, + 0.00048610143130645156, + 0.00048610143130645156, + 0.0005296979798004031, + 0.0005296979798004031 ], [ + 0.0365513414144516, 0.0365513414144516, 0.0452008955180645, + 0.0452008955180645, + 0.0474330373108387, 0.0474330373108387, 0.0513392873108387, - 0.0010027204407379031, - 0.0004207066376693547, - 0.0006583078065887094, - 0.0005296979798004031 - ], - [ - 0.0527343787252903, - 0.0527343787252903, - 0.0449218787252903, - 0.0438058041036129, - 0.0005623953766189516, - 0.00047084264224395156, - 0.0013863700442016125, - 0.0005929129547439516 + 0.0513392873108387, + 0.0008108956972137094, + 0.0008108956972137094, + 0.00045776370097883046, + 0.00045776370097883046, + 0.0006757464143447578, + 0.0006757464143447578, + 0.0005405971314758062, + 0.0005405971314758062 ], [ - 0.0658482164144516, - 0.0703125, - 0.0703125, - 0.0731026828289032, - 0.0007106236298568547, - 0.0012730190064758062, - 0.0006888253847137094, - 0.0009678432252258062 + 0.0532924123108387, + 0.0532924123108387, + 0.0530133955180645, + 0.0530133955180645, + 0.0426897332072258, + 0.0426897332072258, + 0.0446428582072258, + 0.0446428582072258, + 0.00054931640625, + 0.00054931640625, + 0.00047302249004133046, + 0.00047302249004133046, + 0.0013427735539153218, + 0.0013427735539153218, + 0.0005667550722137094, + 0.0005667550722137094 + ], + [ + 0.0647321492433548, + 0.0647321492433548, + 0.0714285746216774, + 0.0714285746216774, + 0.0686383992433548, + 0.0686383992433548, + 0.0719866082072258, + 0.0719866082072258, + 0.0006757464143447578, + 0.0006757464143447578, + 0.0011335101444274187, + 0.0011335101444274187, + 0.0006583078065887094, + 0.0006583078065887094, + 0.0008588518830947578, + 0.0008588518830947578 ], [ - 0.0705915242433548, - 0.0672433078289032, - 0.0482700914144516, - 0.0583147332072258, - 0.0008021763642318547, - 0.0009329660097137094, - 0.0006844656891189516, - 0.0010811942629516125 + 0.0711495578289032, + 0.0711495578289032, + 0.0680803582072258, + 0.0680803582072258, + 0.0477120541036129, + 0.0477120541036129, + 0.0597098246216774, + 0.0597098246216774, + 0.0007542201783508062, + 0.0007542201783508062, + 0.0009765625, + 0.0009765625, + 0.0007716587861068547, + 0.0007716587861068547, + 0.0011858259094879031, + 0.0011858259094879031 ], [ - 0.0468750037252903, - 0.037109375, - 0.0362723246216774, - 0.0429687537252903, + 0.0485491082072258, + 0.0485491082072258, + 0.0355747789144516, + 0.0355747789144516, + 0.0359933041036129, + 0.0359933041036129, + 0.0415736623108387, + 0.0415736623108387, + 0.0005972726503387094, + 0.0005972726503387094, + 0.0006670270813629031, + 0.0006670270813629031, 0.0006147112580947578, - 0.0006495884736068547, - 0.0006234305328689516, - 0.0010986328125 + 0.0006147112580947578, + 0.0008370536379516125, + 0.0008370536379516125 ], [ 0.0521763414144516, - 0.0376674123108387, - 0.0404575914144516, - 0.0499441996216774, - 0.0007890973938629031, - 0.0008021763642318547, - 0.0008806501282379031, - 0.0005296979798004031 + 0.0521763414144516, + 0.0382254496216774, + 0.0382254496216774, + 0.0415736623108387, + 0.0415736623108387, + 0.0507812537252903, + 0.0507812537252903, + 0.0007367815705947578, + 0.0007367815705947578, + 0.0008108956972137094, + 0.0008108956972137094, + 0.0008632115204818547, + 0.0008632115204818547, + 0.0005841936799697578, + 0.0005841936799697578 ], [ 0.0465959832072258, - 0.0421316996216774, - 0.0390625, + 0.0465959832072258, + 0.0424107164144516, + 0.0424107164144516, + 0.0368303582072258, + 0.0368303582072258, 0.0393415205180645, - 0.0010332380188629031, - 0.0005841936799697578, - 0.0009068080689758062, - 0.000640869140625 + 0.0393415205180645, + 0.0009722028626129031, + 0.0009722028626129031, + 0.0006801060517318547, + 0.0006801060517318547, + 0.0008850098238326609, + 0.0008850098238326609, + 0.0006583078065887094, + 0.0006583078065887094 ], [ - 0.0340401791036129, - 0.0398995541036129, + 0.0334821455180645, + 0.0334821455180645, + 0.0396205373108387, + 0.0396205373108387, + 0.0471540205180645, 0.0471540205180645, 0.0443638414144516, - 0.0005405971314758062, - 0.0006713867769576609, - 0.0006539481109939516, - 0.0004926409455947578 + 0.0443638414144516, + 0.0005340576171875, + 0.0005340576171875, + 0.0006670270813629031, + 0.0006670270813629031, + 0.0006888253847137094, + 0.0006888253847137094, + 0.00054931640625, + 0.00054931640625 ], [ - 0.0457589291036129, - 0.0571986623108387, - 0.0385044664144516, + 0.0443638414144516, + 0.0443638414144516, + 0.0563616082072258, + 0.0563616082072258, + 0.0401785746216774, + 0.0401785746216774, 0.0502232164144516, - 0.0013253348879516125, - 0.0006059919251129031, - 0.0007149832672439516, - 0.0006452288362197578 + 0.0502232164144516, + 0.0013078962219879031, + 0.0013078962219879031, + 0.0006190708954818547, + 0.0006190708954818547, + 0.0006321498658508062, + 0.0006321498658508062, + 0.0006713867769576609, + 0.0006713867769576609 ], [ 0.0446428582072258, - 0.0345982164144516, + 0.0446428582072258, + 0.0344587080180645, + 0.0344587080180645, 0.0435267873108387, 0.0435267873108387, + 0.0440848246216774, + 0.0440848246216774, + 0.0007890973938629031, + 0.0007890973938629031, + 0.0006626674439758062, + 0.0006626674439758062, 0.0008021763642318547, - 0.0006147112580947578, - 0.0008239746675826609, - 0.0007280622376129031 + 0.0008021763642318547, + 0.0007542201783508062, + 0.0007542201783508062 ], [ - 0.0853794664144516, - 0.102120541036129, - 0.0613839328289032, - 0.0655691996216774, - 0.0009155274019576609, - 0.0012642997317016125, + 0.0887276828289032, + 0.0887276828289032, + 0.1049107164144516, + 0.1049107164144516, + 0.0608258955180645, + 0.0608258955180645, + 0.0636160746216774, + 0.0636160746216774, 0.0008588518830947578, - 0.0010986328125 + 0.0008588518830947578, + 0.0011858259094879031, + 0.0011858259094879031, + 0.0008283343049697578, + 0.0008283343049697578, + 0.0010288783814758062, + 0.0010288783814758062 ], [ - 0.0336216539144516, + 0.0343191996216774, + 0.0343191996216774, + 0.0387834832072258, 0.0387834832072258, 0.0460379496216774, - 0.02832031436264515, - 0.0008544922457076609, - 0.0008370536379516125, - 0.0007411412079818547, - 0.0019880023319274187 + 0.0460379496216774, + 0.02845982275903225, + 0.02845982275903225, + 0.0008588518830947578, + 0.0008588518830947578, + 0.0008196149719879031, + 0.0008196149719879031, + 0.0007062639924697578, + 0.0007062639924697578, + 0.0027553015388548374, + 0.0027553015388548374 ], [ - 0.0555245578289032, - 0.0597098246216774, - 0.0541294664144516, + 0.0538504496216774, + 0.0538504496216774, + 0.0594308078289032, + 0.0594308078289032, + 0.0560825914144516, + 0.0560825914144516, 0.0477120541036129, - 0.0010506766848266125, - 0.0008937290986068547, - 0.0006452288362197578, - 0.0008457729127258062 + 0.0477120541036129, + 0.0010637555969879031, + 0.0010637555969879031, + 0.0008719308534637094, + 0.0008719308534637094, + 0.0006495884736068547, + 0.0006495884736068547, + 0.0008108956972137094, + 0.0008108956972137094 ], [ - 0.0616629496216774, - 0.0814732164144516, - 0.0758928582072258, - 0.068917416036129, + 0.065011166036129, + 0.065011166036129, + 0.08203125, + 0.08203125, + 0.0775669664144516, + 0.0775669664144516, + 0.0703125, + 0.0703125, + 0.0007280622376129031, 0.0007280622376129031, - 0.0007193429628387094, 0.0007498605409637094, - 0.0008414132753387094 + 0.0007498605409637094, + 0.0007585798157379031, + 0.0007585798157379031, + 0.0007934570894576609, + 0.0007934570894576609 ], [ - 0.0365513414144516, + 0.0368303582072258, + 0.0368303582072258, + 0.0457589291036129, 0.0457589291036129, 0.0485491082072258, + 0.0485491082072258, 0.0513392873108387, - 0.0018310548039153218, - 0.0006103515625, - 0.0014822824159637094, - 0.0007542201783508062 + 0.0513392873108387, + 0.0021275111939758062, + 0.0021275111939758062, + 0.0006321498658508062, + 0.0006321498658508062, + 0.0015607562381774187, + 0.0015607562381774187, + 0.0008283343049697578, + 0.0008283343049697578 ], [ - 0.0613839328289032, - 0.0516183041036129, + 0.0597098246216774, + 0.0597098246216774, 0.0518973246216774, - 0.0488281287252903, - 0.0020054408814758062, - 0.0006626674439758062, + 0.0518973246216774, + 0.0507812537252903, + 0.0507812537252903, + 0.0502232164144516, + 0.0502232164144516, + 0.001979283057153225, + 0.001979283057153225, + 0.0006495884736068547, + 0.0006495884736068547, + 0.003749302588403225, 0.003749302588403225, - 0.0009024484315887094 + 0.0009504046174697578, + 0.0009504046174697578 ], [ - 0.0697544664144516, - 0.0725446492433548, - 0.0691964328289032, - 0.0770089328289032, - 0.0015345982974395156, + 0.0694754496216774, + 0.0694754496216774, + 0.0797991082072258, + 0.0797991082072258, + 0.065011166036129, + 0.065011166036129, + 0.078125, + 0.078125, + 0.0015694755129516125, + 0.0015694755129516125, 0.0028076174203306437, - 0.0014212472597137094, - 0.0021100726444274187 + 0.0028076174203306437, + 0.0014474052004516125, + 0.0014474052004516125, + 0.002101353369653225, + 0.002101353369653225 ], [ - 0.0725446492433548, - 0.0666852742433548, - 0.0471540205180645, - 0.0616629496216774, - 0.0010593959596008062, - 0.0021798270754516125, - 0.002284458838403225, - 0.0024937221314758062 + 0.0731026828289032, + 0.0731026828289032, + 0.0647321492433548, + 0.0647321492433548, + 0.0474330373108387, + 0.0474330373108387, + 0.0608258955180645, + 0.0608258955180645, + 0.0011422294192016125, + 0.0011422294192016125, + 0.002214704407379031, + 0.002214704407379031, + 0.0022495815064758062, + 0.0022495815064758062, + 0.00244140625, + 0.00244140625 ], [ - 0.0457589291036129, - 0.0390625, - 0.0387834832072258, - 0.0446428582072258, - 0.0009111677063629031, - 0.0010768346255645156, - 0.0011509486939758062, - 0.0018310548039153218 + 0.0468750037252903, + 0.0468750037252903, + 0.0382254496216774, + 0.0382254496216774, + 0.0376674123108387, + 0.0376674123108387, + 0.0432477705180645, + 0.0432477705180645, + 0.0010027204407379031, + 0.0010027204407379031, + 0.0009634835878387094, + 0.0009634835878387094, + 0.0011858259094879031, + 0.0011858259094879031, + 0.001918247900903225, + 0.001918247900903225 ], [ 0.0510602705180645, - 0.0407366082072258, + 0.0510602705180645, + 0.0382254496216774, + 0.0382254496216774, 0.0421316996216774, - 0.0516183041036129, - 0.0014648438664153218, - 0.0017089844914153218, - 0.002406529150903225, - 0.0012294225161895156 + 0.0421316996216774, + 0.0505022332072258, + 0.0505022332072258, + 0.0014561244752258062, + 0.0014561244752258062, + 0.0015781947877258062, + 0.0015781947877258062, + 0.0023890906013548374, + 0.0023890906013548374, + 0.0012904576724395156, + 0.0012904576724395156 ], [ 0.0465959832072258, - 0.0396205373108387, - 0.0390625, - 0.0396205373108387, - 0.0030517580453306437, - 0.0014822824159637094, - 0.0021885463502258062, - 0.002162388525903225 + 0.0465959832072258, + 0.0382254496216774, + 0.0382254496216774, + 0.0404575914144516, + 0.0404575914144516, + 0.0376674123108387, + 0.0376674123108387, + 0.0030866351444274187, + 0.0030866351444274187, + 0.0013514928286895156, + 0.0013514928286895156, + 0.002197265625, + 0.002197265625, + 0.002214704407379031, + 0.002214704407379031 ], [ - 0.0326450914144516, - 0.0510602705180645, + 0.0325055830180645, + 0.0325055830180645, + 0.0516183041036129, + 0.0516183041036129, + 0.0479910746216774, 0.0479910746216774, 0.0438058041036129, - 0.0012642997317016125, - 0.0010332380188629031, - 0.0012904576724395156, + 0.0438058041036129, + 0.0011771066347137094, + 0.0011771066347137094, + 0.0010288783814758062, + 0.0010288783814758062, + 0.00128173828125, + 0.00128173828125, + 0.0023890906013548374, 0.0023890906013548374 ], [ 0.0407366082072258, - 0.0513392873108387, - 0.0333426371216774, + 0.0407366082072258, + 0.0521763414144516, + 0.0521763414144516, + 0.033203125, + 0.033203125, 0.0493861623108387, - 0.0038190570194274187, - 0.0024239677004516125, - 0.001953125, - 0.0013078962219879031 + 0.0493861623108387, + 0.003871372900903225, + 0.003871372900903225, + 0.002458845032379031, + 0.002458845032379031, + 0.00201416015625, + 0.00201416015625, + 0.0013427735539153218, + 0.0013427735539153218 ], [ 0.0460379496216774, - 0.037109375, - 0.0452008955180645, + 0.0460379496216774, + 0.0362723246216774, + 0.0362723246216774, + 0.0457589291036129, + 0.0457589291036129, 0.0474330373108387, - 0.0011509486939758062, - 0.0010506766848266125, - 0.0020839148201048374, - 0.0026855471078306437 + 0.0474330373108387, + 0.0011117118410766125, + 0.0011117118410766125, + 0.0010463170474395156, + 0.0010463170474395156, + 0.0021100726444274187, + 0.0021100726444274187, + 0.002825055969879031, + 0.002825055969879031 ], [ - 0.0725446492433548, - 0.094308041036129, - 0.0591517873108387, - 0.0585937537252903, - 0.0024762835819274187, - 0.004063197877258062, - 0.001220703125, - 0.001918247900903225 + 0.0758928582072258, + 0.0758928582072258, + 0.0948660746216774, + 0.0948660746216774, + 0.0613839328289032, + 0.0613839328289032, + 0.0569196455180645, + 0.0569196455180645, + 0.002528599463403225, + 0.002528599463403225, + 0.004167829640209675, + 0.004167829640209675, + 0.0013427735539153218, + 0.0013427735539153218, + 0.0018746512942016125, + 0.0018746512942016125 ], [ - 0.0341796875, + 0.0344587080180645, + 0.0344587080180645, + 0.0393415205180645, 0.0393415205180645, 0.0465959832072258, - 0.0301339291036129, + 0.0465959832072258, + 0.02957589365541935, + 0.02957589365541935, + 0.002458845032379031, 0.002458845032379031, - 0.0020664760377258062, - 0.0016479493351653218, - 0.004167829640209675 + 0.002092634094879031, + 0.002092634094879031, + 0.0015956334536895156, + 0.0015956334536895156, + 0.003435407532379031, + 0.003435407532379031 ], [ 0.0319475457072258, - 0.0343191996216774, + 0.0319475457072258, + 0.033203125, + 0.033203125, + 0.0482700914144516, 0.0482700914144516, 0.0563616082072258, - 0.0016305106692016125, - 0.001953125, - 0.0018223354127258062, - 0.0013602121034637094 + 0.0563616082072258, + 0.0017177037661895156, + 0.0017177037661895156, + 0.0016130720032379031, + 0.0016130720032379031, + 0.0019444057252258062, + 0.0019444057252258062, + 0.0014997210819274187, + 0.0014997210819274187 ], [ - 0.033203125, + 0.0298549123108387, + 0.0298549123108387, 0.0415736623108387, - 0.0368303582072258, - 0.0315290205180645, - 0.0016828265506774187, - 0.0016479493351653218, - 0.0017525809817016125, - 0.002528599463403225 + 0.0415736623108387, + 0.0357142873108387, + 0.0357142873108387, + 0.0326450914144516, + 0.0326450914144516, + 0.0016217913944274187, + 0.0016217913944274187, + 0.0015433175722137094, + 0.0015433175722137094, + 0.0017177037661895156, + 0.0017177037661895156, + 0.0024239677004516125, + 0.0024239677004516125 ], [ + 0.0546875037252903, 0.0546875037252903, 0.0452008955180645, + 0.0452008955180645, 0.0418526791036129, - 0.0313895121216774, - 0.0018659320194274187, - 0.0024937221314758062, - 0.002197265625, - 0.0015171596314758062 - ], - [ - 0.0898437574505806, - 0.08203125, - 0.0976562574505806, - 0.0837053582072258, - 0.0018310548039153218, - 0.0014648438664153218, - 0.002458845032379031, - 0.0013950893189758062 + 0.0418526791036129, + 0.0319475457072258, + 0.0319475457072258, + 0.0019444057252258062, + 0.0019444057252258062, + 0.002528599463403225, + 0.002528599463403225, + 0.0017787389224395156, + 0.0017787389224395156, + 0.0015433175722137094, + 0.0015433175722137094 + ], + [ + 0.0887276828289032, + 0.0887276828289032, + 0.0842633992433548, + 0.0842633992433548, + 0.098214291036129, + 0.098214291036129, + 0.082589291036129, + 0.082589291036129, + 0.0018223354127258062, + 0.0018223354127258062, + 0.0014038087101653218, + 0.0014038087101653218, + 0.0023890906013548374, + 0.0023890906013548374, + 0.0014822824159637094, + 0.0014822824159637094 ], [ 0.0505022332072258, + 0.0505022332072258, + 0.0424107164144516, 0.0424107164144516, 0.0426897332072258, + 0.0426897332072258, 0.0418526791036129, - 0.0024239677004516125, - 0.0017177037661895156, - 0.0014299665344879031, - 0.0014474052004516125 + 0.0418526791036129, + 0.0024937221314758062, + 0.0024937221314758062, + 0.0016915458254516125, + 0.0016915458254516125, + 0.0014212472597137094, + 0.0014212472597137094, + 0.0015345982974395156, + 0.0015345982974395156 ], [ - 0.0330636166036129, - 0.0415736623108387, - 0.0357142873108387, + 0.0347377248108387, + 0.0347377248108387, + 0.0385044664144516, + 0.0385044664144516, + 0.0362723246216774, + 0.0362723246216774, 0.0549665205180645, - 0.0014910016907379031, - 0.0014910016907379031, - 0.0016915458254516125, - 0.002197265625 + 0.0549665205180645, + 0.0017002651002258062, + 0.0017002651002258062, + 0.0014735631411895156, + 0.0014735631411895156, + 0.0019618445076048374, + 0.0019618445076048374, + 0.002223423682153225, + 0.002223423682153225 ], [ - 0.0571986623108387, - 0.0544084832072258, - 0.0560825914144516, - 0.0463169664144516, - 0.0013340541627258062, - 0.0021100726444274187, - 0.0025634765625, - 0.003627232275903225 + 0.0563616082072258, + 0.0563616082072258, + 0.0549665205180645, + 0.0549665205180645, + 0.0552455373108387, + 0.0552455373108387, + 0.0474330373108387, + 0.0474330373108387, + 0.0014735631411895156, + 0.0014735631411895156, + 0.0019356864504516125, + 0.0019356864504516125, + 0.0028599330689758062, + 0.0028599330689758062, + 0.0030866351444274187, + 0.0030866351444274187 ], [ - 0.0591517873108387, - 0.0622209832072258, - 0.066964291036129, - 0.0555245578289032, - 0.0022670202888548374, - 0.002528599463403225, - 0.0020228796638548374, - 0.0023542132694274187 + 0.0619419664144516, + 0.0619419664144516, + 0.06640625, + 0.06640625, + 0.065011166036129, + 0.065011166036129, + 0.0577566996216774, + 0.0577566996216774, + 0.0026681083254516125, + 0.0026681083254516125, + 0.0025460380129516125, + 0.0025460380129516125, + 0.001979283057153225, + 0.001979283057153225, + 0.00244140625, + 0.00244140625 ], [ - 0.03515625, + 0.0362723246216774, + 0.0362723246216774, 0.0465959832072258, + 0.0465959832072258, + 0.0485491082072258, 0.0485491082072258, 0.0530133955180645, - 0.00359235517680645, - 0.0016566686099395156, + 0.0530133955180645, + 0.0040108817629516125, + 0.0040108817629516125, + 0.0015956334536895156, + 0.0015956334536895156, + 0.002197265625, + 0.002197265625, 0.002153669251129031, - 0.0020490374881774187 + 0.002153669251129031 ], [ - 0.0566406287252903, - 0.0479910746216774, - 0.0454799123108387, - 0.0438058041036129, + 0.0574776828289032, + 0.0574776828289032, + 0.0485491082072258, + 0.0485491082072258, + 0.0452008955180645, + 0.0452008955180645, + 0.0449218787252903, + 0.0449218787252903, 0.006417410913854837, - 0.0024937221314758062, - 0.01067243330180645, - 0.002580915344879031 - ], - [ - 0.0700334832072258, - 0.0633370578289032, - 0.0703125, - 0.0666852742433548, - 0.003679548157379031, - 0.00906808115541935, - 0.004603794775903225, - 0.006068638525903225 + 0.006417410913854837, + 0.0025460380129516125, + 0.0025460380129516125, + 0.010811942629516125, + 0.010811942629516125, + 0.0025634765625, + 0.0025634765625 ], [ + 0.0647321492433548, + 0.0647321492433548, + 0.0655691996216774, + 0.0655691996216774, + 0.0691964328289032, + 0.0691964328289032, 0.0694754496216774, - 0.064453125, - 0.0485491082072258, - 0.0577566996216774, - 0.002580915344879031, - 0.006975446827709675, - 0.00701032392680645, - 0.007882255129516125 + 0.0694754496216774, + 0.0036969867069274187, + 0.0036969867069274187, + 0.00927734375, + 0.00927734375, + 0.004673549439758062, + 0.004673549439758062, + 0.00603376142680645, + 0.00603376142680645 + ], + [ + 0.070870541036129, + 0.070870541036129, + 0.06640625, + 0.06640625, + 0.0516183041036129, + 0.0516183041036129, + 0.0585937537252903, + 0.0585937537252903, + 0.0028773718513548374, + 0.0028773718513548374, + 0.006905692163854837, + 0.006905692163854837, + 0.0069405697286129, + 0.0069405697286129, + 0.007952009327709675, + 0.007952009327709675 ], [ - 0.0454799123108387, 0.0457589291036129, - 0.0339006707072258, - 0.0385044664144516, - 0.0020839148201048374, - 0.0021275111939758062, - 0.0042550223879516125, - 0.007777622900903225 + 0.0457589291036129, + 0.0460379496216774, + 0.0460379496216774, + 0.0318080373108387, + 0.0318080373108387, + 0.0390625, + 0.0390625, + 0.0027901786379516125, + 0.0027901786379516125, + 0.0022495815064758062, + 0.0022495815064758062, + 0.004289899952709675, + 0.004289899952709675, + 0.00784737803041935, + 0.00784737803041935 ], [ 0.0541294664144516, - 0.0373883955180645, - 0.0426897332072258, + 0.0541294664144516, + 0.041015625, + 0.041015625, + 0.041015625, + 0.041015625, 0.0485491082072258, - 0.0040283203125, - 0.00371442548930645, - 0.005754743702709675, - 0.00359235517680645 + 0.0485491082072258, + 0.00395856611430645, + 0.00395856611430645, + 0.004534040577709675, + 0.004534040577709675, + 0.0057198661379516125, + 0.0057198661379516125, + 0.0037841799203306437, + 0.0037841799203306437 ], [ 0.0477120541036129, - 0.0407366082072258, - 0.0357142873108387, - 0.041015625, - 0.009905134327709675, - 0.0028599330689758062, - 0.006870815064758062, - 0.005092076025903225 + 0.0477120541036129, + 0.0415736623108387, + 0.0415736623108387, + 0.037109375, + 0.037109375, + 0.0418526791036129, + 0.0418526791036129, + 0.01004464365541935, + 0.01004464365541935, + 0.0030343192629516125, + 0.0030343192629516125, + 0.006975446827709675, + 0.006975446827709675, + 0.00530133955180645, + 0.00530133955180645 ], [ - 0.0344587080180645, - 0.0449218787252903, + 0.0340401791036129, + 0.0340401791036129, + 0.0474330373108387, + 0.0474330373108387, 0.0488281287252903, - 0.0376674123108387, - 0.00347028486430645, - 0.0030866351444274187, - 0.0030343192629516125, - 0.0087890625 + 0.0488281287252903, + 0.0385044664144516, + 0.0385044664144516, + 0.00359235517680645, + 0.00359235517680645, + 0.0032784598879516125, + 0.0032784598879516125, + 0.003941127564758062, + 0.003941127564758062, + 0.009347098879516125, + 0.009347098879516125 ], [ - 0.0493861623108387, - 0.0443638414144516, + 0.0488281287252903, + 0.0488281287252903, + 0.0438058041036129, + 0.0438058041036129, + 0.0327845998108387, 0.0327845998108387, 0.0424107164144516, - 0.009765625, - 0.009347098879516125, + 0.0424107164144516, + 0.0101143978536129, + 0.0101143978536129, + 0.009486607275903225, + 0.009486607275903225, 0.005440848413854837, - 0.0038190570194274187 + 0.005440848413854837, + 0.0036097937263548374, + 0.0036097937263548374 ], [ - 0.0468750037252903, - 0.0396205373108387, - 0.0449218787252903, - 0.0432477705180645, - 0.004063197877258062, + 0.0471540205180645, + 0.0471540205180645, + 0.0387834832072258, + 0.0387834832072258, + 0.0452008955180645, + 0.0452008955180645, + 0.0465959832072258, + 0.0465959832072258, + 0.004098074976354837, 0.004098074976354837, + 0.00390625, + 0.00390625, + 0.00676618330180645, + 0.00676618330180645, 0.006591797340661287, - 0.006835937965661287 + 0.006591797340661287 ], [ - 0.078125, + 0.0775669664144516, + 0.0775669664144516, 0.094308041036129, - 0.0694754496216774, - 0.0571986623108387, - 0.007882255129516125, - 0.01213727705180645, - 0.0048828125, - 0.0048828125 + 0.094308041036129, + 0.066964291036129, + 0.066964291036129, + 0.0524553582072258, + 0.0524553582072258, + 0.008056640625, + 0.008056640625, + 0.0125558041036129, + 0.0125558041036129, + 0.0043770927004516125, + 0.0043770927004516125, + 0.0047433036379516125, + 0.0047433036379516125 ], [ + 0.0359933041036129, 0.0359933041036129, 0.0390625, + 0.0390625, + 0.0491071455180645, 0.0491071455180645, - 0.0297154039144516, - 0.006382533814758062, - 0.0059640067629516125, - 0.005929129663854837, - 0.00749860517680645 + 0.03027343936264515, + 0.03027343936264515, + 0.0069405697286129, + 0.0069405697286129, + 0.005650111939758062, + 0.005650111939758062, + 0.00603376142680645, + 0.00603376142680645, + 0.01164899580180645, + 0.01164899580180645 ], [ - 0.0319475457072258, - 0.0333426371216774, - 0.0482700914144516, + 0.0318080373108387, + 0.0318080373108387, + 0.0339006707072258, + 0.0339006707072258, + 0.0485491082072258, + 0.0485491082072258, + 0.0544084832072258, 0.0544084832072258, - 0.007149832788854837, - 0.01981026865541935, - 0.00871930830180645, - 0.012207032181322575 + 0.007603236939758062, + 0.007603236939758062, + 0.01869419775903225, + 0.01869419775903225, + 0.00920758955180645, + 0.00920758955180645, + 0.010951451025903225, + 0.010951451025903225 ], [ - 0.0327845998108387, + 0.0340401791036129, + 0.0340401791036129, 0.0449218787252903, - 0.0401785746216774, - 0.0345982164144516, - 0.008544921875, - 0.0135323666036129, - 0.0069405697286129, - 0.012207032181322575 + 0.0449218787252903, + 0.0404575914144516, + 0.0404575914144516, + 0.033203125, + 0.033203125, + 0.00847516767680645, + 0.00847516767680645, + 0.0149972103536129, + 0.0149972103536129, + 0.006975446827709675, + 0.006975446827709675, + 0.0110909603536129, + 0.0110909603536129 ], [ 0.0541294664144516, + 0.0541294664144516, + 0.0435267873108387, 0.0435267873108387, 0.0415736623108387, - 0.0325055830180645, - 0.00798688642680645, - 0.007638114038854837, - 0.006801060400903225, - 0.004342215601354837 + 0.0415736623108387, + 0.033203125, + 0.033203125, + 0.00969587080180645, + 0.00969587080180645, + 0.007533482275903225, + 0.007533482275903225, + 0.005998884327709675, + 0.005998884327709675, + 0.003976004663854837, + 0.003976004663854837 ], [ + 0.0387834832072258, + 0.0387834832072258, 0.0385044664144516, 0.0385044664144516, - 0.0387834832072258, - 0.0343191996216774, - 0.004778181202709675, + 0.0368303582072258, + 0.0368303582072258, + 0.0344587080180645, + 0.0344587080180645, + 0.005161830689758062, + 0.005161830689758062, 0.006556919775903225, - 0.012834821827709675, + 0.006556919775903225, + 0.01297433115541935, + 0.01297433115541935, + 0.005894252564758062, 0.005894252564758062 ], [ + 0.0385044664144516, 0.0385044664144516, 0.0412946455180645, - 0.0339006707072258, + 0.0412946455180645, + 0.0301339291036129, + 0.0301339291036129, 0.0460379496216774, - 0.007568359840661287, - 0.01311383955180645, - 0.01897321455180645, - 0.014718192629516125 - ], - [ - 0.0376674123108387, + 0.0460379496216774, + 0.00847516767680645, + 0.00847516767680645, + 0.013741630129516125, + 0.013741630129516125, + 0.0182756707072258, + 0.0182756707072258, + 0.015206473879516125, + 0.015206473879516125 + ], + [ + 0.0379464291036129, + 0.0379464291036129, 0.0393415205180645, - 0.0297154039144516, - 0.0259486623108387, - 0.010742188431322575, - 0.00920758955180645, - 0.01213727705180645, - 0.005196707788854837 + 0.0393415205180645, + 0.0309709832072258, + 0.0309709832072258, + 0.02664620615541935, + 0.02664620615541935, + 0.01053292490541935, + 0.01053292490541935, + 0.008928571827709675, + 0.008928571827709675, + 0.015136719681322575, + 0.015136719681322575, + 0.00554548017680645, + 0.00554548017680645 ], [ 0.0368303582072258, - 0.0299944207072258, - 0.0318080373108387, - 0.03041294775903225, + 0.0368303582072258, + 0.03055245615541935, + 0.03055245615541935, + 0.0329241082072258, + 0.0329241082072258, + 0.0311104916036129, + 0.0311104916036129, 0.012207032181322575, - 0.012276786379516125, - 0.010742188431322575, - 0.011928013525903225 + 0.012207032181322575, + 0.01576451025903225, + 0.01576451025903225, + 0.010951451025903225, + 0.010951451025903225, + 0.013741630129516125, + 0.013741630129516125 ], [ - 0.0299944207072258, - 0.0354352705180645, - 0.0316685289144516, - 0.02748326025903225, - 0.009486607275903225, - 0.0135323666036129, - 0.009416853077709675, - 0.02176339365541935 + 0.02943638525903225, + 0.02943638525903225, + 0.0355747789144516, + 0.0355747789144516, + 0.0311104916036129, + 0.0311104916036129, + 0.02929687686264515, + 0.02929687686264515, + 0.0101143978536129, + 0.0101143978536129, + 0.0130440853536129, + 0.0130440853536129, + 0.011788505129516125, + 0.011788505129516125, + 0.0212053582072258, + 0.0212053582072258 ], [ 0.0326450914144516, + 0.0326450914144516, + 0.0279017873108387, + 0.0279017873108387, + 0.0279017873108387, 0.0279017873108387, 0.02762276865541935, - 0.0269252248108387, - 0.010323661379516125, - 0.008858817629516125, - 0.011439732275903225, - 0.0279017873108387 + 0.02762276865541935, + 0.01067243330180645, + 0.01067243330180645, + 0.00969587080180645, + 0.00969587080180645, + 0.011788505129516125, + 0.011788505129516125, + 0.0281808041036129, + 0.0281808041036129 ], [ - 0.0262276791036129, - 0.0249720998108387, + 0.02929687686264515, + 0.02929687686264515, + 0.0280412957072258, + 0.0280412957072258, + 0.0319475457072258, + 0.0319475457072258, + 0.02539062686264515, + 0.02539062686264515, + 0.0185546875, + 0.0185546875, 0.03125, - 0.0269252248108387, - 0.0140206478536129, - 0.02845982275903225, - 0.0252511166036129, - 0.0133928582072258 + 0.03125, + 0.0251116082072258, + 0.0251116082072258, + 0.014160157181322575, + 0.014160157181322575 ], [ - 0.0524553582072258, - 0.0641741082072258, - 0.0546875037252903, - 0.0496651791036129, - 0.0429687537252903, - 0.0185546875, - 0.0398995541036129, - 0.0115792416036129 + 0.0538504496216774, + 0.0538504496216774, + 0.0647321492433548, + 0.0647321492433548, + 0.0538504496216774, + 0.0538504496216774, + 0.0513392873108387, + 0.0513392873108387, + 0.0426897332072258, + 0.0426897332072258, + 0.0184151791036129, + 0.0184151791036129, + 0.0404575914144516, + 0.0404575914144516, + 0.01150948740541935, + 0.01150948740541935 ], [ - 0.0407366082072258, + 0.0398995541036129, + 0.0398995541036129, 0.0322265625, + 0.0322265625, + 0.0424107164144516, + 0.0424107164144516, 0.0438058041036129, - 0.0421316996216774, - 0.010463169775903225, + 0.0438058041036129, + 0.01164899580180645, + 0.01164899580180645, + 0.0184151791036129, + 0.0184151791036129, + 0.017578125, 0.017578125, - 0.0148577019572258, - 0.0318080373108387 + 0.0319475457072258, + 0.0319475457072258 ], [ - 0.0426897332072258, - 0.0443638414144516, - 0.0319475457072258, - 0.0457589291036129, - 0.0185546875, - 0.013253348879516125, - 0.0220424123108387, - 0.0251116082072258 + 0.0421316996216774, + 0.0421316996216774, + 0.0415736623108387, + 0.0415736623108387, + 0.0326450914144516, + 0.0326450914144516, + 0.0474330373108387, + 0.0474330373108387, + 0.01722935400903225, + 0.01722935400903225, + 0.01443917490541935, + 0.01443917490541935, + 0.02148437686264515, + 0.02148437686264515, + 0.02246093936264515, + 0.02246093936264515 ] ] } \ No newline at end of file diff --git a/test/advanced_config/fp8_calibration_per_tensor/test_kv_cache_calib_per_tensor_qwen3_235b.json b/test/advanced_config/fp8_calibration_per_tensor/test_kv_cache_calib_per_tensor_qwen3_235b.json index 0c0a47e86..6d4692ff5 100644 --- a/test/advanced_config/fp8_calibration_per_tensor/test_kv_cache_calib_per_tensor_qwen3_235b.json +++ b/test/advanced_config/fp8_calibration_per_tensor/test_kv_cache_calib_per_tensor_qwen3_235b.json @@ -5,7 +5,7 @@ "qmin": -448.0, "qmax": 448.0, "num_layers": 94, - "num_head": 4, + "num_head": 8, "scales_shape": [ 94, 2 @@ -16,296 +16,296 @@ 0.00022888185048941523 ], [ - 0.0404575914144516, - 0.0002899169921875 + 0.0407366082072258, + 0.0003073556290473789 ], [ - 0.0792410746216774, - 0.0002855573548004031 + 0.0803571492433548, + 0.0003204345703125 ], [ 0.0705915242433548, - 0.0004512242157943547 + 0.00045558385318145156 ], [ - 0.0719866082072258, - 0.0003923688782379031 + 0.0731026828289032, + 0.0004054478486068547 ], [ - 0.149553582072258, - 0.0006801060517318547 + 0.1484375, + 0.0006626674439758062 ], [ 0.0491071455180645, - 0.0007106236298568547 + 0.0006626674439758062 ], [ - 0.0566406287252903, - 0.0007367815705947578 + 0.0563616082072258, + 0.0007237026002258062 ], [ - 0.1104910746216774, - 0.0007629395113326609 + 0.1127232164144516, + 0.0008370536379516125 ], [ 0.0555245578289032, - 0.0009722028626129031 + 0.0013078962219879031 ], [ - 0.106026791036129, - 0.0005929129547439516 + 0.1049107164144516, + 0.0004599435196723789 ], [ 0.0482700914144516, - 0.0005384173127822578 + 0.00048828125 ], [ 0.0538504496216774, - 0.0005296979798004031 + 0.0005318777984939516 ], [ 0.0583147332072258, - 0.0005318777984939516 + 0.000518798828125 ], [ - 0.0736607164144516, - 0.0005580357392318547 + 0.07421875, + 0.0005972726503387094 ], [ 0.0505022332072258, - 0.00045558385318145156 + 0.0006234305328689516 ], [ - 0.0602678582072258, - 0.0007716587861068547 + 0.0597098246216774, + 0.0007890973938629031 ], [ 0.078683041036129, - 0.0004991804016754031 + 0.0005144391907379031 ], [ - 0.074776791036129, - 0.0004817417939193547 + 0.0753348246216774, + 0.0005318777984939516 ], [ - 0.0446428582072258, - 0.0005296979798004031 + 0.0452008955180645, + 0.0006103515625 ], [ 0.0499441996216774, - 0.0006583078065887094 + 0.0005841936799697578 ], [ - 0.0446428582072258, - 0.00040980748599395156 + 0.0488281287252903, + 0.00045776370097883046 ], [ 0.0479910746216774, - 0.00044904439710080624 + 0.00046430318616330624 ], [ - 0.0499441996216774, - 0.000518798828125 + 0.0521763414144516, + 0.0005166190094314516 ], [ 0.0463169664144516, - 0.0006321498658508062 + 0.0006975446594879031 ], [ - 0.0998883992433548, - 0.0006190708954818547 + 0.0993303582072258, + 0.0006583078065887094 ], [ 0.0460379496216774, - 0.0015520368469879031 + 0.0011073522036895156 ], [ - 0.0560825914144516, - 0.0007498605409637094 + 0.0563616082072258, + 0.0008588518830947578 ], [ 0.0429687537252903, - 0.0005623953766189516 + 0.0006103515625 ], [ 0.0549665205180645, - 0.0013078962219879031 + 0.0013253348879516125 ], [ - 0.109933041036129, - 0.0006365095032379031 + 0.1116071492433548, + 0.000640869140625 ], [ 0.0491071455180645, - 0.000518798828125 + 0.0005471365875564516 ], [ 0.0563616082072258, - 0.0008196149719879031 + 0.0008850098238326609 ], [ - 0.0571986623108387, - 0.0011422294192016125 + 0.0583147332072258, + 0.0009678432252258062 ], [ - 0.078125, - 0.0006975446594879031 + 0.0797991082072258, + 0.0006801060517318547 ], [ 0.0513392873108387, - 0.0008501325501129031 + 0.0008806501282379031 ], [ - 0.0544084832072258, - 0.00128173828125 + 0.0538504496216774, + 0.0014561244752258062 ], [ - 0.0764508992433548, - 0.00115966796875 + 0.074776791036129, + 0.0011335101444274187 ], [ - 0.0711495578289032, - 0.001068115234375 + 0.0714285746216774, + 0.0011771066347137094 ], [ - 0.0460379496216774, - 0.0009024484315887094 + 0.0477120541036129, + 0.0008632115204818547 ], [ 0.0521763414144516, - 0.0008065360016189516 + 0.0008588518830947578 ], [ 0.0465959832072258, - 0.0009329660097137094 + 0.0009983608033508062 ], [ 0.0471540205180645, - 0.0006801060517318547 + 0.0006888253847137094 ], [ - 0.0544084832072258, - 0.001220703125 + 0.0583147332072258, + 0.0012904576724395156 ], [ - 0.0446428582072258, - 0.0008283343049697578 + 0.0460379496216774, + 0.0008632115204818547 ], [ - 0.1037946492433548, - 0.0011945453006774187 + 0.1049107164144516, + 0.0012294225161895156 ], [ 0.0460379496216774, - 0.001970563782379031 + 0.0020839148201048374 ], [ - 0.0591517873108387, - 0.0010942731751129031 + 0.0602678582072258, + 0.0011073522036895156 ], [ - 0.0809151828289032, - 0.0008414132753387094 + 0.0797991082072258, + 0.0008544922457076609 ], [ 0.0513392873108387, - 0.001735142432153225 + 0.0018920899601653218 ], [ - 0.0611049123108387, - 0.0036621096078306437 + 0.0605468787252903, + 0.0036969867069274187 ], [ - 0.078125, - 0.0028076174203306437 + 0.082589291036129, + 0.002825055969879031 ], [ - 0.0725446492433548, - 0.002406529150903225 + 0.0736607164144516, + 0.002458845032379031 ], [ - 0.0452008955180645, - 0.0019880023319274187 + 0.0454799123108387, + 0.0019008092349395156 ], [ - 0.0513392873108387, - 0.0023890906013548374 + 0.0510602705180645, + 0.0024239677004516125 ], [ 0.0465959832072258, - 0.0030866351444274187 + 0.0031738283578306437 ], [ 0.0513392873108387, - 0.0023890906013548374 + 0.00244140625 ], [ - 0.0502232164144516, + 0.0558035746216774, 0.0038888114504516125 ], [ - 0.0465959832072258, - 0.0028424945194274187 + 0.0474330373108387, + 0.0027553015388548374 ], [ - 0.0970982164144516, + 0.094308041036129, 0.004185268189758062 ], [ 0.0465959832072258, - 0.0036097937263548374 + 0.0037841799203306437 ], [ 0.0563616082072258, - 0.0019967216067016125 + 0.0022670202888548374 ], [ 0.0415736623108387, - 0.0025634765625 + 0.0026332312263548374 ], [ 0.0546875037252903, - 0.002580915344879031 + 0.002702985657379031 ], [ 0.0976562574505806, - 0.0023716518189758062 + 0.00244140625 ], [ 0.0505022332072258, - 0.0018833705689758062 + 0.0024937221314758062 ], [ 0.0549665205180645, - 0.0022495815064758062 + 0.002214704407379031 ], [ - 0.0580357164144516, - 0.0034528460819274187 + 0.0571986623108387, + 0.0036446708254516125 ], [ 0.0652901828289032, - 0.002406529150903225 + 0.0027553015388548374 ], [ 0.0530133955180645, - 0.003679548157379031 + 0.00371442548930645 ], [ 0.0569196455180645, - 0.010742188431322575 + 0.010811942629516125 ], [ - 0.0700334832072258, - 0.0088936947286129 + 0.0719866082072258, + 0.00906808115541935 ], [ - 0.0691964328289032, - 0.0079171322286129 + 0.0703125, + 0.007882255129516125 ], [ - 0.0460379496216774, - 0.007638114038854837 + 0.0463169664144516, + 0.007777622900903225 ], [ 0.0541294664144516, @@ -313,79 +313,79 @@ ], [ 0.0477120541036129, - 0.01018415205180645 + 0.009905134327709675 ], [ 0.0488281287252903, - 0.008858817629516125 + 0.008998326025903225 ], [ - 0.0485491082072258, - 0.009905134327709675 + 0.0510602705180645, + 0.00969587080180645 ], [ - 0.0468750037252903, - 0.006835937965661287 + 0.0479910746216774, + 0.006870815064758062 ], [ - 0.094308041036129, - 0.012276786379516125 + 0.0965401828289032, + 0.012346540577709675 ], [ 0.0491071455180645, - 0.010811942629516125 + 0.01025390625 ], [ 0.0544084832072258, - 0.0193917416036129 + 0.02246093936264515 ], [ 0.0449218787252903, - 0.0148577019572258 + 0.0133928582072258 ], [ 0.0541294664144516, - 0.009835380129516125 + 0.0091378353536129 ], [ - 0.0398995541036129, - 0.013253348879516125 + 0.0390625, + 0.013671875931322575 ], [ 0.0460379496216774, - 0.01981026865541935 + 0.0200892873108387 ], [ 0.0393415205180645, - 0.01311383955180645 + 0.0135323666036129 ], [ 0.0368303582072258, - 0.01395089365541935 + 0.0148577019572258 ], [ - 0.0362723246216774, - 0.02064732275903225 + 0.0368303582072258, + 0.0221819207072258 ], [ 0.0326450914144516, - 0.02580915205180645 + 0.02650669775903225 ], [ - 0.0315290205180645, - 0.0290178582072258 + 0.0316685289144516, + 0.03125 ], [ 0.064453125, - 0.0429687537252903 + 0.0424107164144516 ], [ - 0.0426897332072258, - 0.0319475457072258 + 0.0471540205180645, + 0.0323660746216774 ], [ - 0.0463169664144516, - 0.02762276865541935 + 0.0465959832072258, + 0.0223214291036129 ] ] } \ No newline at end of file