From a29ca3561b9a483e16ade9f2c99d9bddad9942cf Mon Sep 17 00:00:00 2001 From: Ofir Gordon Date: Tue, 28 Jan 2025 10:15:46 +0200 Subject: [PATCH] add bounded symmetric selection test and removed all symmetric selection tests --- ...ric_threshold_selection_activation_test.py | 114 ------------------ .../test_features_runner.py | 16 --- .../keras/test_post_training_quantization.py | 11 +- .../test_symmetric_selection_activation.py | 28 ++++- 4 files changed, 33 insertions(+), 136 deletions(-) delete mode 100644 tests/keras_tests/feature_networks_tests/feature_networks/symmetric_threshold_selection_activation_test.py diff --git a/tests/keras_tests/feature_networks_tests/feature_networks/symmetric_threshold_selection_activation_test.py b/tests/keras_tests/feature_networks_tests/feature_networks/symmetric_threshold_selection_activation_test.py deleted file mode 100644 index 0e59acf3c..000000000 --- a/tests/keras_tests/feature_networks_tests/feature_networks/symmetric_threshold_selection_activation_test.py +++ /dev/null @@ -1,114 +0,0 @@ -# Copyright 2022 Sony Semiconductor Israel, Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - - -import tensorflow as tf -import numpy as np - -from mct_quantizers import KerasActivationQuantizationHolder, QuantizationMethod -from model_compression_toolkit.target_platform_capabilities.tpc_models.imx500_tpc.latest import generate_keras_tpc -from tests.common_tests.helpers.generate_test_tpc import generate_test_tpc -from tests.keras_tests.feature_networks_tests.base_keras_feature_test import BaseKerasFeatureNetworkTest -import model_compression_toolkit as mct -from tests.keras_tests.utils import get_layers_from_model_by_type - -keras = tf.keras -layers = keras.layers - - -class SymmetricThresholdSelectionActivationTest(BaseKerasFeatureNetworkTest): - def __init__(self, unit_test, activation_threshold_method): - super().__init__(unit_test ) - self.activation_threshold_method = activation_threshold_method - - def generate_inputs(self): - return [np.random.uniform(low=-7, high=7, size=in_shape) for in_shape in self.get_input_shapes()] - - def get_tpc(self): - tpc = generate_test_tpc({ - 'activation_quantization_method': QuantizationMethod.SYMMETRIC, - 'activation_n_bits': 8}) - return generate_keras_tpc(name="symmetric_threshold_test", tpc=tpc) - - def get_quantization_config(self): - return mct.core.QuantizationConfig(activation_error_method=self.activation_threshold_method) - - def create_networks(self): - inputs = layers.Input(shape=self.get_input_shapes()[0][1:]) - x = layers.ReLU()(inputs) - outputs = tf.add(x, -1) # to get negative values in activation to test signed symmetric quantization - return keras.Model(inputs=inputs, outputs=outputs) - - def compare(self, quantized_model, float_model, input_x=None, quantization_info=None): - # verify threshold not power of 2 and unsigned symmetric range for first two layers' activations - input_holder_layer = get_layers_from_model_by_type(quantized_model, KerasActivationQuantizationHolder)[0] - fake_layer_input_args = input_holder_layer.activation_holder_quantizer.get_config() - relu_holder_layer = get_layers_from_model_by_type(quantized_model, KerasActivationQuantizationHolder)[1] - fake_layer_relu_args = relu_holder_layer.activation_holder_quantizer.get_config() - - threshold_input = fake_layer_input_args['threshold'][0] - threshold_relu = fake_layer_relu_args['threshold'][0] - - self.unit_test.assertFalse( - np.log2(threshold_input).is_integer(), msg=f"Input layer threshold {threshold_input} is a power of 2") - self.unit_test.assertFalse( - np.log2(threshold_relu).is_integer(), msg=f"ReLU layer threshold {threshold_relu} is a power of 2") - - self.unit_test.assertTrue(fake_layer_relu_args['signed'] == False, - msg=f"ReLU expected to have unsigned symmetric quantization param but is signed") - - # verify threshold not power of 2 and signed symmetric range for first Add activation layer - add_holder_layer = get_layers_from_model_by_type(quantized_model, KerasActivationQuantizationHolder)[2] - fake_layer_add_args = add_holder_layer.activation_holder_quantizer.get_config() - - threshold_add = fake_layer_add_args['threshold'][0] - - self.unit_test.assertFalse( - np.log2(threshold_input).is_integer(), msg=f"Add layer threshold {threshold_add} is a power of 2") - self.unit_test.assertTrue(fake_layer_add_args['signed'] == True, - msg=f"Add expected to have signed quantization range but is unsigned") - - -class SymmetricThresholdSelectionBoundedActivationTest(SymmetricThresholdSelectionActivationTest): - def __init__(self, unit_test, activation_threshold_method): - super().__init__(unit_test, activation_threshold_method) - - def create_networks(self): - inputs = layers.Input(shape=self.get_input_shapes()[0][1:]) - x = layers.Softmax()(inputs) - outputs = tf.add(x, 1) - return keras.Model(inputs=inputs, outputs=outputs) - - def compare(self, quantized_model, float_model, input_x=None, quantization_info=None): - holder_layers = get_layers_from_model_by_type(quantized_model, KerasActivationQuantizationHolder) - fake_layer_input_args = holder_layers[0].activation_holder_quantizer.get_config() - fake_layer_softmax_args = holder_layers[1].activation_holder_quantizer.get_config() - - threshold_input = fake_layer_input_args['threshold'][0] - threshold_softmax = fake_layer_softmax_args['threshold'][0] - - # Verify threshold not power of 2 - self.unit_test.assertFalse( - np.log2(threshold_input).is_integer(), msg=f"Input layer threshold {threshold_input} is a power of 2") - - # Verify threshold is 1 - self.unit_test.assertTrue(threshold_softmax == 1.0, - msg=f"Threshold of Softmax layer is {threshold_softmax} (expected: 1)") - - # Verify min/max is bounded by 0 and 1 - self.unit_test.assertTrue(fake_layer_softmax_args['signed'] == False, - msg=f"Softmax layer symmetric range is signed. Expected to be unsigned") - self.unit_test.assertTrue(fake_layer_softmax_args['threshold'] == [1.0], - msg=f"Softmax layer threshold is {fake_layer_softmax_args['threshold']}. Expected to be 1") diff --git a/tests/keras_tests/feature_networks_tests/test_features_runner.py b/tests/keras_tests/feature_networks_tests/test_features_runner.py index 0ca667abf..ab2976faf 100644 --- a/tests/keras_tests/feature_networks_tests/test_features_runner.py +++ b/tests/keras_tests/feature_networks_tests/test_features_runner.py @@ -131,8 +131,6 @@ from tests.keras_tests.feature_networks_tests.feature_networks.softmax_shift_test import SoftmaxShiftTest from tests.keras_tests.feature_networks_tests.feature_networks.split_concatenate_test import SplitConcatenateTest from tests.keras_tests.feature_networks_tests.feature_networks.split_conv_bug_test import SplitConvBugTest -from tests.keras_tests.feature_networks_tests.feature_networks.symmetric_threshold_selection_activation_test import \ - SymmetricThresholdSelectionActivationTest, SymmetricThresholdSelectionBoundedActivationTest from tests.keras_tests.feature_networks_tests.feature_networks.test_depthwise_conv2d_replacement import \ DwConv2dReplacementTest from tests.keras_tests.feature_networks_tests.feature_networks.test_kmeans_quantizer import \ @@ -754,20 +752,6 @@ def test_gptq_conv_group_dilation(self): def test_split_conv_bug(self): SplitConvBugTest(self).run_test() - def test_symmetric_threshold_selection_activation(self): - SymmetricThresholdSelectionActivationTest(self, QuantizationErrorMethod.NOCLIPPING).run_test() - SymmetricThresholdSelectionActivationTest(self, QuantizationErrorMethod.MSE).run_test() - SymmetricThresholdSelectionActivationTest(self, QuantizationErrorMethod.MAE).run_test() - SymmetricThresholdSelectionActivationTest(self, QuantizationErrorMethod.LP).run_test() - SymmetricThresholdSelectionActivationTest(self, QuantizationErrorMethod.KL).run_test() - - def test_symmetric_threshold_selection_softmax_activation(self): - SymmetricThresholdSelectionBoundedActivationTest(self, QuantizationErrorMethod.NOCLIPPING).run_test() - SymmetricThresholdSelectionBoundedActivationTest(self, QuantizationErrorMethod.MSE).run_test() - SymmetricThresholdSelectionBoundedActivationTest(self, QuantizationErrorMethod.MAE).run_test() - SymmetricThresholdSelectionBoundedActivationTest(self, QuantizationErrorMethod.LP).run_test() - SymmetricThresholdSelectionBoundedActivationTest(self, QuantizationErrorMethod.KL).run_test() - def test_uniform_range_selection_activation(self): UniformRangeSelectionActivationTest(self, QuantizationErrorMethod.NOCLIPPING).run_test() UniformRangeSelectionActivationTest(self, QuantizationErrorMethod.MSE).run_test() diff --git a/tests_pytest/integration_tests/keras/test_post_training_quantization.py b/tests_pytest/integration_tests/keras/test_post_training_quantization.py index b53b3d74a..6fe0e0d37 100644 --- a/tests_pytest/integration_tests/keras/test_post_training_quantization.py +++ b/tests_pytest/integration_tests/keras/test_post_training_quantization.py @@ -24,8 +24,8 @@ from model_compression_toolkit.core.common.user_info import UserInformation from model_compression_toolkit.core.keras.constants import KERNEL, DEPTHWISE_KERNEL from model_compression_toolkit.ptq import keras_post_training_quantization -from model_compression_toolkit.target_platform_capabilities import AttributeQuantizationConfig, OpQuantizationConfig, \ - Signedness +from model_compression_toolkit.target_platform_capabilities.schema.mct_current_schema import OpQuantizationConfig, \ + AttributeQuantizationConfig, Signedness from model_compression_toolkit.target_platform_capabilities.constants import KERNEL_ATTR, BIAS_ATTR from tests.common_tests.helpers.tpcs_for_tests.v4.tpc import generate_tpc @@ -120,9 +120,10 @@ def _verify_weights_quantizer_params(quant_method, weights_quantizer, params_sha class TestPostTrainingQuantizationApi: - # TODO: - # [a, w&a] - # extend to also test with different settings? (bc, snc, etc.) + # TODO: add tests for: + # 1) activation only, W&A, LUT quantizer (separate) + # 2) extend to also test with different settings features (bc, snc, etc.) + # 3) advanced models and operators def _verify_quantized_model_structure(self, model, q_model, quantization_info): diff --git a/tests_pytest/unit_tests/common/core/quantization/quantization_params_generation/test_symmetric_selection_activation.py b/tests_pytest/unit_tests/common/core/quantization/quantization_params_generation/test_symmetric_selection_activation.py index a19eb4f3b..3ca066d3f 100644 --- a/tests_pytest/unit_tests/common/core/quantization/quantization_params_generation/test_symmetric_selection_activation.py +++ b/tests_pytest/unit_tests/common/core/quantization/quantization_params_generation/test_symmetric_selection_activation.py @@ -35,6 +35,19 @@ def hist(): return count, bins +@pytest.fixture +def bounded_hist(): + np.random.seed(42) + size = (32, 32, 3) + num_bins = 2048 + x = np.random.uniform(-7, 7, size=size).flatten() + e_x = np.exp(x - np.max(x)) + x = (e_x / e_x.sum()) + 1 + count, bins = np.histogram(x, bins=num_bins) + + return count, bins + + err_methods_to_test = [e.name for e in QuantizationErrorMethod if e != QuantizationErrorMethod.HMSE] @@ -48,4 +61,17 @@ def test_symmetric_threshold_selection(error_method, hist): assert THRESHOLD in search_res assert SIGNED in search_res assert np.isclose(search_res[THRESHOLD], 7, atol=0.4) - assert search_res[SIGNED] is True + assert search_res[SIGNED] + + +@pytest.mark.parametrize("error_method", err_methods_to_test) +def test_symmetric_threshold_selection_bounded_activation(error_method, bounded_hist): + counts, bins = bounded_hist + + search_res = symmetric_selection_histogram(bins, counts, 2, 8, Mock(), Mock(), Mock(), Mock(), + MIN_THRESHOLD, QuantizationErrorMethod[error_method], False) + + assert THRESHOLD in search_res + assert SIGNED in search_res + assert np.isclose(search_res[THRESHOLD], 1, atol=0.4) + assert not search_res[SIGNED]