From c1fb6c6624072cef4785689d4f3ec02bff52266d Mon Sep 17 00:00:00 2001 From: James Zern Date: Tue, 16 Aug 2022 16:48:00 -0700 Subject: [PATCH 1/6] variance_sse2.c: add some missing casts quiets integer sanitizer warnings of the form: ../vpx_dsp/x86/variance_sse2.c:100:10: runtime error: implicit conversion from type 'unsigned int' of value 4294966272 (32-bit, unsigned) to type 'int' changed the value to -1024 (32-bit, signed) Bug: b/229626362 Change-Id: I150cc0a6a6b85143c3bf96886686fe3a40897db5 --- vpx_dsp/x86/variance_sse2.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/vpx_dsp/x86/variance_sse2.c b/vpx_dsp/x86/variance_sse2.c index fedc8b84e51..d6eb12da1ac 100644 --- a/vpx_dsp/x86/variance_sse2.c +++ b/vpx_dsp/x86/variance_sse2.c @@ -85,7 +85,7 @@ static INLINE void variance_final_512_pel_sse2(__m128i vsse, __m128i vsum, vsum = _mm_add_epi16(vsum, _mm_srli_si128(vsum, 8)); vsum = _mm_unpacklo_epi16(vsum, vsum); vsum = _mm_srai_epi32(vsum, 16); - *sum = add32x4_sse2(vsum); + *sum = (int)add32x4_sse2(vsum); } static INLINE __m128i sum_to_32bit_sse2(const __m128i sum) { @@ -97,7 +97,7 @@ static INLINE __m128i sum_to_32bit_sse2(const __m128i sum) { // Can handle 1024 pixels' diff sum (such as 32x32) static INLINE int sum_final_sse2(const __m128i sum) { const __m128i t = sum_to_32bit_sse2(sum); - return add32x4_sse2(t); + return (int)add32x4_sse2(t); } static INLINE void variance4_sse2(const uint8_t *src_ptr, const int src_stride, @@ -349,7 +349,7 @@ unsigned int vpx_variance32x64_sse2(const uint8_t *src_ptr, int src_stride, vsum = _mm_add_epi32(vsum, sum_to_32bit_sse2(vsum16)); } *sse = add32x4_sse2(vsse); - sum = add32x4_sse2(vsum); + sum = (int)add32x4_sse2(vsum); return *sse - (unsigned int)(((int64_t)sum * sum) >> 11); } @@ -369,7 +369,7 @@ unsigned int vpx_variance64x32_sse2(const uint8_t *src_ptr, int src_stride, vsum = _mm_add_epi32(vsum, sum_to_32bit_sse2(vsum16)); } *sse = add32x4_sse2(vsse); - sum = add32x4_sse2(vsum); + sum = (int)add32x4_sse2(vsum); return *sse - (unsigned int)(((int64_t)sum * sum) >> 11); } @@ -389,7 +389,7 @@ unsigned int vpx_variance64x64_sse2(const uint8_t *src_ptr, int src_stride, vsum = _mm_add_epi32(vsum, sum_to_32bit_sse2(vsum16)); } *sse = add32x4_sse2(vsse); - sum = add32x4_sse2(vsum); + sum = (int)add32x4_sse2(vsum); return *sse - (unsigned int)(((int64_t)sum * sum) >> 12); } From d939886809d1bf8428d7bfb515fa8ff544c44fe7 Mon Sep 17 00:00:00 2001 From: James Zern Date: Tue, 16 Aug 2022 16:52:06 -0700 Subject: [PATCH 2/6] load_unaligned_u32: use an int w/_mm_cvtsi32_si128 this matches the type of the function parameter; quiets integer sanitizer warnings of the form: implicit conversion from type 'uint32_t' (aka 'unsigned int') of value 3215646151 (32-bit, unsigned) to type 'int' changed the value to -1079321145 (32-bit, signed) Bug: b/229626362 Change-Id: Ia9a5dc5e1f57cbf4f8f8fa457bb674ef43369d37 --- vpx_dsp/x86/mem_sse2.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vpx_dsp/x86/mem_sse2.h b/vpx_dsp/x86/mem_sse2.h index 6df4773f73a..031f361a41d 100644 --- a/vpx_dsp/x86/mem_sse2.h +++ b/vpx_dsp/x86/mem_sse2.h @@ -27,7 +27,7 @@ static INLINE int32_t loadu_int32(const void *src) { } static INLINE __m128i load_unaligned_u32(const void *a) { - uint32_t val; + int val; memcpy(&val, a, sizeof(val)); return _mm_cvtsi32_si128(val); } From b77b6b68d3e7bf54db7ac2996a6b103cafa7c32c Mon Sep 17 00:00:00 2001 From: James Zern Date: Tue, 16 Aug 2022 17:28:08 -0700 Subject: [PATCH 3/6] highbd_quantize_intrin_sse2: quiet int sanitizer warnings add a missing cast in ^ operations; quiets warnings of the form: implicit conversion from type 'int' of value -1 (32-bit, signed) to type 'unsigned int' changed the value to 4294967295 (32-bit, unsigned) Bug: b/229626362 Change-Id: I56f74981050b2c9d00bad20e68f1b73ce7454729 --- vpx_dsp/x86/highbd_quantize_intrin_sse2.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/vpx_dsp/x86/highbd_quantize_intrin_sse2.c b/vpx_dsp/x86/highbd_quantize_intrin_sse2.c index 4535a0f7a2c..1264fbed227 100644 --- a/vpx_dsp/x86/highbd_quantize_intrin_sse2.c +++ b/vpx_dsp/x86/highbd_quantize_intrin_sse2.c @@ -82,7 +82,8 @@ void vpx_highbd_quantize_b_sse2(const tran_low_t *coeff_ptr, intptr_t count, const int64_t tmp4 = ((tmp3 * quant_ptr[k != 0]) >> 16) + tmp3; const uint32_t abs_qcoeff = (uint32_t)((tmp4 * quant_shift_ptr[k != 0]) >> 16); - qcoeff_ptr[k] = (int)(abs_qcoeff ^ coeff_sign[j]) - coeff_sign[j]; + qcoeff_ptr[k] = + (int)(abs_qcoeff ^ (uint32_t)coeff_sign[j]) - coeff_sign[j]; dqcoeff_ptr[k] = qcoeff_ptr[k] * dequant_ptr[k != 0]; if (abs_qcoeff) eob_i = iscan[k] > eob_i ? iscan[k] : eob_i; } @@ -143,7 +144,7 @@ void vpx_highbd_quantize_b_32x32_sse2( const int64_t tmp2 = ((tmp1 * quant_ptr[rc != 0]) >> 16) + tmp1; const uint32_t abs_qcoeff = (uint32_t)((tmp2 * quant_shift_ptr[rc != 0]) >> 15); - qcoeff_ptr[rc] = (int)(abs_qcoeff ^ coeff_sign) - coeff_sign; + qcoeff_ptr[rc] = (int)(abs_qcoeff ^ (uint32_t)coeff_sign) - coeff_sign; dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0] / 2; if (abs_qcoeff) eob = iscan[idx_arr[i]] > eob ? iscan[idx_arr[i]] : eob; } From cb18d72c306b8400279191940ec0861af338bb5a Mon Sep 17 00:00:00 2001 From: James Zern Date: Tue, 16 Aug 2022 17:54:14 -0700 Subject: [PATCH 4/6] webmdec,WebmInputContext: make timestamp_ns signed this matches the type returned from libwebm, which uses -1 as an error; quiets integer sanitizer warnings of the form: implicit conversion from type 'long long' of value -1 (64-bit, signed) to type 'uint64_t' (aka 'unsigned long') changed the value to 18446744073709551615 (64-bit, unsigned) Bug: b/229626362 Change-Id: Id3966912f802aee3c0f7852225b55f3057c3e76a --- webmdec.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webmdec.h b/webmdec.h index d8618b07d67..6ae7ee16d0b 100644 --- a/webmdec.h +++ b/webmdec.h @@ -27,7 +27,7 @@ struct WebmInputContext { const void *block; int block_frame_index; int video_track_index; - uint64_t timestamp_ns; + int64_t timestamp_ns; int is_key_frame; int reached_eos; }; From a76a0228359723fe8b3c522ea0e7c2e2acb26ca8 Mon Sep 17 00:00:00 2001 From: James Zern Date: Tue, 16 Aug 2022 17:46:24 -0700 Subject: [PATCH 5/6] vp8,VP8_COMP: normalize segment_encode_breakout type use unsigned int as the API value is of this type; this quiets some integer sanitizer warnings of the form: implicit conversion from type 'unsigned int' of value 2147483648 (32-bit, unsigned) to type 'int' changed the value to -2147483648 (32-bit, signed) Bug: b/229626362 Change-Id: I3d1ca618bf1b3cd57a5dca65a3067f351c1473f8 --- test/encode_api_test.cc | 4 ++-- vp8/encoder/onyx_int.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/encode_api_test.cc b/test/encode_api_test.cc index 08159148bd8..ecdf9283431 100644 --- a/test/encode_api_test.cc +++ b/test/encode_api_test.cc @@ -233,8 +233,8 @@ TEST(EncodeAPI, SetRoi) { roi.roi_map = roi_map; // VP8 only. This value isn't range checked. roi.static_threshold[1] = 1000; - roi.static_threshold[2] = INT_MIN; - roi.static_threshold[3] = INT_MAX; + roi.static_threshold[2] = UINT_MAX / 2 + 1; + roi.static_threshold[3] = UINT_MAX; for (const auto delta : { -63, -1, 0, 1, 63 }) { for (int i = 0; i < 8; ++i) { diff --git a/vp8/encoder/onyx_int.h b/vp8/encoder/onyx_int.h index 424f51b180f..726dcc94660 100644 --- a/vp8/encoder/onyx_int.h +++ b/vp8/encoder/onyx_int.h @@ -483,7 +483,7 @@ typedef struct VP8_COMP { unsigned char *segmentation_map; signed char segment_feature_data[MB_LVL_MAX][MAX_MB_SEGMENTS]; - int segment_encode_breakout[MAX_MB_SEGMENTS]; + unsigned int segment_encode_breakout[MAX_MB_SEGMENTS]; unsigned char *active_map; unsigned int active_map_enabled; From 9db0ec67e33fae4cfd066a7cf0fce9f81442c90e Mon Sep 17 00:00:00 2001 From: James Zern Date: Tue, 16 Aug 2022 21:59:20 -0700 Subject: [PATCH 6/6] vpx_encoder.h: make flag constants unsigned this matches the type for vpx_codec_frame_flags_t and vpx_codec_er_flags_t and quiets int sanitizer warnings of the form: implicit conversion from type 'int' of value -9 (32-bit, signed) to type 'unsigned int' changed the value to 4294967287 (32-bit, unsigned) Bug: b/229626362 Change-Id: Icfc5993250f37cedb300c7032cab28ce4bec1f86 --- vpx/vpx_encoder.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/vpx/vpx_encoder.h b/vpx/vpx_encoder.h index 21254bb547f..e776ec8136e 100644 --- a/vpx/vpx_encoder.h +++ b/vpx/vpx_encoder.h @@ -115,14 +115,14 @@ typedef int64_t vpx_codec_pts_t; * support frame types that are codec specific (MPEG-1 D-frames for example) */ typedef uint32_t vpx_codec_frame_flags_t; -#define VPX_FRAME_IS_KEY 0x1 /**< frame is the start of a GOP */ +#define VPX_FRAME_IS_KEY 0x1u /**< frame is the start of a GOP */ /*!\brief frame can be dropped without affecting the stream (no future frame * depends on this one) */ -#define VPX_FRAME_IS_DROPPABLE 0x2 +#define VPX_FRAME_IS_DROPPABLE 0x2u /*!\brief frame should be decoded but will not be shown */ -#define VPX_FRAME_IS_INVISIBLE 0x4 +#define VPX_FRAME_IS_INVISIBLE 0x4u /*!\brief this is a fragment of the encoded frame */ -#define VPX_FRAME_IS_FRAGMENT 0x8 +#define VPX_FRAME_IS_FRAGMENT 0x8u /*!\brief Error Resilient flags * @@ -132,12 +132,12 @@ typedef uint32_t vpx_codec_frame_flags_t; */ typedef uint32_t vpx_codec_er_flags_t; /*!\brief Improve resiliency against losses of whole frames */ -#define VPX_ERROR_RESILIENT_DEFAULT 0x1 +#define VPX_ERROR_RESILIENT_DEFAULT 0x1u /*!\brief The frame partitions are independently decodable by the bool decoder, * meaning that partitions can be decoded even though earlier partitions have * been lost. Note that intra prediction is still done over the partition * boundary. */ -#define VPX_ERROR_RESILIENT_PARTITIONS 0x2 +#define VPX_ERROR_RESILIENT_PARTITIONS 0x2u /*!\brief Encoder output packet variants *