From a9090c99b5f33fa2d0b40591e3f7973f70047d00 Mon Sep 17 00:00:00 2001 From: siivonek Date: Fri, 5 Aug 2022 14:16:32 +0300 Subject: [PATCH] [isp] Fix error in inverse transform shifting. --- src/strategies/generic/dct-generic.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/strategies/generic/dct-generic.c b/src/strategies/generic/dct-generic.c index 9e410541..717f1b5f 100644 --- a/src/strategies/generic/dct-generic.c +++ b/src/strategies/generic/dct-generic.c @@ -2584,8 +2584,11 @@ static void mts_idct_generic( partial_tr_func* idct_ver = idct_table[type_ver][log2_width_minus2]; int16_t tmp[32 * 32]; - const int32_t shift_1st = log2_width_minus2 - 7; - const int32_t shift_2nd = log2_height_minus2 + 8; + const int max_log2_tr_dynamic_range = 15; + const int transform_matrix_shift = 6; + + const int32_t shift_1st = transform_matrix_shift + 1; + const int32_t shift_2nd = (transform_matrix_shift + max_log2_tr_dynamic_range - 1) - bitdepth; idct_ver(input, tmp, shift_1st, width, skip_width, skip_height); idct_hor(tmp, output, shift_2nd, height, 0, skip_width);