hashirama/uvg266
1
0
Fork 0
mirror of https://github.com/ultravideo/uvg266.git synced 2024-11-27 19:24:06 +00:00
Code Issues Actions Projects Releases Wiki Activity

[isp] Fix error in inverse transform shifting.

Browse source
This commit is contained in:
siivonek 2022-08-05 14:16:32 +03:00 committed by Marko Viitanen
parent cd7e091992
commit a9090c99b5
1 changed files with 5 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
src/strategies/generic/dct-generic.c
View file

@ -2584,8 +2584,11 @@ static void mts_idct_generic(
partial_tr_func* idct_ver = idct_table[type_ver][log2_width_minus2]; partial_tr_func* idct_ver = idct_table[type_ver][log2_width_minus2];
int16_t tmp[32 * 32]; int16_t tmp[32 * 32];
const int32_t shift_1st = log2_width_minus2 - 7; const int max_log2_tr_dynamic_range = 15;
const int32_t shift_2nd = log2_height_minus2 + 8; const int transform_matrix_shift = 6;
const int32_t shift_1st = transform_matrix_shift + 1;
const int32_t shift_2nd = (transform_matrix_shift + max_log2_tr_dynamic_range - 1) - bitdepth;
idct_ver(input, tmp, shift_1st, width, skip_width, skip_height); idct_ver(input, tmp, shift_1st, width, skip_width, skip_height);
idct_hor(tmp, output, shift_2nd, height, 0, skip_width); idct_hor(tmp, output, shift_2nd, height, 0, skip_width);