hashirama/uvg266
1
0
Fork 0
mirror of https://github.com/ultravideo/uvg266.git synced 2024-11-27 19:24:06 +00:00
Code Issues Actions Projects Releases Wiki Activity

[LMCS] Fix casting of the chroma scaled residual

Browse source
This commit is contained in:
Marko Viitanen 2021-06-15 18:32:34 +03:00
parent b22fd61c7f
commit c004735821
2 changed files with 2 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/strategies/avx2/quant-avx2.c
View file

@ -677,7 +677,7 @@ int kvz_quantize_residual_avx2(encoder_state_t *const state,
for (x = 0; x < width; ++x) {
sign = residual[x + y * width] >= 0 ? 1 : -1;
absval = sign * residual[x + y * width];
residual[x + y * width] = (int8_t)CLIP(-maxAbsclipBD, maxAbsclipBD, sign * (((absval << CSCALE_FP_PREC) + (lmcs_chroma_adj >> 1)) / lmcs_chroma_adj));
residual[x + y * width] = (int16_t)CLIP(-maxAbsclipBD, maxAbsclipBD, sign * (((absval << CSCALE_FP_PREC) + (lmcs_chroma_adj >> 1)) / lmcs_chroma_adj));
}
}
}

2
src/strategies/generic/quant-generic.c
View file

@ -223,7 +223,7 @@ int kvz_quantize_residual_generic(encoder_state_t *const state,
for (x = 0; x < width; ++x) {
sign = residual[x + y * width] >= 0 ? 1 : -1;
absval = sign * residual[x + y * width];
residual[x + y * width] = (int8_t)CLIP(-maxAbsclipBD, maxAbsclipBD, sign * (((absval << CSCALE_FP_PREC) + (lmcs_chroma_adj >> 1)) / lmcs_chroma_adj));
residual[x + y * width] = (int16_t)CLIP(-maxAbsclipBD, maxAbsclipBD, sign * (((absval << CSCALE_FP_PREC) + (lmcs_chroma_adj >> 1)) / lmcs_chroma_adj));
}
}
}