[lfnst] Remove unused stuff from lfnst constraint check. Use temp values for storing lfnst constraints during search instead of updating values inside pred cu.

This commit is contained in:
siivonek 2022-05-16 13:34:10 +03:00 committed by Joose Sainio
parent 7b6b6586c6
commit 896bc093d3

View file

@ -250,39 +250,30 @@ static void derive_mts_constraints(cu_info_t *const pred_cu,
*/
static void derive_lfnst_constraints(cu_info_t* const pred_cu,
lcu_t* const lcu, const int depth,
const vector2d_t lcu_px, const int color)
const vector2d_t lcu_px, bool *constraints)
{
const int width = LCU_WIDTH >> depth;
const int height = width; // TODO: height for non-square blocks.
int8_t scan_idx = kvz_get_scan_order(pred_cu->type, pred_cu->intra.mode, depth);
// ToDo: large block support in VVC?
uint32_t sig_coeffgroup_flag[32 * 32] = { 0 };
const uint32_t log2_block_size = kvz_g_convert_to_bit[width] + 2;
const uint32_t log2_cg_size = kvz_g_log2_sbb_size[log2_block_size][log2_block_size][0]
+ kvz_g_log2_sbb_size[log2_block_size][log2_block_size][1];
const uint32_t* scan = kvz_g_sig_last_scan[scan_idx][log2_block_size - 1];
const uint32_t* scan_cg = g_sig_last_scan_cg[log2_block_size - 1][scan_idx];
const coeff_t* coeff = &lcu->coeff.y[xy_to_zorder(LCU_WIDTH, lcu_px.x, lcu_px.y)];
signed scan_cg_last = -1;
signed scan_pos_last = -1;
for (int i = 0; i < width * width; i++) {
for (int i = 0; i < width * height; i++) {
if (coeff[scan[i]]) {
scan_pos_last = i;
sig_coeffgroup_flag[scan_cg[i >> log2_cg_size]] = 1;
}
}
if (scan_pos_last < 0) return;
scan_cg_last = scan_pos_last >> log2_cg_size;
bool is_chroma = color != COLOR_Y;
if (pred_cu != NULL && pred_cu->tr_idx != MTS_SKIP && height >= 4 && width >= 4) {
const int max_lfnst_pos = ((height == 4 && width == 4) || (height == 8 && width == 8)) ? 7 : 15;
pred_cu->violates_lfnst_constrained[is_chroma] |= scan_pos_last > max_lfnst_pos;
pred_cu->lfnst_last_scan_pos |= scan_pos_last >= 1;
constraints[0] |= scan_pos_last > max_lfnst_pos;
constraints[1] |= scan_pos_last >= 1;
}
}
@ -419,9 +410,13 @@ static double search_intra_trdepth(
continue;
}
}
derive_lfnst_constraints(pred_cu, lcu, depth, lcu_px, COLOR_Y); // TODO: pass proper color if it is passed into search_trdepth in the future
if (pred_cu->violates_lfnst_constrained[is_chroma] || !pred_cu->lfnst_last_scan_pos) {
continue;
if (pred_cu->lfnst_idx > 0) {
// Temp constraints. Updating the actual pred_cu constraints here will break things later
bool constraints[2] = {pred_cu->violates_lfnst_constrained[is_chroma], pred_cu->lfnst_last_scan_pos};
derive_lfnst_constraints(pred_cu, lcu, depth, lcu_px, constraints);
if (constraints[0] || !constraints[1]) {
continue;
}
}
double rd_cost = uvg_cu_rd_cost_luma(state, lcu_px.x, lcu_px.y, depth, pred_cu, lcu);