From 93c7e9c2969de3e322edce4f97cb5fd30e3b0f9a Mon Sep 17 00:00:00 2001 From: Joose Sainio Date: Mon, 23 Jan 2023 13:39:22 +0200 Subject: [PATCH] [DepQuant] Fix for mts and lfnst being quantized incorrectly during search --- src/dep_quant.c | 4 ++-- src/search_intra.c | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/dep_quant.c b/src/dep_quant.c index b5fe6c55..cd461815 100644 --- a/src/dep_quant.c +++ b/src/dep_quant.c @@ -1145,7 +1145,7 @@ int uvg_dep_quant( const bool is_mts = compID == COLOR_Y && cur_tu->tr_idx > MTS_SKIP; const bool is_ts = cur_tu->tr_skip >> compID & 1; - const uint32_t log2_tr_width = uvg_g_convert_to_log2[width]; + const uint32_t log2_tr_width = uvg_g_convert_to_log2[width]; const uint32_t log2_tr_height = uvg_g_convert_to_log2[height]; const uint32_t* const scan = uvg_get_scan_order_table(SCAN_GROUP_4X4,0,log2_tr_width,log2_tr_height); const uint32_t* const cg_scan = uvg_get_scan_order_table(SCAN_GROUP_UNGROUPED,0,log2_tr_width,log2_tr_height); @@ -1155,7 +1155,7 @@ int uvg_dep_quant( bool needs_block_size_trafo_scale = is_ts && ((log2_tr_height + log2_tr_width) % 2 == 1); needs_block_size_trafo_scale |= 0; // Non log2 block size - const int32_t scalinglist_type = (cur_tu->type == CU_INTRA ? 0 : 3) + (int8_t)compID; + const int32_t scalinglist_type = (cur_tu->type == CU_INTRA ? 0 : 3) + (int8_t)compID; const int32_t *q_coeff = encoder->scaling_list.quant_coeff[log2_tr_width][log2_tr_height][scalinglist_type][qp_scaled % 6]; const int32_t transform_shift = MAX_TR_DYNAMIC_RANGE - encoder->bitdepth - ((log2_tr_height + log2_tr_width) >> 1) - needs_block_size_trafo_scale; //!< Represents scaling through forward transform const int64_t q_bits = QUANT_SHIFT + qp_scaled / 6 + (is_ts ? 0 : transform_shift ); diff --git a/src/search_intra.c b/src/search_intra.c index f05aa208..8067772a 100644 --- a/src/search_intra.c +++ b/src/search_intra.c @@ -390,7 +390,12 @@ static double search_intra_trdepth( if (pred_cu->lfnst_idx > 0 && pred_cu->tr_idx > 0) { continue; } - + + if (!has_been_split) { + memcpy(&state->search_cabac, &cabac_data, sizeof(cabac_data)); + state->search_cabac.update = 1; + } + uvg_intra_recon_cu( state, search_data, @@ -435,12 +440,7 @@ static double search_intra_trdepth( continue; } } - - if (!has_been_split) { - memcpy(&state->search_cabac, &cabac_data, sizeof(cabac_data)); - state->search_cabac.update = 1; - } - + double rd_cost = uvg_cu_rd_cost_luma( state, cu_loc,