From 0f50caa2d0c558d07175d6f19dd34ab85920a294 Mon Sep 17 00:00:00 2001 From: Joose Sainio Date: Wed, 15 Feb 2023 14:23:55 +0200 Subject: [PATCH] [mtt] Fix various small issues and DepQuant for non-square blocks --- src/dep_quant.c | 11 ++++------- src/intra.c | 2 +- src/search.c | 2 +- src/search_intra.c | 9 ++++++--- src/transform.c | 6 +++--- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/dep_quant.c b/src/dep_quant.c index 103e02d4..a41bf6c5 100644 --- a/src/dep_quant.c +++ b/src/dep_quant.c @@ -1153,7 +1153,7 @@ int uvg_dep_quant( int32_t qp_scaled = uvg_get_scaled_qp(compID, state->qp, (encoder->bitdepth - 8) * 6, encoder->qp_map[0]); qp_scaled = is_ts ? MAX(qp_scaled, 4 + 6 * MIN_QP_PRIME_TS) : qp_scaled; - bool needs_block_size_trafo_scale = is_ts && ((log2_tr_height + log2_tr_width) % 2 == 1); + bool needs_block_size_trafo_scale = !is_ts && ((log2_tr_height + log2_tr_width) % 2 == 1); needs_block_size_trafo_scale |= 0; // Non log2 block size const int32_t scalinglist_type = (cur_tu->type == CU_INTRA ? 0 : 3) + (int8_t)compID; @@ -1252,8 +1252,8 @@ int uvg_dep_quant( uint32_t pos_y_next = blkpos_next >> log2_tr_width; uint32_t pos_x_next = blkpos_next - (pos_y_next << log2_tr_width); uint32_t cg_blockpos_next = scanIdx ? cg_scan[(scanIdx -1) >> 4] : 0; - uint32_t cg_pos_y_next = cg_blockpos_next / height_in_sbb; - uint32_t cg_pos_x_next = cg_blockpos_next - (cg_pos_y_next * height_in_sbb); + uint32_t cg_pos_y_next = cg_blockpos_next / width_in_sbb; + uint32_t cg_pos_x_next = cg_blockpos_next - (cg_pos_y_next * width_in_sbb); uint32_t diag = pos_y_next + pos_x_next; uint32_t sig_ctx_offset = compID == COLOR_Y ? (diag < 2 ? 8 : diag < 5 ? 4 : 0) : (diag < 2 ? 4 : 0); @@ -1308,10 +1308,7 @@ int uvg_dep_quant( width, height); //tu.cu->slice->getReverseLastSigCoeffFlag()); } - Decision temp[8]; - Decision* decisions = ctxs->m_trellis[scanIdx]; - memcpy(temp, decisions, sizeof(Decision) * 8); - decisions++; + } //===== find best path ===== diff --git a/src/intra.c b/src/intra.c index 2dae1a6c..3d2c1f81 100644 --- a/src/intra.c +++ b/src/intra.c @@ -2070,7 +2070,7 @@ double uvg_recon_and_estimate_cost_isp(encoder_state_t* const state, int cbf = cbf_is_set(search_data->pred_cu.cbf, COLOR_Y); - if (i + 1 != split_limit && search_data->best_isp_cbfs != 0) { + if (i + 1 != split_limit || search_data->best_isp_cbfs != 1 << (split_limit - 1)) { CABAC_FBITS_UPDATE(&state->search_cabac, &state->search_cabac.ctx.qt_cbf_model_luma[cbf_context], cbf, coeff_bits, "cbf_luma_isp_recon"); } cost += ssd + coeff_bits * state->lambda; diff --git a/src/search.c b/src/search.c index f5f6e044..8d7e343d 100644 --- a/src/search.c +++ b/src/search.c @@ -1798,7 +1798,7 @@ static double search_cu( // 3.9 const double factor = state->qp > 30 ? 1.1 : 1.075; - if (split_bits * state->frame->lambda + cost / factor > cost) { + if (split_bits * state->lambda + cost / factor > cost) { can_split[split_type] = false; continue; } diff --git a/src/search_intra.c b/src/search_intra.c index 689d872d..83ec950d 100644 --- a/src/search_intra.c +++ b/src/search_intra.c @@ -457,7 +457,7 @@ static double search_intra_trdepth( double transform_bits = 0; if (state->encoder_control->cfg.lfnst && PU_IS_TU(pred_cu) && trafo != MTS_SKIP && end_lfnst_idx != 0) { - if (!constraints[0] && constraints[1]) { + if ((!constraints[0] && constraints[1]) || lfnst_idx != 0) { transform_bits += CTX_ENTROPY_FBITS( &state->search_cabac.ctx.lfnst_idx_model[tree_type == UVG_LUMA_T], lfnst_idx != 0); @@ -468,7 +468,10 @@ static double search_intra_trdepth( } } } - if (num_transforms > 2 && trafo != MTS_SKIP && width <= 32 + if (num_transforms > 2 && trafo != MTS_SKIP + && pred_cu->intra.isp_mode == ISP_MODE_NO_ISP + && lfnst_idx == 0 + && width <= 32 && height <= 32 && !pred_cu->violates_mts_coeff_constraint && pred_cu-> mts_last_scan_pos) { @@ -488,7 +491,7 @@ static double search_intra_trdepth( } } - rd_cost += transform_bits * state->frame->lambda; + rd_cost += transform_bits * state->lambda; search_data->lfnst_costs[lfnst_idx] = MIN( search_data->lfnst_costs[lfnst_idx], diff --git a/src/transform.c b/src/transform.c index 2ee2fc32..783d9f2b 100644 --- a/src/transform.c +++ b/src/transform.c @@ -782,8 +782,8 @@ void uvg_chroma_transform_search( pred_cu->violates_lfnst_constrained_chroma = false; } if (!is_jccr) { - double u_cost = UVG_CHROMA_MULT * ssd_u + u_bits * state->frame->lambda; - double v_cost = UVG_CHROMA_MULT * ssd_v + v_bits * state->frame->lambda; + double u_cost = UVG_CHROMA_MULT * ssd_u + u_bits * state->c_lambda; + double v_cost = UVG_CHROMA_MULT * ssd_v + v_bits * state->c_lambda; if (u_cost < chorma_ts_out->best_u_cost) { chorma_ts_out->best_u_cost = u_cost; chorma_ts_out->best_u_index = u_has_coeffs ? transforms[i] : NO_RESIDUAL; @@ -794,7 +794,7 @@ void uvg_chroma_transform_search( } } else { - double cost = UVG_CHROMA_MULT * (ssd_u + ssd_v) + (u_bits + v_bits) * state->frame->lambda; + double cost = UVG_CHROMA_MULT * (ssd_u + ssd_v) + (u_bits + v_bits) * state->c_lambda; if (cost < chorma_ts_out->best_combined_cost) { chorma_ts_out->best_combined_cost = cost; chorma_ts_out->best_combined_index = transforms[i];