hashirama/uvg266
1
0
Fork 0
mirror of https://github.com/ultravideo/uvg266.git synced 2024-11-23 18:14:06 +00:00
Code Issues Actions Projects Releases Wiki Activity

[isp] Fix isp bitcost calculation

Browse source
This commit is contained in:
Joose Sainio 2022-12-20 09:13:08 +02:00 committed by Marko Viitanen
parent f3c8a4f5db
commit 73956a9a46
1 changed files with 13 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
src/search.c
View file

@ -538,8 +538,9 @@ double uvg_cu_rd_cost_luma(
return sum + tr_tree_bits * state->lambda;
}
const bool is_not_isp = pred_cu->type == CU_INTER || pred_cu->intra.isp_mode == ISP_MODE_NO_ISP;
// Add transform_tree cbf_luma bit cost.
if (pred_cu->type == CU_INTER || pred_cu->intra.isp_mode == ISP_MODE_NO_ISP) {
if (is_not_isp) {
const int depth = 6 - uvg_g_convert_to_log2[cu_loc->width];
int is_set = cbf_is_set(pred_cu->cbf, COLOR_Y);
if (pred_cu->type == CU_INTRA ||
@ -562,8 +563,9 @@ double uvg_cu_rd_cost_luma(
// TODO: 8x4 CUs
const int split_limit = uvg_get_isp_split_num(cu_loc->width, cu_loc->height, pred_cu->intra.isp_mode, true);
int luma_ctx = 2;
const int split_limit_minus_one = split_limit - 1;
for (int i = 0; i < split_limit; i++) {
if (i != 3 && isp_cbf != 0x8) {
if (i != split_limit_minus_one || isp_cbf != 1 << split_limit_minus_one) {
const int flag = (isp_cbf >> i) & 1;
CABAC_FBITS_UPDATE(cabac, &(cabac->ctx.qt_cbf_model_luma[luma_ctx]), flag, tr_tree_bits, "cbf_y_search");
luma_ctx = 2 + flag;
@ -583,7 +585,7 @@ double uvg_cu_rd_cost_luma(
if (!skip_residual_coding) {
int8_t luma_scan_mode = SCAN_DIAG;
if (pred_cu->type == CU_INTER || pred_cu->intra.isp_mode == ISP_MODE_NO_ISP) {
if (is_not_isp) {
//const coeff_t* coeffs = &lcu->coeff.y[xy_to_zorder(LCU_WIDTH, x_px, y_px)];
const coeff_t* coeffs = lcu->coeff.y;
@ -795,9 +797,10 @@ static double cu_rd_cost_tr_split_accurate(
else {
// TODO: 8x4 CUs
const int split_limit = uvg_get_isp_split_num(width, height, pred_cu->intra.isp_mode, true);
for (int i = 0; i < split_limit; i++) {
int luma_ctx = 2;
if (i != 3 && isp_cbf != 0x8) {
const int split_limit_minus_one = split_limit - 1;
for (int i = 0; i < split_limit; i++) {
if (i != split_limit_minus_one || isp_cbf != 1 << split_limit_minus_one) {
const int flag = (isp_cbf >> i) & 1;
CABAC_FBITS_UPDATE(cabac, &(cabac->ctx.qt_cbf_model_luma[luma_ctx]), flag, tr_tree_bits, "cbf_y_search");
luma_ctx = 2 + flag;
@ -829,7 +832,7 @@ static double cu_rd_cost_tr_split_accurate(
&& height <= (1 << state->encoder_control->cfg.trskip_max_size)
&& !is_isp;
if(cb_flag_y){
if(cb_flag_y || is_isp){
if (can_use_tr_skip) {
CABAC_FBITS_UPDATE(cabac, &cabac->ctx.transform_skip_model_luma, tr_cu->tr_idx == MTS_SKIP, tr_tree_bits, "transform_skip_flag");
}
@ -936,7 +939,7 @@ static double cu_rd_cost_tr_split_accurate(
}
const bool is_chroma_tree = is_local_sep_tree || tree_type == UVG_CHROMA_T;
if (uvg_is_lfnst_allowed(state, tr_cu, is_local_sep_tree ? UVG_CHROMA_T : tree_type, is_chroma_tree ? COLOR_UV : COLOR_Y, is_chroma_tree ? chroma_loc : cu_loc, lcu)) {
if (uvg_is_lfnst_allowed(state, tr_cu, is_local_sep_tree ? UVG_CHROMA_T : tree_type, is_chroma_tree ? COLOR_UV : COLOR_Y, is_chroma_tree ? chroma_loc : cu_loc, lcu) && tree_type != UVG_LUMA_T) {
const int lfnst_idx = is_chroma_tree ? tr_cu->cr_lfnst_idx : tr_cu->lfnst_idx;
CABAC_FBITS_UPDATE(
cabac,
@ -1255,6 +1258,8 @@ static double search_cu(
cur_cu->log2_chroma_width = uvg_g_convert_to_log2[chroma_loc->chroma_width];
}
intra_search_data_t intra_search;
// If the CU is completely inside the frame at this depth, search for
// prediction modes at this depth.
if ( x + luma_width <= frame_width && y + luma_height <= frame_height)
@ -1304,7 +1309,6 @@ static double search_cu(
(y & ~(cu_width_intra_min - 1)) + cu_width_intra_min > frame_height) &&
!(state->encoder_control->cfg.force_inter && state->frame->slicetype != UVG_SLICE_I);
intra_search_data_t intra_search;
intra_search.cost = 0;
if (can_use_intra && !skip_intra) {
intra_search.pred_cu = *cur_cu;
@ -1553,7 +1557,7 @@ static double search_cu(
cost = bits * state->lambda;
cost += cu_rd_cost_tr_split_accurate(state, cur_cu, lcu, tree_type, 0, cu_loc, chroma_loc, has_chroma);
cost += cu_rd_cost_tr_split_accurate(state, cur_cu, lcu, tree_type, intra_search.best_isp_cbfs, cu_loc, chroma_loc, has_chroma);
//fprintf(stderr, "%4d %4d %2d %2d %d %d %f\n", x, y, cu_width, cu_height, has_chroma, cur_cu->split_tree, cost);
//if (ctrl->cfg.zero_coeff_rdo && inter_zero_coeff_cost <= cost) {