hashirama/uvg266
1
0
Fork 0
mirror of https://github.com/ultravideo/uvg266.git synced 2024-11-27 19:24:06 +00:00
Code Issues Actions Projects Releases Wiki Activity

[isp] Fix assert. Implement coef cost calculation for isp splits.

Browse source
This commit is contained in:
siivonek 2022-08-18 15:22:17 +03:00
parent 943ae415db
commit fae833c4d2
2 changed files with 42 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/intra.c
View file

@ -1502,7 +1502,7 @@ int uvg_get_isp_split_dim(const int width, const int height, const int split_typ
const int factor_to_min_samples = non_split_dim_size < min_num_samples ? min_num_samples >> uvg_math_floor_log2(non_split_dim_size) : 1;
partition_size = (split_dim_size >> div_shift) < factor_to_min_samples ? factor_to_min_samples : (split_dim_size >> div_shift);
assert((uvg_math_floor_log2(partition_size) + uvg_math_floor_log2(non_split_dim_size) < uvg_math_floor_log2(min_num_samples)) &&
assert((uvg_math_floor_log2(partition_size) + uvg_math_floor_log2(non_split_dim_size) >= uvg_math_floor_log2(min_num_samples)) &&
"Partition has less than allowed minimum number of samples.");
return partition_size;
}

46
src/search.c
View file

@ -339,7 +339,7 @@ double uvg_cu_rd_cost_luma(const encoder_state_t *const state,
}
// Add transform_tree cbf_luma bit cost.
if (pred_cu->intra.isp_mode == ISP_MODE_NO_ISP) {
if (pred_cu->type == CU_INTER || pred_cu->intra.isp_mode == ISP_MODE_NO_ISP) {
const int is_tr_split = tr_cu->tr_depth - tr_cu->depth;
int is_set = cbf_is_set(pred_cu->cbf, depth, COLOR_Y);
if (pred_cu->type == CU_INTRA ||
@ -378,9 +378,27 @@ double uvg_cu_rd_cost_luma(const encoder_state_t *const state,
if (!skip_residual_coding) {
int8_t luma_scan_mode = uvg_get_scan_order(pred_cu->type, pred_cu->intra.mode, depth);
const coeff_t *coeffs = &lcu->coeff.y[xy_to_zorder(LCU_WIDTH, x_px, y_px)];
if (pred_cu->type == CU_INTER || pred_cu->intra.isp_mode == ISP_MODE_NO_ISP) {
const coeff_t* coeffs = &lcu->coeff.y[xy_to_zorder(LCU_WIDTH, x_px, y_px)];
coeff_bits += uvg_get_coeff_cost(state, coeffs, NULL, width, height, 0, luma_scan_mode, pred_cu->tr_idx == MTS_SKIP);
coeff_bits += uvg_get_coeff_cost(state, coeffs, NULL, width, height, 0, luma_scan_mode, pred_cu->tr_idx == MTS_SKIP);
}
else {
int split_type = pred_cu->intra.isp_mode;
int part_dim = uvg_get_isp_split_dim(width, height, split_type);
int limit = split_type == ISP_MODE_HOR ? height : width;
int split_num = 0;
for (int part = 0; part < limit; part += part_dim) {
const int part_x = split_type == ISP_MODE_HOR ? x_px : x_px + part;
const int part_y = split_type == ISP_MODE_HOR ? y_px + part : y_px;
const int part_w = split_type == ISP_MODE_HOR ? part_dim : width;
const int part_h = split_type == ISP_MODE_HOR ? height : part_dim;
const coeff_t* coeffs = &lcu->coeff.y[xy_to_zorder(LCU_WIDTH, part_x, part_y)];
coeff_bits += uvg_get_coeff_cost(state, coeffs, NULL, part_w, part_h, 0, luma_scan_mode, pred_cu->tr_idx == MTS_SKIP);
}
}
}
double bits = tr_tree_bits + coeff_bits;
@ -597,9 +615,27 @@ static double cu_rd_cost_tr_split_accurate(
CABAC_FBITS_UPDATE(cabac, &cabac->ctx.transform_skip_model_luma, tr_cu->tr_idx == MTS_SKIP, tr_tree_bits, "transform_skip_flag");
}
int8_t luma_scan_mode = uvg_get_scan_order(pred_cu->type, pred_cu->intra.mode, depth);
const coeff_t* coeffs = &lcu->coeff.y[xy_to_zorder(LCU_WIDTH, x_px, y_px)];
if (pred_cu->type == CU_INTER || pred_cu->intra.isp_mode == ISP_MODE_NO_ISP) {
const coeff_t* coeffs = &lcu->coeff.y[xy_to_zorder(LCU_WIDTH, x_px, y_px)];
coeff_bits += uvg_get_coeff_cost(state, coeffs, tr_cu, width, height, 0, luma_scan_mode, tr_cu->tr_skip & 1);
coeff_bits += uvg_get_coeff_cost(state, coeffs, NULL, width, height, 0, luma_scan_mode, pred_cu->tr_idx == MTS_SKIP);
}
else {
int split_type = pred_cu->intra.isp_mode;
int part_dim = uvg_get_isp_split_dim(width, height, split_type);
int limit = split_type == ISP_MODE_HOR ? height : width;
int split_num = 0;
for (int part = 0; part < limit; part += part_dim) {
const int part_x = split_type == ISP_MODE_HOR ? x_px : x_px + part;
const int part_y = split_type == ISP_MODE_HOR ? y_px + part : y_px;
const int part_w = split_type == ISP_MODE_HOR ? part_dim : width;
const int part_h = split_type == ISP_MODE_HOR ? height : part_dim;
const coeff_t* coeffs = &lcu->coeff.y[xy_to_zorder(LCU_WIDTH, part_x, part_y)];
coeff_bits += uvg_get_coeff_cost(state, coeffs, NULL, part_w, part_h, 0, luma_scan_mode, pred_cu->tr_idx == MTS_SKIP);
}
}
}
if(depth == 4 || tree_type == UVG_LUMA_T) {