hashirama/uvg266
1
0
Fork 0
mirror of https://github.com/ultravideo/uvg266.git synced 2024-11-30 12:44:07 +00:00
Code Issues Actions Projects Releases Wiki Activity

Fix crash when PU depth is limited

Browse source 
When video width or height was not a multiple of the smallest CU size,
no prediction would be performed at the border CUs. Kvazaar would later
crash at an assertion failure when attempting to write the bitstream for
the CU.

Fixed by permitting inter and intra prediction when the CU split is
forced, even if CUs of that size would otherwise be disabled.
This commit is contained in:
Arttu Ylä-Outinen 2017-04-27 10:07:45 +03:00
parent 9130b5107c
commit 2f2405dfe6
1 changed files with 33 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

44
src/search.c
View file

@ -508,15 +508,21 @@ static double search_cu(encoder_state_t * const state, int x, int y, int depth,
cur_cu->tr_depth = depth > 0 ? depth : 1; cur_cu->tr_depth = depth > 0 ? depth : 1;
cur_cu->type = CU_NOTSET; cur_cu->type = CU_NOTSET;
cur_cu->part_size = SIZE_2Nx2N; cur_cu->part_size = SIZE_2Nx2N;
// If the CU is completely inside the frame at this depth, search for // If the CU is completely inside the frame at this depth, search for
// prediction modes at this depth. // prediction modes at this depth.
if (x + cu_width <= frame->width && if (x + cu_width <= frame->width &&
y + cu_width <= frame->height) y + cu_width <= frame->height)
{ {
int cu_width_inter_min = LCU_WIDTH >> ctrl->cfg.pu_depth_inter.max;
bool can_use_inter = bool can_use_inter = state->frame->slicetype != KVZ_SLICE_I && (
state->frame->slicetype != KVZ_SLICE_I WITHIN(depth, ctrl->cfg.pu_depth_inter.min, ctrl->cfg.pu_depth_inter.max) ||
&& WITHIN(depth, ctrl->cfg.pu_depth_inter.min, ctrl->cfg.pu_depth_inter.max); // When the split was forced because the CTU is partially outside the
// frame, we permit inter coding even if pu_depth_inter would
// otherwise forbid it.
(x & ~(cu_width_inter_min - 1)) + cu_width_inter_min > frame->width ||
(y & ~(cu_width_inter_min - 1)) + cu_width_inter_min > frame->height
);
if (can_use_inter) { if (can_use_inter) {
double mode_cost; double mode_cost;
@ -566,9 +572,17 @@ static double search_cu(encoder_state_t * const state, int x, int y, int depth,
bool skip_intra = state->encoder_control->cfg.rdo == 0 bool skip_intra = state->encoder_control->cfg.rdo == 0
&& cur_cu->type != CU_NOTSET && cur_cu->type != CU_NOTSET
&& cost / (cu_width * cu_width) < INTRA_TRESHOLD; && cost / (cu_width * cu_width) < INTRA_TRESHOLD;
if (!skip_intra
&& WITHIN(depth, ctrl->cfg.pu_depth_intra.min, ctrl->cfg.pu_depth_intra.max)) int32_t cu_width_intra_min = LCU_WIDTH >> ctrl->cfg.pu_depth_intra.max;
{ bool can_use_intra =
WITHIN(depth, ctrl->cfg.pu_depth_intra.min, ctrl->cfg.pu_depth_intra.max) ||
// When the split was forced because the CTU is partially outside
// the frame, we permit intra coding even if pu_depth_intra would
// otherwise forbid it.
(x & ~(cu_width_intra_min - 1)) + cu_width_intra_min > frame->width ||
(y & ~(cu_width_intra_min - 1)) + cu_width_intra_min > frame->height;
if (can_use_intra && !skip_intra) {
int8_t intra_mode; int8_t intra_mode;
double intra_cost; double intra_cost;
kvz_search_cu_intra(state, x, y, depth, &work_tree[depth], kvz_search_cu_intra(state, x, y, depth, &work_tree[depth],
@ -684,11 +698,17 @@ static double search_cu(encoder_state_t * const state, int x, int y, int depth,
cost += mode_bits * state->lambda; cost += mode_bits * state->lambda;
} }
bool can_split_cu =
// If the CU is partially outside the frame, we need to split it even
// if pu_depth_intra and pu_depth_inter would not permit it.
cur_cu->type == CU_NOTSET ||
depth < ctrl->cfg.pu_depth_intra.max ||
(state->frame->slicetype != KVZ_SLICE_I &&
depth < ctrl->cfg.pu_depth_inter.max);
// Recursively split all the way to max search depth. // Recursively split all the way to max search depth.
if (depth < ctrl->cfg.pu_depth_intra.max || if (can_split_cu) {
(depth < ctrl->cfg.pu_depth_inter.max && state->frame->slicetype != KVZ_SLICE_I))
{
int half_cu = cu_width / 2; int half_cu = cu_width / 2;
double split_cost = 0.0; double split_cost = 0.0;
int cbf = cbf_is_set_any(cur_cu->cbf, depth); int cbf = cbf_is_set_any(cur_cu->cbf, depth);
@ -787,6 +807,8 @@ static double search_cu(encoder_state_t * const state, int x, int y, int depth,
(state->tile->lcu_offset_y * LCU_WIDTH) + y + (LCU_WIDTH >> depth), (state->tile->lcu_offset_y * LCU_WIDTH) + y + (LCU_WIDTH >> depth),
depth, debug_split, (cur_cu->type==CU_INTRA)?1:0); depth, debug_split, (cur_cu->type==CU_INTRA)?1:0);
assert(cur_cu->type != CU_NOTSET);
return cost; return cost;
} }