diff --git a/src/cu.c b/src/cu.c index a64ed4be..40fce65e 100644 --- a/src/cu.c +++ b/src/cu.c @@ -114,6 +114,7 @@ const cu_info_t* uvg_cu_array_at_const(const cu_array_t *cua, unsigned x_px, uns cu_array_t * uvg_cu_array_alloc(const int width, const int height) { cu_array_t *cua = MALLOC(cu_array_t, 1); + if (cua == NULL) return NULL; // Round up to a multiple of LCU width and divide by cell width. const int width_scu = CEILDIV(width, LCU_WIDTH) * LCU_WIDTH / SCU_WIDTH; @@ -132,6 +133,7 @@ cu_array_t * uvg_cu_array_alloc(const int width, const int height) cu_array_t * uvg_cu_array_chroma_alloc(const int width, const int height, enum uvg_chroma_format chroma) { cu_array_t *cua = MALLOC(cu_array_t, 1); + if (cua == NULL) return NULL; // Round up to a multiple of LCU width and divide by cell width. const int chroma_height = chroma == UVG_CSP_444 ? LCU_WIDTH : LCU_WIDTH_C; @@ -168,6 +170,7 @@ cu_array_t * uvg_cu_subarray(cu_array_t *base, } cu_array_t *cua = MALLOC(cu_array_t, 1); + if (cua == NULL) return NULL; // Find the real base array. cu_array_t *real_base = base; diff --git a/src/cu.h b/src/cu.h index bad9a044..74ff25a6 100644 --- a/src/cu.h +++ b/src/cu.h @@ -238,10 +238,10 @@ typedef struct { typedef struct cu_array_t { struct cu_array_t *base; //!< \brief base cu array or NULL cu_info_t *data; //!< \brief cu array - int32_t width; //!< \brief width of the array in pixels - int32_t height; //!< \brief height of the array in pixels - int32_t stride; //!< \brief stride of the array in pixels - int32_t refcount; //!< \brief number of references to this cu_array + uint32_t width; //!< \brief width of the array in pixels + uint32_t height; //!< \brief height of the array in pixels + uint32_t stride; //!< \brief stride of the array in pixels + uint32_t refcount; //!< \brief number of references to this cu_array } cu_array_t; cu_info_t* uvg_cu_array_at(cu_array_t *cua, unsigned x_px, unsigned y_px); diff --git a/src/encode_coding_tree.c b/src/encode_coding_tree.c index fa3eaa36..4404bc63 100644 --- a/src/encode_coding_tree.c +++ b/src/encode_coding_tree.c @@ -131,8 +131,8 @@ static int get_isp_split_dim(const int width, const int height, const int isp_sp non_split_dim_size = height; } - const int min_num_samples_cu = 1 << ((uvg_math_floor_log2(MIN_TB_SIZE_Y) << 1)); - const int factor_to_min_samples = non_split_dim_size < min_num_samples_cu ? min_num_samples_cu >> uvg_math_floor_log2(non_split_dim_size) : 1; + const unsigned min_num_samples_cu = 1 << ((uvg_math_floor_log2(MIN_TB_SIZE_Y) << 1)); + const unsigned factor_to_min_samples = non_split_dim_size < min_num_samples_cu ? min_num_samples_cu >> uvg_math_floor_log2(non_split_dim_size) : 1; partition_size = (split_dim_size >> div_shift) < factor_to_min_samples ? factor_to_min_samples : (split_dim_size >> div_shift); assert(!(uvg_math_floor_log2(partition_size) + uvg_math_floor_log2(non_split_dim_size) < uvg_math_floor_log2(min_num_samples_cu)) && "Partition has less than minimum amount of samples."); @@ -518,7 +518,7 @@ static void encode_chroma_tu( int y, int depth, const uint8_t width_c, - const cu_info_t* cur_pu, + cu_info_t* cur_pu, int8_t* scan_idx, lcu_coeff_t* coeff, uint8_t joint_chroma, @@ -631,7 +631,8 @@ static void encode_transform_unit( bool chroma_cbf_set = cbf_is_set(cur_pu->cbf, depth, COLOR_U) || cbf_is_set(cur_pu->cbf, depth, COLOR_V); if (chroma_cbf_set || joint_chroma) { - encode_chroma_tu(state, x, y, depth, width_c, cur_pu, &scan_idx, coeff, joint_chroma, tree_type); + //Need to drop const to get lfnst constraints + encode_chroma_tu(state, x, y, depth, width_c, (cu_info_t*)cur_pu, &scan_idx, coeff, joint_chroma, tree_type); } } diff --git a/src/search.c b/src/search.c index e47d5ad1..3e2ae87b 100644 --- a/src/search.c +++ b/src/search.c @@ -133,11 +133,11 @@ static void work_tree_copy_down(int x_local, int y_local, int depth, lcu_t *work } } -void uvg_lcu_fill_trdepth(lcu_t *lcu, int x_px, int y_px, int depth, int tr_depth) +void uvg_lcu_fill_trdepth(lcu_t *lcu, int x_px, int y_px, int depth, uint8_t tr_depth) { const int x_local = SUB_SCU(x_px); const int y_local = SUB_SCU(y_px); - const uint32_t width = LCU_WIDTH >> depth; + const unsigned width = LCU_WIDTH >> depth; for (unsigned y = 0; y < width; y += SCU_WIDTH) { for (unsigned x = 0; x < width; x += SCU_WIDTH) { @@ -146,7 +146,7 @@ void uvg_lcu_fill_trdepth(lcu_t *lcu, int x_px, int y_px, int depth, int tr_dept } } -static void lcu_fill_cu_info(lcu_t *lcu, int x_local, int y_local, int width, int height, cu_info_t *cu) +static void lcu_fill_cu_info(lcu_t *lcu, int x_local, int y_local, int width, int height, const cu_info_t *cu) { // Set mode in every CU covered by part_mode in this depth. for (int y = y_local; y < y_local + height; y += SCU_WIDTH) { @@ -195,7 +195,7 @@ static void lcu_fill_inter(lcu_t *lcu, int x_local, int y_local, int cu_width) } } -static void lcu_fill_cbf(lcu_t *lcu, uint32_t x_local, uint32_t y_local, uint32_t width, cu_info_t *cur_cu) +static void lcu_fill_cbf(lcu_t *lcu, int x_local, unsigned y_local, unsigned width, const cu_info_t *cur_cu) { const uint32_t tr_split = cur_cu->tr_depth - cur_cu->depth; const uint32_t mask = ~((width >> tr_split)-1); diff --git a/src/search.h b/src/search.h index 7703bd5b..60dea9e3 100644 --- a/src/search.h +++ b/src/search.h @@ -93,7 +93,7 @@ double uvg_cu_rd_cost_chroma(const encoder_state_t *const state, cu_info_t *const pred_cu, lcu_t *const lcu); -void uvg_lcu_fill_trdepth(lcu_t *lcu, int x_px, int y_px, int depth, int tr_depth); +void uvg_lcu_fill_trdepth(lcu_t *lcu, int x_px, int y_px, int depth, uint8_t tr_depth); void uvg_intra_recon_lcu_luma(encoder_state_t * const state, int x, int y, int depth, int8_t intra_mode, cu_info_t *cur_cu, lcu_t *lcu); void uvg_intra_recon_lcu_chroma(encoder_state_t * const state, int x, int y, int depth, int8_t intra_mode, cu_info_t *cur_cu, lcu_t *lcu); diff --git a/src/search_intra.c b/src/search_intra.c index b4c997d5..7812e2c6 100644 --- a/src/search_intra.c +++ b/src/search_intra.c @@ -1816,8 +1816,8 @@ void uvg_search_cu_intra( // Find modes with multiple reference lines if in use. Do not use if CU in first row. uint8_t lines = state->encoder_control->cfg.mrl && (y_px % LCU_WIDTH) != 0 ? MAX_REF_LINE_IDX : 1; - int16_t number_of_modes; - int16_t num_regular_modes; + uint8_t number_of_modes; + uint8_t num_regular_modes; bool skip_rough_search = (depth == 0 || state->encoder_control->cfg.rdo >= 4); if (!skip_rough_search) { num_regular_modes = number_of_modes = search_intra_rough( @@ -1843,7 +1843,7 @@ void uvg_search_cu_intra( number_of_modes = UVG_NUM_INTRA_MODES; } - int16_t num_mrl_modes = 0; + uint8_t num_mrl_modes = 0; for(int line = 1; line < lines; ++line) { uvg_pixel extra_refs[128 * MAX_REF_LINE_IDX] = { 0 }; diff --git a/src/strategies/avx2/quant-avx2.c b/src/strategies/avx2/quant-avx2.c index 9fdc7acb..4d4f15c1 100644 --- a/src/strategies/avx2/quant-avx2.c +++ b/src/strategies/avx2/quant-avx2.c @@ -390,7 +390,7 @@ void uvg_quant_avx2(const encoder_state_t * const state, const coeff_t * __restr const int32_t scalinglist_type = (block_type == CU_INTRA ? 0 : 3) + (int8_t)color; const int32_t *quant_coeff = encoder->scaling_list.quant_coeff[log2_tr_width][log2_tr_height][scalinglist_type][qp_scaled % 6]; const int32_t transform_shift = MAX_TR_DYNAMIC_RANGE - encoder->bitdepth - ((log2_tr_width + log2_tr_height) >> 1); //!< Represents scaling through forward transform - const int32_t q_bits = QUANT_SHIFT + qp_scaled / 6 + (transform_skip ? 0 : transform_shift); + const int64_t q_bits = QUANT_SHIFT + qp_scaled / 6 + (transform_skip ? 0 : transform_shift); const int32_t add = ((state->frame->slicetype == UVG_SLICE_I) ? 171 : 85) << (q_bits - 9); const int32_t q_bits8 = q_bits - 8; diff --git a/src/strategies/generic/quant-generic.c b/src/strategies/generic/quant-generic.c index fda29c71..2eb73cdb 100644 --- a/src/strategies/generic/quant-generic.c +++ b/src/strategies/generic/quant-generic.c @@ -72,7 +72,7 @@ void uvg_quant_generic( const int32_t scalinglist_type = (block_type == CU_INTRA ? 0 : 3) + (int8_t)color; const int32_t *quant_coeff = encoder->scaling_list.quant_coeff[log2_tr_width][log2_tr_height][scalinglist_type][qp_scaled % 6]; const int32_t transform_shift = MAX_TR_DYNAMIC_RANGE - encoder->bitdepth - ((log2_tr_height + log2_tr_width) >> 1); //!< Represents scaling through forward transform - const int32_t q_bits = QUANT_SHIFT + qp_scaled / 6 + (transform_skip ? 0 : transform_shift); + const int64_t q_bits = QUANT_SHIFT + qp_scaled / 6 + (transform_skip ? 0 : transform_shift); const int32_t add = ((state->frame->slicetype == UVG_SLICE_I) ? 171 : 85) << (q_bits - 9); const int32_t q_bits8 = q_bits - 8; @@ -482,7 +482,7 @@ int uvg_quantize_residual_generic(encoder_state_t *const state, uvg_transform2d(state->encoder_control, residual, coeff, width, color, cur_cu); } - const uint16_t lfnst_index = color == COLOR_Y ? cur_cu->lfnst_idx : cur_cu->cr_lfnst_idx; + const uint8_t lfnst_index = color == COLOR_Y ? cur_cu->lfnst_idx : cur_cu->cr_lfnst_idx; if (state->encoder_control->cfg.lfnst && cur_cu->type == CU_INTRA) { // Forward low frequency non-separable transform diff --git a/src/transform.c b/src/transform.c index 47fd0cd4..99aeca3e 100644 --- a/src/transform.c +++ b/src/transform.c @@ -524,9 +524,9 @@ void uvg_chroma_transform_search( trans_offset, &num_transforms); } - chorma_ts_out->best_u_cost = MAX_INT64; - chorma_ts_out->best_v_cost = MAX_INT64; - chorma_ts_out->best_combined_cost = MAX_INT64; + chorma_ts_out->best_u_cost = MAX_DOUBLE; + chorma_ts_out->best_v_cost = MAX_DOUBLE; + chorma_ts_out->best_combined_cost = MAX_DOUBLE; chorma_ts_out->best_u_index = -1; chorma_ts_out->best_v_index = -1; chorma_ts_out->best_combined_index = -1;