[cleanup] fix warnings

src/cu.c

@@ -114,6 +114,7 @@ const cu_info_t* uvg_cu_array_at_const(const cu_array_t *cua, unsigned x_px, uns
 cu_array_t * uvg_cu_array_alloc(const int width, const int height)
 {
   cu_array_t *cua = MALLOC(cu_array_t, 1);
+  if (cua == NULL) return NULL;
 
   // Round up to a multiple of LCU width and divide by cell width.
   const int width_scu  = CEILDIV(width,  LCU_WIDTH) * LCU_WIDTH / SCU_WIDTH;
@@ -132,6 +133,7 @@ cu_array_t * uvg_cu_array_alloc(const int width, const int height)
 cu_array_t * uvg_cu_array_chroma_alloc(const int width, const int height, enum uvg_chroma_format chroma)
 {
   cu_array_t *cua = MALLOC(cu_array_t, 1);
+  if (cua == NULL) return NULL;
 
   // Round up to a multiple of LCU width and divide by cell width.
   const int chroma_height = chroma == UVG_CSP_444 ? LCU_WIDTH : LCU_WIDTH_C;
@@ -168,6 +170,7 @@ cu_array_t * uvg_cu_subarray(cu_array_t *base,
   }
 
   cu_array_t *cua = MALLOC(cu_array_t, 1);
+  if (cua == NULL) return NULL;
 
   // Find the real base array.
   cu_array_t *real_base = base;

src/cu.h

@@ -238,10 +238,10 @@ typedef struct {
 typedef struct cu_array_t {
   struct cu_array_t *base; //!< \brief base cu array or NULL
   cu_info_t *data;  //!< \brief cu array
-  int32_t width;    //!< \brief width of the array in pixels
+  uint32_t width;    //!< \brief width of the array in pixels
-  int32_t height;   //!< \brief height of the array in pixels
+  uint32_t height;   //!< \brief height of the array in pixels
-  int32_t stride;   //!< \brief stride of the array in pixels
+  uint32_t stride;   //!< \brief stride of the array in pixels
-  int32_t refcount; //!< \brief number of references to this cu_array
+  uint32_t refcount; //!< \brief number of references to this cu_array
 } cu_array_t;
 
 cu_info_t* uvg_cu_array_at(cu_array_t *cua, unsigned x_px, unsigned y_px);

src/encode_coding_tree.c

@@ -131,8 +131,8 @@ static int get_isp_split_dim(const int width, const int height, const int isp_sp
     non_split_dim_size = height;
   }
   
-  const int min_num_samples_cu = 1 << ((uvg_math_floor_log2(MIN_TB_SIZE_Y) << 1));
+  const unsigned min_num_samples_cu = 1 << ((uvg_math_floor_log2(MIN_TB_SIZE_Y) << 1));
-  const int factor_to_min_samples = non_split_dim_size < min_num_samples_cu ? min_num_samples_cu >> uvg_math_floor_log2(non_split_dim_size) : 1;
+  const unsigned factor_to_min_samples = non_split_dim_size < min_num_samples_cu ? min_num_samples_cu >> uvg_math_floor_log2(non_split_dim_size) : 1;
   partition_size = (split_dim_size >> div_shift) < factor_to_min_samples ? factor_to_min_samples : (split_dim_size >> div_shift);
 
   assert(!(uvg_math_floor_log2(partition_size) + uvg_math_floor_log2(non_split_dim_size) < uvg_math_floor_log2(min_num_samples_cu)) && "Partition has less than minimum amount of samples.");
@@ -518,7 +518,7 @@ static void encode_chroma_tu(
   int y,
   int depth,
   const uint8_t width_c,
-  const cu_info_t* cur_pu,
+  cu_info_t* cur_pu,
   int8_t* scan_idx,
   lcu_coeff_t* coeff,
   uint8_t joint_chroma,
@@ -631,7 +631,8 @@ static void encode_transform_unit(
   bool chroma_cbf_set = cbf_is_set(cur_pu->cbf, depth, COLOR_U) ||
                         cbf_is_set(cur_pu->cbf, depth, COLOR_V);
   if (chroma_cbf_set || joint_chroma) {
-    encode_chroma_tu(state, x, y, depth, width_c, cur_pu, &scan_idx, coeff, joint_chroma, tree_type);
+    //Need to drop const to get lfnst constraints
+    encode_chroma_tu(state, x, y, depth, width_c, (cu_info_t*)cur_pu, &scan_idx, coeff, joint_chroma, tree_type);
   }
 }
 

src/search.c

@@ -133,11 +133,11 @@ static void work_tree_copy_down(int x_local, int y_local, int depth, lcu_t *work
   }
 }
 
-void uvg_lcu_fill_trdepth(lcu_t *lcu, int x_px, int y_px, int depth, int tr_depth)
+void uvg_lcu_fill_trdepth(lcu_t *lcu, int x_px, int y_px, int depth, uint8_t tr_depth)
 {
   const int x_local = SUB_SCU(x_px);
   const int y_local = SUB_SCU(y_px);
-  const uint32_t width = LCU_WIDTH >> depth;
+  const unsigned width = LCU_WIDTH >> depth;
 
   for (unsigned y = 0; y < width; y += SCU_WIDTH) {
     for (unsigned x = 0; x < width; x += SCU_WIDTH) {
@@ -146,7 +146,7 @@ void uvg_lcu_fill_trdepth(lcu_t *lcu, int x_px, int y_px, int depth, int tr_dept
   }
 }
 
-static void lcu_fill_cu_info(lcu_t *lcu, int x_local, int y_local, int width, int height, cu_info_t *cu)
+static void lcu_fill_cu_info(lcu_t *lcu, int x_local, int y_local, int width, int height, const cu_info_t *cu)
 {
   // Set mode in every CU covered by part_mode in this depth.
   for (int y = y_local; y < y_local + height; y += SCU_WIDTH) {
@@ -195,7 +195,7 @@ static void lcu_fill_inter(lcu_t *lcu, int x_local, int y_local, int cu_width)
   }
 }
 
-static void lcu_fill_cbf(lcu_t *lcu, uint32_t x_local, uint32_t y_local, uint32_t width, cu_info_t *cur_cu)
+static void lcu_fill_cbf(lcu_t *lcu, int x_local, unsigned y_local, unsigned width, const cu_info_t *cur_cu)
 {
   const uint32_t tr_split = cur_cu->tr_depth - cur_cu->depth;
   const uint32_t mask = ~((width >> tr_split)-1);

src/search.h

@@ -93,7 +93,7 @@ double uvg_cu_rd_cost_chroma(const encoder_state_t *const state,
                              cu_info_t *const pred_cu,
                              lcu_t *const lcu);
 
-void uvg_lcu_fill_trdepth(lcu_t *lcu, int x_px, int y_px, int depth, int tr_depth);
+void uvg_lcu_fill_trdepth(lcu_t *lcu, int x_px, int y_px, int depth, uint8_t tr_depth);
 
 void uvg_intra_recon_lcu_luma(encoder_state_t * const state, int x, int y, int depth, int8_t intra_mode, cu_info_t *cur_cu, lcu_t *lcu);
 void uvg_intra_recon_lcu_chroma(encoder_state_t * const state, int x, int y, int depth, int8_t intra_mode, cu_info_t *cur_cu, lcu_t *lcu);

src/search_intra.c

@@ -1816,8 +1816,8 @@ void uvg_search_cu_intra(
   // Find modes with multiple reference lines if in use. Do not use if CU in first row.
   uint8_t lines = state->encoder_control->cfg.mrl && (y_px % LCU_WIDTH) != 0 ? MAX_REF_LINE_IDX : 1;
 
-  int16_t number_of_modes;
+  uint8_t number_of_modes;
-  int16_t num_regular_modes;
+  uint8_t num_regular_modes;
   bool skip_rough_search = (depth == 0 || state->encoder_control->cfg.rdo >= 4);
   if (!skip_rough_search) {
     num_regular_modes = number_of_modes = search_intra_rough(
@@ -1843,7 +1843,7 @@ void uvg_search_cu_intra(
     number_of_modes = UVG_NUM_INTRA_MODES;
   }
 
-  int16_t num_mrl_modes = 0;
+  uint8_t num_mrl_modes = 0;
   for(int line = 1; line < lines; ++line) {
     uvg_pixel extra_refs[128 * MAX_REF_LINE_IDX] = { 0 };
 

src/strategies/avx2/quant-avx2.c

@@ -390,7 +390,7 @@ void uvg_quant_avx2(const encoder_state_t * const state, const coeff_t * __restr
   const int32_t scalinglist_type = (block_type == CU_INTRA ? 0 : 3) + (int8_t)color;
   const int32_t *quant_coeff = encoder->scaling_list.quant_coeff[log2_tr_width][log2_tr_height][scalinglist_type][qp_scaled % 6];
   const int32_t transform_shift = MAX_TR_DYNAMIC_RANGE - encoder->bitdepth - ((log2_tr_width + log2_tr_height) >> 1); //!< Represents scaling through forward transform
-  const int32_t q_bits = QUANT_SHIFT + qp_scaled / 6 + (transform_skip ? 0 : transform_shift);
+  const int64_t q_bits = QUANT_SHIFT + qp_scaled / 6 + (transform_skip ? 0 : transform_shift);
   const int32_t add = ((state->frame->slicetype == UVG_SLICE_I) ? 171 : 85) << (q_bits - 9);
   const int32_t q_bits8 = q_bits - 8;
 

src/strategies/generic/quant-generic.c

@@ -72,7 +72,7 @@ void uvg_quant_generic(
   const int32_t scalinglist_type = (block_type == CU_INTRA ? 0 : 3) + (int8_t)color;
   const int32_t *quant_coeff = encoder->scaling_list.quant_coeff[log2_tr_width][log2_tr_height][scalinglist_type][qp_scaled % 6];
   const int32_t transform_shift = MAX_TR_DYNAMIC_RANGE - encoder->bitdepth - ((log2_tr_height + log2_tr_width) >> 1); //!< Represents scaling through forward transform
-  const int32_t q_bits = QUANT_SHIFT + qp_scaled / 6 + (transform_skip ? 0 : transform_shift);
+  const int64_t q_bits = QUANT_SHIFT + qp_scaled / 6 + (transform_skip ? 0 : transform_shift);
   const int32_t add = ((state->frame->slicetype == UVG_SLICE_I) ? 171 : 85) << (q_bits - 9);
   const int32_t q_bits8 = q_bits - 8;
 
@@ -482,7 +482,7 @@ int uvg_quantize_residual_generic(encoder_state_t *const state,
     uvg_transform2d(state->encoder_control, residual, coeff, width, color, cur_cu);
   }
 
-  const uint16_t lfnst_index = color == COLOR_Y ? cur_cu->lfnst_idx : cur_cu->cr_lfnst_idx;
+  const uint8_t lfnst_index = color == COLOR_Y ? cur_cu->lfnst_idx : cur_cu->cr_lfnst_idx;
 
   if (state->encoder_control->cfg.lfnst && cur_cu->type == CU_INTRA) {
     // Forward low frequency non-separable transform

src/transform.c

@@ -524,9 +524,9 @@ void uvg_chroma_transform_search(
       trans_offset,
       &num_transforms);
   }
-  chorma_ts_out->best_u_cost = MAX_INT64;
+  chorma_ts_out->best_u_cost = MAX_DOUBLE;
-  chorma_ts_out->best_v_cost = MAX_INT64;
+  chorma_ts_out->best_v_cost = MAX_DOUBLE;
-  chorma_ts_out->best_combined_cost = MAX_INT64;
+  chorma_ts_out->best_combined_cost = MAX_DOUBLE;
   chorma_ts_out->best_u_index = -1;
   chorma_ts_out->best_v_index = -1;
   chorma_ts_out->best_combined_index = -1;