Implemented proper intra mode cost calculation

This does not currently affect the search primary search defining the used block sizes, only the refining second intra search. Gain 1.9% BD-rate on All Intra 600f of BQMall QP 22,27,32,37.

src/encoder.c

@@ -2623,6 +2623,12 @@ void encode_block_residual(encoder_control *encoder,
     static vector2d offsets[4] = {{0,0},{1,0},{0,1},{1,1}};
     int num_pu = (cur_cu->part_size == SIZE_2Nx2N ? 1 : 4);
     int i;
+    int8_t merge[3];
+
+    intra_get_dir_luma_predictor(encoder->in.cur_pic,
+                                   x_ctb * 2,
+                                   y_ctb * 2,
+                                   merge);
 
     if (cur_cu->part_size == SIZE_NxN) {
       width = width_c;
@@ -2645,7 +2651,7 @@ void encode_block_residual(encoder_control *encoder,
                                                   x_ctb * (LCU_WIDTH >> (MAX_DEPTH)),
                                                   y_ctb * (LCU_WIDTH >> (MAX_DEPTH)),
                                                   width, pred_y, width,
-                                                  &cur_cu->intra[0].cost);
+                                                  &cur_cu->intra[0].cost,merge);
     intra_set_block_mode(encoder->in.cur_pic, x_ctb, y_ctb, depth,
                          cur_cu->intra[0].mode, cur_cu->part_size);
 

src/intra.c

@@ -233,6 +233,25 @@ void intra_filter(pixel *ref, int32_t stride,int32_t width, int8_t mode)
   #undef FWIDTH
 }
 
+/** 
+ * \brief Helper function to find intra merge costs
+ * \returns intra mode coding cost in bits
+ */
+static uint32_t intra_merge_cost(int8_t mode, int8_t *merge_modes)
+{
+   // merge mode -1 means they are not used -> cost 0
+   if(merge_modes[0] == -1) return 0;
+
+   // First candidate needs only one bit and two other need two
+   if(merge_modes[0] == mode) {
+     return 1; 
+   } else if(merge_modes[1] == mode || merge_modes[2] == mode) {
+     return 2;
+   }
+   // Without merging the cost is 5 bits
+   return 5;
+}
+
 /**
  * \brief Function to test best intra prediction mode
  * \param orig original picture data
@@ -250,13 +269,14 @@ void intra_filter(pixel *ref, int32_t stride,int32_t width, int8_t mode)
  This function derives the prediction samples for planar mode (intra coding).
 */
 int16_t intra_prediction(pixel *orig, int32_t origstride, pixel *rec, int16_t recstride, uint16_t xpos,
-                         uint16_t ypos, uint8_t width, pixel *dst, int32_t dststride, uint32_t *sad_out)
+                         uint16_t ypos, uint8_t width, pixel *dst, int32_t dststride, uint32_t *sad_out, int8_t *merge)
 {
   uint32_t best_sad = 0xffffffff;
   uint32_t sad = 0;
   int16_t best_mode = 1;
   int32_t x,y;
   int16_t i;
+  uint32_t ratecost = 0;
 
   cost_16bit_nxn_func cost_func = get_sad_16bit_nxn_func(width);
 
@@ -306,7 +326,8 @@ int16_t intra_prediction(pixel *orig, int32_t origstride, pixel *rec, int16_t re
     for (i = 0; i < (int32_t)(width*width); i++) {
       pred[i] = val;
     }
-    CHECK_FOR_BEST(1,0);
+    ratecost = intra_merge_cost(1,merge)*(int)(g_cur_lambda_cost+0.5);
+    CHECK_FOR_BEST(1,ratecost);
   }
 
   // Check angular not requiring filtering
@@ -314,7 +335,8 @@ int16_t intra_prediction(pixel *orig, int32_t origstride, pixel *rec, int16_t re
     int distance = MIN(abs(i - 26),abs(i - 10)); //!< Distance from top and left predictions
     if(distance <= threshold) {
       intra_get_angular_pred(rec, recstride, pred, width, width, i, filter);
-      CHECK_FOR_BEST(i,0);
+      ratecost = intra_merge_cost(i,merge)*(int)(g_cur_lambda_cost+0.5);
+      CHECK_FOR_BEST(i,ratecost);
     }
   }
 
@@ -322,7 +344,8 @@ int16_t intra_prediction(pixel *orig, int32_t origstride, pixel *rec, int16_t re
 
   // Test planar mode (always filtered)
   intra_get_planar_pred(rec_filtered, recstride, width, pred, width);
-  CHECK_FOR_BEST(0,0);
+  ratecost = intra_merge_cost(0,merge)*(int)(g_cur_lambda_cost+0.5);
+  CHECK_FOR_BEST(0,ratecost);
 
   // Check angular predictions which require filtered samples
   // TODO: add conditions to skip some modes on borders
@@ -331,7 +354,8 @@ int16_t intra_prediction(pixel *orig, int32_t origstride, pixel *rec, int16_t re
     int distance = MIN(abs(i-26),abs(i-10)); //!< Distance from top and left predictions
     if(distance > threshold) {
       intra_get_angular_pred(rec_filtered, recstride, pred, width, width, i, filter);
-      CHECK_FOR_BEST(i,0);
+      ratecost = intra_merge_cost(i,merge)*(int)(g_cur_lambda_cost+0.5);
+      CHECK_FOR_BEST(i,ratecost);
     }
   }
 

src/intra.h

@@ -38,7 +38,7 @@ void intra_build_reference_border(picture* pic, const pixel *src, int32_t x_ctb,
 void intra_filter(pixel* ref, int32_t stride, int32_t width, int8_t mode);
 
 /* Predictions */
-int16_t intra_prediction(pixel* orig, int32_t orig_stride, pixel* rec, int16_t rec_stride,  uint16_t x_pos, uint16_t ypos, uint8_t width, pixel* dst, int32_t dst_stride, uint32_t *sad);
+int16_t intra_prediction(pixel* orig, int32_t orig_stride, pixel* rec, int16_t rec_stride,  uint16_t x_pos, uint16_t ypos, uint8_t width, pixel* dst, int32_t dst_stride, uint32_t *sad, int8_t *merge);
 
 pixel intra_get_dc_pred(pixel* pic, uint16_t pic_width, uint8_t width);
 void intra_get_planar_pred(pixel* src,int32_t srcstride, uint32_t width, pixel* dst, int32_t dststride);

src/search.c

@@ -664,13 +664,15 @@ static void search_intra(encoder_control *encoder, uint16_t x_ctb,
   pixel rec[(LCU_WIDTH * 2 + 1) * (LCU_WIDTH * 2 + 1)];
   pixel *recShift = &rec[(LCU_WIDTH >> (depth)) * 2 + 8 + 1];
 
+  int8_t merge[3] = {-1,-1,-1};
+
   // Build reconstructed block to use in prediction with extrapolated borders
   intra_build_reference_border(cur_pic, cur_pic->y_data,
                                x, y,
                                (int16_t)width * 2 + 8, rec, (int16_t)width * 2 + 8, 0);
   cur_cu->intra[0].mode = (int8_t)intra_prediction(encoder->in.cur_pic->y_data,
       encoder->in.width, recShift, width * 2 + 8, x, y,
-      width, pred, width, &cur_cu->intra[0].cost);
+      width, pred, width, &cur_cu->intra[0].cost,merge);
   cur_cu->part_size = SIZE_2Nx2N;
 
   // Do search for NxN split.
@@ -692,7 +694,7 @@ static void search_intra(encoder_control *encoder, uint16_t x_ctb,
                                    (int16_t)width * 2 + 8, rec, (int16_t)width * 2 + 8, 0);
       cur_cu->intra[i].mode = (int8_t)intra_prediction(encoder->in.cur_pic->y_data,
           encoder->in.width, recShift, width * 2 + 8, (int16_t)x_pos, (int16_t)y_pos,
-          width, pred, width, &cur_cu->intra[i].cost);
+          width, pred, width, &cur_cu->intra[i].cost,merge);
       cost += cur_cu->intra[i].cost;
     }