diff --git a/src/inter.c b/src/inter.c
index ea21b390..3b435da6 100644
--- a/src/inter.c
+++ b/src/inter.c
@@ -514,7 +514,7 @@ void inter_get_mv_cand(const encoder_state_t * const state, int32_t x, int32_t y
  * \param depth current block depth
  * \param mv_pred[MRG_MAX_NUM_CANDS][2] MRG_MAX_NUM_CANDS motion vector prediction
  */
-uint8_t inter_get_merge_cand(int32_t x, int32_t y, int8_t depth, int16_t mv_cand[MRG_MAX_NUM_CANDS][3], lcu_t *lcu)
+uint8_t inter_get_merge_cand(int32_t x, int32_t y, int8_t depth, inter_merge_cand_t mv_cand[MRG_MAX_NUM_CANDS], lcu_t *lcu, bool inter_b)
 {
   uint8_t candidates = 0;
   int8_t duplicate = 0;
@@ -530,19 +530,21 @@ uint8_t inter_get_merge_cand(int32_t x, int32_t y, int8_t depth, int16_t mv_cand
                                                      (CU1)->inter.mv[1] == (CU2)->inter.mv[1] && \
                                                      (CU1)->inter.mv_ref == (CU2)->inter.mv_ref) duplicate = 1; }
 
-  if (a1 && a1->type == CU_INTER) {
-      mv_cand[candidates][0] = a1->inter.mv[0];
-      mv_cand[candidates][1] = a1->inter.mv[1];
-      mv_cand[candidates][2] = a1->inter.mv_ref;
-      candidates++;
+  if (a1 && a1->type == CU_INTER) {      
+    mv_cand[candidates].mv[a1->inter.mv_dir - 1][0] = a1->inter.mv[0];
+    mv_cand[candidates].mv[a1->inter.mv_dir - 1][1] = a1->inter.mv[1];
+    mv_cand[candidates].ref = a1->inter.mv_ref;
+    mv_cand[candidates].dir = a1->inter.mv_dir;
+    candidates++;
   }
 
   if (b1 && b1->type == CU_INTER) {
     if(candidates) CHECK_DUPLICATE(b1, a1);
     if(!duplicate) {
-      mv_cand[candidates][0] = b1->inter.mv[0];
-      mv_cand[candidates][1] = b1->inter.mv[1];
-      mv_cand[candidates][2] = b1->inter.mv_ref;
+      mv_cand[candidates].mv[b1->inter.mv_dir - 1][0] = b1->inter.mv[0];
+      mv_cand[candidates].mv[b1->inter.mv_dir - 1][1] = b1->inter.mv[1];
+      mv_cand[candidates].ref = b1->inter.mv_ref;
+      mv_cand[candidates].dir = b1->inter.mv_dir;
       candidates++;
     }
   }
@@ -550,9 +552,10 @@ uint8_t inter_get_merge_cand(int32_t x, int32_t y, int8_t depth, int16_t mv_cand
   if (b0 && b0->type == CU_INTER) {
     if(candidates) CHECK_DUPLICATE(b0,b1);
     if(!duplicate) {
-      mv_cand[candidates][0] = b0->inter.mv[0];
-      mv_cand[candidates][1] = b0->inter.mv[1];
-      mv_cand[candidates][2] = b0->inter.mv_ref;
+      mv_cand[candidates].mv[b0->inter.mv_dir - 1][0] = b0->inter.mv[0];
+      mv_cand[candidates].mv[b0->inter.mv_dir - 1][1] = b0->inter.mv[1];
+      mv_cand[candidates].ref = b0->inter.mv_ref;
+      mv_cand[candidates].dir = b0->inter.mv_dir;
       candidates++;
     }
   }
@@ -560,9 +563,10 @@ uint8_t inter_get_merge_cand(int32_t x, int32_t y, int8_t depth, int16_t mv_cand
   if (a0 && a0->type == CU_INTER) {
     if(candidates) CHECK_DUPLICATE(a0,a1);
     if(!duplicate) {
-      mv_cand[candidates][0] = a0->inter.mv[0];
-      mv_cand[candidates][1] = a0->inter.mv[1];
-      mv_cand[candidates][2] = a0->inter.mv_ref;
+      mv_cand[candidates].mv[a0->inter.mv_dir - 1][0] = a0->inter.mv[0];
+      mv_cand[candidates].mv[a0->inter.mv_dir - 1][1] = a0->inter.mv[1];
+      mv_cand[candidates].ref = a0->inter.mv_ref;
+      mv_cand[candidates].dir = a0->inter.mv_dir;
       candidates++;
     }
   }
@@ -573,9 +577,10 @@ uint8_t inter_get_merge_cand(int32_t x, int32_t y, int8_t depth, int16_t mv_cand
       if(!duplicate) {
         CHECK_DUPLICATE(b2,b1);
         if(!duplicate) {
-          mv_cand[candidates][0] = b2->inter.mv[0];
-          mv_cand[candidates][1] = b2->inter.mv[1];
-          mv_cand[candidates][2] = b2->inter.mv_ref;
+          mv_cand[candidates].mv[b2->inter.mv_dir - 1][0] = b2->inter.mv[0];
+          mv_cand[candidates].mv[b2->inter.mv_dir - 1][1] = b2->inter.mv[1];
+          mv_cand[candidates].ref = b2->inter.mv_ref;
+          mv_cand[candidates].dir = b2->inter.mv_dir;
           candidates++;
         }
       }
@@ -589,11 +594,49 @@ uint8_t inter_get_merge_cand(int32_t x, int32_t y, int8_t depth, int16_t mv_cand
   }
 #endif
 
+  if (candidates == MRG_MAX_NUM_CANDS) return MRG_MAX_NUM_CANDS;
+
+  if (inter_b) {
+    #define NUM_PRIORITY_LIST 12;
+    static const uint8_t priorityList0[] = { 0, 1, 0, 2, 1, 2, 0, 3, 1, 3, 2, 3 };
+    static const uint8_t priorityList1[] = { 1, 0, 2, 0, 2, 1, 3, 0, 3, 1, 3, 2 };
+    uint8_t cutoff = candidates;
+    for (int32_t idx = 0; idx<cutoff*(cutoff - 1) && candidates != MRG_MAX_NUM_CANDS; idx++) {
+      uint8_t i = priorityList0[idx];
+      uint8_t j = priorityList1[idx];
+
+      // Find one L0 and L1 candidate according to the priority list
+      if ((mv_cand[i].dir & 0x1) && (mv_cand[j].dir & 0x2)) {
+        mv_cand[candidates].dir = 3;
+
+        // get Mv from cand[i] and cand[j]
+        mv_cand[candidates].mv[0][0] = mv_cand[i].mv[0][0];
+        mv_cand[candidates].mv[0][1] = mv_cand[i].mv[0][1];
+        mv_cand[candidates].mv[1][0] = mv_cand[j].mv[1][0];
+        mv_cand[candidates].mv[1][1] = mv_cand[j].mv[1][1];
+
+        if (mv_cand[i].ref == mv_cand[j].ref &&
+          mv_cand[i].mv[0][0] == mv_cand[j].mv[1][0] && 
+          mv_cand[i].mv[0][1] == mv_cand[j].mv[1][1]) {
+          // Not a candidate
+        } else {
+          candidates++;
+        }
+      }
+    }
+  }
+
   // Add (0,0) prediction
-  if (candidates != 5) {
-    mv_cand[candidates][0] = 0;
-    mv_cand[candidates][1] = 0;
-    mv_cand[candidates][2] = zero_idx;
+  if (candidates != MRG_MAX_NUM_CANDS) {
+    mv_cand[candidates].mv[0][0] = 0;
+    mv_cand[candidates].mv[0][1] = 0;
+    mv_cand[candidates].ref = zero_idx;
+    mv_cand[candidates].dir = 1;
+    if (inter_b) {
+      mv_cand[candidates].mv[1][0] = 0;
+      mv_cand[candidates].mv[1][1] = 0;
+      mv_cand[candidates].dir = 3;
+    }
     zero_idx++;
     candidates++;
   }
diff --git a/src/inter.h b/src/inter.h
index 80e7e208..f60a80ec 100644
--- a/src/inter.h
+++ b/src/inter.h
@@ -31,6 +31,13 @@
 #include "encoder.h"
 #include "encoderstate.h"
 
+typedef struct {
+  uint8_t dir;
+  uint8_t ref;
+  int16_t mv[2][2];
+
+} inter_merge_cand_t;
+
 
 //void inter_set_block(image* im,uint32_t x_cu, uint32_t y_cu, uint8_t depth, cu_info *cur_cu);
 void inter_recon_lcu(const encoder_state_t * const state, const image_t * ref, int32_t xpos, int32_t ypos, int32_t width, const int16_t mv_param[2], lcu_t* lcu);
@@ -38,5 +45,5 @@ void inter_recon_lcu(const encoder_state_t * const state, const image_t * ref, i
 void inter_get_spatial_merge_candidates(int32_t x, int32_t y, int8_t depth, cu_info_t **b0, cu_info_t **b1,
                                         cu_info_t **b2, cu_info_t **a0, cu_info_t **a1, lcu_t *lcu);
 void inter_get_mv_cand(const encoder_state_t *state, int32_t x, int32_t y, int8_t depth, int16_t mv_cand[2][2], cu_info_t* cur_cu, lcu_t *lcu);
-uint8_t inter_get_merge_cand(int32_t x, int32_t y, int8_t depth, int16_t mv_cand[MRG_MAX_NUM_CANDS][3], lcu_t *lcu);
+uint8_t inter_get_merge_cand(int32_t x, int32_t y, int8_t depth, inter_merge_cand_t mv_cand[MRG_MAX_NUM_CANDS], lcu_t *lcu, bool inter_b);
 #endif
diff --git a/src/search.c b/src/search.c
index 277ddae2..cb62f587 100644
--- a/src/search.c
+++ b/src/search.c
@@ -162,7 +162,7 @@ static uint32_t get_mvd_coding_cost(vector2d_t *mvd)
 }
 
 static int calc_mvd_cost(const encoder_state_t * const state, int x, int y, int mv_shift,
-                         int16_t mv_cand[2][2], int16_t merge_cand[MRG_MAX_NUM_CANDS][3],
+                         int16_t mv_cand[2][2], inter_merge_cand_t merge_cand[MRG_MAX_NUM_CANDS],
                          int16_t num_cand,int32_t ref_idx, uint32_t *bitcost)
 {
   uint32_t temp_bitcost = 0;
@@ -177,9 +177,9 @@ static int calc_mvd_cost(const encoder_state_t * const state, int x, int y, int
 
   // Check every candidate to find a match
   for(merge_idx = 0; merge_idx < (uint32_t)num_cand; merge_idx++) {
-    if (merge_cand[merge_idx][0] == x &&
-        merge_cand[merge_idx][1] == y &&
-        merge_cand[merge_idx][2] == ref_idx) {
+    if (merge_cand[merge_idx].mv[merge_cand[merge_idx].dir-1][0] == x &&
+        merge_cand[merge_idx].mv[merge_cand[merge_idx].dir - 1][1] == y &&
+        merge_cand[merge_idx].ref == ref_idx) {
       temp_bitcost += merge_idx;
       merged = 1;
       break;
@@ -230,7 +230,7 @@ static int calc_mvd_cost(const encoder_state_t * const state, int x, int y, int
 static unsigned hexagon_search(const encoder_state_t * const state, unsigned depth,
                                const image_t *pic, const image_t *ref,
                                const vector2d_t *orig, vector2d_t *mv_in_out,
-                               int16_t mv_cand[2][2], int16_t merge_cand[MRG_MAX_NUM_CANDS][3],
+                               int16_t mv_cand[2][2], inter_merge_cand_t merge_cand[MRG_MAX_NUM_CANDS],
                                int16_t num_cand, int32_t ref_idx, uint32_t *bitcost_out)
 {
   vector2d_t mv = { mv_in_out->x >> 2, mv_in_out->y >> 2 };
@@ -267,8 +267,8 @@ static unsigned hexagon_search(const encoder_state_t * const state, unsigned dep
   // Select starting point from among merge candidates. These should include
   // both mv_cand vectors and (0, 0).
   for (i = 0; i < num_cand; ++i) {
-    mv.x = merge_cand[i][0] >> 2;
-    mv.y = merge_cand[i][1] >> 2;
+    mv.x = merge_cand[i].mv[merge_cand[i].dir - 1][0] >> 2;
+    mv.y = merge_cand[i].mv[merge_cand[i].dir - 1][1] >> 2;
 
     PERFORMANCE_MEASURE_START(_DEBUG_PERF_SEARCH_PIXELS);
 
@@ -291,8 +291,8 @@ static unsigned hexagon_search(const encoder_state_t * const state, unsigned dep
     }
   }
   if (best_index < num_cand) {
-    mv.x = merge_cand[best_index][0] >> 2;
-    mv.y = merge_cand[best_index][1] >> 2;
+    mv.x = merge_cand[best_index].mv[merge_cand[best_index].dir - 1][0] >> 2;
+    mv.y = merge_cand[best_index].mv[merge_cand[best_index].dir - 1][1] >> 2;
   } else {
     mv.x = mv_in_out->x >> 2;
     mv.y = mv_in_out->y >> 2;
@@ -483,7 +483,7 @@ static unsigned search_frac(const encoder_state_t * const state,
                             unsigned depth,
                             const image_t *pic, const image_t *ref,
                             const vector2d_t *orig, vector2d_t *mv_in_out,
-                            int16_t mv_cand[2][2], int16_t merge_cand[MRG_MAX_NUM_CANDS][3],
+                            int16_t mv_cand[2][2], inter_merge_cand_t merge_cand[MRG_MAX_NUM_CANDS],
                             int16_t num_cand, int32_t ref_idx, uint32_t *bitcost_out)
 {
 
@@ -626,9 +626,9 @@ static int search_cu_inter(const encoder_state_t * const state, int x, int y, in
 
   int16_t mv_cand[2][2];
   // Search for merge mode candidate
-  int16_t merge_cand[MRG_MAX_NUM_CANDS][3];
+  inter_merge_cand_t merge_cand[MRG_MAX_NUM_CANDS];
   // Get list of candidates
-  int16_t num_cand = inter_get_merge_cand(x, y, depth, merge_cand, lcu);
+  int16_t num_cand = inter_get_merge_cand(x, y, depth, merge_cand, lcu, (state->global->pictype == SLICE_B));
 
   // Select better candidate
   cur_cu->inter.mv_cand = 0; // Default to candidate 0
@@ -670,9 +670,10 @@ static int search_cu_inter(const encoder_state_t * const state, int x, int y, in
     merged = 0;
     // Check every candidate to find a match
     for(merge_idx = 0; merge_idx < num_cand; merge_idx++) {
-      if (merge_cand[merge_idx][0] == mv.x &&
-          merge_cand[merge_idx][1] == mv.y &&
-          (uint32_t)merge_cand[merge_idx][2] == ref_idx) {
+      if (merge_cand[merge_idx].mv[merge_cand[merge_idx].dir - 1][0] == mv.x &&
+          merge_cand[merge_idx].mv[merge_cand[merge_idx].dir - 1][1] == mv.y &&
+          merge_cand[merge_idx].dir != 3 &&
+          (uint32_t)merge_cand[merge_idx].ref == ref_idx) {
         merged = 1;
         break;
       }
@@ -705,13 +706,15 @@ static int search_cu_inter(const encoder_state_t * const state, int x, int y, in
         if (state->global->ref->images[j]->poc < state->global->poc) {
           if (ref_idx == j) {
             cur_cu->inter.mv_dir = 1;            
-            cur_cu->inter.mv_ref_coded = ref_list[0];            
+            cur_cu->inter.mv_ref_coded = ref_list[0]; 
+            break;
           }
           ref_list[0]++;
         } else {          
           if (ref_idx == j) {
             cur_cu->inter.mv_dir = 2;
             cur_cu->inter.mv_ref_coded = ref_list[1];
+            break;
           }
           ref_list[1]++;
         }
@@ -719,13 +722,12 @@ static int search_cu_inter(const encoder_state_t * const state, int x, int y, in
       cur_cu->merged        = merged;
       cur_cu->merge_idx     = merge_idx;
       cur_cu->inter.mv_ref  = ref_idx;
-      //cur_cu->inter.mv_dir  = 1;
       cur_cu->inter.mv[0]   = (int16_t)mv.x;
       cur_cu->inter.mv[1]   = (int16_t)mv.y;
       cur_cu->inter.mvd[0]  = (int16_t)mvd.x;
       cur_cu->inter.mvd[1]  = (int16_t)mvd.y;
       cur_cu->inter.cost    = temp_cost;
-      cur_cu->inter.bitcost = temp_bitcost + ref_idx;
+      cur_cu->inter.bitcost = temp_bitcost + cur_cu->inter.mv_dir - 1 + cur_cu->inter.mv_ref_coded;
       cur_cu->inter.mv_cand = cu_mv_cand;
     }
   }