diff --git a/src/encoderstate.c b/src/encoderstate.c
index 4f9df5d0..df5add86 100644
--- a/src/encoderstate.c
+++ b/src/encoderstate.c
@@ -949,6 +949,59 @@ void encoder_compute_stats(encoder_state_t *state, FILE * const recout, uint32_t
   *bitstream_length += state->stats_bitstream_length;
 }
 
+static void encoder_ref_insertion_sort(int reflist[16], int length) {
+
+  for (uint8_t i = 1; i < length; ++i) {
+    const int16_t cur_poc = reflist[i];
+    int16_t j = i;
+    while (j > 0 && cur_poc < reflist[j - 1]) {
+      reflist[j] = reflist[j - 1];
+      --j;
+    }
+    reflist[j] = cur_poc;
+  }
+}
+static void encoder_state_ref_sort(encoder_state_t *state) {
+  int j, ref_list[2] = { 0, 0 },ref_list_poc[2][16];
+
+  // List all pocs of lists
+  for (j = 0; j < state->global->ref->used_size; j++) {
+    if (state->global->ref->images[j]->poc < state->global->poc) {
+      ref_list_poc[0][ref_list[0]] = state->global->ref->images[j]->poc;
+      ref_list[0]++;
+    } else {
+      ref_list_poc[1][ref_list[1]] = state->global->ref->images[j]->poc;
+      ref_list[1]++;
+    }
+  }
+
+  encoder_ref_insertion_sort(ref_list_poc[0], ref_list[0]);
+  encoder_ref_insertion_sort(ref_list_poc[1], ref_list[1]);
+
+  for (j = 0; j < state->global->ref->used_size; j++) {
+    if (state->global->ref->images[j]->poc < state->global->poc) {
+      int idx = ref_list[0];
+      for (int ref_idx = 0; ref_idx < ref_list[0]; ref_idx++) {
+        if (ref_list_poc[0][ref_idx] == state->global->ref->images[j]->poc) {
+          state->global->refmap[j].idx = ref_list[0] - ref_idx - 1;
+          break;
+        }
+      }
+      state->global->refmap[j].list = 1;
+      
+    } else {
+      int idx = ref_list[1];
+      for (int ref_idx = 0; ref_idx < ref_list[1]; ref_idx++) {
+        if (ref_list_poc[1][ref_idx] == state->global->ref->images[j]->poc) {
+          state->global->refmap[j].idx = ref_idx;
+          break;
+        }
+      }
+      state->global->refmap[j].list = 2;
+    }
+    state->global->refmap[j].poc = state->global->ref->images[j]->poc;
+  }
+}
 
 void encoder_next_frame(encoder_state_t *state) {
   const encoder_control_t * const encoder = state->encoder_control;
@@ -999,6 +1052,8 @@ void encoder_next_frame(encoder_state_t *state) {
         image_list_rem(state->global->ref, state->global->ref->used_size - 1);
       }
     }
+
+    encoder_state_ref_sort(state);
     return; //FIXME reference frames
   }
 
@@ -1026,6 +1081,7 @@ void encoder_next_frame(encoder_state_t *state) {
   
   state->tile->frame->rec = image_alloc(state->tile->frame->width, state->tile->frame->height, state->global->poc);
   videoframe_set_poc(state->tile->frame, state->global->poc);
+  encoder_state_ref_sort(state);
 }
 
 
diff --git a/src/encoderstate.h b/src/encoderstate.h
index 44f9da93..c97874ac 100644
--- a/src/encoderstate.h
+++ b/src/encoderstate.h
@@ -71,7 +71,12 @@ typedef struct {
   //Current picture available references
   image_list_t *ref;
   int8_t ref_list;
-  //int8_t ref_idx_num[2];
+
+  struct {
+    int32_t poc;
+    int8_t list;
+    int8_t idx;
+  } refmap[16];
   
   int is_radl_frame;
   uint8_t pictype;
diff --git a/src/search.c b/src/search.c
index f803f3f6..d179a828 100644
--- a/src/search.c
+++ b/src/search.c
@@ -701,24 +701,9 @@ static int search_cu_inter(const encoder_state_t * const state, int x, int y, in
     mvd.y = mv.y - mv_cand[cu_mv_cand][1];
 
     if(temp_cost < cur_cu->inter.cost) {
-      int j, ref_list[2] = { 0, 0 };
-      for (j = 0; j < state->global->ref->used_size; j++) {
-        if (state->global->ref->images[j]->poc < state->global->poc) {
-          if (ref_idx == j) {
-            cur_cu->inter.mv_dir = 1;            
-            cur_cu->inter.mv_ref_coded = ref_list[0]; 
-            break;
-          }
-          ref_list[0]++;
-        } else {          
-          if (ref_idx == j) {
-            cur_cu->inter.mv_dir = 2;
-            cur_cu->inter.mv_ref_coded = ref_list[1];
-            break;
-          }
-          ref_list[1]++;
-        }
-      }
+      cur_cu->inter.mv_dir = state->global->refmap[ref_idx].list;
+      cur_cu->inter.mv_ref_coded = state->global->refmap[ref_idx].idx;
+
       cur_cu->merged        = merged;
       cur_cu->merge_idx     = merge_idx;
       cur_cu->inter.mv_ref  = ref_idx;