diff --git a/src/cu.c b/src/cu.c
index 569fd302..83778b04 100644
--- a/src/cu.c
+++ b/src/cu.c
@@ -25,6 +25,7 @@
 #include <stdlib.h>
 
 #include "cu.h"
+#include "threads.h"
 
 
 void coefficients_blit(const coefficient * const orig, coefficient * const dst,
@@ -51,4 +52,26 @@ unsigned coefficients_calc_abs(const coefficient *const buf, const int buf_strid
   }
 
   return sum;
-}
\ No newline at end of file
+}
+
+cu_array * cu_array_alloc(const int width_in_scu, const int height_in_scu) {
+  unsigned cu_array_size = height_in_scu * width_in_scu;
+  cu_array *cua;
+  cua = MALLOC(cu_array, 1);
+  cua->data = (cu_info*)malloc(sizeof(cu_info) * cu_array_size);
+  cua->refcount = 1;
+  memset(cua->data, 0, sizeof(cu_info) * cu_array_size);
+  return cua;
+}
+
+int cu_array_free(cu_array * const cua)
+{
+  int32_t new_refcount = ATOMIC_DEC(&(cua->refcount));
+  //Still we have some references, do nothing
+  if (new_refcount > 0) return 1;
+  
+  FREE_POINTER(cua->data);
+  free(cua);
+
+  return 1;
+}
diff --git a/src/cu.h b/src/cu.h
index 54bf6bcf..3518743e 100644
--- a/src/cu.h
+++ b/src/cu.h
@@ -111,6 +111,15 @@ typedef struct
   (cu).inter.cost, (cu).inter.bitcost, (cu).inter.mv[0], (cu).inter.mv[1], (cu).inter.mvd[0], (cu).inter.mvd[1], \
   (cu).inter.mv_cand, (cu).inter.mv_ref, (cu).inter.mv_dir, (cu).inter.mode)
 
+typedef struct {
+  cu_info *data;           //!< \brief cu_info data
+  int32_t refcount;        //!< \brief number of references in reflists to this cu_array
+} cu_array;
+
+cu_array * cu_array_alloc(int width_in_scu, int height_in_scu);
+int cu_array_free(cu_array *cua);
+  
+
 #define SUB_SCU_BIT_MASK (64 - 1)
 #define SUB_SCU(xy) (xy & SUB_SCU_BIT_MASK)
 #define LCU_CU_WIDTH 8
diff --git a/src/encoderstate.c b/src/encoderstate.c
index 8db1b007..c3e7df35 100644
--- a/src/encoderstate.c
+++ b/src/encoderstate.c
@@ -791,6 +791,13 @@ void encoder_next_frame(encoder_state *encoder_state) {
     
     encoder_state->tile->frame->rec = image_alloc(encoder_state->tile->frame->width, encoder_state->tile->frame->height, encoder_state->global->poc);
     videoframe_set_poc(encoder_state->tile->frame, encoder_state->global->poc);
+    
+    image_list_copy_contents(encoder_state->global->ref, encoder_state->previous_encoder_state->global->ref);
+    image_list_add(encoder_state->global->ref, encoder_state->previous_encoder_state->tile->frame->rec, encoder_state->previous_encoder_state->tile->frame->cu_array);
+    // Remove the ref pics in excess
+    while (encoder_state->global->ref->used_size > (uint32_t)encoder->cfg->ref_frames) {
+      image_list_rem(encoder_state->global->ref, encoder_state->global->ref->used_size-1);
+    }
     return; //FIXME reference frames
   }
 
diff --git a/src/imagelist.c b/src/imagelist.c
index fe85a09c..cb00712f 100644
--- a/src/imagelist.c
+++ b/src/imagelist.c
@@ -43,7 +43,7 @@ image_list * image_list_alloc(int size)
   list->size = size;
   if (size > 0) {
     list->images = (image**)malloc(sizeof(image*) * size);
-    list->cu_arrays = (cu_info**)malloc(sizeof(cu_info*) * size);
+    list->cu_arrays = (cu_array**)malloc(sizeof(cu_array*) * size);
   }
 
   list->used_size = 0;
@@ -61,7 +61,7 @@ int image_list_resize(image_list *list, unsigned size)
 {
   unsigned int i;
   image** old_images = NULL;
-  cu_info** old_cu_arrays = NULL;
+  cu_array** old_cu_arrays = NULL;
   
   //FIXME This could be done in a simple way using realloc...
 
@@ -78,7 +78,7 @@ int image_list_resize(image_list *list, unsigned size)
 
   // allocate space for the new list
   list->images = (image**)malloc(sizeof(image*)*size);
-  list->cu_arrays = (cu_info**)malloc(sizeof(cu_info*)*size);
+  list->cu_arrays = (cu_array**)malloc(sizeof(cu_array*)*size);
 
   // Copy everything from the old list to the new if needed.
   if (old_images != NULL) {
@@ -106,7 +106,7 @@ int image_list_destroy(image_list *list)
     for (i = 0; i < list->used_size; ++i) {
       image_free(list->images[i]);
       list->images[i] = NULL;
-      free(list->cu_arrays[i]);
+      cu_array_free(list->cu_arrays[i]);
       list->cu_arrays[i] = NULL;
     }
   }
@@ -125,7 +125,7 @@ int image_list_destroy(image_list *list)
  * \param picture_list list to use
  * \return 1 on success
  */
-int image_list_add(image_list *list, image* im, cu_info* cu_array)
+int image_list_add(image_list *list, image* im, cu_array* cua)
 {
   int i = 0;
   if (ATOMIC_INC(&(im->refcount)) == 1) {
@@ -133,6 +133,12 @@ int image_list_add(image_list *list, image* im, cu_info* cu_array)
     assert(0); //Stop for debugging
     return 0;
   }
+  
+  if (ATOMIC_INC(&(cua->refcount)) == 1) {
+    fprintf(stderr, "Tried to add an unreferenced cu_array. This is a bug!\n");
+    assert(0); //Stop for debugging
+    return 0;
+  }
 
   if (list->size == list->used_size) {
     if (!image_list_resize(list, list->size*2)) return 0;
@@ -144,20 +150,8 @@ int image_list_add(image_list *list, image* im, cu_info* cu_array)
   }
 
   list->images[0] = im;
-  //We need (only here, for malloc/memcpy) to compute the size of the image in SCU
-  {
-    //FIXME FIXME FIXME Do like images, use a pointer instead of copying
-    unsigned int width_in_lcu, height_in_lcu, width_in_scu, height_in_scu;
-    width_in_lcu  = im->width / LCU_WIDTH;
-    if (width_in_lcu * LCU_WIDTH < im->width) width_in_lcu++;
-    height_in_lcu = im->height / LCU_WIDTH;
-    if (height_in_lcu * LCU_WIDTH < im->height) height_in_lcu++;
-    height_in_scu = height_in_lcu << MAX_DEPTH;
-    width_in_scu = width_in_lcu << MAX_DEPTH;
-    
-    list->cu_arrays[0] = (cu_info*)malloc(sizeof(cu_info) * width_in_scu * height_in_scu);
-    memcpy(list->cu_arrays[0], cu_array, sizeof(cu_info) * width_in_scu * height_in_scu);
-  }
+  list->cu_arrays[0] = cua;
+  
   list->used_size++;
   return 1;
 }
@@ -181,7 +175,12 @@ int image_list_rem(image_list * const list, const unsigned n)
     assert(0); //Stop here
     return 0;
   }
-  free(list->cu_arrays[n]);
+  
+  if (!cu_array_free(list->cu_arrays[n])) {
+    fprintf(stderr, "Could not free cu_array!\n");
+    assert(0); //Stop here
+    return 0;
+  }
 
   // The last item is easy to remove
   if (n == list->used_size - 1) {
diff --git a/src/imagelist.h b/src/imagelist.h
index 75d04a99..138a22fb 100644
--- a/src/imagelist.h
+++ b/src/imagelist.h
@@ -32,8 +32,8 @@
  */
 typedef struct
 {
-  struct image** images;          //!< \brief Pointer to array of picture pointers.
-  cu_info** cu_arrays;
+  struct image* *images;          //!< \brief Pointer to array of picture pointers.
+  cu_array* *cu_arrays;
   uint32_t size;       //!< \brief Array size.
   uint32_t used_size;
 } image_list;
@@ -41,7 +41,7 @@ typedef struct
 image_list * image_list_alloc(int size);
 int image_list_resize(image_list *list, unsigned size);
 int image_list_destroy(image_list *list);
-int image_list_add(image_list *list, image *im, cu_info* cu_array);
+int image_list_add(image_list *list, image *im, cu_array* cua);
 int image_list_rem(image_list *list, unsigned n);
 
 enum { REF_PIC_LIST_0 = 0, REF_PIC_LIST_1 = 1, REF_PIC_LIST_X = 100 };
diff --git a/src/search.c b/src/search.c
index ff2c9ee6..72363d4e 100644
--- a/src/search.c
+++ b/src/search.c
@@ -398,7 +398,7 @@ static int search_cu_inter(const encoder_state * const encoder_state, int x, int
 
   for (ref_idx = 0; ref_idx < encoder_state->global->ref->used_size; ref_idx++) {
     image *ref_image = encoder_state->global->ref->images[ref_idx];
-    const cu_info *ref_cu = &encoder_state->global->ref->cu_arrays[ref_idx][x_cu + y_cu * (frame->width_in_lcu << MAX_DEPTH)];
+    const cu_info *ref_cu = &encoder_state->global->ref->cu_arrays[ref_idx]->data[x_cu + y_cu * (frame->width_in_lcu << MAX_DEPTH)];
     uint32_t temp_bitcost = 0;
     uint32_t temp_cost = 0;
     vector2d orig, mv, mvd;
diff --git a/src/videoframe.c b/src/videoframe.c
index fa1c2a6e..afe6766b 100644
--- a/src/videoframe.c
+++ b/src/videoframe.c
@@ -56,9 +56,7 @@ videoframe *videoframe_alloc(const int32_t width, const int32_t height, const in
     // Allocate height_in_scu x width_in_scu x sizeof(CU_info)
     unsigned height_in_scu = frame->height_in_lcu << MAX_DEPTH;
     unsigned width_in_scu = frame->width_in_lcu << MAX_DEPTH;
-    unsigned cu_array_size = height_in_scu * width_in_scu;
-    frame->cu_array = (cu_info*)malloc(sizeof(cu_info) * cu_array_size);
-    memset(frame->cu_array, 0, sizeof(cu_info) * cu_array_size);
+    frame->cu_array = cu_array_alloc(width_in_scu, height_in_scu);
   }
 
   frame->coeff_y = NULL; frame->coeff_u = NULL; frame->coeff_v = NULL;
@@ -103,14 +101,14 @@ const cu_info* videoframe_get_cu_const(const videoframe * const frame, unsigned
   assert(x_in_scu < (frame->width_in_lcu << MAX_DEPTH));
   assert(y_in_scu < (frame->height_in_lcu << MAX_DEPTH));
   
-  return &frame->cu_array[x_in_scu + y_in_scu * (frame->width_in_lcu << MAX_DEPTH)];
+  return &frame->cu_array->data[x_in_scu + y_in_scu * (frame->width_in_lcu << MAX_DEPTH)];
 }
 
 cu_info* videoframe_get_cu(videoframe * const frame, const unsigned int x_in_scu, const unsigned int y_in_scu) {
   assert(x_in_scu < (frame->width_in_lcu << MAX_DEPTH));
   assert(y_in_scu < (frame->height_in_lcu << MAX_DEPTH));
   
-  return &frame->cu_array[x_in_scu + y_in_scu * (frame->width_in_lcu << MAX_DEPTH)];
+  return &frame->cu_array->data[x_in_scu + y_in_scu * (frame->width_in_lcu << MAX_DEPTH)];
 }
 
 #define PSNRMAX (255.0 * 255.0)
diff --git a/src/videoframe.h b/src/videoframe.h
index 2d868f3f..a8624630 100644
--- a/src/videoframe.h
+++ b/src/videoframe.h
@@ -47,7 +47,7 @@ typedef struct videoframe
   int32_t height_in_lcu;  //!< \brief Picture width in number of LCU's.
   int32_t width_in_lcu;   //!< \brief Picture height in number of LCU's.
 
-  cu_info* cu_array;     //!< \brief Info for each CU at each depth.
+  cu_array* cu_array;     //!< \brief Info for each CU at each depth.
   struct sao_info_struct *sao_luma;   //!< \brief Array of sao parameters for every LCU.
   struct sao_info_struct *sao_chroma;   //!< \brief Array of sao parameters for every LCU.
   int32_t poc;           //!< \brief Picture order count