diff --git a/src/encoderstate.c b/src/encoderstate.c
index f85224f2..c2880a30 100644
--- a/src/encoderstate.c
+++ b/src/encoderstate.c
@@ -720,7 +720,7 @@ static void encoder_state_remove_refs(encoder_state_t *state) {
     refnumber = encoder->cfg->gop[state->global->gop_offset].ref_neg_count + encoder->cfg->gop[state->global->gop_offset].ref_pos_count;
     check_refs = 1;
   } else if (state->global->slicetype == SLICE_I) {
-    refnumber = 1;
+    refnumber = 0;
   }
   // Remove the ref pic (if present)
   while (check_refs || state->global->ref->used_size > (uint32_t)refnumber) {
@@ -798,11 +798,11 @@ static void encoder_state_new_frame(encoder_state_t * const state) {
 
     if (state->global->is_radl_frame) {
       encoder_state_clear_refs(state);
-    } else {
-      encoder_state_remove_refs(state);
-      encoder_state_ref_sort(state);
     }
 
+    encoder_state_remove_refs(state);
+    encoder_state_ref_sort(state);
+
     if (state->encoder_control->cfg->gop_len) {
       if (state->global->slicetype == SLICE_I) {
         state->global->QP = state->encoder_control->cfg->qp;