diff --git a/src/encmain.c b/src/encmain.c
index 7c2f541b..33f939fe 100644
--- a/src/encmain.c
+++ b/src/encmain.c
@@ -59,7 +59,6 @@ int main(int argc, char *argv[])
   config *cfg  = NULL; //!< Global configuration
   FILE *input  = NULL; //!< input file (YUV)
   FILE *output = NULL; //!< output file (HEVC NAL stream)
-  FILE *cqmfile = NULL; //!< HM-compatible CQM file
   encoder_control *encoder = NULL; //!< Encoder control struct
   double psnr[3] = { 0.0, 0.0, 0.0 };
   uint64_t curpos  = 0;
@@ -250,9 +249,6 @@ int main(int argc, char *argv[])
   encoder->vui.chroma_loc  = (int8_t)encoder->cfg->vui.chroma_loc;
   // AUD
   encoder->aud_enable = (int8_t)encoder->cfg->aud_enable;
-  // CQM
-  cqmfile = cfg->cqmfile ? fopen(cfg->cqmfile, "rb") : NULL;
-  encoder->cqmfile = cqmfile;
 
   init_encoder_input(&encoder->in, input, cfg->width, cfg->height);
 
@@ -392,7 +388,6 @@ int main(int argc, char *argv[])
 
   fclose(input);
   fclose(output);
-  if(cqmfile != NULL) fclose(cqmfile);
   if(recout != NULL) fclose(recout);
 
   // Deallocating
diff --git a/src/encoder.c b/src/encoder.c
index 1a87c07a..9623f65f 100644
--- a/src/encoder.c
+++ b/src/encoder.c
@@ -311,7 +311,17 @@ encoder_control *init_encoder_control(config *cfg)
 
   enc_c->ref = pic_list;
   enc_c->ref_list = REF_PIC_LIST_0;
-
+  
+  // CQM
+  {
+    FILE* cqmfile;
+    cqmfile = cfg->cqmfile ? fopen(cfg->cqmfile, "rb") : NULL;
+    if (cqmfile) {
+      scalinglist_parse(enc_c, cqmfile);
+      fclose(cqmfile);
+    }
+  }
+  
   return enc_c;
 
 init_failure:
@@ -409,9 +419,6 @@ void encode_one_frame(encoder_control* encoder)
       picture_list_rem(encoder->ref, encoder->ref->used_size - 1, 1);
     }
 
-    if (encoder->frame == 0 && encoder->cqmfile)
-      scalinglist_parse(encoder, encoder->cqmfile);
-
     encoder->poc = 0;
 
     encoder->in.cur_pic->slicetype = SLICE_I;
diff --git a/src/encoder.h b/src/encoder.h
index e5b18263..4fdfea00 100644
--- a/src/encoder.h
+++ b/src/encoder.h
@@ -106,7 +106,6 @@ typedef struct
   } vui;
 
   int8_t aud_enable;
-  FILE *cqmfile; // \brief Costum Quantization Matrices
 
   //scaling list
   uint8_t      scaling_list_enable;