reconst optimation doesn't work yet

src/strategies/avx2/sao-avx2.c

@@ -35,9 +35,12 @@
 // Might be useful to check that if (when) this file
 // is difficult to understand.
 
-static INLINE __m128i load_6_pixels(const kvz_pixel* data)
+static INLINE __m128i load_14_pixels(const kvz_pixel* data)
 {
-  return _mm_insert_epi16(_mm_cvtsi32_si128(*(int32_t*)&(data[0])), *(int16_t*)&(data[4]), 2);
+ __m128i temp = _mm_loadl_epi64((__m128i*)&data[0]);
+ temp = _mm_insert_epi32(temp, *(int32_t*)&data[8], 2);
+ temp = _mm_insert_epi16(temp, *(int16_t*)&data[12], 6);
+ return temp;
 }
 
 static INLINE __m256i load_5_offsets(const int* offsets)
@@ -60,7 +63,7 @@ static __m128i sao_calc_eo_cat_avx2_256(__m128i* a, __m128i* b, __m128i* c)
  v_eo_idx = _mm256_add_epi16(v_eo_idx, temp_b);
 
  v_eo_idx = _mm256_packus_epi16(v_eo_idx, v_eo_idx);
- v_eo_idx = _mm256_permute4x64_epi64(v_eo_idx, 0b11011000);
+ v_eo_idx = _mm256_permute4x64_epi64(v_eo_idx, _MM_SHUFFLE(3,1,2,0));
 
  __m128i v_cat_lookup = _mm_setr_epi8(1, 2, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
  __m128i v_cat = _mm_shuffle_epi8(v_cat_lookup, _mm256_extracti128_si256(v_eo_idx, 1));
@@ -123,7 +126,7 @@ static int sao_edge_ddistortion_avx2(const kvz_pixel *orig_data,
     __m128i temp_cat = sao_calc_eo_cat_avx2_256(&vector_a, &vector_b, &vector_c_data);
 
     //Split temp_cat vector to higher and upper parts
-    __m256i vector_cat_lower = _mm256_cvtepu8_epi32(_mm_cvtsi64_si128(_mm_extract_epi64(temp_cat, 0)));
+    __m256i vector_cat_lower = _mm256_cvtepu8_epi32(_mm_cvtsi64_si128(*(int64_t*)&(temp_cat)));
     __m256i vector_cat_upper = _mm256_cvtepu8_epi32(_mm_cvtsi64_si128(_mm_extract_epi64(temp_cat, 1)));
 
     // Re-arrenge offsets to right order
@@ -147,30 +150,20 @@ static int sao_edge_ddistortion_avx2(const kvz_pixel *orig_data,
     v_temp_sum = _mm256_sub_epi32(_mm256_mullo_epi32(v_diff_minus_offset, v_diff_minus_offset), _mm256_mullo_epi32(v_diff, v_diff));
     v_accum = _mm256_add_epi32(v_accum, v_temp_sum);
    }
-   /*
-
-   Make 14 pixel load here
-
-   static INLINE __m128i load_14_pixels(const kvz_pixel* data) {
-    __m128i temp;
-    temp = _mm_loadl_epi64((__m128i*)data);
-    _mm_insert_epi32((int32_t*)&)
-     *
-   }*/
-
-
-   // After x> (block_width-16) handle 8 pixels and after that the last 6 pixels
    const kvz_pixel *c_data = &rec_data[y * block_width + x];
 
-   vector_c_data = _mm_loadl_epi64((__m128i*)c_data);
+   vector_c_data = load_14_pixels(c_data);
-   vector_a = _mm_loadl_epi64((__m128i*)(&c_data[a_ofs.y * block_width + a_ofs.x]));
+   vector_a = load_14_pixels((&c_data[a_ofs.y * block_width + a_ofs.x]));
-   vector_b = _mm_loadl_epi64((__m128i*)(&c_data[b_ofs.y * block_width + b_ofs.x]));
 
-   __m256i v_cat = _mm256_cvtepu8_epi32(sao_calc_eo_cat_avx2_256(&vector_a, &vector_b, &vector_c_data));
+   vector_b = load_14_pixels((&c_data[b_ofs.y * block_width + b_ofs.x]));
-
+   __m128i temp_cat = sao_calc_eo_cat_avx2_256(&vector_a, &vector_b, &vector_c_data);
-   temp_v_offset = _mm256_permutevar8x32_epi32(v_offset, v_cat);
 
+   //Split temp_cat vector to higher and upper parts
+   __m256i vector_cat_lower = _mm256_cvtepu8_epi32(_mm_cvtsi64_si128(*(int64_t*)&(temp_cat)));
+   __m256i vector_cat_upper = _mm256_cvtepu8_epi32(_mm_cvtsi64_si128(_mm_extract_epi64(temp_cat, 1)));
 
+   // Re-arrenge offsets to right order
+   temp_v_offset = _mm256_permutevar8x32_epi32(v_offset, vector_cat_lower);
    __m256i v_diff = _mm256_cvtepu8_epi32(_mm_loadl_epi64((__m128i*)&(orig_data[y * block_width + x])));
 
    v_diff = _mm256_sub_epi32(v_diff, _mm256_cvtepu8_epi32(vector_c_data));
@@ -180,28 +173,16 @@ static int sao_edge_ddistortion_avx2(const kvz_pixel *orig_data,
    __m256i v_temp_sum = _mm256_sub_epi32(_mm256_mullo_epi32(v_diff_minus_offset, v_diff_minus_offset), _mm256_mullo_epi32(v_diff, v_diff));
    v_accum = _mm256_add_epi32(v_accum, v_temp_sum);
 
-   //Handle last 6 pixels separately to prevent reading over boundary
+   temp_v_offset = _mm256_permutevar8x32_epi32(v_offset, vector_cat_upper);
-   x += 8;
+   v_diff = _mm256_cvtepu8_epi32(_mm_loadl_epi64((__m128i*)&(orig_data[y * block_width + x + 8])));
-   c_data = &rec_data[y * block_width + x];
+   int64_t*c_pointer = (int64_t*)&vector_c_data;
-   vector_c_data = load_6_pixels(c_data);
+   v_diff = _mm256_sub_epi32(v_diff, _mm256_cvtepu8_epi32(_mm_cvtsi64_si128(c_pointer[1])));
-   const kvz_pixel *a_ptr = &c_data[a_ofs.y * block_width + a_ofs.x];
-   const kvz_pixel *b_ptr = &c_data[b_ofs.y * block_width + b_ofs.x];
-   vector_a = load_6_pixels(a_ptr);
-   vector_b = load_6_pixels(b_ptr);
-
-   v_cat = _mm256_cvtepu8_epi32(sao_calc_eo_cat_avx2_256(&vector_a, &vector_b, &vector_c_data));
-
-   temp_v_offset = _mm256_permutevar8x32_epi32(v_offset, v_cat);
-
-   const kvz_pixel *orig_ptr = &(orig_data[y * block_width + x]);
-   v_diff = _mm256_cvtepu8_epi32(load_6_pixels(orig_ptr));
-
-   v_diff = _mm256_sub_epi32(v_diff, _mm256_cvtepu8_epi32(vector_c_data));
-
 
    v_diff_minus_offset = _mm256_sub_epi32(v_diff, temp_v_offset);
+
    v_temp_sum = _mm256_sub_epi32(_mm256_mullo_epi32(v_diff_minus_offset, v_diff_minus_offset), _mm256_mullo_epi32(v_diff, v_diff));
    v_accum = _mm256_add_epi32(v_accum, v_temp_sum);
+
   }
 
   //Full horizontal sum
@@ -243,6 +224,24 @@ static INLINE void accum_count_eo_cat_avx2(__m256i*  __restrict v_diff_accum,
   accum_count_eo_cat_avx2(&(v_diff_accum[ EO_CAT ]), &(v_count[ EO_CAT ]), &V_CAT , &v_diff, EO_CAT);
 
 
+
+static INLINE void accum_count_eo_cat_avx2_256(__m256i*  __restrict v_diff_accum,
+ __m256i* __restrict v_count,
+ __m256i* __restrict v_cat,
+ __m256i* __restrict v_diff,
+ int eo_cat)
+{
+ __m256i v_mask = _mm256_cmpeq_epi32(*v_cat, _mm256_set1_epi32(eo_cat));
+ *v_diff_accum = _mm256_add_epi32(*v_diff_accum, _mm256_and_si256(*v_diff, v_mask));
+ *v_count = _mm256_sub_epi32(*v_count, v_mask);
+}
+
+
+#define ACCUM_COUNT_EO_CAT_AVX2_256(EO_CAT, V_CAT) \
+  \
+  accum_count_eo_cat_avx2(&(v_diff_accum[ EO_CAT ]), &(v_count[ EO_CAT ]), &V_CAT , &v_diff, EO_CAT);
+
+
 /*
 static void calc_sao_edge_dir_generic(const kvz_pixel *orig_data,
                                       const kvz_pixel *rec_data,
@@ -331,51 +330,36 @@ static void calc_sao_edge_dir_avx2(const kvz_pixel *orig_data,
 
   const kvz_pixel *c_data = &rec_data[y * block_width + x];
 
-  __m128i v_c_data = _mm_loadl_epi64((__m128i* __restrict)c_data);
+  __m128i v_c_data = load_14_pixels(c_data);
-  __m128i v_a = _mm_loadl_epi64((__m128i* __restrict)(&c_data[a_ofs.y * block_width + a_ofs.x]));
+  __m128i v_a = load_14_pixels(&c_data[a_ofs.y * block_width + a_ofs.x]);
   __m128i v_c = v_c_data;
-  __m128i v_b = _mm_loadl_epi64((__m128i* __restrict)(&c_data[b_ofs.y * block_width + b_ofs.x]));
+  __m128i v_b = load_14_pixels(&c_data[b_ofs.y * block_width + b_ofs.x]);
 
-  __m256i v_cat = _mm256_cvtepu8_epi32(sao_calc_eo_cat_avx2(&v_a, &v_b, &v_c));
+  temp_cat = sao_calc_eo_cat_avx2_256(&v_a, &v_b, &v_c);
+  __m256i cat_lower = _mm256_cvtepu8_epi32(_mm_cvtsi64_si128(_mm_extract_epi64(temp_cat, 0)));
+  __m256i cat_upper = _mm256_cvtepu8_epi32(_mm_cvtsi64_si128(_mm_extract_epi64(temp_cat, 1)));
 
   __m256i v_diff = _mm256_cvtepu8_epi32(_mm_loadl_epi64((__m128i* __restrict)&(orig_data[y * block_width + x])));
   v_diff = _mm256_sub_epi32(v_diff, _mm256_cvtepu8_epi32(v_c));
 
   //Accumulate differences and occurrences for each category
-  ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT0, v_cat);
+  ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT0, cat_lower);
-  ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT1, v_cat);
+  ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT1, cat_lower);
-  ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT2, v_cat);
+  ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT2, cat_lower);
-  ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT3, v_cat);
+  ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT3, cat_lower);
-  ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT4, v_cat);
+  ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT4, cat_lower);
 
-  x += 8;
 
-  //Handle last 6 pixels separately to prevent reading over boundary
+  v_diff = _mm256_cvtepu8_epi32(_mm_loadl_epi64((__m128i* __restrict)&(orig_data[y * block_width + x + 8])));
-  c_data = &rec_data[y * block_width + x];
+  int64_t*c_pointer = (int64_t*)&v_c;
-  v_c_data = load_6_pixels(c_data);
+  v_diff = _mm256_sub_epi32(v_diff, _mm256_cvtepu8_epi32(_mm_cvtsi64_si128(c_pointer[1])));
-  const kvz_pixel* a_ptr = &c_data[a_ofs.y * block_width + a_ofs.x];
-  const kvz_pixel* b_ptr = &c_data[b_ofs.y * block_width + b_ofs.x];
-  v_a = load_6_pixels(a_ptr);
-  v_c = v_c_data;
-  v_b = load_6_pixels(b_ptr);
-
-  v_cat = _mm256_cvtepu8_epi32(sao_calc_eo_cat_avx2(&v_a, &v_b, &v_c));
-
-  //Set the last two elements to a non-existing category to cause
-  //the accumulate-count macro to discard those values.
-  __m256i v_mask = _mm256_setr_epi32(0, 0, 0, 0, 0, 0, -1, -1);
-  v_cat = _mm256_or_si256(v_cat, v_mask);
-
-  const kvz_pixel* orig_ptr = &(orig_data[y * block_width + x]);
-  v_diff = _mm256_cvtepu8_epi32(load_6_pixels(orig_ptr));
-  v_diff = _mm256_sub_epi32(v_diff, _mm256_cvtepu8_epi32(v_c));
 
   //Accumulate differences and occurrences for each category
-  ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT0, v_cat);
+  ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT0, cat_upper);
-  ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT1, v_cat);
+  ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT1, cat_upper);
-  ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT2, v_cat);
+  ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT2, cat_upper);
-  ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT3, v_cat);
+  ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT3, cat_upper);
-  ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT4, v_cat);
+  ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT4, cat_upper);
  }
 
  for (int eo_cat = 0; eo_cat < NUM_SAO_EDGE_CATEGORIES; ++eo_cat) {
@@ -402,6 +386,65 @@ static void calc_sao_edge_dir_avx2(const kvz_pixel *orig_data,
 
 }
 
+static int sao_calc_eo_cat(kvz_pixel a, kvz_pixel b, kvz_pixel c)
+{
+ // Mapping relationships between a, b and c to eo_idx.
+ static const int sao_eo_idx_to_eo_category[] = { 1, 2, 0, 3, 4 };
+
+ int eo_idx = 2 + SIGN3((int)c - (int)a) + SIGN3((int)c - (int)b);
+
+ return sao_eo_idx_to_eo_category[eo_idx];
+}
+
+static void sao_reconstruct_color_avx(const encoder_control_t * const encoder,
+ const kvz_pixel *rec_data,
+ kvz_pixel *new_rec_data,
+ const sao_info_t *sao,
+ int stride,
+ int new_stride,
+ int block_width,
+ int block_height,
+ color_t color_i)
+{
+ // Arrays orig_data and rec_data are quarter size for chroma.
+ int offset_v = color_i == COLOR_V ? 5 : 0;
+
+ if (sao->type == SAO_TYPE_BAND) {
+  int offsets[1 << KVZ_BIT_DEPTH];
+  kvz_calc_sao_offset_array(encoder, sao, offsets, color_i);
+  for (int y = 0; y < block_height; ++y) {
+   for (int x = 0; x < block_width; ++x) {
+    new_rec_data[y * new_stride + x] = offsets[rec_data[y * stride + x]];
+   }
+  }
+ }
+ else {
+  // Don't sample the edge pixels because this function doesn't have access to
+  // their neighbours.
+  for (int y = 0; y < block_height; ++y) {
+   for (int x = 0; x < block_width; ++x) {
+    vector2d_t a_ofs = g_sao_edge_offsets[sao->eo_class][0];
+    vector2d_t b_ofs = g_sao_edge_offsets[sao->eo_class][1];
+    const kvz_pixel *c_data = &rec_data[y * stride + x];
+    kvz_pixel *new_data = &new_rec_data[y * new_stride + x];
+    kvz_pixel a = c_data[a_ofs.y * stride + a_ofs.x];
+    kvz_pixel c = c_data[0];
+    kvz_pixel b = c_data[b_ofs.y * stride + b_ofs.x];
+
+    int eo_cat = sao_calc_eo_cat(a, b, c);
+
+    printf("%d", eo_cat);
+
+    new_data[0] = (kvz_pixel)CLIP(0, (1 << KVZ_BIT_DEPTH) - 1, c_data[0] + sao->offsets[eo_cat + offset_v]);
+   }
+  }
+ }
+}
+/*
+Does't work
+
+
+*/
 static void sao_reconstruct_color_avx2(const encoder_control_t * const encoder,
                                        const kvz_pixel *rec_data, kvz_pixel *new_rec_data,
                                        const sao_info_t *sao,
@@ -423,34 +466,67 @@ static void sao_reconstruct_color_avx2(const encoder_control_t * const encoder,
   } else {
     // Don't sample the edge pixels because this function doesn't have access to
     // their neighbours.
+
+   vector2d_t a_ofs = g_sao_edge_offsets[sao->eo_class][0];
+   vector2d_t b_ofs = g_sao_edge_offsets[sao->eo_class][1];
     for (int y = 0; y < block_height; ++y) {
-      for (int x = 0; x < block_width; x+=8) {
+      for (int x = 0; x < block_width; x+=16) {
-        vector2d_t a_ofs = g_sao_edge_offsets[sao->eo_class][0];
+
-        vector2d_t b_ofs = g_sao_edge_offsets[sao->eo_class][1];
         const kvz_pixel *c_data = &rec_data[y * stride + x];
         kvz_pixel *new_data = &new_rec_data[y * new_stride + x];
         const kvz_pixel* a_ptr = &c_data[a_ofs.y * stride + a_ofs.x];
         const kvz_pixel* c_ptr = &c_data[0];
         const kvz_pixel* b_ptr = &c_data[b_ofs.y * stride + b_ofs.x];
 
+        
+        /*
         __m128i v_a = _mm_loadl_epi64((__m128i*)a_ptr);
         __m128i v_b = _mm_loadl_epi64((__m128i*)b_ptr);
-        __m128i v_c = _mm_loadl_epi64((__m128i*)c_ptr);
+        __m128i v_c = _mm_loadl_epi64((__m128i*)c_ptr);*/
+
+        __m128i v_a = _mm_loadu_si128((__m128i*)a_ptr);
+        __m128i v_b = _mm_loadu_si128((__m128i*)b_ptr);
+        __m128i v_c = _mm_loadu_si128((__m128i*)c_ptr);
 
         __m256i v_cat = _mm256_cvtepu8_epi32(sao_calc_eo_cat_avx2(&v_a, &v_b, &v_c) );
 
+        __m128i temp_cat = sao_calc_eo_cat_avx2_256(&v_a, &v_b, &v_c);
+        
+        int8_t* test = (int8_t*)&temp_cat;
+        for (int i = 0; i < 16; i++) {
+         printf("%d", test[i]);
+        }
+
+        __m256i cat_lower = _mm256_cvtepu8_epi32(_mm_cvtsi64_si128(_mm_extract_epi64(temp_cat, 0)));
+        __m256i cat_upper = _mm256_cvtepu8_epi32(_mm_cvtsi64_si128(_mm_extract_epi64(temp_cat, 1)));
+
         __m256i v_offset_v = load_5_offsets(sao->offsets + offset_v);
-        __m256i v_new_data = _mm256_permutevar8x32_epi32(v_offset_v, v_cat);
+        __m256i v_new_data = _mm256_permutevar8x32_epi32(v_offset_v, cat_lower);
         v_new_data = _mm256_add_epi32(v_new_data, _mm256_cvtepu8_epi32(v_c));
+
+        __m256i v_new_data_2 = _mm256_permutevar8x32_epi32(v_offset_v, cat_upper);
+        v_new_data_2 = _mm256_permutevar8x32_epi32(v_offset_v, cat_upper);
+        v_new_data_2 = _mm256_add_epi32(v_new_data, _mm256_cvtepu8_epi32(v_c));
+
+        __m256i v_new_data_256 = _mm256_packus_epi32(v_new_data, v_new_data_2);
+
+        v_new_data_256 = _mm256_packus_epi16(v_new_data_256, v_new_data_256);
+
+        __m128i v_new_data_128 = _mm256_castsi256_si128(v_new_data_256);
+        v_new_data_128 = _mm_shuffle_epi32(v_new_data_128, _MM_SHUFFLE(3,1,2,0));
+        
+
+        /*
         __m128i v_new_data_128 = _mm_packus_epi32(_mm256_castsi256_si128(v_new_data), _mm256_extracti128_si256(v_new_data, 1));
-        v_new_data_128 = _mm_packus_epi16(v_new_data_128, v_new_data_128);
+        v_new_data_128 = _mm_packus_epi16(v_new_data_128, v_new_data_128);*/
         
         if ((block_width - x) >= 8) {
-          _mm_storel_epi64((__m128i*)new_data, v_new_data_128);
+         _mm_storeu_si128((__m128i*)new_data, v_new_data_128);
+
         } else {
           
           kvz_pixel arr[8];
-          _mm_storel_epi64((__m128i*)arr, v_new_data_128);
+          _mm_storeu_si128((__m128i*)arr, v_new_data_128);
           for (int i = 0; i < block_width - x; ++i) new_data[i] = arr[i];
         }