diff --git a/src/strategies/avx2/sao-avx2.c b/src/strategies/avx2/sao-avx2.c index d7ba7bcb..a41d3be3 100644 --- a/src/strategies/avx2/sao-avx2.c +++ b/src/strategies/avx2/sao-avx2.c @@ -158,6 +158,16 @@ static int sao_edge_ddistortion_avx2(const kvz_pixel *orig_data, v_temp_sum = _mm256_sub_epi32(_mm256_mullo_epi32(v_diff_minus_offset, v_diff_minus_offset), _mm256_mullo_epi32(v_diff, v_diff)); v_accum = _mm256_add_epi32(v_accum, v_temp_sum); } + /* + + Make 14 pixel load here + + static INLINE __m128i load_14_pixels(const kvz_pixel* data) { + __m128i temp; + temp = _mm_loadl_epi64((__m128i*)data); + _mm_insert_epi32((int32_t*)&) + * + }*/ // After x> (block_width-16) handle 8 pixels and after that the last 6 pixels @@ -274,7 +284,7 @@ static void calc_sao_edge_dir_generic(const kvz_pixel *orig_data, } }*/ -static void calc_sao_edge_dir_avx2_test(const kvz_pixel *orig_data, +static void calc_sao_edge_dir_avx2(const kvz_pixel *orig_data, const kvz_pixel *rec_data, int eo_class, int block_width, @@ -284,179 +294,125 @@ static void calc_sao_edge_dir_avx2_test(const kvz_pixel *orig_data, int y, x; vector2d_t a_ofs = g_sao_edge_offsets[eo_class][0]; vector2d_t b_ofs = g_sao_edge_offsets[eo_class][1]; + + __m256i v_diff_accum[NUM_SAO_EDGE_CATEGORIES] = { { 0 } }; + __m256i v_count[NUM_SAO_EDGE_CATEGORIES] = { { 0 } }; // Arrays orig_data and rec_data are quarter size for chroma. // Don't sample the edge pixels because this function doesn't have access to // their neighbours. - __m256i v_table[NUM_SAO_EDGE_CATEGORIES] = { { 0 } }; - __m256i v_count[NUM_SAO_EDGE_CATEGORIES] = { { 0 } }; - __m256i increase = _mm256_set1_epi32(1); + __m128i temp_cat; for (y = 1; y < block_height - 1; ++y) { - for (x = 1; x < block_width - 8; x+=8) { + for (x = 1; x < block_width - 16; x+=16) { const kvz_pixel *c_data = &rec_data[y * block_width + x]; - __m128i v_c_data = _mm_loadl_epi64((__m128i*)c_data); - - __m128i v_a = _mm_loadl_epi64((__m128i*)(&c_data[a_ofs.y * block_width + a_ofs.x])); + __m128i v_c_data = _mm_loadu_si128((__m128i*)c_data); + __m128i v_a = _mm_loadu_si128((__m128i*)(&c_data[a_ofs.y * block_width + a_ofs.x])); __m128i v_c = v_c_data; - __m128i v_b = _mm_loadl_epi64((__m128i*)(&c_data[b_ofs.y * block_width + b_ofs.x])); + __m128i v_b = _mm_loadu_si128((__m128i*)(&c_data[b_ofs.y * block_width + b_ofs.x])); - __m256i v_cat = _mm256_cvtepu8_epi32(sao_calc_eo_cat_avx2(&v_a, &v_b, &v_c)); + temp_cat = sao_calc_eo_cat_avx2_256(&v_a, &v_b, &v_c); + __m256i cat_lower = _mm256_cvtepu8_epi32(_mm_cvtsi64_si128(_mm_extract_epi64(temp_cat, 0))); + __m256i cat_upper = _mm256_cvtepu8_epi32(_mm_cvtsi64_si128(_mm_extract_epi64(temp_cat, 1))); - __m256i v_data = _mm256_cvtepu8_epi32(_mm_loadl_epi64((__m128i*)&(orig_data[y * block_width + x]))); - v_data = _mm256_sub_epi32(v_data, _mm256_cvtepu8_epi32(v_c)); + __m256i v_diff = _mm256_cvtepu8_epi32(_mm_loadl_epi64((__m128i* __restrict)&(orig_data[y * block_width + x]))); + v_diff = _mm256_sub_epi32(v_diff, _mm256_cvtepu8_epi32(v_c)); - __m256i compare_mask = _mm256_cmpeq_epi32(v_cat, _mm256_set1_epi32(SAO_EO_CAT0)); - v_table[SAO_EO_CAT0] = _mm256_add_epi32(v_table[SAO_EO_CAT0], _mm256_and_si256(v_data, compare_mask)); - v_count[SAO_EO_CAT0] = _mm256_add_epi32(v_table[SAO_EO_CAT0], _mm256_and_si256(increase, compare_mask)); + //Accumulate differences and occurrences for each category + ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT0, cat_lower); + ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT1, cat_lower); + ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT2, cat_lower); + ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT3, cat_lower); + ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT4, cat_lower); - compare_mask = _mm256_cmpeq_epi32(v_cat, _mm256_set1_epi32(SAO_EO_CAT1)); - v_table[SAO_EO_CAT1] = _mm256_add_epi32(v_table[SAO_EO_CAT1], _mm256_and_si256(v_data, compare_mask)); - v_count[SAO_EO_CAT1] = _mm256_add_epi32(v_table[SAO_EO_CAT1], _mm256_and_si256(increase, compare_mask)); - compare_mask = _mm256_cmpeq_epi32(v_cat, _mm256_set1_epi32(SAO_EO_CAT2)); - v_table[SAO_EO_CAT2] = _mm256_add_epi32(v_table[SAO_EO_CAT2], _mm256_and_si256(v_data, compare_mask)); - v_count[SAO_EO_CAT2] = _mm256_add_epi32(v_table[SAO_EO_CAT2], _mm256_and_si256(increase, compare_mask)); + v_diff = _mm256_cvtepu8_epi32(_mm_loadl_epi64((__m128i* __restrict)&(orig_data[y * block_width + x+8]))); + int64_t*c_pointer = (int64_t*)&v_c; + v_diff = _mm256_sub_epi32(v_diff, _mm256_cvtepu8_epi32(_mm_cvtsi64_si128(c_pointer[1]))); - compare_mask = _mm256_cmpeq_epi32(v_cat, _mm256_set1_epi32(SAO_EO_CAT3)); - v_table[SAO_EO_CAT3] = _mm256_add_epi32(v_table[SAO_EO_CAT3], _mm256_and_si256(v_data, compare_mask)); - v_count[SAO_EO_CAT3] = _mm256_add_epi32(v_table[SAO_EO_CAT3], _mm256_and_si256(increase, compare_mask)); - - compare_mask = _mm256_cmpeq_epi32(v_cat, _mm256_set1_epi32(SAO_EO_CAT4)); - v_table[SAO_EO_CAT4] = _mm256_add_epi32(v_table[SAO_EO_CAT4], _mm256_and_si256(v_data, compare_mask)); - v_count[SAO_EO_CAT4] = _mm256_add_epi32(v_table[SAO_EO_CAT4], _mm256_and_si256(increase, compare_mask)); + //Accumulate differences and occurrences for each category + ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT0, cat_upper); + ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT1, cat_upper); + ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT2, cat_upper); + ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT3, cat_upper); + ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT4, cat_upper); } + const kvz_pixel *c_data = &rec_data[y * block_width + x]; + __m128i v_c_data = _mm_loadl_epi64((__m128i* __restrict)c_data); + __m128i v_a = _mm_loadl_epi64((__m128i* __restrict)(&c_data[a_ofs.y * block_width + a_ofs.x])); + __m128i v_c = v_c_data; + __m128i v_b = _mm_loadl_epi64((__m128i* __restrict)(&c_data[b_ofs.y * block_width + b_ofs.x])); + __m256i v_cat = _mm256_cvtepu8_epi32(sao_calc_eo_cat_avx2(&v_a, &v_b, &v_c)); - // Vikat 6 pikseliä käsittelyyn tänne + __m256i v_diff = _mm256_cvtepu8_epi32(_mm_loadl_epi64((__m128i* __restrict)&(orig_data[y * block_width + x]))); + v_diff = _mm256_sub_epi32(v_diff, _mm256_cvtepu8_epi32(v_c)); + + //Accumulate differences and occurrences for each category + ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT0, v_cat); + ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT1, v_cat); + ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT2, v_cat); + ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT3, v_cat); + ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT4, v_cat); + + x += 8; + + //Handle last 6 pixels separately to prevent reading over boundary + c_data = &rec_data[y * block_width + x]; + v_c_data = load_6_pixels(c_data); + const kvz_pixel* a_ptr = &c_data[a_ofs.y * block_width + a_ofs.x]; + const kvz_pixel* b_ptr = &c_data[b_ofs.y * block_width + b_ofs.x]; + v_a = load_6_pixels(a_ptr); + v_c = v_c_data; + v_b = load_6_pixels(b_ptr); + + v_cat = _mm256_cvtepu8_epi32(sao_calc_eo_cat_avx2(&v_a, &v_b, &v_c)); + + //Set the last two elements to a non-existing category to cause + //the accumulate-count macro to discard those values. + __m256i v_mask = _mm256_setr_epi32(0, 0, 0, 0, 0, 0, -1, -1); + v_cat = _mm256_or_si256(v_cat, v_mask); + + const kvz_pixel* orig_ptr = &(orig_data[y * block_width + x]); + v_diff = _mm256_cvtepu8_epi32(load_6_pixels(orig_ptr)); + v_diff = _mm256_sub_epi32(v_diff, _mm256_cvtepu8_epi32(v_c)); + + //Accumulate differences and occurrences for each category + ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT0, v_cat); + ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT1, v_cat); + ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT2, v_cat); + ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT3, v_cat); + ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT4, v_cat); } - // Tästä eteenpäin pitäisi tehdä loitsuja - //cat_sum_cnt[0][eo_cat] += orig_data[y * block_width + x] - c; - //cat_sum_cnt[1][eo_cat] += 1; + for (int eo_cat = 0; eo_cat < NUM_SAO_EDGE_CATEGORIES; ++eo_cat) { + int accum = 0; + int count = 0; + + //Full horizontal sum of accumulated values + + v_diff_accum[eo_cat] = _mm256_hadd_epi32(v_diff_accum[eo_cat], v_diff_accum[eo_cat]); + v_diff_accum[eo_cat] = _mm256_hadd_epi32(v_diff_accum[eo_cat], v_diff_accum[eo_cat]); + accum += _mm256_extract_epi32(v_diff_accum[eo_cat], 0) + _mm256_extract_epi32(v_diff_accum[eo_cat], 4); + + + //Full horizontal sum of accumulated values + v_count[eo_cat] = _mm256_hadd_epi32(v_count[eo_cat], v_count[eo_cat]); + v_count[eo_cat] = _mm256_hadd_epi32(v_count[eo_cat], v_count[eo_cat]); + count += _mm256_extract_epi32(v_count[eo_cat], 0) + _mm256_extract_epi32(v_count[eo_cat], 4); + + + cat_sum_cnt[0][eo_cat] += accum; + cat_sum_cnt[1][eo_cat] += count; + + } } - - -static __m128i sao_calc_eo_cat_avx2_test(__m128i* a, __m128i* b, __m128i* c) -{ - __m128i v_eo_idx = _mm_set1_epi16(2); - __m128i v_a = _mm_cvtepu8_epi16(*a); - __m128i v_c = _mm_cvtepu8_epi16(*c); - __m128i v_b = _mm_cvtepu8_epi16(*b); - - __m128i temp_a = _mm_sign_epi16(_mm_set1_epi16(1), _mm_sub_epi16(v_c, v_a)); - __m128i temp_b = _mm_sign_epi16(_mm_set1_epi16(1), _mm_sub_epi16(v_c, v_b)); - v_eo_idx = _mm_add_epi16(v_eo_idx, temp_a); - v_eo_idx = _mm_add_epi16(v_eo_idx, temp_b); - - v_eo_idx = _mm_packus_epi16(v_eo_idx, v_eo_idx); - - __m128i v_cat_lookup = _mm_setr_epi8(1, 2, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); - __m128i v_cat = _mm_shuffle_epi8(v_cat_lookup, v_eo_idx); - - - return v_cat; -} - -static void calc_sao_edge_dir_avx2(const kvz_pixel *orig_data, - const kvz_pixel *rec_data, - int eo_class, - int block_width, - int block_height, - int cat_sum_cnt[2][NUM_SAO_EDGE_CATEGORIES]) -{ - int y, x; - vector2d_t a_ofs = g_sao_edge_offsets[eo_class][0]; - vector2d_t b_ofs = g_sao_edge_offsets[eo_class][1]; - - // Don't sample the edge pixels because this function doesn't have access to - // their neighbours. - - __m256i v_diff_accum[NUM_SAO_EDGE_CATEGORIES] = { { 0 } }; - __m256i v_count[NUM_SAO_EDGE_CATEGORIES] = { { 0 } }; - - for (y = 1; y < block_height - 1; ++y) { - - //Calculation for 8 pixels per round - for (x = 1; x < block_width - 8; x += 8) { - const kvz_pixel *c_data = &rec_data[y * block_width + x]; - - __m128i v_c_data = _mm_loadl_epi64((__m128i* __restrict)c_data); - __m128i v_a = _mm_loadl_epi64((__m128i* __restrict)(&c_data[a_ofs.y * block_width + a_ofs.x])); - __m128i v_c = v_c_data; - __m128i v_b = _mm_loadl_epi64((__m128i* __restrict)(&c_data[b_ofs.y * block_width + b_ofs.x])); - - __m256i v_cat = _mm256_cvtepu8_epi32(sao_calc_eo_cat_avx2_test(&v_a, &v_b, &v_c)); - - __m256i v_diff = _mm256_cvtepu8_epi32(_mm_loadl_epi64((__m128i* __restrict)&(orig_data[y * block_width + x]))); - v_diff = _mm256_sub_epi32(v_diff, _mm256_cvtepu8_epi32(v_c)); - - //Accumulate differences and occurrences for each category - ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT0, v_cat); - ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT1, v_cat); - ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT2, v_cat); - ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT3, v_cat); - ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT4, v_cat); - } - - //Handle last 6 pixels separately to prevent reading over boundary - const kvz_pixel *c_data = &rec_data[y * block_width + x]; - __m128i v_c_data = load_6_pixels(c_data); - const kvz_pixel* a_ptr = &c_data[a_ofs.y * block_width + a_ofs.x]; - const kvz_pixel* b_ptr = &c_data[b_ofs.y * block_width + b_ofs.x]; - __m128i v_a = load_6_pixels(a_ptr); - __m128i v_c = v_c_data; - __m128i v_b = load_6_pixels(b_ptr); - - __m256i v_cat = _mm256_cvtepu8_epi32(sao_calc_eo_cat_avx2(&v_a, &v_b, &v_c)); - - //Set the last two elements to a non-existing category to cause - //the accumulate-count macro to discard those values. - __m256i v_mask = _mm256_setr_epi32(0, 0, 0, 0, 0, 0, -1, -1); - v_cat = _mm256_or_si256(v_cat, v_mask); - - const kvz_pixel* orig_ptr = &(orig_data[y * block_width + x]); - __m256i v_diff = _mm256_cvtepu8_epi32(load_6_pixels(orig_ptr)); - v_diff = _mm256_sub_epi32(v_diff, _mm256_cvtepu8_epi32(v_c)); - - //Accumulate differences and occurrences for each category - ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT0, v_cat); - ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT1, v_cat); - ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT2, v_cat); - ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT3, v_cat); - ACCUM_COUNT_EO_CAT_AVX2(SAO_EO_CAT4, v_cat); - } - - for (int eo_cat = 0; eo_cat < NUM_SAO_EDGE_CATEGORIES; ++eo_cat) { - int accum = 0; - int count = 0; - - //Full horizontal sum of accumulated values - - v_diff_accum[eo_cat] = _mm256_hadd_epi32(v_diff_accum[eo_cat], v_diff_accum[eo_cat]); - v_diff_accum[eo_cat] = _mm256_hadd_epi32(v_diff_accum[eo_cat], v_diff_accum[eo_cat]); - accum += _mm256_extract_epi32(v_diff_accum[eo_cat], 0) + _mm256_extract_epi32(v_diff_accum[eo_cat], 4); - - - //Full horizontal sum of accumulated values - v_count[eo_cat] = _mm256_hadd_epi32(v_count[eo_cat], v_count[eo_cat]); - v_count[eo_cat] = _mm256_hadd_epi32(v_count[eo_cat], v_count[eo_cat]); - count += _mm256_extract_epi32(v_count[eo_cat], 0) + _mm256_extract_epi32(v_count[eo_cat], 4); - - - cat_sum_cnt[0][eo_cat] += accum; - cat_sum_cnt[1][eo_cat] += count; - - } -} - - static void sao_reconstruct_color_avx2(const encoder_control_t * const encoder, const kvz_pixel *rec_data, kvz_pixel *new_rec_data, const sao_info_t *sao,