diff --git a/src/strategies/avx2/picture-avx2.c b/src/strategies/avx2/picture-avx2.c index ac1cfc26..75be6f2a 100644 --- a/src/strategies/avx2/picture-avx2.c +++ b/src/strategies/avx2/picture-avx2.c @@ -193,6 +193,18 @@ static void hor_add_sub_avx2(__m128i *row0, __m128i *row1){ *row1 = _mm_hsub_epi16(c, d); } +static void hor_add_sub_dual_avx2(__m256i *row0, __m256i *row1){ + + __m256i a = _mm256_hadd_epi16(*row0, *row1); + __m256i b = _mm256_hsub_epi16(*row0, *row1); + + __m256i c = _mm256_hadd_epi16(a, b); + __m256i d = _mm256_hsub_epi16(a, b); + + *row0 = _mm256_hadd_epi16(c, d); + *row1 = _mm256_hsub_epi16(c, d); +} + static INLINE void ver_add_sub_avx2(__m128i (*temp_hor)[8], __m128i (*temp_ver)[8]){ // First stage @@ -216,12 +228,41 @@ static INLINE void ver_add_sub_avx2(__m128i (*temp_hor)[8], __m128i (*temp_ver)[ } } +static INLINE void ver_add_sub_dual_avx2(__m256i (*temp_hor)[8], __m256i (*temp_ver)[8]){ + + // First stage + for (int i = 0; i < 8; i += 2){ + (*temp_ver)[i+0] = _mm256_hadd_epi16((*temp_hor)[i + 0], (*temp_hor)[i + 1]); + (*temp_ver)[i+1] = _mm256_hsub_epi16((*temp_hor)[i + 0], (*temp_hor)[i + 1]); + } + + // Second stage + for (int i = 0; i < 8; i += 4){ + (*temp_hor)[i + 0] = _mm256_add_epi16((*temp_ver)[i + 0], (*temp_ver)[i + 2]); + (*temp_hor)[i + 1] = _mm256_add_epi16((*temp_ver)[i + 1], (*temp_ver)[i + 3]); + (*temp_hor)[i + 2] = _mm256_sub_epi16((*temp_ver)[i + 0], (*temp_ver)[i + 2]); + (*temp_hor)[i + 3] = _mm256_sub_epi16((*temp_ver)[i + 1], (*temp_ver)[i + 3]); + } + + // Third stage + for (int i = 0; i < 4; ++i){ + (*temp_ver)[i + 0] = _mm256_add_epi16((*temp_hor)[0 + i], (*temp_hor)[4 + i]); + (*temp_ver)[i + 4] = _mm256_sub_epi16((*temp_hor)[0 + i], (*temp_hor)[4 + i]); + } +} + INLINE static void haddwd_accumulate_avx2(__m128i *accumulate, __m128i *ver_row) { __m128i abs_value = _mm_abs_epi16(*ver_row); *accumulate = _mm_add_epi32(*accumulate, _mm_madd_epi16(abs_value, _mm_set1_epi16(1))); } +INLINE static void haddwd_accumulate_dual_avx2(__m256i *accumulate, __m256i *ver_row) +{ + __m256i abs_value = _mm256_abs_epi16(*ver_row); + *accumulate = _mm256_add_epi32(*accumulate, _mm256_madd_epi16(abs_value, _mm256_set1_epi16(1))); +} + INLINE static unsigned sum_block_avx2(__m128i *ver_row) { __m128i sad = _mm_setzero_si128(); @@ -240,6 +281,25 @@ INLINE static unsigned sum_block_avx2(__m128i *ver_row) return _mm_cvtsi128_si32(sad); } +INLINE static void sum_block_dual_avx2(__m256i *ver_row, unsigned *sum0, unsigned *sum1) +{ + __m256i sad = _mm256_setzero_si256(); + haddwd_accumulate_dual_avx2(&sad, ver_row + 0); + haddwd_accumulate_dual_avx2(&sad, ver_row + 1); + haddwd_accumulate_dual_avx2(&sad, ver_row + 2); + haddwd_accumulate_dual_avx2(&sad, ver_row + 3); + haddwd_accumulate_dual_avx2(&sad, ver_row + 4); + haddwd_accumulate_dual_avx2(&sad, ver_row + 5); + haddwd_accumulate_dual_avx2(&sad, ver_row + 6); + haddwd_accumulate_dual_avx2(&sad, ver_row + 7); + + sad = _mm256_add_epi32(sad, _mm256_shuffle_epi32(sad, KVZ_PERMUTE(2, 3, 0, 1))); + sad = _mm256_add_epi32(sad, _mm256_shuffle_epi32(sad, KVZ_PERMUTE(1, 0, 1, 0))); + + *sum0 = _mm_cvtsi128_si32(_mm256_extracti128_si256(sad, 0)); + *sum1 = _mm_cvtsi128_si32(_mm256_extracti128_si256(sad, 1)); +} + INLINE static __m128i diff_row_avx2(const kvz_pixel *buf1, const kvz_pixel *buf2) { __m128i buf1_row = _mm_cvtepu8_epi16(_mm_loadl_epi64((__m128i*)buf1)); @@ -247,6 +307,17 @@ INLINE static __m128i diff_row_avx2(const kvz_pixel *buf1, const kvz_pixel *buf2 return _mm_sub_epi16(buf1_row, buf2_row); } +INLINE static __m256i diff_row_dual_avx2(const kvz_pixel *buf1, const kvz_pixel *buf2, const kvz_pixel *orig) +{ + __m128i temp1 = _mm_loadl_epi64((__m128i*)buf1); + __m128i temp2 = _mm_loadl_epi64((__m128i*)buf2); + __m128i temp3 = _mm_loadl_epi64((__m128i*)orig); + __m256i buf1_row = _mm256_cvtepu8_epi16(_mm_unpacklo_epi64(temp1, temp2)); + __m256i buf2_row = _mm256_cvtepu8_epi16(_mm_broadcastq_epi64(temp3)); + + return _mm256_sub_epi16(buf1_row, buf2_row); +} + INLINE static void diff_blocks_and_hor_transform_avx2(__m128i (*row_diff)[8], const kvz_pixel * buf1, unsigned stride1, const kvz_pixel * buf2, unsigned stride2) { (*row_diff)[0] = diff_row_avx2(buf1 + 0 * stride1, buf2 + 0 * stride2); @@ -266,6 +337,28 @@ INLINE static void diff_blocks_and_hor_transform_avx2(__m128i (*row_diff)[8], co hor_add_sub_avx2((*row_diff) + 6, (*row_diff) + 7); } +INLINE static void diff_blocks_and_hor_transform_dual_avx2(__m256i (*row_diff)[8], + const kvz_pixel * buf1, unsigned stride1, + const kvz_pixel * buf2, unsigned stride2, + const kvz_pixel * orig, unsigned stride_orig) +{ + (*row_diff)[0] = diff_row_dual_avx2(buf1 + 0 * stride1, buf2 + 0 * stride2, orig + 0 * stride_orig); + (*row_diff)[1] = diff_row_dual_avx2(buf1 + 1 * stride1, buf2 + 1 * stride2, orig + 1 * stride_orig); + hor_add_sub_dual_avx2((*row_diff) + 0, (*row_diff) + 1); + + (*row_diff)[2] = diff_row_dual_avx2(buf1 + 2 * stride1, buf2 + 2 * stride2, orig + 2 * stride_orig); + (*row_diff)[3] = diff_row_dual_avx2(buf1 + 3 * stride1, buf2 + 3 * stride2, orig + 3 * stride_orig); + hor_add_sub_dual_avx2((*row_diff) + 2, (*row_diff) + 3); + + (*row_diff)[4] = diff_row_dual_avx2(buf1 + 4 * stride1, buf2 + 4 * stride2, orig + 4 * stride_orig); + (*row_diff)[5] = diff_row_dual_avx2(buf1 + 5 * stride1, buf2 + 5 * stride2, orig + 5 * stride_orig); + hor_add_sub_dual_avx2((*row_diff) + 4, (*row_diff) + 5); + + (*row_diff)[6] = diff_row_dual_avx2(buf1 + 6 * stride1, buf2 + 6 * stride2, orig + 6 * stride_orig); + (*row_diff)[7] = diff_row_dual_avx2(buf1 + 7 * stride1, buf2 + 7 * stride2, orig + 7 * stride_orig); + hor_add_sub_dual_avx2((*row_diff) + 6, (*row_diff) + 7); +} + static unsigned kvz_satd_8bit_8x8_general_avx2(const kvz_pixel * buf1, unsigned stride1, const kvz_pixel * buf2, unsigned stride2) { __m128i temp_hor[8]; @@ -318,6 +411,68 @@ SATD_NXN_AVX2(16) SATD_NXN_AVX2(32) SATD_NXN_AVX2(64) +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +static void kvz_satd_8bit_8x8_general_dual_avx2(const kvz_pixel * buf1, unsigned stride1, + const kvz_pixel * buf2, unsigned stride2, + const kvz_pixel * orig, unsigned stride_orig, + unsigned *sum0, unsigned *sum1) +{ + __m256i temp_hor[8]; + __m256i temp_ver[8]; + + diff_blocks_and_hor_transform_dual_avx2(&temp_hor, buf1, stride1, buf2, stride2, orig, stride_orig); + + ver_add_sub_dual_avx2(&temp_hor, &temp_ver); + + sum_block_dual_avx2(temp_ver, sum0, sum1); + + *sum0 = (*sum0 + 2) >> 2; + *sum1 = (*sum1 + 2) >> 2; +} +/* +// Function macro for defining hadamard calculating functions +// for fixed size blocks. They calculate hadamard for integer +// multiples of 8x8 with the 8x8 hadamard function. +#define SATD_NXN_AVX2(n) \ +static unsigned satd_8bit_ ## n ## x ## n ## _dual_avx2( \ + const kvz_pixel * const block1, const kvz_pixel * const block2) \ +{ \ + unsigned x, y; \ + unsigned sum = 0; \ + for (y = 0; y < (n); y += 8) { \ + unsigned row = y * (n); \ + for (x = 0; x < (n); x += 8) { \ + sum += kvz_satd_8bit_8x8_general_avx2(&block1[row + x], (n), &block2[row + x], (n); \ + } \ + } \ + return sum>>(KVZ_BIT_DEPTH-8); \ +} +*/ +static void satd_8bit_8x8_dual_avx2( + const pred_buffer preds, const kvz_pixel * const orig, unsigned num_modes, unsigned *satds_out) +{ + unsigned x, y; + satds_out[0] = 0; + satds_out[1] = 0; + unsigned sum1 = 0; + unsigned sum2 = 0; + for (y = 0; y < (8); y += 8) { + unsigned row = y * (8); + for (x = 0; x < (8); x += 8) { + kvz_satd_8bit_8x8_general_dual_avx2(&preds[0][row + x], (8), &preds[1][row + x], (8), &orig[row + x], (8), &sum1, &sum2); + satds_out[0] += sum1; + satds_out[1] += sum2; + } + } + satds_out[0] = satds_out[0] >>(KVZ_BIT_DEPTH-8); + satds_out[1] = satds_out[1] >>(KVZ_BIT_DEPTH-8); +} +/* +//SATD_NXN_AVX2(8) //Use the non-macro version +SATD_NXN_AVX2(16) +SATD_NXN_AVX2(32) +SATD_NXN_AVX2(64) +*/ void kvz_pixels_blit_avx2(const kvz_pixel * const orig, kvz_pixel * const dst, const unsigned width, const unsigned height, const unsigned orig_stride, const unsigned dst_stride) @@ -422,6 +577,11 @@ int kvz_strategy_register_picture_avx2(void* opaque, uint8_t bitdepth) success &= kvz_strategyselector_register(opaque, "satd_32x32", "avx2", 40, &satd_8bit_32x32_avx2); success &= kvz_strategyselector_register(opaque, "satd_64x64", "avx2", 40, &satd_8bit_64x64_avx2); + success &= kvz_strategyselector_register(opaque, "satd_8x8_dual", "avx2", 40, &satd_8bit_8x8_dual_avx2); + //success &= kvz_strategyselector_register(opaque, "satd_16x16_dual", "avx2", 40, &satd_8bit_16x16_dual_avx2); + //success &= kvz_strategyselector_register(opaque, "satd_32x32_dual", "avx2", 40, &satd_8bit_32x32_dual_avx2); + //success &= kvz_strategyselector_register(opaque, "satd_64x64_dual", "avx2", 40, &satd_8bit_64x64_dual_avx2); + success &= kvz_strategyselector_register(opaque, "pixels_blit", "avx2", 40, &kvz_pixels_blit_avx2); } #endif