From 57db234d95c7d3dd54b83898f2c49facd6560299 Mon Sep 17 00:00:00 2001 From: Pauli Oikkonen Date: Thu, 31 Jan 2019 21:04:30 +0200 Subject: [PATCH] Move 32-wide SSE4.1 hor_sad to picture-sse41.c It's not used by picture-avx2.c that also includes the header, so it should not be in the header --- src/strategies/sse41/picture-sse41.c | 79 +++++++++++++++++++ .../sse41/reg_sad_pow2_widths-sse41.h | 79 ------------------- 2 files changed, 79 insertions(+), 79 deletions(-) diff --git a/src/strategies/sse41/picture-sse41.c b/src/strategies/sse41/picture-sse41.c index cf22ddcb..0493d596 100644 --- a/src/strategies/sse41/picture-sse41.c +++ b/src/strategies/sse41/picture-sse41.c @@ -83,6 +83,85 @@ static uint32_t ver_sad_sse41(const kvz_pixel *pic_data, const kvz_pixel *ref_da return ver_sad_arbitrary(pic_data, ref_data, width, height, stride); } +static uint32_t hor_sad_sse41_w32(const kvz_pixel *pic_data, const kvz_pixel *ref_data, + int32_t width, int32_t height, uint32_t pic_stride, + uint32_t ref_stride, uint32_t left, uint32_t right) +{ + const int32_t height_twoline_groups = height & ~1; + const int32_t height_residual_lines = height & 1; + + __m128i nslo = _mm_setr_epi8(0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15); + __m128i nshi = _mm_setr_epi8(16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31); + __m128i epol_masklo, epol_maskhi; + int32_t border_pix_off; + + if (left) { + border_pix_off = left; + __m128i first_valid_idx = _mm_set1_epi8(left); + + epol_masklo = _mm_cmpgt_epi8(first_valid_idx, nslo); + epol_maskhi = _mm_cmpgt_epi8(first_valid_idx, nshi); + } else { + border_pix_off = width - (right + 1); + __m128i last_valid_idx = _mm_set1_epi8(border_pix_off); + + epol_masklo = _mm_cmpgt_epi8(nslo, last_valid_idx); + epol_maskhi = _mm_cmpgt_epi8(nshi, last_valid_idx); + } + + __m128i sse_inc = _mm_setzero_si128(); + int32_t y; + for (y = 0; y < height_twoline_groups; y += 2) { + __m128i a = _mm_loadu_si128((__m128i *)(pic_data + (y + 0) * pic_stride + 0)); + __m128i b = _mm_loadu_si128((__m128i *)(ref_data + (y + 0) * ref_stride + 0)); + __m128i c = _mm_loadu_si128((__m128i *)(pic_data + (y + 0) * pic_stride + 16)); + __m128i d = _mm_loadu_si128((__m128i *)(ref_data + (y + 0) * ref_stride + 16)); + __m128i e = _mm_loadu_si128((__m128i *)(pic_data + (y + 1) * pic_stride + 0)); + __m128i f = _mm_loadu_si128((__m128i *)(ref_data + (y + 1) * ref_stride + 0)); + __m128i g = _mm_loadu_si128((__m128i *)(pic_data + (y + 1) * pic_stride + 16)); + __m128i h = _mm_loadu_si128((__m128i *)(ref_data + (y + 1) * ref_stride + 16)); + + __m128i border_px_lo = _mm_set1_epi8 (*(uint8_t *)(ref_data + (y + 0) * ref_stride + border_pix_off)); + __m128i border_px_hi = _mm_set1_epi8 (*(uint8_t *)(ref_data + (y + 1) * ref_stride + border_pix_off)); + __m128i b_epol = _mm_blendv_epi8(b, border_px_lo, epol_masklo); + __m128i d_epol = _mm_blendv_epi8(d, border_px_lo, epol_maskhi); + __m128i f_epol = _mm_blendv_epi8(f, border_px_hi, epol_masklo); + __m128i h_epol = _mm_blendv_epi8(h, border_px_hi, epol_maskhi); + + __m128i curr_sads_ab = _mm_sad_epu8(a, b_epol); + __m128i curr_sads_cd = _mm_sad_epu8(c, d_epol); + __m128i curr_sads_ef = _mm_sad_epu8(e, f_epol); + __m128i curr_sads_gh = _mm_sad_epu8(g, h_epol); + + sse_inc = _mm_add_epi64(sse_inc, curr_sads_ab); + sse_inc = _mm_add_epi64(sse_inc, curr_sads_cd); + sse_inc = _mm_add_epi64(sse_inc, curr_sads_ef); + sse_inc = _mm_add_epi64(sse_inc, curr_sads_gh); + } + if (height_residual_lines) { + __m128i a = _mm_loadu_si128((__m128i *)(pic_data + (y + 0) * pic_stride + 0)); + __m128i b = _mm_loadu_si128((__m128i *)(ref_data + (y + 0) * ref_stride + 0)); + __m128i c = _mm_loadu_si128((__m128i *)(pic_data + (y + 0) * pic_stride + 16)); + __m128i d = _mm_loadu_si128((__m128i *)(ref_data + (y + 0) * ref_stride + 16)); + + __m128i border_px = _mm_set1_epi8 (*(uint8_t *)(ref_data + (y + 0) * ref_stride + border_pix_off)); + __m128i b_epol = _mm_blendv_epi8(b, border_px, epol_masklo); + __m128i d_epol = _mm_blendv_epi8(d, border_px, epol_maskhi); + + __m128i curr_sads_ab = _mm_sad_epu8(a, b_epol); + __m128i curr_sads_cd = _mm_sad_epu8(c, d_epol); + + sse_inc = _mm_add_epi64(sse_inc, curr_sads_ab); + sse_inc = _mm_add_epi64(sse_inc, curr_sads_cd); + } + + __m128i sse_inc_2 = _mm_shuffle_epi32(sse_inc, _MM_SHUFFLE(1, 0, 3, 2)); + __m128i sad = _mm_add_epi64 (sse_inc, sse_inc_2); + return _mm_cvtsi128_si32(sad); +} + static uint32_t hor_sad_sse41(const kvz_pixel *pic_data, const kvz_pixel *ref_data, int32_t width, int32_t height, uint32_t pic_stride, uint32_t ref_stride, uint32_t left, uint32_t right) diff --git a/src/strategies/sse41/reg_sad_pow2_widths-sse41.h b/src/strategies/sse41/reg_sad_pow2_widths-sse41.h index 812bbcd1..6a3c1e9c 100644 --- a/src/strategies/sse41/reg_sad_pow2_widths-sse41.h +++ b/src/strategies/sse41/reg_sad_pow2_widths-sse41.h @@ -788,85 +788,6 @@ static uint32_t hor_sad_sse41_w16(const kvz_pixel *pic_data, const kvz_pixel *re return _mm_cvtsi128_si32(sad); } -static uint32_t hor_sad_sse41_w32(const kvz_pixel *pic_data, const kvz_pixel *ref_data, - int32_t width, int32_t height, uint32_t pic_stride, - uint32_t ref_stride, uint32_t left, uint32_t right) -{ - const int32_t height_twoline_groups = height & ~1; - const int32_t height_residual_lines = height & 1; - - __m128i nslo = _mm_setr_epi8(0, 1, 2, 3, 4, 5, 6, 7, - 8, 9, 10, 11, 12, 13, 14, 15); - __m128i nshi = _mm_setr_epi8(16, 17, 18, 19, 20, 21, 22, 23, - 24, 25, 26, 27, 28, 29, 30, 31); - __m128i epol_masklo, epol_maskhi; - int32_t border_pix_off; - - if (left) { - border_pix_off = left; - __m128i first_valid_idx = _mm_set1_epi8(left); - - epol_masklo = _mm_cmpgt_epi8(first_valid_idx, nslo); - epol_maskhi = _mm_cmpgt_epi8(first_valid_idx, nshi); - } else { - border_pix_off = width - (right + 1); - __m128i last_valid_idx = _mm_set1_epi8(border_pix_off); - - epol_masklo = _mm_cmpgt_epi8(nslo, last_valid_idx); - epol_maskhi = _mm_cmpgt_epi8(nshi, last_valid_idx); - } - - __m128i sse_inc = _mm_setzero_si128(); - int32_t y; - for (y = 0; y < height_twoline_groups; y += 2) { - __m128i a = _mm_loadu_si128((__m128i *)(pic_data + (y + 0) * pic_stride + 0)); - __m128i b = _mm_loadu_si128((__m128i *)(ref_data + (y + 0) * ref_stride + 0)); - __m128i c = _mm_loadu_si128((__m128i *)(pic_data + (y + 0) * pic_stride + 16)); - __m128i d = _mm_loadu_si128((__m128i *)(ref_data + (y + 0) * ref_stride + 16)); - __m128i e = _mm_loadu_si128((__m128i *)(pic_data + (y + 1) * pic_stride + 0)); - __m128i f = _mm_loadu_si128((__m128i *)(ref_data + (y + 1) * ref_stride + 0)); - __m128i g = _mm_loadu_si128((__m128i *)(pic_data + (y + 1) * pic_stride + 16)); - __m128i h = _mm_loadu_si128((__m128i *)(ref_data + (y + 1) * ref_stride + 16)); - - __m128i border_px_lo = _mm_set1_epi8 (*(uint8_t *)(ref_data + (y + 0) * ref_stride + border_pix_off)); - __m128i border_px_hi = _mm_set1_epi8 (*(uint8_t *)(ref_data + (y + 1) * ref_stride + border_pix_off)); - __m128i b_epol = _mm_blendv_epi8(b, border_px_lo, epol_masklo); - __m128i d_epol = _mm_blendv_epi8(d, border_px_lo, epol_maskhi); - __m128i f_epol = _mm_blendv_epi8(f, border_px_hi, epol_masklo); - __m128i h_epol = _mm_blendv_epi8(h, border_px_hi, epol_maskhi); - - __m128i curr_sads_ab = _mm_sad_epu8(a, b_epol); - __m128i curr_sads_cd = _mm_sad_epu8(c, d_epol); - __m128i curr_sads_ef = _mm_sad_epu8(e, f_epol); - __m128i curr_sads_gh = _mm_sad_epu8(g, h_epol); - - sse_inc = _mm_add_epi64(sse_inc, curr_sads_ab); - sse_inc = _mm_add_epi64(sse_inc, curr_sads_cd); - sse_inc = _mm_add_epi64(sse_inc, curr_sads_ef); - sse_inc = _mm_add_epi64(sse_inc, curr_sads_gh); - } - if (height_residual_lines) { - __m128i a = _mm_loadu_si128((__m128i *)(pic_data + (y + 0) * pic_stride + 0)); - __m128i b = _mm_loadu_si128((__m128i *)(ref_data + (y + 0) * ref_stride + 0)); - __m128i c = _mm_loadu_si128((__m128i *)(pic_data + (y + 0) * pic_stride + 16)); - __m128i d = _mm_loadu_si128((__m128i *)(ref_data + (y + 0) * ref_stride + 16)); - - __m128i border_px = _mm_set1_epi8 (*(uint8_t *)(ref_data + (y + 0) * ref_stride + border_pix_off)); - __m128i b_epol = _mm_blendv_epi8(b, border_px, epol_masklo); - __m128i d_epol = _mm_blendv_epi8(d, border_px, epol_maskhi); - - __m128i curr_sads_ab = _mm_sad_epu8(a, b_epol); - __m128i curr_sads_cd = _mm_sad_epu8(c, d_epol); - - sse_inc = _mm_add_epi64(sse_inc, curr_sads_ab); - sse_inc = _mm_add_epi64(sse_inc, curr_sads_cd); - } - - __m128i sse_inc_2 = _mm_shuffle_epi32(sse_inc, _MM_SHUFFLE(1, 0, 3, 2)); - __m128i sad = _mm_add_epi64 (sse_inc, sse_inc_2); - return _mm_cvtsi128_si32(sad); -} - static uint32_t hor_sad_sse41_arbitrary(const kvz_pixel *pic_data, const kvz_pixel *ref_data, int32_t width, int32_t height, uint32_t pic_stride, uint32_t ref_stride, uint32_t left, uint32_t right)