King_DuckZ/libbpg
1
0
Fork 0
forked from mirror/libbpg
Code Issues Pull requests Releases Wiki Activity
libbpg/x265/source/common/vec/dct-ssse3.cpp
King_DuckZ 35a8402710 libbpg-0.9.6
2015-10-27 11:46:00 +01:00

1124 lines
55 KiB
C++
Raw Blame History

/*****************************************************************************
* Copyright (C) 2013 x265 project
*
* Authors: Steve Borho <steve@borho.org>
* Mandar Gurav <mandar@multicorewareinc.com>
* Deepthi Devaki Akkoorath <deepthidevaki@multicorewareinc.com>
* Mahesh Pittala <mahesh@multicorewareinc.com>
* Rajesh Paulraj <rajesh@multicorewareinc.com>
* Min Chen <min.chen@multicorewareinc.com>
* Praveen Kumar Tiwari <praveen@multicorewareinc.com>
* Nabajit Deka <nabajit@multicorewareinc.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111, USA.
*
* This program is also available under a commercial proprietary license.
* For more information, contact us at license @ x265.com.
*****************************************************************************/
#include "common.h"
#include "primitives.h"
#include <xmmintrin.h> // SSE
#include <pmmintrin.h> // SSE3
#include <tmmintrin.h> // SSSE3
#define DCT16_SHIFT1 (3 + X265_DEPTH - 8)
#define DCT16_ADD1 (1 << ((DCT16_SHIFT1) - 1))
#define DCT16_SHIFT2 10
#define DCT16_ADD2 (1 << ((DCT16_SHIFT2) - 1))
#define DCT32_SHIFT1 (DCT16_SHIFT1 + 1)
#define DCT32_ADD1 (1 << ((DCT32_SHIFT1) - 1))
#define DCT32_SHIFT2 (DCT16_SHIFT2 + 1)
#define DCT32_ADD2 (1 << ((DCT32_SHIFT2) - 1))
using namespace X265_NS;
ALIGN_VAR_32(static const int16_t, tab_dct_8[][8]) =
{
{ 0x0100, 0x0F0E, 0x0706, 0x0908, 0x0302, 0x0D0C, 0x0504, 0x0B0A },
{ 64, 64, 64, 64, 64, 64, 64, 64 },
{ 64, -64, 64, -64, 64, -64, 64, -64 },
{ 83, 36, 83, 36, 83, 36, 83, 36 },
{ 36, -83, 36, -83, 36, -83, 36, -83 },
{ 89, 18, 75, 50, 89, 18, 75, 50 },
{ 75, -50, -18, -89, 75, -50, -18, -89 },
{ 50, 75, -89, 18, 50, 75, -89, 18 },
{ 18, -89, -50, 75, 18, -89, -50, 75 },
{ 83, 83, -83, -83, 36, 36, -36, -36 },
{ 36, 36, -36, -36, -83, -83, 83, 83 },
{ 89, -89, 18, -18, 75, -75, 50, -50 },
{ 75, -75, -50, 50, -18, 18, -89, 89 },
{ 50, -50, 75, -75, -89, 89, 18, -18 },
{ 18, -18, -89, 89, -50, 50, 75, -75 },
};
ALIGN_VAR_32(static const int16_t, tab_dct_16_0[][8]) =
{
{ 0x0F0E, 0x0D0C, 0x0B0A, 0x0908, 0x0706, 0x0504, 0x0302, 0x0100 }, // 0
{ 0x0100, 0x0F0E, 0x0706, 0x0908, 0x0302, 0x0D0C, 0x0504, 0x0B0A }, // 1
{ 0x0100, 0x0706, 0x0302, 0x0504, 0x0F0E, 0x0908, 0x0D0C, 0x0B0A }, // 2
{ 0x0F0E, 0x0908, 0x0D0C, 0x0B0A, 0x0100, 0x0706, 0x0302, 0x0504 }, // 3
};
ALIGN_VAR_32(static const int16_t, tab_dct_16_1[][8]) =
{
{ 90, 87, 80, 70, 57, 43, 25, 9 }, // 0
{ 87, 57, 9, -43, -80, -90, -70, -25 }, // 1
{ 80, 9, -70, -87, -25, 57, 90, 43 }, // 2
{ 70, -43, -87, 9, 90, 25, -80, -57 }, // 3
{ 57, -80, -25, 90, -9, -87, 43, 70 }, // 4
{ 43, -90, 57, 25, -87, 70, 9, -80 }, // 5
{ 25, -70, 90, -80, 43, 9, -57, 87 }, // 6
{ 9, -25, 43, -57, 70, -80, 87, -90 }, // 7
{ 83, 83, -83, -83, 36, 36, -36, -36 }, // 8
{ 36, 36, -36, -36, -83, -83, 83, 83 }, // 9
{ 89, 89, 18, 18, 75, 75, 50, 50 }, // 10
{ 75, 75, -50, -50, -18, -18, -89, -89 }, // 11
{ 50, 50, 75, 75, -89, -89, 18, 18 }, // 12
{ 18, 18, -89, -89, -50, -50, 75, 75 }, // 13
#define MAKE_COEF(a0, a1, a2, a3, a4, a5, a6, a7) \
{ (a0), -(a0), (a3), -(a3), (a1), -(a1), (a2), -(a2) \
}, \
{ (a7), -(a7), (a4), -(a4), (a6), -(a6), (a5), -(a5) },
MAKE_COEF(90, 87, 80, 70, 57, 43, 25, 9)
MAKE_COEF(87, 57, 9, -43, -80, -90, -70, -25)
MAKE_COEF(80, 9, -70, -87, -25, 57, 90, 43)
MAKE_COEF(70, -43, -87, 9, 90, 25, -80, -57)
MAKE_COEF(57, -80, -25, 90, -9, -87, 43, 70)
MAKE_COEF(43, -90, 57, 25, -87, 70, 9, -80)
MAKE_COEF(25, -70, 90, -80, 43, 9, -57, 87)
MAKE_COEF(9, -25, 43, -57, 70, -80, 87, -90)
#undef MAKE_COEF
};
static void dct16(const int16_t *src, int16_t *dst, intptr_t stride)
{
// Const
__m128i c_4 = _mm_set1_epi32(DCT16_ADD1);
__m128i c_512 = _mm_set1_epi32(DCT16_ADD2);
int i;
ALIGN_VAR_32(int16_t, tmp[16 * 16]);
__m128i T00A, T01A, T02A, T03A, T04A, T05A, T06A, T07A;
__m128i T00B, T01B, T02B, T03B, T04B, T05B, T06B, T07B;
__m128i T10, T11, T12, T13, T14, T15, T16, T17;
__m128i T20, T21, T22, T23, T24, T25, T26, T27;
__m128i T30, T31, T32, T33, T34, T35, T36, T37;
__m128i T40, T41, T42, T43, T44, T45, T46, T47;
__m128i T50, T51, T52, T53;
__m128i T60, T61, T62, T63, T64, T65, T66, T67;
__m128i T70;
// DCT1
for (i = 0; i < 16; i += 8)
{
T00A = _mm_load_si128((__m128i*)&src[(i + 0) * stride + 0]); // [07 06 05 04 03 02 01 00]
T00B = _mm_load_si128((__m128i*)&src[(i + 0) * stride + 8]); // [0F 0E 0D 0C 0B 0A 09 08]
T01A = _mm_load_si128((__m128i*)&src[(i + 1) * stride + 0]); // [17 16 15 14 13 12 11 10]
T01B = _mm_load_si128((__m128i*)&src[(i + 1) * stride + 8]); // [1F 1E 1D 1C 1B 1A 19 18]
T02A = _mm_load_si128((__m128i*)&src[(i + 2) * stride + 0]); // [27 26 25 24 23 22 21 20]
T02B = _mm_load_si128((__m128i*)&src[(i + 2) * stride + 8]); // [2F 2E 2D 2C 2B 2A 29 28]
T03A = _mm_load_si128((__m128i*)&src[(i + 3) * stride + 0]); // [37 36 35 34 33 32 31 30]
T03B = _mm_load_si128((__m128i*)&src[(i + 3) * stride + 8]); // [3F 3E 3D 3C 3B 3A 39 38]
T04A = _mm_load_si128((__m128i*)&src[(i + 4) * stride + 0]); // [47 46 45 44 43 42 41 40]
T04B = _mm_load_si128((__m128i*)&src[(i + 4) * stride + 8]); // [4F 4E 4D 4C 4B 4A 49 48]
T05A = _mm_load_si128((__m128i*)&src[(i + 5) * stride + 0]); // [57 56 55 54 53 52 51 50]
T05B = _mm_load_si128((__m128i*)&src[(i + 5) * stride + 8]); // [5F 5E 5D 5C 5B 5A 59 58]
T06A = _mm_load_si128((__m128i*)&src[(i + 6) * stride + 0]); // [67 66 65 64 63 62 61 60]
T06B = _mm_load_si128((__m128i*)&src[(i + 6) * stride + 8]); // [6F 6E 6D 6C 6B 6A 69 68]
T07A = _mm_load_si128((__m128i*)&src[(i + 7) * stride + 0]); // [77 76 75 74 73 72 71 70]
T07B = _mm_load_si128((__m128i*)&src[(i + 7) * stride + 8]); // [7F 7E 7D 7C 7B 7A 79 78]
T00B = _mm_shuffle_epi8(T00B, _mm_load_si128((__m128i*)tab_dct_16_0[0]));
T01B = _mm_shuffle_epi8(T01B, _mm_load_si128((__m128i*)tab_dct_16_0[0]));
T02B = _mm_shuffle_epi8(T02B, _mm_load_si128((__m128i*)tab_dct_16_0[0]));
T03B = _mm_shuffle_epi8(T03B, _mm_load_si128((__m128i*)tab_dct_16_0[0]));
T04B = _mm_shuffle_epi8(T04B, _mm_load_si128((__m128i*)tab_dct_16_0[0]));
T05B = _mm_shuffle_epi8(T05B, _mm_load_si128((__m128i*)tab_dct_16_0[0]));
T06B = _mm_shuffle_epi8(T06B, _mm_load_si128((__m128i*)tab_dct_16_0[0]));
T07B = _mm_shuffle_epi8(T07B, _mm_load_si128((__m128i*)tab_dct_16_0[0]));
T10 = _mm_add_epi16(T00A, T00B);
T11 = _mm_add_epi16(T01A, T01B);
T12 = _mm_add_epi16(T02A, T02B);
T13 = _mm_add_epi16(T03A, T03B);
T14 = _mm_add_epi16(T04A, T04B);
T15 = _mm_add_epi16(T05A, T05B);
T16 = _mm_add_epi16(T06A, T06B);
T17 = _mm_add_epi16(T07A, T07B);
T20 = _mm_sub_epi16(T00A, T00B);
T21 = _mm_sub_epi16(T01A, T01B);
T22 = _mm_sub_epi16(T02A, T02B);
T23 = _mm_sub_epi16(T03A, T03B);
T24 = _mm_sub_epi16(T04A, T04B);
T25 = _mm_sub_epi16(T05A, T05B);
T26 = _mm_sub_epi16(T06A, T06B);
T27 = _mm_sub_epi16(T07A, T07B);
T30 = _mm_shuffle_epi8(T10, _mm_load_si128((__m128i*)tab_dct_16_0[1]));
T31 = _mm_shuffle_epi8(T11, _mm_load_si128((__m128i*)tab_dct_16_0[1]));
T32 = _mm_shuffle_epi8(T12, _mm_load_si128((__m128i*)tab_dct_16_0[1]));
T33 = _mm_shuffle_epi8(T13, _mm_load_si128((__m128i*)tab_dct_16_0[1]));
T34 = _mm_shuffle_epi8(T14, _mm_load_si128((__m128i*)tab_dct_16_0[1]));
T35 = _mm_shuffle_epi8(T15, _mm_load_si128((__m128i*)tab_dct_16_0[1]));
T36 = _mm_shuffle_epi8(T16, _mm_load_si128((__m128i*)tab_dct_16_0[1]));
T37 = _mm_shuffle_epi8(T17, _mm_load_si128((__m128i*)tab_dct_16_0[1]));
T40 = _mm_hadd_epi16(T30, T31);
T41 = _mm_hadd_epi16(T32, T33);
T42 = _mm_hadd_epi16(T34, T35);
T43 = _mm_hadd_epi16(T36, T37);
T44 = _mm_hsub_epi16(T30, T31);
T45 = _mm_hsub_epi16(T32, T33);
T46 = _mm_hsub_epi16(T34, T35);
T47 = _mm_hsub_epi16(T36, T37);
T50 = _mm_hadd_epi16(T40, T41);
T51 = _mm_hadd_epi16(T42, T43);
T52 = _mm_hsub_epi16(T40, T41);
T53 = _mm_hsub_epi16(T42, T43);
T60 = _mm_madd_epi16(T50, _mm_load_si128((__m128i*)tab_dct_8[1]));
T61 = _mm_madd_epi16(T51, _mm_load_si128((__m128i*)tab_dct_8[1]));
T60 = _mm_srai_epi32(_mm_add_epi32(T60, c_4), DCT16_SHIFT1);
T61 = _mm_srai_epi32(_mm_add_epi32(T61, c_4), DCT16_SHIFT1);
T70 = _mm_packs_epi32(T60, T61);
_mm_store_si128((__m128i*)&tmp[0 * 16 + i], T70);
T60 = _mm_madd_epi16(T50, _mm_load_si128((__m128i*)tab_dct_8[2]));
T61 = _mm_madd_epi16(T51, _mm_load_si128((__m128i*)tab_dct_8[2]));
T60 = _mm_srai_epi32(_mm_add_epi32(T60, c_4), DCT16_SHIFT1);
T61 = _mm_srai_epi32(_mm_add_epi32(T61, c_4), DCT16_SHIFT1);
T70 = _mm_packs_epi32(T60, T61);
_mm_store_si128((__m128i*)&tmp[8 * 16 + i], T70);
T60 = _mm_madd_epi16(T52, _mm_load_si128((__m128i*)tab_dct_8[3]));
T61 = _mm_madd_epi16(T53, _mm_load_si128((__m128i*)tab_dct_8[3]));
T60 = _mm_srai_epi32(_mm_add_epi32(T60, c_4), DCT16_SHIFT1);
T61 = _mm_srai_epi32(_mm_add_epi32(T61, c_4), DCT16_SHIFT1);
T70 = _mm_packs_epi32(T60, T61);
_mm_store_si128((__m128i*)&tmp[4 * 16 + i], T70);
T60 = _mm_madd_epi16(T52, _mm_load_si128((__m128i*)tab_dct_8[4]));
T61 = _mm_madd_epi16(T53, _mm_load_si128((__m128i*)tab_dct_8[4]));
T60 = _mm_srai_epi32(_mm_add_epi32(T60, c_4), DCT16_SHIFT1);
T61 = _mm_srai_epi32(_mm_add_epi32(T61, c_4), DCT16_SHIFT1);
T70 = _mm_packs_epi32(T60, T61);
_mm_store_si128((__m128i*)&tmp[12 * 16 + i], T70);
T60 = _mm_madd_epi16(T44, _mm_load_si128((__m128i*)tab_dct_8[5]));
T61 = _mm_madd_epi16(T45, _mm_load_si128((__m128i*)tab_dct_8[5]));
T62 = _mm_madd_epi16(T46, _mm_load_si128((__m128i*)tab_dct_8[5]));
T63 = _mm_madd_epi16(T47, _mm_load_si128((__m128i*)tab_dct_8[5]));
T60 = _mm_hadd_epi32(T60, T61);
T61 = _mm_hadd_epi32(T62, T63);
T60 = _mm_srai_epi32(_mm_add_epi32(T60, c_4), DCT16_SHIFT1);
T61 = _mm_srai_epi32(_mm_add_epi32(T61, c_4), DCT16_SHIFT1);
T70 = _mm_packs_epi32(T60, T61);
_mm_store_si128((__m128i*)&tmp[2 * 16 + i], T70);
T60 = _mm_madd_epi16(T44, _mm_load_si128((__m128i*)tab_dct_8[6]));
T61 = _mm_madd_epi16(T45, _mm_load_si128((__m128i*)tab_dct_8[6]));
T62 = _mm_madd_epi16(T46, _mm_load_si128((__m128i*)tab_dct_8[6]));
T63 = _mm_madd_epi16(T47, _mm_load_si128((__m128i*)tab_dct_8[6]));
T60 = _mm_hadd_epi32(T60, T61);
T61 = _mm_hadd_epi32(T62, T63);
T60 = _mm_srai_epi32(_mm_add_epi32(T60, c_4), DCT16_SHIFT1);
T61 = _mm_srai_epi32(_mm_add_epi32(T61, c_4), DCT16_SHIFT1);
T70 = _mm_packs_epi32(T60, T61);
_mm_store_si128((__m128i*)&tmp[6 * 16 + i], T70);
T60 = _mm_madd_epi16(T44, _mm_load_si128((__m128i*)tab_dct_8[7]));
T61 = _mm_madd_epi16(T45, _mm_load_si128((__m128i*)tab_dct_8[7]));
T62 = _mm_madd_epi16(T46, _mm_load_si128((__m128i*)tab_dct_8[7]));
T63 = _mm_madd_epi16(T47, _mm_load_si128((__m128i*)tab_dct_8[7]));
T60 = _mm_hadd_epi32(T60, T61);
T61 = _mm_hadd_epi32(T62, T63);
T60 = _mm_srai_epi32(_mm_add_epi32(T60, c_4), DCT16_SHIFT1);
T61 = _mm_srai_epi32(_mm_add_epi32(T61, c_4), DCT16_SHIFT1);
T70 = _mm_packs_epi32(T60, T61);
_mm_store_si128((__m128i*)&tmp[10 * 16 + i], T70);
T60 = _mm_madd_epi16(T44, _mm_load_si128((__m128i*)tab_dct_8[8]));
T61 = _mm_madd_epi16(T45, _mm_load_si128((__m128i*)tab_dct_8[8]));
T62 = _mm_madd_epi16(T46, _mm_load_si128((__m128i*)tab_dct_8[8]));
T63 = _mm_madd_epi16(T47, _mm_load_si128((__m128i*)tab_dct_8[8]));
T60 = _mm_hadd_epi32(T60, T61);
T61 = _mm_hadd_epi32(T62, T63);
T60 = _mm_srai_epi32(_mm_add_epi32(T60, c_4), DCT16_SHIFT1);
T61 = _mm_srai_epi32(_mm_add_epi32(T61, c_4), DCT16_SHIFT1);
T70 = _mm_packs_epi32(T60, T61);
_mm_store_si128((__m128i*)&tmp[14 * 16 + i], T70);
#define MAKE_ODD(tab, dstPos) \
T60 = _mm_madd_epi16(T20, _mm_load_si128((__m128i*)tab_dct_16_1[(tab)])); \
T61 = _mm_madd_epi16(T21, _mm_load_si128((__m128i*)tab_dct_16_1[(tab)])); \
T62 = _mm_madd_epi16(T22, _mm_load_si128((__m128i*)tab_dct_16_1[(tab)])); \
T63 = _mm_madd_epi16(T23, _mm_load_si128((__m128i*)tab_dct_16_1[(tab)])); \
T64 = _mm_madd_epi16(T24, _mm_load_si128((__m128i*)tab_dct_16_1[(tab)])); \
T65 = _mm_madd_epi16(T25, _mm_load_si128((__m128i*)tab_dct_16_1[(tab)])); \
T66 = _mm_madd_epi16(T26, _mm_load_si128((__m128i*)tab_dct_16_1[(tab)])); \
T67 = _mm_madd_epi16(T27, _mm_load_si128((__m128i*)tab_dct_16_1[(tab)])); \
T60 = _mm_hadd_epi32(T60, T61); \
T61 = _mm_hadd_epi32(T62, T63); \
T62 = _mm_hadd_epi32(T64, T65); \
T63 = _mm_hadd_epi32(T66, T67); \
T60 = _mm_hadd_epi32(T60, T61); \
T61 = _mm_hadd_epi32(T62, T63); \
T60 = _mm_srai_epi32(_mm_add_epi32(T60, c_4), DCT16_SHIFT1); \
T61 = _mm_srai_epi32(_mm_add_epi32(T61, c_4), DCT16_SHIFT1); \
T70 = _mm_packs_epi32(T60, T61); \
_mm_store_si128((__m128i*)&tmp[(dstPos) * 16 + i], T70);
MAKE_ODD(0, 1);
MAKE_ODD(1, 3);
MAKE_ODD(2, 5);
MAKE_ODD(3, 7);
MAKE_ODD(4, 9);
MAKE_ODD(5, 11);
MAKE_ODD(6, 13);
MAKE_ODD(7, 15);
#undef MAKE_ODD
}
// DCT2
for (i = 0; i < 16; i += 4)
{
T00A = _mm_load_si128((__m128i*)&tmp[(i + 0) * 16 + 0]); // [07 06 05 04 03 02 01 00]
T00B = _mm_load_si128((__m128i*)&tmp[(i + 0) * 16 + 8]); // [0F 0E 0D 0C 0B 0A 09 08]
T01A = _mm_load_si128((__m128i*)&tmp[(i + 1) * 16 + 0]); // [17 16 15 14 13 12 11 10]
T01B = _mm_load_si128((__m128i*)&tmp[(i + 1) * 16 + 8]); // [1F 1E 1D 1C 1B 1A 19 18]
T02A = _mm_load_si128((__m128i*)&tmp[(i + 2) * 16 + 0]); // [27 26 25 24 23 22 21 20]
T02B = _mm_load_si128((__m128i*)&tmp[(i + 2) * 16 + 8]); // [2F 2E 2D 2C 2B 2A 29 28]
T03A = _mm_load_si128((__m128i*)&tmp[(i + 3) * 16 + 0]); // [37 36 35 34 33 32 31 30]
T03B = _mm_load_si128((__m128i*)&tmp[(i + 3) * 16 + 8]); // [3F 3E 3D 3C 3B 3A 39 38]
T00A = _mm_shuffle_epi8(T00A, _mm_load_si128((__m128i*)tab_dct_16_0[2]));
T00B = _mm_shuffle_epi8(T00B, _mm_load_si128((__m128i*)tab_dct_16_0[3]));
T01A = _mm_shuffle_epi8(T01A, _mm_load_si128((__m128i*)tab_dct_16_0[2]));
T01B = _mm_shuffle_epi8(T01B, _mm_load_si128((__m128i*)tab_dct_16_0[3]));
T02A = _mm_shuffle_epi8(T02A, _mm_load_si128((__m128i*)tab_dct_16_0[2]));
T02B = _mm_shuffle_epi8(T02B, _mm_load_si128((__m128i*)tab_dct_16_0[3]));
T03A = _mm_shuffle_epi8(T03A, _mm_load_si128((__m128i*)tab_dct_16_0[2]));
T03B = _mm_shuffle_epi8(T03B, _mm_load_si128((__m128i*)tab_dct_16_0[3]));
T10 = _mm_unpacklo_epi16(T00A, T00B);
T11 = _mm_unpackhi_epi16(T00A, T00B);
T12 = _mm_unpacklo_epi16(T01A, T01B);
T13 = _mm_unpackhi_epi16(T01A, T01B);
T14 = _mm_unpacklo_epi16(T02A, T02B);
T15 = _mm_unpackhi_epi16(T02A, T02B);
T16 = _mm_unpacklo_epi16(T03A, T03B);
T17 = _mm_unpackhi_epi16(T03A, T03B);
T20 = _mm_madd_epi16(T10, _mm_load_si128((__m128i*)tab_dct_8[1]));
T21 = _mm_madd_epi16(T11, _mm_load_si128((__m128i*)tab_dct_8[1]));
T22 = _mm_madd_epi16(T12, _mm_load_si128((__m128i*)tab_dct_8[1]));
T23 = _mm_madd_epi16(T13, _mm_load_si128((__m128i*)tab_dct_8[1]));
T24 = _mm_madd_epi16(T14, _mm_load_si128((__m128i*)tab_dct_8[1]));
T25 = _mm_madd_epi16(T15, _mm_load_si128((__m128i*)tab_dct_8[1]));
T26 = _mm_madd_epi16(T16, _mm_load_si128((__m128i*)tab_dct_8[1]));
T27 = _mm_madd_epi16(T17, _mm_load_si128((__m128i*)tab_dct_8[1]));
T30 = _mm_add_epi32(T20, T21);
T31 = _mm_add_epi32(T22, T23);
T32 = _mm_add_epi32(T24, T25);
T33 = _mm_add_epi32(T26, T27);
T30 = _mm_hadd_epi32(T30, T31);
T31 = _mm_hadd_epi32(T32, T33);
T40 = _mm_hadd_epi32(T30, T31);
T41 = _mm_hsub_epi32(T30, T31);
T40 = _mm_srai_epi32(_mm_add_epi32(T40, c_512), DCT16_SHIFT2);
T41 = _mm_srai_epi32(_mm_add_epi32(T41, c_512), DCT16_SHIFT2);
T40 = _mm_packs_epi32(T40, T40);
T41 = _mm_packs_epi32(T41, T41);
_mm_storel_epi64((__m128i*)&dst[0 * 16 + i], T40);
_mm_storel_epi64((__m128i*)&dst[8 * 16 + i], T41);
T20 = _mm_madd_epi16(T10, _mm_load_si128((__m128i*)tab_dct_16_1[8]));
T21 = _mm_madd_epi16(T11, _mm_load_si128((__m128i*)tab_dct_16_1[8]));
T22 = _mm_madd_epi16(T12, _mm_load_si128((__m128i*)tab_dct_16_1[8]));
T23 = _mm_madd_epi16(T13, _mm_load_si128((__m128i*)tab_dct_16_1[8]));
T24 = _mm_madd_epi16(T14, _mm_load_si128((__m128i*)tab_dct_16_1[8]));
T25 = _mm_madd_epi16(T15, _mm_load_si128((__m128i*)tab_dct_16_1[8]));
T26 = _mm_madd_epi16(T16, _mm_load_si128((__m128i*)tab_dct_16_1[8]));
T27 = _mm_madd_epi16(T17, _mm_load_si128((__m128i*)tab_dct_16_1[8]));
T30 = _mm_add_epi32(T20, T21);
T31 = _mm_add_epi32(T22, T23);
T32 = _mm_add_epi32(T24, T25);
T33 = _mm_add_epi32(T26, T27);
T30 = _mm_hadd_epi32(T30, T31);
T31 = _mm_hadd_epi32(T32, T33);
T40 = _mm_hadd_epi32(T30, T31);
T40 = _mm_srai_epi32(_mm_add_epi32(T40, c_512), DCT16_SHIFT2);
T40 = _mm_packs_epi32(T40, T40);
_mm_storel_epi64((__m128i*)&dst[4 * 16 + i], T40);
T20 = _mm_madd_epi16(T10, _mm_load_si128((__m128i*)tab_dct_16_1[9]));
T21 = _mm_madd_epi16(T11, _mm_load_si128((__m128i*)tab_dct_16_1[9]));
T22 = _mm_madd_epi16(T12, _mm_load_si128((__m128i*)tab_dct_16_1[9]));
T23 = _mm_madd_epi16(T13, _mm_load_si128((__m128i*)tab_dct_16_1[9]));
T24 = _mm_madd_epi16(T14, _mm_load_si128((__m128i*)tab_dct_16_1[9]));
T25 = _mm_madd_epi16(T15, _mm_load_si128((__m128i*)tab_dct_16_1[9]));
T26 = _mm_madd_epi16(T16, _mm_load_si128((__m128i*)tab_dct_16_1[9]));
T27 = _mm_madd_epi16(T17, _mm_load_si128((__m128i*)tab_dct_16_1[9]));
T30 = _mm_add_epi32(T20, T21);
T31 = _mm_add_epi32(T22, T23);
T32 = _mm_add_epi32(T24, T25);
T33 = _mm_add_epi32(T26, T27);
T30 = _mm_hadd_epi32(T30, T31);
T31 = _mm_hadd_epi32(T32, T33);
T40 = _mm_hadd_epi32(T30, T31);
T40 = _mm_srai_epi32(_mm_add_epi32(T40, c_512), DCT16_SHIFT2);
T40 = _mm_packs_epi32(T40, T40);
_mm_storel_epi64((__m128i*)&dst[12 * 16 + i], T40);
T20 = _mm_madd_epi16(T10, _mm_load_si128((__m128i*)tab_dct_16_1[10]));
T21 = _mm_madd_epi16(T11, _mm_load_si128((__m128i*)tab_dct_16_1[10]));
T22 = _mm_madd_epi16(T12, _mm_load_si128((__m128i*)tab_dct_16_1[10]));
T23 = _mm_madd_epi16(T13, _mm_load_si128((__m128i*)tab_dct_16_1[10]));
T24 = _mm_madd_epi16(T14, _mm_load_si128((__m128i*)tab_dct_16_1[10]));
T25 = _mm_madd_epi16(T15, _mm_load_si128((__m128i*)tab_dct_16_1[10]));
T26 = _mm_madd_epi16(T16, _mm_load_si128((__m128i*)tab_dct_16_1[10]));
T27 = _mm_madd_epi16(T17, _mm_load_si128((__m128i*)tab_dct_16_1[10]));
T30 = _mm_sub_epi32(T20, T21);
T31 = _mm_sub_epi32(T22, T23);
T32 = _mm_sub_epi32(T24, T25);
T33 = _mm_sub_epi32(T26, T27);
T30 = _mm_hadd_epi32(T30, T31);
T31 = _mm_hadd_epi32(T32, T33);
T40 = _mm_hadd_epi32(T30, T31);
T40 = _mm_srai_epi32(_mm_add_epi32(T40, c_512), DCT16_SHIFT2);
T40 = _mm_packs_epi32(T40, T40);
_mm_storel_epi64((__m128i*)&dst[2 * 16 + i], T40);
T20 = _mm_madd_epi16(T10, _mm_load_si128((__m128i*)tab_dct_16_1[11]));
T21 = _mm_madd_epi16(T11, _mm_load_si128((__m128i*)tab_dct_16_1[11]));
T22 = _mm_madd_epi16(T12, _mm_load_si128((__m128i*)tab_dct_16_1[11]));
T23 = _mm_madd_epi16(T13, _mm_load_si128((__m128i*)tab_dct_16_1[11]));
T24 = _mm_madd_epi16(T14, _mm_load_si128((__m128i*)tab_dct_16_1[11]));
T25 = _mm_madd_epi16(T15, _mm_load_si128((__m128i*)tab_dct_16_1[11]));
T26 = _mm_madd_epi16(T16, _mm_load_si128((__m128i*)tab_dct_16_1[11]));
T27 = _mm_madd_epi16(T17, _mm_load_si128((__m128i*)tab_dct_16_1[11]));
T30 = _mm_sub_epi32(T20, T21);
T31 = _mm_sub_epi32(T22, T23);
T32 = _mm_sub_epi32(T24, T25);
T33 = _mm_sub_epi32(T26, T27);
T30 = _mm_hadd_epi32(T30, T31);
T31 = _mm_hadd_epi32(T32, T33);
T40 = _mm_hadd_epi32(T30, T31);
T40 = _mm_srai_epi32(_mm_add_epi32(T40, c_512), DCT16_SHIFT2);
T40 = _mm_packs_epi32(T40, T40);
_mm_storel_epi64((__m128i*)&dst[6 * 16 + i], T40);
T20 = _mm_madd_epi16(T10, _mm_load_si128((__m128i*)tab_dct_16_1[12]));
T21 = _mm_madd_epi16(T11, _mm_load_si128((__m128i*)tab_dct_16_1[12]));
T22 = _mm_madd_epi16(T12, _mm_load_si128((__m128i*)tab_dct_16_1[12]));
T23 = _mm_madd_epi16(T13, _mm_load_si128((__m128i*)tab_dct_16_1[12]));
T24 = _mm_madd_epi16(T14, _mm_load_si128((__m128i*)tab_dct_16_1[12]));
T25 = _mm_madd_epi16(T15, _mm_load_si128((__m128i*)tab_dct_16_1[12]));
T26 = _mm_madd_epi16(T16, _mm_load_si128((__m128i*)tab_dct_16_1[12]));
T27 = _mm_madd_epi16(T17, _mm_load_si128((__m128i*)tab_dct_16_1[12]));
T30 = _mm_sub_epi32(T20, T21);
T31 = _mm_sub_epi32(T22, T23);
T32 = _mm_sub_epi32(T24, T25);
T33 = _mm_sub_epi32(T26, T27);
T30 = _mm_hadd_epi32(T30, T31);
T31 = _mm_hadd_epi32(T32, T33);
T40 = _mm_hadd_epi32(T30, T31);
T40 = _mm_srai_epi32(_mm_add_epi32(T40, c_512), DCT16_SHIFT2);
T40 = _mm_packs_epi32(T40, T40);
_mm_storel_epi64((__m128i*)&dst[10 * 16 + i], T40);
T20 = _mm_madd_epi16(T10, _mm_load_si128((__m128i*)tab_dct_16_1[13]));
T21 = _mm_madd_epi16(T11, _mm_load_si128((__m128i*)tab_dct_16_1[13]));
T22 = _mm_madd_epi16(T12, _mm_load_si128((__m128i*)tab_dct_16_1[13]));
T23 = _mm_madd_epi16(T13, _mm_load_si128((__m128i*)tab_dct_16_1[13]));
T24 = _mm_madd_epi16(T14, _mm_load_si128((__m128i*)tab_dct_16_1[13]));
T25 = _mm_madd_epi16(T15, _mm_load_si128((__m128i*)tab_dct_16_1[13]));
T26 = _mm_madd_epi16(T16, _mm_load_si128((__m128i*)tab_dct_16_1[13]));
T27 = _mm_madd_epi16(T17, _mm_load_si128((__m128i*)tab_dct_16_1[13]));
T30 = _mm_sub_epi32(T20, T21);
T31 = _mm_sub_epi32(T22, T23);
T32 = _mm_sub_epi32(T24, T25);
T33 = _mm_sub_epi32(T26, T27);
T30 = _mm_hadd_epi32(T30, T31);
T31 = _mm_hadd_epi32(T32, T33);
T40 = _mm_hadd_epi32(T30, T31);
T40 = _mm_srai_epi32(_mm_add_epi32(T40, c_512), DCT16_SHIFT2);
T40 = _mm_packs_epi32(T40, T40);
_mm_storel_epi64((__m128i*)&dst[14 * 16 + i], T40);
#define MAKE_ODD(tab, dstPos) \
T20 = _mm_madd_epi16(T10, _mm_load_si128((__m128i*)tab_dct_16_1[(tab)])); /* [*O2_0 *O1_0 *O3_0 *O0_0] */ \
T21 = _mm_madd_epi16(T11, _mm_load_si128((__m128i*)tab_dct_16_1[(tab) + 1])); /* [*O5_0 *O6_0 *O4_0 *O7_0] */ \
T22 = _mm_madd_epi16(T12, _mm_load_si128((__m128i*)tab_dct_16_1[(tab)])); \
T23 = _mm_madd_epi16(T13, _mm_load_si128((__m128i*)tab_dct_16_1[(tab) + 1])); \
T24 = _mm_madd_epi16(T14, _mm_load_si128((__m128i*)tab_dct_16_1[(tab)])); \
T25 = _mm_madd_epi16(T15, _mm_load_si128((__m128i*)tab_dct_16_1[(tab) + 1])); \
T26 = _mm_madd_epi16(T16, _mm_load_si128((__m128i*)tab_dct_16_1[(tab)])); \
T27 = _mm_madd_epi16(T17, _mm_load_si128((__m128i*)tab_dct_16_1[(tab) + 1])); \
\
T30 = _mm_add_epi32(T20, T21); \
T31 = _mm_add_epi32(T22, T23); \
T32 = _mm_add_epi32(T24, T25); \
T33 = _mm_add_epi32(T26, T27); \
\
T30 = _mm_hadd_epi32(T30, T31); \
T31 = _mm_hadd_epi32(T32, T33); \
\
T40 = _mm_hadd_epi32(T30, T31); \
T40 = _mm_srai_epi32(_mm_add_epi32(T40, c_512), DCT16_SHIFT2); \
T40 = _mm_packs_epi32(T40, T40); \
_mm_storel_epi64((__m128i*)&dst[(dstPos) * 16 + i], T40);
MAKE_ODD(14, 1);
MAKE_ODD(16, 3);
MAKE_ODD(18, 5);
MAKE_ODD(20, 7);
MAKE_ODD(22, 9);
MAKE_ODD(24, 11);
MAKE_ODD(26, 13);
MAKE_ODD(28, 15);
#undef MAKE_ODD
}
}
ALIGN_VAR_32(static const int16_t, tab_dct_32_0[][8]) =
{
{ 0x0F0E, 0x0100, 0x0908, 0x0706, 0x0D0C, 0x0302, 0x0B0A, 0x0504 }, // 0
};
ALIGN_VAR_32(static const int16_t, tab_dct_32_1[][8]) =
{
{ 89, -89, 18, -18, 75, -75, 50, -50 }, // 0
{ 75, -75, -50, 50, -18, 18, -89, 89 }, // 1
{ 50, -50, 75, -75, -89, 89, 18, -18 }, // 2
{ 18, -18, -89, 89, -50, 50, 75, -75 }, // 3
#define MAKE_COEF8(a0, a1, a2, a3, a4, a5, a6, a7) \
{ (a0), (a7), (a3), (a4), (a1), (a6), (a2), (a5) \
}, \
MAKE_COEF8(90, 87, 80, 70, 57, 43, 25, 9) // 4
MAKE_COEF8(87, 57, 9, -43, -80, -90, -70, -25) // 5
MAKE_COEF8(80, 9, -70, -87, -25, 57, 90, 43) // 6
MAKE_COEF8(70, -43, -87, 9, 90, 25, -80, -57) // 7
MAKE_COEF8(57, -80, -25, 90, -9, -87, 43, 70) // 8
MAKE_COEF8(43, -90, 57, 25, -87, 70, 9, -80) // 9
MAKE_COEF8(25, -70, 90, -80, 43, 9, -57, 87) // 10
MAKE_COEF8(9, -25, 43, -57, 70, -80, 87, -90) // 11
#undef MAKE_COEF8
#define MAKE_COEF16(a00, a01, a02, a03, a04, a05, a06, a07, a08, a09, a10, a11, a12, a13, a14, a15) \
{ (a00), (a07), (a03), (a04), (a01), (a06), (a02), (a05) }, \
{ (a15), (a08), (a12), (a11), (a14), (a09), (a13), (a10) },
MAKE_COEF16(90, 90, 88, 85, 82, 78, 73, 67, 61, 54, 46, 38, 31, 22, 13, 4) // 12
MAKE_COEF16(90, 82, 67, 46, 22, -4, -31, -54, -73, -85, -90, -88, -78, -61, -38, -13) // 14
MAKE_COEF16(88, 67, 31, -13, -54, -82, -90, -78, -46, -4, 38, 73, 90, 85, 61, 22) // 16
MAKE_COEF16(85, 46, -13, -67, -90, -73, -22, 38, 82, 88, 54, -4, -61, -90, -78, -31) // 18
MAKE_COEF16(82, 22, -54, -90, -61, 13, 78, 85, 31, -46, -90, -67, 4, 73, 88, 38) // 20
MAKE_COEF16(78, -4, -82, -73, 13, 85, 67, -22, -88, -61, 31, 90, 54, -38, -90, -46) // 22
MAKE_COEF16(73, -31, -90, -22, 78, 67, -38, -90, -13, 82, 61, -46, -88, -4, 85, 54) // 24
MAKE_COEF16(67, -54, -78, 38, 85, -22, -90, 4, 90, 13, -88, -31, 82, 46, -73, -61) // 26
MAKE_COEF16(61, -73, -46, 82, 31, -88, -13, 90, -4, -90, 22, 85, -38, -78, 54, 67) // 28
MAKE_COEF16(54, -85, -4, 88, -46, -61, 82, 13, -90, 38, 67, -78, -22, 90, -31, -73) // 30
MAKE_COEF16(46, -90, 38, 54, -90, 31, 61, -88, 22, 67, -85, 13, 73, -82, 4, 78) // 32
MAKE_COEF16(38, -88, 73, -4, -67, 90, -46, -31, 85, -78, 13, 61, -90, 54, 22, -82) // 34
MAKE_COEF16(31, -78, 90, -61, 4, 54, -88, 82, -38, -22, 73, -90, 67, -13, -46, 85) // 36
MAKE_COEF16(22, -61, 85, -90, 73, -38, -4, 46, -78, 90, -82, 54, -13, -31, 67, -88) // 38
MAKE_COEF16(13, -38, 61, -78, 88, -90, 85, -73, 54, -31, 4, 22, -46, 67, -82, 90) // 40
MAKE_COEF16(4, -13, 22, -31, 38, -46, 54, -61, 67, -73, 78, -82, 85, -88, 90, -90) // 42
#undef MAKE_COEF16
{
64, 64, 64, 64, 64, 64, 64, 64
}, // 44
{ 64, 64, -64, -64, -64, -64, 64, 64 }, // 45
{ 83, 83, 36, 36, -36, -36, -83, -83 }, // 46
{ -83, -83, -36, -36, 36, 36, 83, 83 }, // 47
{ 36, 36, -83, -83, 83, 83, -36, -36 }, // 48
{ -36, -36, 83, 83, -83, -83, 36, 36 }, // 49
#define MAKE_COEF16(a00, a01, a02, a03, a04, a05, a06, a07, a08, a09, a10, a11, a12, a13, a14, a15) \
{ (a00), (a00), (a01), (a01), (a02), (a02), (a03), (a03) }, \
{ (a04), (a04), (a05), (a05), (a06), (a06), (a07), (a07) }, \
{ (a08), (a08), (a09), (a09), (a10), (a10), (a11), (a11) }, \
{ (a12), (a12), (a13), (a13), (a14), (a14), (a15), (a15) },
MAKE_COEF16(89, 75, 50, 18, -18, -50, -75, -89, -89, -75, -50, -18, 18, 50, 75, 89) // 50
MAKE_COEF16(75, -18, -89, -50, 50, 89, 18, -75, -75, 18, 89, 50, -50, -89, -18, 75) // 54
// TODO: convert below table here
#undef MAKE_COEF16
{
50, 50, -89, -89, 18, 18, 75, 75
}, // 58
{ -75, -75, -18, -18, 89, 89, -50, -50 }, // 59
{ -50, -50, 89, 89, -18, -18, -75, -75 }, // 60
{ 75, 75, 18, 18, -89, -89, 50, 50 }, // 61
{ 18, 18, -50, -50, 75, 75, -89, -89 }, // 62
{ 89, 89, -75, -75, 50, 50, -18, -18 }, // 63
{ -18, -18, 50, 50, -75, -75, 89, 89 }, // 64
{ -89, -89, 75, 75, -50, -50, 18, 18 }, // 65
{ 90, 90, 87, 87, 80, 80, 70, 70 }, // 66
{ 57, 57, 43, 43, 25, 25, 9, 9 }, // 67
{ -9, -9, -25, -25, -43, -43, -57, -57 }, // 68
{ -70, -70, -80, -80, -87, -87, -90, -90 }, // 69
{ 87, 87, 57, 57, 9, 9, -43, -43 }, // 70
{ -80, -80, -90, -90, -70, -70, -25, -25 }, // 71
{ 25, 25, 70, 70, 90, 90, 80, 80 }, // 72
{ 43, 43, -9, -9, -57, -57, -87, -87 }, // 73
{ 80, 80, 9, 9, -70, -70, -87, -87 }, // 74
{ -25, -25, 57, 57, 90, 90, 43, 43 }, // 75
{ -43, -43, -90, -90, -57, -57, 25, 25 }, // 76
{ 87, 87, 70, 70, -9, -9, -80, -80 }, // 77
{ 70, 70, -43, -43, -87, -87, 9, 9 }, // 78
{ 90, 90, 25, 25, -80, -80, -57, -57 }, // 79
{ 57, 57, 80, 80, -25, -25, -90, -90 }, // 80
{ -9, -9, 87, 87, 43, 43, -70, -70 }, // 81
{ 57, 57, -80, -80, -25, -25, 90, 90 }, // 82
{ -9, -9, -87, -87, 43, 43, 70, 70 }, // 83
{ -70, -70, -43, -43, 87, 87, 9, 9 }, // 84
{ -90, -90, 25, 25, 80, 80, -57, -57 }, // 85
{ 43, 43, -90, -90, 57, 57, 25, 25 }, // 86
{ -87, -87, 70, 70, 9, 9, -80, -80 }, // 87
{ 80, 80, -9, -9, -70, -70, 87, 87 }, // 88
{ -25, -25, -57, -57, 90, 90, -43, -43 }, // 89
{ 25, 25, -70, -70, 90, 90, -80, -80 }, // 90
{ 43, 43, 9, 9, -57, -57, 87, 87 }, // 91
{ -87, -87, 57, 57, -9, -9, -43, -43 }, // 92
{ 80, 80, -90, -90, 70, 70, -25, -25 }, // 93
{ 9, 9, -25, -25, 43, 43, -57, -57 }, // 94
{ 70, 70, -80, -80, 87, 87, -90, -90 }, // 95
{ 90, 90, -87, -87, 80, 80, -70, -70 }, // 96
{ 57, 57, -43, -43, 25, 25, -9, -9 }, // 97
#define MAKE_COEF16(a00, a01, a02, a03, a04, a05, a06, a07, a08, a09, a10, a11, a12, a13, a14, a15) \
{ (a00), -(a00), (a01), -(a01), (a02), -(a02), (a03), -(a03) }, \
{ (a04), -(a04), (a05), -(a05), (a06), -(a06), (a07), -(a07) }, \
{ (a08), -(a08), (a09), -(a09), (a10), -(a10), (a11), -(a11) }, \
{ (a12), -(a12), (a13), -(a13), (a14), -(a14), (a15), -(a15) },
MAKE_COEF16(90, 90, 88, 85, 82, 78, 73, 67, 61, 54, 46, 38, 31, 22, 13, 4) // 98
MAKE_COEF16(90, 82, 67, 46, 22, -4, -31, -54, -73, -85, -90, -88, -78, -61, -38, -13) //102
MAKE_COEF16(88, 67, 31, -13, -54, -82, -90, -78, -46, -4, 38, 73, 90, 85, 61, 22) //106
MAKE_COEF16(85, 46, -13, -67, -90, -73, -22, 38, +82, 88, 54, -4, -61, -90, -78, -31) //110
MAKE_COEF16(82, 22, -54, -90, -61, 13, 78, 85, +31, -46, -90, -67, 4, 73, 88, 38) //114
MAKE_COEF16(78, -4, -82, -73, 13, 85, 67, -22, -88, -61, 31, 90, 54, -38, -90, -46) //118
MAKE_COEF16(73, -31, -90, -22, 78, 67, -38, -90, -13, 82, 61, -46, -88, -4, 85, 54) //122
MAKE_COEF16(67, -54, -78, 38, 85, -22, -90, 4, +90, 13, -88, -31, 82, 46, -73, -61) //126
MAKE_COEF16(61, -73, -46, 82, 31, -88, -13, 90, -4, -90, 22, 85, -38, -78, 54, 67) //130
MAKE_COEF16(54, -85, -4, 88, -46, -61, 82, 13, -90, 38, 67, -78, -22, 90, -31, -73) //134
MAKE_COEF16(46, -90, 38, 54, -90, 31, 61, -88, +22, 67, -85, 13, 73, -82, 4, 78) //138
MAKE_COEF16(38, -88, 73, -4, -67, 90, -46, -31, +85, -78, 13, 61, -90, 54, 22, -82) //142
MAKE_COEF16(31, -78, 90, -61, 4, 54, -88, 82, -38, -22, 73, -90, 67, -13, -46, 85) //146
MAKE_COEF16(22, -61, 85, -90, 73, -38, -4, 46, -78, 90, -82, 54, -13, -31, 67, -88) //150
MAKE_COEF16(13, -38, 61, -78, 88, -90, 85, -73, +54, -31, 4, 22, -46, 67, -82, 90) //154
MAKE_COEF16(4, -13, 22, -31, 38, -46, 54, -61, +67, -73, 78, -82, 85, -88, 90, -90) //158
#undef MAKE_COEF16
};
static void dct32(const int16_t *src, int16_t *dst, intptr_t stride)
{
// Const
__m128i c_8 = _mm_set1_epi32(DCT32_ADD1);
__m128i c_1024 = _mm_set1_epi32(DCT32_ADD2);
int i;
__m128i T00A, T01A, T02A, T03A, T04A, T05A, T06A, T07A;
__m128i T00B, T01B, T02B, T03B, T04B, T05B, T06B, T07B;
__m128i T00C, T01C, T02C, T03C, T04C, T05C, T06C, T07C;
__m128i T00D, T01D, T02D, T03D, T04D, T05D, T06D, T07D;
__m128i T10A, T11A, T12A, T13A, T14A, T15A, T16A, T17A;
__m128i T10B, T11B, T12B, T13B, T14B, T15B, T16B, T17B;
__m128i T20, T21, T22, T23, T24, T25, T26, T27;
__m128i T30, T31, T32, T33, T34, T35, T36, T37;
__m128i T40, T41, T42, T43, T44, T45, T46, T47;
__m128i T50, T51, T52, T53;
__m128i T60, T61, T62, T63, T64, T65, T66, T67;
__m128i im[32][4];
// DCT1
for (i = 0; i < 32 / 8; i++)
{
T00A = _mm_load_si128((__m128i*)&src[(i * 8 + 0) * stride + 0]); // [07 06 05 04 03 02 01 00]
T00B = _mm_load_si128((__m128i*)&src[(i * 8 + 0) * stride + 8]); // [15 14 13 12 11 10 09 08]
T00C = _mm_load_si128((__m128i*)&src[(i * 8 + 0) * stride + 16]); // [23 22 21 20 19 18 17 16]
T00D = _mm_load_si128((__m128i*)&src[(i * 8 + 0) * stride + 24]); // [31 30 29 28 27 26 25 24]
T01A = _mm_load_si128((__m128i*)&src[(i * 8 + 1) * stride + 0]);
T01B = _mm_load_si128((__m128i*)&src[(i * 8 + 1) * stride + 8]);
T01C = _mm_load_si128((__m128i*)&src[(i * 8 + 1) * stride + 16]);
T01D = _mm_load_si128((__m128i*)&src[(i * 8 + 1) * stride + 24]);
T02A = _mm_load_si128((__m128i*)&src[(i * 8 + 2) * stride + 0]);
T02B = _mm_load_si128((__m128i*)&src[(i * 8 + 2) * stride + 8]);
T02C = _mm_load_si128((__m128i*)&src[(i * 8 + 2) * stride + 16]);
T02D = _mm_load_si128((__m128i*)&src[(i * 8 + 2) * stride + 24]);
T03A = _mm_load_si128((__m128i*)&src[(i * 8 + 3) * stride + 0]);
T03B = _mm_load_si128((__m128i*)&src[(i * 8 + 3) * stride + 8]);
T03C = _mm_load_si128((__m128i*)&src[(i * 8 + 3) * stride + 16]);
T03D = _mm_load_si128((__m128i*)&src[(i * 8 + 3) * stride + 24]);
T04A = _mm_load_si128((__m128i*)&src[(i * 8 + 4) * stride + 0]);
T04B = _mm_load_si128((__m128i*)&src[(i * 8 + 4) * stride + 8]);
T04C = _mm_load_si128((__m128i*)&src[(i * 8 + 4) * stride + 16]);
T04D = _mm_load_si128((__m128i*)&src[(i * 8 + 4) * stride + 24]);
T05A = _mm_load_si128((__m128i*)&src[(i * 8 + 5) * stride + 0]);
T05B = _mm_load_si128((__m128i*)&src[(i * 8 + 5) * stride + 8]);
T05C = _mm_load_si128((__m128i*)&src[(i * 8 + 5) * stride + 16]);
T05D = _mm_load_si128((__m128i*)&src[(i * 8 + 5) * stride + 24]);
T06A = _mm_load_si128((__m128i*)&src[(i * 8 + 6) * stride + 0]);
T06B = _mm_load_si128((__m128i*)&src[(i * 8 + 6) * stride + 8]);
T06C = _mm_load_si128((__m128i*)&src[(i * 8 + 6) * stride + 16]);
T06D = _mm_load_si128((__m128i*)&src[(i * 8 + 6) * stride + 24]);
T07A = _mm_load_si128((__m128i*)&src[(i * 8 + 7) * stride + 0]);
T07B = _mm_load_si128((__m128i*)&src[(i * 8 + 7) * stride + 8]);
T07C = _mm_load_si128((__m128i*)&src[(i * 8 + 7) * stride + 16]);
T07D = _mm_load_si128((__m128i*)&src[(i * 8 + 7) * stride + 24]);
T00A = _mm_shuffle_epi8(T00A, _mm_load_si128((__m128i*)tab_dct_16_0[1])); // [05 02 06 01 04 03 07 00]
T00B = _mm_shuffle_epi8(T00B, _mm_load_si128((__m128i*)tab_dct_32_0[0])); // [10 13 09 14 11 12 08 15]
T00C = _mm_shuffle_epi8(T00C, _mm_load_si128((__m128i*)tab_dct_16_0[1])); // [21 18 22 17 20 19 23 16]
T00D = _mm_shuffle_epi8(T00D, _mm_load_si128((__m128i*)tab_dct_32_0[0])); // [26 29 25 30 27 28 24 31]
T01A = _mm_shuffle_epi8(T01A, _mm_load_si128((__m128i*)tab_dct_16_0[1]));
T01B = _mm_shuffle_epi8(T01B, _mm_load_si128((__m128i*)tab_dct_32_0[0]));
T01C = _mm_shuffle_epi8(T01C, _mm_load_si128((__m128i*)tab_dct_16_0[1]));
T01D = _mm_shuffle_epi8(T01D, _mm_load_si128((__m128i*)tab_dct_32_0[0]));
T02A = _mm_shuffle_epi8(T02A, _mm_load_si128((__m128i*)tab_dct_16_0[1]));
T02B = _mm_shuffle_epi8(T02B, _mm_load_si128((__m128i*)tab_dct_32_0[0]));
T02C = _mm_shuffle_epi8(T02C, _mm_load_si128((__m128i*)tab_dct_16_0[1]));
T02D = _mm_shuffle_epi8(T02D, _mm_load_si128((__m128i*)tab_dct_32_0[0]));
T03A = _mm_shuffle_epi8(T03A, _mm_load_si128((__m128i*)tab_dct_16_0[1]));
T03B = _mm_shuffle_epi8(T03B, _mm_load_si128((__m128i*)tab_dct_32_0[0]));
T03C = _mm_shuffle_epi8(T03C, _mm_load_si128((__m128i*)tab_dct_16_0[1]));
T03D = _mm_shuffle_epi8(T03D, _mm_load_si128((__m128i*)tab_dct_32_0[0]));
T04A = _mm_shuffle_epi8(T04A, _mm_load_si128((__m128i*)tab_dct_16_0[1]));
T04B = _mm_shuffle_epi8(T04B, _mm_load_si128((__m128i*)tab_dct_32_0[0]));
T04C = _mm_shuffle_epi8(T04C, _mm_load_si128((__m128i*)tab_dct_16_0[1]));
T04D = _mm_shuffle_epi8(T04D, _mm_load_si128((__m128i*)tab_dct_32_0[0]));
T05A = _mm_shuffle_epi8(T05A, _mm_load_si128((__m128i*)tab_dct_16_0[1]));
T05B = _mm_shuffle_epi8(T05B, _mm_load_si128((__m128i*)tab_dct_32_0[0]));
T05C = _mm_shuffle_epi8(T05C, _mm_load_si128((__m128i*)tab_dct_16_0[1]));
T05D = _mm_shuffle_epi8(T05D, _mm_load_si128((__m128i*)tab_dct_32_0[0]));
T06A = _mm_shuffle_epi8(T06A, _mm_load_si128((__m128i*)tab_dct_16_0[1]));
T06B = _mm_shuffle_epi8(T06B, _mm_load_si128((__m128i*)tab_dct_32_0[0]));
T06C = _mm_shuffle_epi8(T06C, _mm_load_si128((__m128i*)tab_dct_16_0[1]));
T06D = _mm_shuffle_epi8(T06D, _mm_load_si128((__m128i*)tab_dct_32_0[0]));
T07A = _mm_shuffle_epi8(T07A, _mm_load_si128((__m128i*)tab_dct_16_0[1]));
T07B = _mm_shuffle_epi8(T07B, _mm_load_si128((__m128i*)tab_dct_32_0[0]));
T07C = _mm_shuffle_epi8(T07C, _mm_load_si128((__m128i*)tab_dct_16_0[1]));
T07D = _mm_shuffle_epi8(T07D, _mm_load_si128((__m128i*)tab_dct_32_0[0]));
T10A = _mm_add_epi16(T00A, T00D); // [E05 E02 E06 E01 E04 E03 E07 E00]
T10B = _mm_add_epi16(T00B, T00C); // [E10 E13 E09 E14 E11 E12 E08 E15]
T11A = _mm_add_epi16(T01A, T01D);
T11B = _mm_add_epi16(T01B, T01C);
T12A = _mm_add_epi16(T02A, T02D);
T12B = _mm_add_epi16(T02B, T02C);
T13A = _mm_add_epi16(T03A, T03D);
T13B = _mm_add_epi16(T03B, T03C);
T14A = _mm_add_epi16(T04A, T04D);
T14B = _mm_add_epi16(T04B, T04C);
T15A = _mm_add_epi16(T05A, T05D);
T15B = _mm_add_epi16(T05B, T05C);
T16A = _mm_add_epi16(T06A, T06D);
T16B = _mm_add_epi16(T06B, T06C);
T17A = _mm_add_epi16(T07A, T07D);
T17B = _mm_add_epi16(T07B, T07C);
T00A = _mm_sub_epi16(T00A, T00D); // [O05 O02 O06 O01 O04 O03 O07 O00]
T00B = _mm_sub_epi16(T00B, T00C); // [O10 O13 O09 O14 O11 O12 O08 O15]
T01A = _mm_sub_epi16(T01A, T01D);
T01B = _mm_sub_epi16(T01B, T01C);
T02A = _mm_sub_epi16(T02A, T02D);
T02B = _mm_sub_epi16(T02B, T02C);
T03A = _mm_sub_epi16(T03A, T03D);
T03B = _mm_sub_epi16(T03B, T03C);
T04A = _mm_sub_epi16(T04A, T04D);
T04B = _mm_sub_epi16(T04B, T04C);
T05A = _mm_sub_epi16(T05A, T05D);
T05B = _mm_sub_epi16(T05B, T05C);
T06A = _mm_sub_epi16(T06A, T06D);
T06B = _mm_sub_epi16(T06B, T06C);
T07A = _mm_sub_epi16(T07A, T07D);
T07B = _mm_sub_epi16(T07B, T07C);
T20 = _mm_add_epi16(T10A, T10B); // [EE5 EE2 EE6 EE1 EE4 EE3 EE7 EE0]
T21 = _mm_add_epi16(T11A, T11B);
T22 = _mm_add_epi16(T12A, T12B);
T23 = _mm_add_epi16(T13A, T13B);
T24 = _mm_add_epi16(T14A, T14B);
T25 = _mm_add_epi16(T15A, T15B);
T26 = _mm_add_epi16(T16A, T16B);
T27 = _mm_add_epi16(T17A, T17B);
T30 = _mm_madd_epi16(T20, _mm_load_si128((__m128i*)tab_dct_8[1]));
T31 = _mm_madd_epi16(T21, _mm_load_si128((__m128i*)tab_dct_8[1]));
T32 = _mm_madd_epi16(T22, _mm_load_si128((__m128i*)tab_dct_8[1]));
T33 = _mm_madd_epi16(T23, _mm_load_si128((__m128i*)tab_dct_8[1]));
T34 = _mm_madd_epi16(T24, _mm_load_si128((__m128i*)tab_dct_8[1]));
T35 = _mm_madd_epi16(T25, _mm_load_si128((__m128i*)tab_dct_8[1]));
T36 = _mm_madd_epi16(T26, _mm_load_si128((__m128i*)tab_dct_8[1]));
T37 = _mm_madd_epi16(T27, _mm_load_si128((__m128i*)tab_dct_8[1]));
T40 = _mm_hadd_epi32(T30, T31);
T41 = _mm_hadd_epi32(T32, T33);
T42 = _mm_hadd_epi32(T34, T35);
T43 = _mm_hadd_epi32(T36, T37);
T50 = _mm_hadd_epi32(T40, T41);
T51 = _mm_hadd_epi32(T42, T43);
T50 = _mm_srai_epi32(_mm_add_epi32(T50, c_8), DCT32_SHIFT1);
T51 = _mm_srai_epi32(_mm_add_epi32(T51, c_8), DCT32_SHIFT1);
T60 = _mm_packs_epi32(T50, T51);
im[0][i] = T60;
T50 = _mm_hsub_epi32(T40, T41);
T51 = _mm_hsub_epi32(T42, T43);
T50 = _mm_srai_epi32(_mm_add_epi32(T50, c_8), DCT32_SHIFT1);
T51 = _mm_srai_epi32(_mm_add_epi32(T51, c_8), DCT32_SHIFT1);
T60 = _mm_packs_epi32(T50, T51);
im[16][i] = T60;
T30 = _mm_madd_epi16(T20, _mm_load_si128((__m128i*)tab_dct_16_1[8]));
T31 = _mm_madd_epi16(T21, _mm_load_si128((__m128i*)tab_dct_16_1[8]));
T32 = _mm_madd_epi16(T22, _mm_load_si128((__m128i*)tab_dct_16_1[8]));
T33 = _mm_madd_epi16(T23, _mm_load_si128((__m128i*)tab_dct_16_1[8]));
T34 = _mm_madd_epi16(T24, _mm_load_si128((__m128i*)tab_dct_16_1[8]));
T35 = _mm_madd_epi16(T25, _mm_load_si128((__m128i*)tab_dct_16_1[8]));
T36 = _mm_madd_epi16(T26, _mm_load_si128((__m128i*)tab_dct_16_1[8]));
T37 = _mm_madd_epi16(T27, _mm_load_si128((__m128i*)tab_dct_16_1[8]));
T40 = _mm_hadd_epi32(T30, T31);
T41 = _mm_hadd_epi32(T32, T33);
T42 = _mm_hadd_epi32(T34, T35);
T43 = _mm_hadd_epi32(T36, T37);
T50 = _mm_hadd_epi32(T40, T41);
T51 = _mm_hadd_epi32(T42, T43);
T50 = _mm_srai_epi32(_mm_add_epi32(T50, c_8), DCT32_SHIFT1);
T51 = _mm_srai_epi32(_mm_add_epi32(T51, c_8), DCT32_SHIFT1);
T60 = _mm_packs_epi32(T50, T51);
im[8][i] = T60;
T30 = _mm_madd_epi16(T20, _mm_load_si128((__m128i*)tab_dct_16_1[9]));
T31 = _mm_madd_epi16(T21, _mm_load_si128((__m128i*)tab_dct_16_1[9]));
T32 = _mm_madd_epi16(T22, _mm_load_si128((__m128i*)tab_dct_16_1[9]));
T33 = _mm_madd_epi16(T23, _mm_load_si128((__m128i*)tab_dct_16_1[9]));
T34 = _mm_madd_epi16(T24, _mm_load_si128((__m128i*)tab_dct_16_1[9]));
T35 = _mm_madd_epi16(T25, _mm_load_si128((__m128i*)tab_dct_16_1[9]));
T36 = _mm_madd_epi16(T26, _mm_load_si128((__m128i*)tab_dct_16_1[9]));
T37 = _mm_madd_epi16(T27, _mm_load_si128((__m128i*)tab_dct_16_1[9]));
T40 = _mm_hadd_epi32(T30, T31);
T41 = _mm_hadd_epi32(T32, T33);
T42 = _mm_hadd_epi32(T34, T35);
T43 = _mm_hadd_epi32(T36, T37);
T50 = _mm_hadd_epi32(T40, T41);
T51 = _mm_hadd_epi32(T42, T43);
T50 = _mm_srai_epi32(_mm_add_epi32(T50, c_8), DCT32_SHIFT1);
T51 = _mm_srai_epi32(_mm_add_epi32(T51, c_8), DCT32_SHIFT1);
T60 = _mm_packs_epi32(T50, T51);
im[24][i] = T60;
#define MAKE_ODD(tab, dstPos) \
T30 = _mm_madd_epi16(T20, _mm_load_si128((__m128i*)tab_dct_32_1[(tab)])); \
T31 = _mm_madd_epi16(T21, _mm_load_si128((__m128i*)tab_dct_32_1[(tab)])); \
T32 = _mm_madd_epi16(T22, _mm_load_si128((__m128i*)tab_dct_32_1[(tab)])); \
T33 = _mm_madd_epi16(T23, _mm_load_si128((__m128i*)tab_dct_32_1[(tab)])); \
T34 = _mm_madd_epi16(T24, _mm_load_si128((__m128i*)tab_dct_32_1[(tab)])); \
T35 = _mm_madd_epi16(T25, _mm_load_si128((__m128i*)tab_dct_32_1[(tab)])); \
T36 = _mm_madd_epi16(T26, _mm_load_si128((__m128i*)tab_dct_32_1[(tab)])); \
T37 = _mm_madd_epi16(T27, _mm_load_si128((__m128i*)tab_dct_32_1[(tab)])); \
\
T40 = _mm_hadd_epi32(T30, T31); \
T41 = _mm_hadd_epi32(T32, T33); \
T42 = _mm_hadd_epi32(T34, T35); \
T43 = _mm_hadd_epi32(T36, T37); \
\
T50 = _mm_hadd_epi32(T40, T41); \
T51 = _mm_hadd_epi32(T42, T43); \
T50 = _mm_srai_epi32(_mm_add_epi32(T50, c_8), DCT32_SHIFT1); \
T51 = _mm_srai_epi32(_mm_add_epi32(T51, c_8), DCT32_SHIFT1); \
T60 = _mm_packs_epi32(T50, T51); \
im[(dstPos)][i] = T60;
MAKE_ODD(0, 4);
MAKE_ODD(1, 12);
MAKE_ODD(2, 20);
MAKE_ODD(3, 28);
T20 = _mm_sub_epi16(T10A, T10B); // [EO5 EO2 EO6 EO1 EO4 EO3 EO7 EO0]
T21 = _mm_sub_epi16(T11A, T11B);
T22 = _mm_sub_epi16(T12A, T12B);
T23 = _mm_sub_epi16(T13A, T13B);
T24 = _mm_sub_epi16(T14A, T14B);
T25 = _mm_sub_epi16(T15A, T15B);
T26 = _mm_sub_epi16(T16A, T16B);
T27 = _mm_sub_epi16(T17A, T17B);
MAKE_ODD(4, 2);
MAKE_ODD(5, 6);
MAKE_ODD(6, 10);
MAKE_ODD(7, 14);
MAKE_ODD(8, 18);
MAKE_ODD(9, 22);
MAKE_ODD(10, 26);
MAKE_ODD(11, 30);
#undef MAKE_ODD
#define MAKE_ODD(tab, dstPos) \
T20 = _mm_madd_epi16(T00A, _mm_load_si128((__m128i*)tab_dct_32_1[(tab)])); \
T21 = _mm_madd_epi16(T00B, _mm_load_si128((__m128i*)tab_dct_32_1[(tab) + 1])); \
T22 = _mm_madd_epi16(T01A, _mm_load_si128((__m128i*)tab_dct_32_1[(tab)])); \
T23 = _mm_madd_epi16(T01B, _mm_load_si128((__m128i*)tab_dct_32_1[(tab) + 1])); \
T24 = _mm_madd_epi16(T02A, _mm_load_si128((__m128i*)tab_dct_32_1[(tab)])); \
T25 = _mm_madd_epi16(T02B, _mm_load_si128((__m128i*)tab_dct_32_1[(tab) + 1])); \
T26 = _mm_madd_epi16(T03A, _mm_load_si128((__m128i*)tab_dct_32_1[(tab)])); \
T27 = _mm_madd_epi16(T03B, _mm_load_si128((__m128i*)tab_dct_32_1[(tab) + 1])); \
T30 = _mm_madd_epi16(T04A, _mm_load_si128((__m128i*)tab_dct_32_1[(tab)])); \
T31 = _mm_madd_epi16(T04B, _mm_load_si128((__m128i*)tab_dct_32_1[(tab) + 1])); \
T32 = _mm_madd_epi16(T05A, _mm_load_si128((__m128i*)tab_dct_32_1[(tab)])); \
T33 = _mm_madd_epi16(T05B, _mm_load_si128((__m128i*)tab_dct_32_1[(tab) + 1])); \
T34 = _mm_madd_epi16(T06A, _mm_load_si128((__m128i*)tab_dct_32_1[(tab)])); \
T35 = _mm_madd_epi16(T06B, _mm_load_si128((__m128i*)tab_dct_32_1[(tab) + 1])); \
T36 = _mm_madd_epi16(T07A, _mm_load_si128((__m128i*)tab_dct_32_1[(tab)])); \
T37 = _mm_madd_epi16(T07B, _mm_load_si128((__m128i*)tab_dct_32_1[(tab) + 1])); \
\
T40 = _mm_hadd_epi32(T20, T21); \
T41 = _mm_hadd_epi32(T22, T23); \
T42 = _mm_hadd_epi32(T24, T25); \
T43 = _mm_hadd_epi32(T26, T27); \
T44 = _mm_hadd_epi32(T30, T31); \
T45 = _mm_hadd_epi32(T32, T33); \
T46 = _mm_hadd_epi32(T34, T35); \
T47 = _mm_hadd_epi32(T36, T37); \
\
T50 = _mm_hadd_epi32(T40, T41); \
T51 = _mm_hadd_epi32(T42, T43); \
T52 = _mm_hadd_epi32(T44, T45); \
T53 = _mm_hadd_epi32(T46, T47); \
\
T50 = _mm_hadd_epi32(T50, T51); \
T51 = _mm_hadd_epi32(T52, T53); \
T50 = _mm_srai_epi32(_mm_add_epi32(T50, c_8), DCT32_SHIFT1); \
T51 = _mm_srai_epi32(_mm_add_epi32(T51, c_8), DCT32_SHIFT1); \
T60 = _mm_packs_epi32(T50, T51); \
im[(dstPos)][i] = T60;
MAKE_ODD(12, 1);
MAKE_ODD(14, 3);
MAKE_ODD(16, 5);
MAKE_ODD(18, 7);
MAKE_ODD(20, 9);
MAKE_ODD(22, 11);
MAKE_ODD(24, 13);
MAKE_ODD(26, 15);
MAKE_ODD(28, 17);
MAKE_ODD(30, 19);
MAKE_ODD(32, 21);
MAKE_ODD(34, 23);
MAKE_ODD(36, 25);
MAKE_ODD(38, 27);
MAKE_ODD(40, 29);
MAKE_ODD(42, 31);
#undef MAKE_ODD
}
// DCT2
for (i = 0; i < 32 / 4; i++)
{
// OPT_ME: to avoid register spill, I use matrix multiply, have other way?
T00A = im[i * 4 + 0][0]; // [07 06 05 04 03 02 01 00]
T00B = im[i * 4 + 0][1]; // [15 14 13 12 11 10 09 08]
T00C = im[i * 4 + 0][2]; // [23 22 21 20 19 18 17 16]
T00D = im[i * 4 + 0][3]; // [31 30 29 28 27 26 25 24]
T01A = im[i * 4 + 1][0];
T01B = im[i * 4 + 1][1];
T01C = im[i * 4 + 1][2];
T01D = im[i * 4 + 1][3];
T02A = im[i * 4 + 2][0];
T02B = im[i * 4 + 2][1];
T02C = im[i * 4 + 2][2];
T02D = im[i * 4 + 2][3];
T03A = im[i * 4 + 3][0];
T03B = im[i * 4 + 3][1];
T03C = im[i * 4 + 3][2];
T03D = im[i * 4 + 3][3];
T00C = _mm_shuffle_epi8(T00C, _mm_load_si128((__m128i*)tab_dct_16_0[0])); // [16 17 18 19 20 21 22 23]
T00D = _mm_shuffle_epi8(T00D, _mm_load_si128((__m128i*)tab_dct_16_0[0])); // [24 25 26 27 28 29 30 31]
T01C = _mm_shuffle_epi8(T01C, _mm_load_si128((__m128i*)tab_dct_16_0[0]));
T01D = _mm_shuffle_epi8(T01D, _mm_load_si128((__m128i*)tab_dct_16_0[0]));
T02C = _mm_shuffle_epi8(T02C, _mm_load_si128((__m128i*)tab_dct_16_0[0]));
T02D = _mm_shuffle_epi8(T02D, _mm_load_si128((__m128i*)tab_dct_16_0[0]));
T03C = _mm_shuffle_epi8(T03C, _mm_load_si128((__m128i*)tab_dct_16_0[0]));
T03D = _mm_shuffle_epi8(T03D, _mm_load_si128((__m128i*)tab_dct_16_0[0]));
T10A = _mm_unpacklo_epi16(T00A, T00D); // [28 03 29 02 30 01 31 00]
T10B = _mm_unpackhi_epi16(T00A, T00D); // [24 07 25 06 26 05 27 04]
T00A = _mm_unpacklo_epi16(T00B, T00C); // [20 11 21 10 22 09 23 08]
T00B = _mm_unpackhi_epi16(T00B, T00C); // [16 15 17 14 18 13 19 12]
T11A = _mm_unpacklo_epi16(T01A, T01D);
T11B = _mm_unpackhi_epi16(T01A, T01D);
T01A = _mm_unpacklo_epi16(T01B, T01C);
T01B = _mm_unpackhi_epi16(T01B, T01C);
T12A = _mm_unpacklo_epi16(T02A, T02D);
T12B = _mm_unpackhi_epi16(T02A, T02D);
T02A = _mm_unpacklo_epi16(T02B, T02C);
T02B = _mm_unpackhi_epi16(T02B, T02C);
T13A = _mm_unpacklo_epi16(T03A, T03D);
T13B = _mm_unpackhi_epi16(T03A, T03D);
T03A = _mm_unpacklo_epi16(T03B, T03C);
T03B = _mm_unpackhi_epi16(T03B, T03C);
#define MAKE_ODD(tab0, tab1, tab2, tab3, dstPos) \
T20 = _mm_madd_epi16(T10A, _mm_load_si128((__m128i*)tab_dct_32_1[(tab0)])); \
T21 = _mm_madd_epi16(T10B, _mm_load_si128((__m128i*)tab_dct_32_1[(tab1)])); \
T22 = _mm_madd_epi16(T00A, _mm_load_si128((__m128i*)tab_dct_32_1[(tab2)])); \
T23 = _mm_madd_epi16(T00B, _mm_load_si128((__m128i*)tab_dct_32_1[(tab3)])); \
T24 = _mm_madd_epi16(T11A, _mm_load_si128((__m128i*)tab_dct_32_1[(tab0)])); \
T25 = _mm_madd_epi16(T11B, _mm_load_si128((__m128i*)tab_dct_32_1[(tab1)])); \
T26 = _mm_madd_epi16(T01A, _mm_load_si128((__m128i*)tab_dct_32_1[(tab2)])); \
T27 = _mm_madd_epi16(T01B, _mm_load_si128((__m128i*)tab_dct_32_1[(tab3)])); \
T30 = _mm_madd_epi16(T12A, _mm_load_si128((__m128i*)tab_dct_32_1[(tab0)])); \
T31 = _mm_madd_epi16(T12B, _mm_load_si128((__m128i*)tab_dct_32_1[(tab1)])); \
T32 = _mm_madd_epi16(T02A, _mm_load_si128((__m128i*)tab_dct_32_1[(tab2)])); \
T33 = _mm_madd_epi16(T02B, _mm_load_si128((__m128i*)tab_dct_32_1[(tab3)])); \
T34 = _mm_madd_epi16(T13A, _mm_load_si128((__m128i*)tab_dct_32_1[(tab0)])); \
T35 = _mm_madd_epi16(T13B, _mm_load_si128((__m128i*)tab_dct_32_1[(tab1)])); \
T36 = _mm_madd_epi16(T03A, _mm_load_si128((__m128i*)tab_dct_32_1[(tab2)])); \
T37 = _mm_madd_epi16(T03B, _mm_load_si128((__m128i*)tab_dct_32_1[(tab3)])); \
\
T60 = _mm_hadd_epi32(T20, T21); \
T61 = _mm_hadd_epi32(T22, T23); \
T62 = _mm_hadd_epi32(T24, T25); \
T63 = _mm_hadd_epi32(T26, T27); \
T64 = _mm_hadd_epi32(T30, T31); \
T65 = _mm_hadd_epi32(T32, T33); \
T66 = _mm_hadd_epi32(T34, T35); \
T67 = _mm_hadd_epi32(T36, T37); \
\
T60 = _mm_hadd_epi32(T60, T61); \
T61 = _mm_hadd_epi32(T62, T63); \
T62 = _mm_hadd_epi32(T64, T65); \
T63 = _mm_hadd_epi32(T66, T67); \
\
T60 = _mm_hadd_epi32(T60, T61); \
T61 = _mm_hadd_epi32(T62, T63); \
\
T60 = _mm_hadd_epi32(T60, T61); \
\
T60 = _mm_srai_epi32(_mm_add_epi32(T60, c_1024), DCT32_SHIFT2); \
T60 = _mm_packs_epi32(T60, T60); \
_mm_storel_epi64((__m128i*)&dst[(dstPos) * 32 + (i * 4) + 0], T60); \
MAKE_ODD(44, 44, 44, 44, 0);
MAKE_ODD(45, 45, 45, 45, 16);
MAKE_ODD(46, 47, 46, 47, 8);
MAKE_ODD(48, 49, 48, 49, 24);
MAKE_ODD(50, 51, 52, 53, 4);
MAKE_ODD(54, 55, 56, 57, 12);
MAKE_ODD(58, 59, 60, 61, 20);
MAKE_ODD(62, 63, 64, 65, 28);
MAKE_ODD(66, 67, 68, 69, 2);
MAKE_ODD(70, 71, 72, 73, 6);
MAKE_ODD(74, 75, 76, 77, 10);
MAKE_ODD(78, 79, 80, 81, 14);
MAKE_ODD(82, 83, 84, 85, 18);
MAKE_ODD(86, 87, 88, 89, 22);
MAKE_ODD(90, 91, 92, 93, 26);
MAKE_ODD(94, 95, 96, 97, 30);
MAKE_ODD(98, 99, 100, 101, 1);
MAKE_ODD(102, 103, 104, 105, 3);
MAKE_ODD(106, 107, 108, 109, 5);
MAKE_ODD(110, 111, 112, 113, 7);
MAKE_ODD(114, 115, 116, 117, 9);
MAKE_ODD(118, 119, 120, 121, 11);
MAKE_ODD(122, 123, 124, 125, 13);
MAKE_ODD(126, 127, 128, 129, 15);
MAKE_ODD(130, 131, 132, 133, 17);
MAKE_ODD(134, 135, 136, 137, 19);
MAKE_ODD(138, 139, 140, 141, 21);
MAKE_ODD(142, 143, 144, 145, 23);
MAKE_ODD(146, 147, 148, 149, 25);
MAKE_ODD(150, 151, 152, 153, 27);
MAKE_ODD(154, 155, 156, 157, 29);
MAKE_ODD(158, 159, 160, 161, 31);
#undef MAKE_ODD
}
}
namespace X265_NS {
void setupIntrinsicDCT_ssse3(EncoderPrimitives &p)
{
/* Note: We have AVX2 assembly for these two functions, but since AVX2 is
* still somewhat rare on end-user PCs we still compile and link these SSSE3
* intrinsic SIMD functions */
p.cu[BLOCK_16x16].dct = dct16;
p.cu[BLOCK_32x32].dct = dct32;
}
}
Reference in a new issue View git blame Copy permalink