King_DuckZ/libbpg
1
0
Fork 0
forked from mirror/libbpg
Code Issues Pull requests Releases Wiki Activity
libbpg/x265/source/common/x86/sad-a.asm
King_DuckZ 35a8402710 libbpg-0.9.6
2015-10-27 11:46:00 +01:00

4573 lines
106 KiB
NASM
Raw Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

;*****************************************************************************
;* sad-a.asm: x86 sad functions
;*****************************************************************************
;* Copyright (C) 2003-2013 x264 project
;*
;* Authors: Loren Merritt <lorenm@u.washington.edu>
;* Fiona Glaser <fiona@x264.com>
;* Laurent Aimar <fenrir@via.ecp.fr>
;* Alex Izvorski <aizvorksi@gmail.com>
;* Min Chen <chenm003@163.com>
;*
;* This program is free software; you can redistribute it and/or modify
;* it under the terms of the GNU General Public License as published by
;* the Free Software Foundation; either version 2 of the License, or
;* (at your option) any later version.
;*
;* This program is distributed in the hope that it will be useful,
;* but WITHOUT ANY WARRANTY; without even the implied warranty of
;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
;* GNU General Public License for more details.
;*
;* You should have received a copy of the GNU General Public License
;* along with this program; if not, write to the Free Software
;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111, USA.
;*
;* This program is also available under a commercial proprietary license.
;* For more information, contact us at license @ x265.com.
;*****************************************************************************
%include "x86inc.asm"
%include "x86util.asm"
SECTION_RODATA 32
MSK: db 255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0
SECTION .text
cextern pb_3
cextern pb_shuf8x8c
cextern pw_8
cextern pd_64
;=============================================================================
; SAD MMX
;=============================================================================
%macro SAD_INC_2x16P 0
movq mm1, [r0]
movq mm2, [r0+8]
movq mm3, [r0+r1]
movq mm4, [r0+r1+8]
psadbw mm1, [r2]
psadbw mm2, [r2+8]
psadbw mm3, [r2+r3]
psadbw mm4, [r2+r3+8]
lea r0, [r0+2*r1]
paddw mm1, mm2
paddw mm3, mm4
lea r2, [r2+2*r3]
paddw mm0, mm1
paddw mm0, mm3
%endmacro
%macro SAD_INC_2x8P 0
movq mm1, [r0]
movq mm2, [r0+r1]
psadbw mm1, [r2]
psadbw mm2, [r2+r3]
lea r0, [r0+2*r1]
paddw mm0, mm1
paddw mm0, mm2
lea r2, [r2+2*r3]
%endmacro
%macro SAD_INC_2x4P 0
movd mm1, [r0]
movd mm2, [r2]
punpckldq mm1, [r0+r1]
punpckldq mm2, [r2+r3]
psadbw mm1, mm2
paddw mm0, mm1
lea r0, [r0+2*r1]
lea r2, [r2+2*r3]
%endmacro
;-----------------------------------------------------------------------------
; int pixel_sad_16x16( uint8_t *, intptr_t, uint8_t *, intptr_t )
;-----------------------------------------------------------------------------
%macro SAD 2
cglobal pixel_sad_%1x%2_mmx2, 4,4
pxor mm0, mm0
%rep %2/2
SAD_INC_2x%1P
%endrep
movd eax, mm0
RET
%endmacro
SAD 16, 16
SAD 16, 8
SAD 8, 16
SAD 8, 8
SAD 8, 4
SAD 4, 16
SAD 4, 8
SAD 4, 4
;=============================================================================
; SAD XMM
;=============================================================================
%macro SAD_END_SSE2 0
movhlps m1, m0
paddw m0, m1
movd eax, m0
RET
%endmacro
%macro PROCESS_SAD_12x4 0
movu m1, [r2]
movu m2, [r0]
pand m1, m4
pand m2, m4
psadbw m1, m2
paddd m0, m1
lea r2, [r2 + r3]
lea r0, [r0 + r1]
movu m1, [r2]
movu m2, [r0]
pand m1, m4
pand m2, m4
psadbw m1, m2
paddd m0, m1
lea r2, [r2 + r3]
lea r0, [r0 + r1]
movu m1, [r2]
movu m2, [r0]
pand m1, m4
pand m2, m4
psadbw m1, m2
paddd m0, m1
lea r2, [r2 + r3]
lea r0, [r0 + r1]
movu m1, [r2]
movu m2, [r0]
pand m1, m4
pand m2, m4
psadbw m1, m2
paddd m0, m1
%endmacro
%macro PROCESS_SAD_16x4 0
movu m1, [r2]
movu m2, [r2 + r3]
psadbw m1, [r0]
psadbw m2, [r0 + r1]
paddd m1, m2
paddd m0, m1
lea r2, [r2 + 2 * r3]
lea r0, [r0 + 2 * r1]
movu m1, [r2]
movu m2, [r2 + r3]
psadbw m1, [r0]
psadbw m2, [r0 + r1]
paddd m1, m2
paddd m0, m1
lea r2, [r2 + 2 * r3]
lea r0, [r0 + 2 * r1]
%endmacro
%macro PROCESS_SAD_24x4 0
movu m1, [r2]
movq m2, [r2 + 16]
lea r2, [r2 + r3]
movu m3, [r2]
movq m4, [r2 + 16]
psadbw m1, [r0]
psadbw m3, [r0 + r1]
paddd m0, m1
paddd m0, m3
movq m1, [r0 + 16]
lea r0, [r0 + r1]
movq m3, [r0 + 16]
punpcklqdq m2, m4
punpcklqdq m1, m3
psadbw m2, m1
paddd m0, m2
lea r2, [r2 + r3]
lea r0, [r0 + r1]
movu m1, [r2]
movq m2, [r2 + 16]
lea r2, [r2 + r3]
movu m3, [r2]
movq m4, [r2 + 16]
psadbw m1, [r0]
psadbw m3, [r0 + r1]
paddd m0, m1
paddd m0, m3
movq m1, [r0 + 16]
lea r0, [r0 + r1]
movq m3, [r0 + 16]
punpcklqdq m2, m4
punpcklqdq m1, m3
psadbw m2, m1
paddd m0, m2
%endmacro
%macro PROCESS_SAD_32x4 0
movu m1, [r2]
movu m2, [r2 + 16]
psadbw m1, [r0]
psadbw m2, [r0 + 16]
paddd m1, m2
paddd m0, m1
lea r2, [r2 + r3]
lea r0, [r0 + r1]
movu m1, [r2]
movu m2, [r2 + 16]
psadbw m1, [r0]
psadbw m2, [r0 + 16]
paddd m1, m2
paddd m0, m1
lea r2, [r2 + r3]
lea r0, [r0 + r1]
movu m1, [r2]
movu m2, [r2 + 16]
psadbw m1, [r0]
psadbw m2, [r0 + 16]
paddd m1, m2
paddd m0, m1
lea r2, [r2 + r3]
lea r0, [r0 + r1]
movu m1, [r2]
movu m2, [r2 + 16]
psadbw m1, [r0]
psadbw m2, [r0 + 16]
paddd m1, m2
paddd m0, m1
lea r2, [r2 + r3]
lea r0, [r0 + r1]
%endmacro
%macro PROCESS_SAD_48x4 0
movu m1, [r2]
movu m2, [r2 + 16]
movu m3, [r2 + 32]
psadbw m1, [r0]
psadbw m2, [r0 + 16]
psadbw m3, [r0 + 32]
paddd m1, m2
paddd m0, m1
paddd m0, m3
lea r2, [r2 + r3]
lea r0, [r0 + r1]
movu m1, [r2]
movu m2, [r2 + 16]
movu m3, [r2 + 32]
psadbw m1, [r0]
psadbw m2, [r0 + 16]
psadbw m3, [r0 + 32]
paddd m1, m2
paddd m0, m1
paddd m0, m3
lea r2, [r2 + r3]
lea r0, [r0 + r1]
movu m1, [r2]
movu m2, [r2 + 16]
movu m3, [r2 + 32]
psadbw m1, [r0]
psadbw m2, [r0 + 16]
psadbw m3, [r0 + 32]
paddd m1, m2
paddd m0, m1
paddd m0, m3
lea r2, [r2 + r3]
lea r0, [r0 + r1]
movu m1, [r2]
movu m2, [r2 + 16]
movu m3, [r2 + 32]
psadbw m1, [r0]
psadbw m2, [r0 + 16]
psadbw m3, [r0 + 32]
paddd m1, m2
paddd m0, m1
paddd m0, m3
%endmacro
%macro PROCESS_SAD_8x4 0
movq m1, [r2]
movq m2, [r2 + r3]
lea r2, [r2 + 2 * r3]
movq m3, [r0]
movq m4, [r0 + r1]
lea r0, [r0 + 2 * r1]
punpcklqdq m1, m2
punpcklqdq m3, m4
psadbw m1, m3
paddd m0, m1
movq m1, [r2]
movq m2, [r2 + r3]
lea r2, [r2 + 2 * r3]
movq m3, [r0]
movq m4, [r0 + r1]
lea r0, [r0 + 2 * r1]
punpcklqdq m1, m2
punpcklqdq m3, m4
psadbw m1, m3
paddd m0, m1
%endmacro
%macro PROCESS_SAD_64x4 0
movu m1, [r2]
movu m2, [r2 + 16]
movu m3, [r2 + 32]
movu m4, [r2 + 48]
psadbw m1, [r0]
psadbw m2, [r0 + 16]
psadbw m3, [r0 + 32]
psadbw m4, [r0 + 48]
paddd m1, m2
paddd m3, m4
paddd m0, m1
paddd m0, m3
lea r2, [r2 + r3]
lea r0, [r0 + r1]
movu m1, [r2]
movu m2, [r2 + 16]
movu m3, [r2 + 32]
movu m4, [r2 + 48]
psadbw m1, [r0]
psadbw m2, [r0 + 16]
psadbw m3, [r0 + 32]
psadbw m4, [r0 + 48]
paddd m1, m2
paddd m3, m4
paddd m0, m1
paddd m0, m3
lea r2, [r2 + r3]
lea r0, [r0 + r1]
movu m1, [r2]
movu m2, [r2 + 16]
movu m3, [r2 + 32]
movu m4, [r2 + 48]
psadbw m1, [r0]
psadbw m2, [r0 + 16]
psadbw m3, [r0 + 32]
psadbw m4, [r0 + 48]
paddd m1, m2
paddd m3, m4
paddd m0, m1
paddd m0, m3
lea r2, [r2 + r3]
lea r0, [r0 + r1]
movu m1, [r2]
movu m2, [r2 + 16]
movu m3, [r2 + 32]
movu m4, [r2 + 48]
psadbw m1, [r0]
psadbw m2, [r0 + 16]
psadbw m3, [r0 + 32]
psadbw m4, [r0 + 48]
paddd m1, m2
paddd m3, m4
paddd m0, m1
paddd m0, m3
lea r2, [r2 + r3]
lea r0, [r0 + r1]
%endmacro
%macro SAD_W16 0
;-----------------------------------------------------------------------------
; int pixel_sad_16x16( uint8_t *, intptr_t, uint8_t *, intptr_t )
;-----------------------------------------------------------------------------
cglobal pixel_sad_16x16, 4,4,8
movu m0, [r2]
movu m1, [r2+r3]
lea r2, [r2+2*r3]
movu m2, [r2]
movu m3, [r2+r3]
lea r2, [r2+2*r3]
psadbw m0, [r0]
psadbw m1, [r0+r1]
lea r0, [r0+2*r1]
movu m4, [r2]
paddw m0, m1
psadbw m2, [r0]
psadbw m3, [r0+r1]
lea r0, [r0+2*r1]
movu m5, [r2+r3]
lea r2, [r2+2*r3]
paddw m2, m3
movu m6, [r2]
movu m7, [r2+r3]
lea r2, [r2+2*r3]
paddw m0, m2
psadbw m4, [r0]
psadbw m5, [r0+r1]
lea r0, [r0+2*r1]
movu m1, [r2]
paddw m4, m5
psadbw m6, [r0]
psadbw m7, [r0+r1]
lea r0, [r0+2*r1]
movu m2, [r2+r3]
lea r2, [r2+2*r3]
paddw m6, m7
movu m3, [r2]
paddw m0, m4
movu m4, [r2+r3]
lea r2, [r2+2*r3]
paddw m0, m6
psadbw m1, [r0]
psadbw m2, [r0+r1]
lea r0, [r0+2*r1]
movu m5, [r2]
paddw m1, m2
psadbw m3, [r0]
psadbw m4, [r0+r1]
lea r0, [r0+2*r1]
movu m6, [r2+r3]
lea r2, [r2+2*r3]
paddw m3, m4
movu m7, [r2]
paddw m0, m1
movu m1, [r2+r3]
paddw m0, m3
psadbw m5, [r0]
psadbw m6, [r0+r1]
lea r0, [r0+2*r1]
paddw m5, m6
psadbw m7, [r0]
psadbw m1, [r0+r1]
paddw m7, m1
paddw m0, m5
paddw m0, m7
SAD_END_SSE2
;-----------------------------------------------------------------------------
; int pixel_sad_16x8( uint8_t *, intptr_t, uint8_t *, intptr_t )
;-----------------------------------------------------------------------------
cglobal pixel_sad_16x8, 4,4
movu m0, [r2]
movu m2, [r2+r3]
lea r2, [r2+2*r3]
movu m3, [r2]
movu m4, [r2+r3]
psadbw m0, [r0]
psadbw m2, [r0+r1]
lea r0, [r0+2*r1]
psadbw m3, [r0]
psadbw m4, [r0+r1]
lea r0, [r0+2*r1]
lea r2, [r2+2*r3]
paddw m0, m2
paddw m3, m4
paddw m0, m3
movu m1, [r2]
movu m2, [r2+r3]
lea r2, [r2+2*r3]
movu m3, [r2]
movu m4, [r2+r3]
psadbw m1, [r0]
psadbw m2, [r0+r1]
lea r0, [r0+2*r1]
psadbw m3, [r0]
psadbw m4, [r0+r1]
lea r0, [r0+2*r1]
lea r2, [r2+2*r3]
paddw m1, m2
paddw m3, m4
paddw m0, m1
paddw m0, m3
SAD_END_SSE2
;-----------------------------------------------------------------------------
; int pixel_sad_16x12( uint8_t *, intptr_t, uint8_t *, intptr_t )
;-----------------------------------------------------------------------------
cglobal pixel_sad_16x12, 4,4,3
pxor m0, m0
PROCESS_SAD_16x4
PROCESS_SAD_16x4
PROCESS_SAD_16x4
movhlps m1, m0
paddd m0, m1
movd eax, m0
RET
;-----------------------------------------------------------------------------
; int pixel_sad_16x32( uint8_t *, intptr_t, uint8_t *, intptr_t )
;-----------------------------------------------------------------------------
cglobal pixel_sad_16x32, 4,5,3
pxor m0, m0
mov r4d, 4
.loop:
PROCESS_SAD_16x4
PROCESS_SAD_16x4
dec r4d
jnz .loop
movhlps m1, m0
paddd m0, m1
movd eax, m0
RET
;-----------------------------------------------------------------------------
; int pixel_sad_16x64( uint8_t *, intptr_t, uint8_t *, intptr_t )
;-----------------------------------------------------------------------------
cglobal pixel_sad_16x64, 4,5,3
pxor m0, m0
mov r4d, 8
.loop:
PROCESS_SAD_16x4
PROCESS_SAD_16x4
dec r4d
jnz .loop
movhlps m1, m0
paddd m0, m1
movd eax, m0
RET
;-----------------------------------------------------------------------------
; int pixel_sad_16x4( uint8_t *, intptr_t, uint8_t *, intptr_t )
;-----------------------------------------------------------------------------
cglobal pixel_sad_16x4, 4,4,3
movu m0, [r2]
movu m1, [r2 + r3]
psadbw m0, [r0]
psadbw m1, [r0 + r1]
paddd m0, m1
lea r2, [r2 + 2 * r3]
lea r0, [r0 + 2 * r1]
movu m1, [r2]
movu m2, [r2 + r3]
psadbw m1, [r0]
psadbw m2, [r0 + r1]
paddd m1, m2
paddd m0, m1
movhlps m1, m0
paddd m0, m1
movd eax, m0
RET
;-----------------------------------------------------------------------------
; int pixel_sad_32x8( uint8_t *, intptr_t, uint8_t *, intptr_t )
;-----------------------------------------------------------------------------
cglobal pixel_sad_32x8, 4,4,3
pxor m0, m0
PROCESS_SAD_32x4
PROCESS_SAD_32x4
movhlps m1, m0
paddd m0, m1
movd eax, m0
RET
;-----------------------------------------------------------------------------
; int pixel_sad_32x24( uint8_t *, intptr_t, uint8_t *, intptr_t )
;-----------------------------------------------------------------------------
cglobal pixel_sad_32x24, 4,5,3
pxor m0, m0
mov r4d, 3
.loop:
PROCESS_SAD_32x4
PROCESS_SAD_32x4
dec r4d
jnz .loop
movhlps m1, m0
paddd m0, m1
movd eax, m0
RET
;-----------------------------------------------------------------------------
; int pixel_sad_32x32( uint8_t *, intptr_t, uint8_t *, intptr_t )
;-----------------------------------------------------------------------------
cglobal pixel_sad_32x32, 4,5,3
pxor m0, m0
mov r4d, 4
.loop:
PROCESS_SAD_32x4
PROCESS_SAD_32x4
dec r4d
jnz .loop
movhlps m1, m0
paddd m0, m1
movd eax, m0
RET
;-----------------------------------------------------------------------------
; int pixel_sad_32x16( uint8_t *, intptr_t, uint8_t *, intptr_t )
;-----------------------------------------------------------------------------
cglobal pixel_sad_32x16, 4,4,3
pxor m0, m0
PROCESS_SAD_32x4
PROCESS_SAD_32x4
PROCESS_SAD_32x4
PROCESS_SAD_32x4
movhlps m1, m0
paddd m0, m1
movd eax, m0
RET
;-----------------------------------------------------------------------------
; int pixel_sad_32x64( uint8_t *, intptr_t, uint8_t *, intptr_t )
;-----------------------------------------------------------------------------
cglobal pixel_sad_32x64, 4,5,3
pxor m0, m0
mov r4d, 8
.loop:
PROCESS_SAD_32x4
PROCESS_SAD_32x4
dec r4d
jnz .loop
movhlps m1, m0
paddd m0, m1
movd eax, m0
RET
;-----------------------------------------------------------------------------
; int pixel_sad_8x32( uint8_t *, intptr_t, uint8_t *, intptr_t )
;-----------------------------------------------------------------------------
cglobal pixel_sad_8x32, 4,5,3
pxor m0, m0
mov r4d, 4
.loop:
PROCESS_SAD_8x4
PROCESS_SAD_8x4
dec r4d
jnz .loop
movhlps m1, m0
paddd m0, m1
movd eax, m0
RET
;-----------------------------------------------------------------------------
; int pixel_sad_64x16( uint8_t *, intptr_t, uint8_t *, intptr_t )
;-----------------------------------------------------------------------------
cglobal pixel_sad_64x16, 4,4,5
pxor m0, m0
PROCESS_SAD_64x4
PROCESS_SAD_64x4
PROCESS_SAD_64x4
PROCESS_SAD_64x4
movhlps m1, m0
paddd m0, m1
movd eax, m0
RET
;-----------------------------------------------------------------------------
; int pixel_sad_64x32( uint8_t *, intptr_t, uint8_t *, intptr_t )
;-----------------------------------------------------------------------------
cglobal pixel_sad_64x32, 4,5,5
pxor m0, m0
mov r4, 4
.loop:
PROCESS_SAD_64x4
PROCESS_SAD_64x4
dec r4
jnz .loop
movhlps m1, m0
paddd m0, m1
movd eax, m0
RET
;-----------------------------------------------------------------------------
; int pixel_sad_64x48( uint8_t *, intptr_t, uint8_t *, intptr_t )
;-----------------------------------------------------------------------------
cglobal pixel_sad_64x48, 4,5,5
pxor m0, m0
mov r4, 6
.loop:
PROCESS_SAD_64x4
PROCESS_SAD_64x4
dec r4d
jnz .loop
movhlps m1, m0
paddd m0, m1
movd eax, m0
RET
;-----------------------------------------------------------------------------
; int pixel_sad_64x64( uint8_t *, intptr_t, uint8_t *, intptr_t )
;-----------------------------------------------------------------------------
cglobal pixel_sad_64x64, 4,5,5
pxor m0, m0
mov r4, 8
.loop:
PROCESS_SAD_64x4
PROCESS_SAD_64x4
dec r4
jnz .loop
movhlps m1, m0
paddd m0, m1
movd eax, m0
RET
;-----------------------------------------------------------------------------
; int pixel_sad_48x64( uint8_t *, intptr_t, uint8_t *, intptr_t )
;-----------------------------------------------------------------------------
cglobal pixel_sad_48x64, 4,5,5
pxor m0, m0
mov r4, 64
.loop:
PROCESS_SAD_48x4
lea r2, [r2 + r3]
lea r0, [r0 + r1]
PROCESS_SAD_48x4
lea r2, [r2 + r3]
lea r0, [r0 + r1]
sub r4, 8
cmp r4, 8
jnz .loop
PROCESS_SAD_48x4
lea r2, [r2 + r3]
lea r0, [r0 + r1]
PROCESS_SAD_48x4
movhlps m1, m0
paddd m0, m1
movd eax, m0
RET
;-----------------------------------------------------------------------------
; int pixel_sad_24x32( uint8_t *, intptr_t, uint8_t *, intptr_t )
;-----------------------------------------------------------------------------
cglobal pixel_sad_24x32, 4,5,4
pxor m0, m0
mov r4, 32
.loop:
PROCESS_SAD_24x4
lea r2, [r2 + r3]
lea r0, [r0 + r1]
PROCESS_SAD_24x4
lea r2, [r2 + r3]
lea r0, [r0 + r1]
sub r4, 8
cmp r4, 8
jnz .loop
PROCESS_SAD_24x4
lea r2, [r2 + r3]
lea r0, [r0 + r1]
PROCESS_SAD_24x4
movhlps m1, m0
paddd m0, m1
movd eax, m0
RET
;-----------------------------------------------------------------------------
; int pixel_sad_12x16( uint8_t *, intptr_t, uint8_t *, intptr_t )
;-----------------------------------------------------------------------------
cglobal pixel_sad_12x16, 4,4,4
mova m4, [MSK]
pxor m0, m0
PROCESS_SAD_12x4
lea r2, [r2 + r3]
lea r0, [r0 + r1]
PROCESS_SAD_12x4
lea r2, [r2 + r3]
lea r0, [r0 + r1]
PROCESS_SAD_12x4
lea r2, [r2 + r3]
lea r0, [r0 + r1]
PROCESS_SAD_12x4
movhlps m1, m0
paddd m0, m1
movd eax, m0
RET
%endmacro
INIT_XMM sse2
SAD_W16
INIT_XMM sse3
SAD_W16
INIT_XMM sse2, aligned
SAD_W16
%macro SAD_INC_4x8P_SSE 1
movq m1, [r0]
movq m2, [r0+r1]
lea r0, [r0+2*r1]
movq m3, [r2]
movq m4, [r2+r3]
lea r2, [r2+2*r3]
movhps m1, [r0]
movhps m2, [r0+r1]
movhps m3, [r2]
movhps m4, [r2+r3]
lea r0, [r0+2*r1]
psadbw m1, m3
psadbw m2, m4
lea r2, [r2+2*r3]
ACCUM paddw, 0, 1, %1
paddw m0, m2
%endmacro
INIT_XMM
;Even on Nehalem, no sizes other than 8x16 benefit from this method.
cglobal pixel_sad_8x16_sse2, 4,4
SAD_INC_4x8P_SSE 0
SAD_INC_4x8P_SSE 1
SAD_INC_4x8P_SSE 1
SAD_INC_4x8P_SSE 1
SAD_END_SSE2
RET
;=============================================================================
; SAD x3/x4 MMX
;=============================================================================
%macro SAD_X3_START_1x8P 0
movq mm3, [r0]
movq mm0, [r1]
movq mm1, [r2]
movq mm2, [r3]
psadbw mm0, mm3
psadbw mm1, mm3
psadbw mm2, mm3
%endmacro
%macro SAD_X3_1x8P 2
movq mm3, [r0+%1]
movq mm4, [r1+%2]
movq mm5, [r2+%2]
movq mm6, [r3+%2]
psadbw mm4, mm3
psadbw mm5, mm3
psadbw mm6, mm3
paddw mm0, mm4
paddw mm1, mm5
paddw mm2, mm6
%endmacro
%macro SAD_X3_START_2x4P 3
movd mm3, [r0]
movd %1, [r1]
movd %2, [r2]
movd %3, [r3]
punpckldq mm3, [r0+FENC_STRIDE]
punpckldq %1, [r1+r4]
punpckldq %2, [r2+r4]
punpckldq %3, [r3+r4]
psadbw %1, mm3
psadbw %2, mm3
psadbw %3, mm3
%endmacro
%macro SAD_X3_2x16P 1
%if %1
SAD_X3_START_1x8P
%else
SAD_X3_1x8P 0, 0
%endif
SAD_X3_1x8P 8, 8
SAD_X3_1x8P FENC_STRIDE, r4
SAD_X3_1x8P FENC_STRIDE+8, r4+8
add r0, 2*FENC_STRIDE
lea r1, [r1+2*r4]
lea r2, [r2+2*r4]
lea r3, [r3+2*r4]
%endmacro
%macro SAD_X3_2x8P 1
%if %1
SAD_X3_START_1x8P
%else
SAD_X3_1x8P 0, 0
%endif
SAD_X3_1x8P FENC_STRIDE, r4
add r0, 2*FENC_STRIDE
lea r1, [r1+2*r4]
lea r2, [r2+2*r4]
lea r3, [r3+2*r4]
%endmacro
%macro SAD_X3_2x4P 1
%if %1
SAD_X3_START_2x4P mm0, mm1, mm2
%else
SAD_X3_START_2x4P mm4, mm5, mm6
paddw mm0, mm4
paddw mm1, mm5
paddw mm2, mm6
%endif
add r0, 2*FENC_STRIDE
lea r1, [r1+2*r4]
lea r2, [r2+2*r4]
lea r3, [r3+2*r4]
%endmacro
%macro SAD_X4_START_1x8P 0
movq mm7, [r0]
movq mm0, [r1]
movq mm1, [r2]
movq mm2, [r3]
movq mm3, [r4]
psadbw mm0, mm7
psadbw mm1, mm7
psadbw mm2, mm7
psadbw mm3, mm7
%endmacro
%macro SAD_X4_1x8P 2
movq mm7, [r0+%1]
movq mm4, [r1+%2]
movq mm5, [r2+%2]
movq mm6, [r3+%2]
psadbw mm4, mm7
psadbw mm5, mm7
psadbw mm6, mm7
psadbw mm7, [r4+%2]
paddw mm0, mm4
paddw mm1, mm5
paddw mm2, mm6
paddw mm3, mm7
%endmacro
%macro SAD_X4_START_2x4P 0
movd mm7, [r0]
movd mm0, [r1]
movd mm1, [r2]
movd mm2, [r3]
movd mm3, [r4]
punpckldq mm7, [r0+FENC_STRIDE]
punpckldq mm0, [r1+r5]
punpckldq mm1, [r2+r5]
punpckldq mm2, [r3+r5]
punpckldq mm3, [r4+r5]
psadbw mm0, mm7
psadbw mm1, mm7
psadbw mm2, mm7
psadbw mm3, mm7
%endmacro
%macro SAD_X4_INC_2x4P 0
movd mm7, [r0]
movd mm4, [r1]
movd mm5, [r2]
punpckldq mm7, [r0+FENC_STRIDE]
punpckldq mm4, [r1+r5]
punpckldq mm5, [r2+r5]
psadbw mm4, mm7
psadbw mm5, mm7
paddw mm0, mm4
paddw mm1, mm5
movd mm4, [r3]
movd mm5, [r4]
punpckldq mm4, [r3+r5]
punpckldq mm5, [r4+r5]
psadbw mm4, mm7
psadbw mm5, mm7
paddw mm2, mm4
paddw mm3, mm5
%endmacro
%macro SAD_X4_2x16P 1
%if %1
SAD_X4_START_1x8P
%else
SAD_X4_1x8P 0, 0
%endif
SAD_X4_1x8P 8, 8
SAD_X4_1x8P FENC_STRIDE, r5
SAD_X4_1x8P FENC_STRIDE+8, r5+8
add r0, 2*FENC_STRIDE
lea r1, [r1+2*r5]
lea r2, [r2+2*r5]
lea r3, [r3+2*r5]
lea r4, [r4+2*r5]
%endmacro
%macro SAD_X4_2x8P 1
%if %1
SAD_X4_START_1x8P
%else
SAD_X4_1x8P 0, 0
%endif
SAD_X4_1x8P FENC_STRIDE, r5
add r0, 2*FENC_STRIDE
lea r1, [r1+2*r5]
lea r2, [r2+2*r5]
lea r3, [r3+2*r5]
lea r4, [r4+2*r5]
%endmacro
%macro SAD_X4_2x4P 1
%if %1
SAD_X4_START_2x4P
%else
SAD_X4_INC_2x4P
%endif
add r0, 2*FENC_STRIDE
lea r1, [r1+2*r5]
lea r2, [r2+2*r5]
lea r3, [r3+2*r5]
lea r4, [r4+2*r5]
%endmacro
%macro SAD_X3_END 0
%if UNIX64
movd [r5+0], mm0
movd [r5+4], mm1
movd [r5+8], mm2
%else
mov r0, r5mp
movd [r0+0], mm0
movd [r0+4], mm1
movd [r0+8], mm2
%endif
RET
%endmacro
%macro SAD_X4_END 0
mov r0, r6mp
movd [r0+0], mm0
movd [r0+4], mm1
movd [r0+8], mm2
movd [r0+12], mm3
RET
%endmacro
%macro SAD_X3_12x4 0
mova m3, [r0]
movu m5, [r1]
pand m3, m4
pand m5, m4
psadbw m5, m3
paddd m0, m5
movu m5, [r2]
pand m5, m4
psadbw m5, m3
paddd m1, m5
movu m5, [r3]
pand m5, m4
psadbw m5, m3
paddd m2, m5
mova m3, [r0 + FENC_STRIDE]
movu m5, [r1 + r4]
pand m3, m4
pand m5, m4
psadbw m5, m3
paddd m0, m5
movu m5, [r2 + r4]
pand m5, m4
psadbw m5, m3
paddd m1, m5
movu m5, [r3 + r4]
pand m5, m4
psadbw m5, m3
paddd m2, m5
mova m3, [r0 + FENC_STRIDE * 2]
movu m5, [r1 + r4 * 2]
pand m3, m4
pand m5, m4
psadbw m5, m3
paddd m0, m5
movu m5, [r2 + r4 * 2]
pand m5, m4
psadbw m5, m3
paddd m1, m5
movu m5, [r3 + r4 * 2]
pand m5, m4
psadbw m5, m3
paddd m2, m5
lea r1, [r1 + r4 * 2]
lea r2, [r2 + r4 * 2]
lea r3, [r3 + r4 * 2]
mova m3, [r0 + FENC_STRIDE + FENC_STRIDE * 2]
movu m5, [r1 + r4]
pand m3, m4
pand m5, m4
psadbw m5, m3
paddd m0, m5
movu m5, [r2 + r4]
pand m5, m4
psadbw m5, m3
paddd m1, m5
movu m5, [r3 + r4]
pand m5, m4
psadbw m5, m3
paddd m2, m5
lea r0, [r0 + FENC_STRIDE * 4]
lea r1, [r1 + r4 * 2]
lea r2, [r2 + r4 * 2]
lea r3, [r3 + r4 * 2]
%endmacro
%macro SAD_X4_12x4 0
mova m4, [r0]
movu m5, [r1]
pand m4, m6
pand m5, m6
psadbw m5, m4
paddd m0, m5
movu m5, [r2]
pand m5, m6
psadbw m5, m4
paddd m1, m5
movu m5, [r3]
pand m5, m6
psadbw m5, m4
paddd m2, m5
movu m5, [r4]
pand m5, m6
psadbw m5, m4
paddd m3, m5
mova m4, [r0 + FENC_STRIDE]
movu m5, [r1 + r5]
pand m4, m6
pand m5, m6
psadbw m5, m4
paddd m0, m5
movu m5, [r2 + r5]
pand m5, m6
psadbw m5, m4
paddd m1, m5
movu m5, [r3 + r5]
pand m5, m6
psadbw m5, m4
paddd m2, m5
movu m5, [r4 + r5]
pand m5, m6
psadbw m5, m4
paddd m3, m5
mova m4, [r0 + FENC_STRIDE * 2]
movu m5, [r1 + r5 * 2]
pand m4, m6
pand m5, m6
psadbw m5, m4
paddd m0, m5
movu m5, [r2 + r5 * 2]
pand m5, m6
psadbw m5, m4
paddd m1, m5
movu m5, [r3 + r5 * 2]
pand m5, m6
psadbw m5, m4
paddd m2, m5
movu m5, [r4 + r5 * 2]
pand m5, m6
psadbw m5, m4
paddd m3, m5
lea r1, [r1 + r5 * 2]
lea r2, [r2 + r5 * 2]
lea r3, [r3 + r5 * 2]
lea r4, [r4 + r5 * 2]
mova m4, [r0 + FENC_STRIDE + FENC_STRIDE * 2]
movu m5, [r1 + r5]
pand m4, m6
pand m5, m6
psadbw m5, m4
paddd m0, m5
movu m5, [r2 + r5]
pand m5, m6
psadbw m5, m4
paddd m1, m5
movu m5, [r3 + r5]
pand m5, m6
psadbw m5, m4
paddd m2, m5
movu m5, [r4 + r5]
pand m5, m6
psadbw m5, m4
paddd m3, m5
lea r0, [r0 + FENC_STRIDE * 4]
lea r1, [r1 + r5 * 2]
lea r2, [r2 + r5 * 2]
lea r3, [r3 + r5 * 2]
lea r4, [r4 + r5 * 2]
%endmacro
%macro SAD_X3_24x4 0
mova m3, [r0]
mova m4, [r0 + 16]
movu m5, [r1]
movu m6, [r1 + 16]
psadbw m5, m3
psadbw m6, m4
pshufd m6, m6, 84
paddd m5, m6
paddd m0, m5
movu m5, [r2]
movu m6, [r2 + 16]
psadbw m5, m3
psadbw m6, m4
pshufd m6, m6, 84
paddd m5, m6
paddd m1, m5
movu m5, [r3]
movu m6, [r3 + 16]
psadbw m5, m3
psadbw m6, m4
pshufd m6, m6, 84
paddd m5, m6
paddd m2, m5
mova m3, [r0 + FENC_STRIDE]
mova m4, [r0 + 16 + FENC_STRIDE]
movu m5, [r1 + r4]
movu m6, [r1 + 16 + r4]
psadbw m5, m3
psadbw m6, m4
pshufd m6, m6, 84
paddd m5, m6
paddd m0, m5
movu m5, [r2 + r4]
movu m6, [r2 + 16 + r4]
psadbw m5, m3
psadbw m6, m4
pshufd m6, m6, 84
paddd m5, m6
paddd m1, m5
movu m5, [r3 + r4]
movu m6, [r3 + 16 + r4]
psadbw m5, m3
psadbw m6, m4
pshufd m6, m6, 84
paddd m5, m6
paddd m2, m5
mova m3, [r0 + FENC_STRIDE * 2]
mova m4, [r0 + 16 + FENC_STRIDE * 2]
movu m5, [r1 + r4 * 2]
movu m6, [r1 + 16 + r4 * 2]
psadbw m5, m3
psadbw m6, m4
pshufd m6, m6, 84
paddd m5, m6
paddd m0, m5
movu m5, [r2 + r4 * 2]
movu m6, [r2 + 16 + r4 * 2]
psadbw m5, m3
psadbw m6, m4
pshufd m6, m6, 84
paddd m5, m6
paddd m1, m5
movu m5, [r3 + r4 * 2]
movu m6, [r3 + 16 + r4 * 2]
psadbw m5, m3
psadbw m6, m4
pshufd m6, m6, 84
paddd m5, m6
paddd m2, m5
lea r0, [r0 + FENC_STRIDE * 2]
lea r1, [r1 + r4 * 2]
lea r2, [r2 + r4 * 2]
lea r3, [r3 + r4 * 2]
mova m3, [r0 + FENC_STRIDE]
mova m4, [r0 + 16 + FENC_STRIDE]
movu m5, [r1 + r4]
movu m6, [r1 + 16 + r4]
psadbw m5, m3
psadbw m6, m4
pshufd m6, m6, 84
paddd m5, m6
paddd m0, m5
movu m5, [r2 + r4]
movu m6, [r2 + 16 + r4]
psadbw m5, m3
psadbw m6, m4
pshufd m6, m6, 84
paddd m5, m6
paddd m1, m5
movu m5, [r3 + r4]
movu m6, [r3 + 16 + r4]
psadbw m5, m3
psadbw m6, m4
pshufd m6, m6, 84
paddd m5, m6
paddd m2, m5
lea r0, [r0 + FENC_STRIDE * 2]
lea r1, [r1 + r4 * 2]
lea r2, [r2 + r4 * 2]
lea r3, [r3 + r4 * 2]
%endmacro
%macro SAD_X4_24x4 0
mova m4, [r0]
mova m5, [r0 + 16]
movu m6, [r1]
movu m7, [r1 + 16]
psadbw m6, m4
psadbw m7, m5
pshufd m7, m7, 84
paddd m6, m7
paddd m0, m6
movu m6, [r2]
movu m7, [r2 + 16]
psadbw m6, m4
psadbw m7, m5
pshufd m7, m7, 84
paddd m6, m7
paddd m1, m6
movu m6, [r3]
movu m7, [r3 + 16]
psadbw m6, m4
psadbw m7, m5
pshufd m7, m7, 84
paddd m6, m7
paddd m2, m6
movu m6, [r4]
movu m7, [r4 + 16]
psadbw m6, m4
psadbw m7, m5
pshufd m7, m7, 84
paddd m6, m7
paddd m3, m6
mova m4, [r0 + FENC_STRIDE]
mova m5, [r0 + 16 + FENC_STRIDE]
movu m6, [r1 + r5]
movu m7, [r1 + 16 + r5]
psadbw m6, m4
psadbw m7, m5
pshufd m7, m7, 84
paddd m6, m7
paddd m0, m6
movu m6, [r2 + r5]
movu m7, [r2 + 16 + r5]
psadbw m6, m4
psadbw m7, m5
pshufd m7, m7, 84
paddd m6, m7
paddd m1, m6
movu m6, [r3 + r5]
movu m7, [r3 + 16 + r5]
psadbw m6, m4
psadbw m7, m5
pshufd m7, m7, 84
paddd m6, m7
paddd m2, m6
movu m6, [r4 + r5]
movu m7, [r4 + 16 + r5]
psadbw m6, m4
psadbw m7, m5
pshufd m7, m7, 84
paddd m6, m7
paddd m3, m6
mova m4, [r0 + FENC_STRIDE * 2]
mova m5, [r0 + 16 + FENC_STRIDE * 2]
movu m6, [r1 + r5 * 2]
movu m7, [r1 + 16 + r5 * 2]
psadbw m6, m4
psadbw m7, m5
pshufd m7, m7, 84
paddd m6, m7
paddd m0, m6
movu m6, [r2 + r5 * 2]
movu m7, [r2 + 16 + r5 * 2]
psadbw m6, m4
psadbw m7, m5
pshufd m7, m7, 84
paddd m6, m7
paddd m1, m6
movu m6, [r3 + r5 * 2]
movu m7, [r3 + 16 + r5 * 2]
psadbw m6, m4
psadbw m7, m5
pshufd m7, m7, 84
paddd m6, m7
paddd m2, m6
movu m6, [r4 + r5 * 2]
movu m7, [r4 + 16 + r5 * 2]
psadbw m6, m4
psadbw m7, m5
pshufd m7, m7, 84
paddd m6, m7
paddd m3, m6
lea r0, [r0 + FENC_STRIDE * 2]
lea r1, [r1 + r5 * 2]
lea r2, [r2 + r5 * 2]
lea r3, [r3 + r5 * 2]
lea r4, [r4 + r5 * 2]
mova m4, [r0 + FENC_STRIDE]
mova m5, [r0 + 16 + FENC_STRIDE]
movu m6, [r1 + r5]
movu m7, [r1 + 16 + r5]
psadbw m6, m4
psadbw m7, m5
pshufd m7, m7, 84
paddd m6, m7
paddd m0, m6
movu m6, [r2 + r5]
movu m7, [r2 + 16 + r5]
psadbw m6, m4
psadbw m7, m5
pshufd m7, m7, 84
paddd m6, m7
paddd m1, m6
movu m6, [r3 + r5]
movu m7, [r3 + 16 + r5]
psadbw m6, m4
psadbw m7, m5
pshufd m7, m7, 84
paddd m6, m7
paddd m2, m6
movu m6, [r4 + r5]
movu m7, [r4 + 16 + r5]
psadbw m6, m4
psadbw m7, m5
pshufd m7, m7, 84
paddd m6, m7
paddd m3, m6
lea r0, [r0 + FENC_STRIDE * 2]
lea r1, [r1 + r5 * 2]
lea r2, [r2 + r5 * 2]
lea r3, [r3 + r5 * 2]
lea r4, [r4 + r5 * 2]
%endmacro
%macro SAD_X3_32x4 0
mova m3, [r0]
mova m4, [r0 + 16]
movu m5, [r1]
movu m6, [r1 + 16]
psadbw m5, m3
psadbw m6, m4
paddd m5, m6
paddd m0, m5
movu m5, [r2]
movu m6, [r2 + 16]
psadbw m5, m3
psadbw m6, m4
paddd m5, m6
paddd m1, m5
movu m5, [r3]
movu m6, [r3 + 16]
psadbw m5, m3
psadbw m6, m4
paddd m5, m6
paddd m2, m5
lea r0, [r0 + FENC_STRIDE]
lea r1, [r1 + r4]
lea r2, [r2 + r4]
lea r3, [r3 + r4]
mova m3, [r0]
mova m4, [r0 + 16]
movu m5, [r1]
movu m6, [r1 + 16]
psadbw m5, m3
psadbw m6, m4
paddd m5, m6
paddd m0, m5
movu m5, [r2]
movu m6, [r2 + 16]
psadbw m5, m3
psadbw m6, m4
paddd m5, m6
paddd m1, m5
movu m5, [r3]
movu m6, [r3 + 16]
psadbw m5, m3
psadbw m6, m4
paddd m5, m6
paddd m2, m5
lea r0, [r0 + FENC_STRIDE]
lea r1, [r1 + r4]
lea r2, [r2 + r4]
lea r3, [r3 + r4]
mova m3, [r0]
mova m4, [r0 + 16]
movu m5, [r1]
movu m6, [r1 + 16]
psadbw m5, m3
psadbw m6, m4
paddd m5, m6
paddd m0, m5
movu m5, [r2]
movu m6, [r2 + 16]
psadbw m5, m3
psadbw m6, m4
paddd m5, m6
paddd m1, m5
movu m5, [r3]
movu m6, [r3 + 16]
psadbw m5, m3
psadbw m6, m4
paddd m5, m6
paddd m2, m5
lea r0, [r0 + FENC_STRIDE]
lea r1, [r1 + r4]
lea r2, [r2 + r4]
lea r3, [r3 + r4]
mova m3, [r0]
mova m4, [r0 + 16]
movu m5, [r1]
movu m6, [r1 + 16]
psadbw m5, m3
psadbw m6, m4
paddd m5, m6
paddd m0, m5
movu m5, [r2]
movu m6, [r2 + 16]
psadbw m5, m3
psadbw m6, m4
paddd m5, m6
paddd m1, m5
movu m5, [r3]
movu m6, [r3 + 16]
psadbw m5, m3
psadbw m6, m4
paddd m5, m6
paddd m2, m5
lea r0, [r0 + FENC_STRIDE]
lea r1, [r1 + r4]
lea r2, [r2 + r4]
lea r3, [r3 + r4]
%endmacro
%macro SAD_X4_32x4 0
mova m4, [r0]
mova m5, [r0 + 16]
movu m6, [r1]
movu m7, [r1 + 16]
psadbw m6, m4
psadbw m7, m5
paddd m6, m7
paddd m0, m6
movu m6, [r2]
movu m7, [r2 + 16]
psadbw m6, m4
psadbw m7, m5
paddd m6, m7
paddd m1, m6
movu m6, [r3]
movu m7, [r3 + 16]
psadbw m6, m4
psadbw m7, m5
paddd m6, m7
paddd m2, m6
movu m6, [r4]
movu m7, [r4 + 16]
psadbw m6, m4
psadbw m7, m5
paddd m6, m7
paddd m3, m6
lea r0, [r0 + FENC_STRIDE]
lea r1, [r1 + r5]
lea r2, [r2 + r5]
lea r3, [r3 + r5]
lea r4, [r4 + r5]
mova m4, [r0]
mova m5, [r0 + 16]
movu m6, [r1]
movu m7, [r1 + 16]
psadbw m6, m4
psadbw m7, m5
paddd m6, m7
paddd m0, m6
movu m6, [r2]
movu m7, [r2 + 16]
psadbw m6, m4
psadbw m7, m5
paddd m6, m7
paddd m1, m6
movu m6, [r3]
movu m7, [r3 + 16]
psadbw m6, m4
psadbw m7, m5
paddd m6, m7
paddd m2, m6
movu m6, [r4]
movu m7, [r4 + 16]
psadbw m6, m4
psadbw m7, m5
paddd m6, m7
paddd m3, m6
lea r0, [r0 + FENC_STRIDE]
lea r1, [r1 + r5]
lea r2, [r2 + r5]
lea r3, [r3 + r5]
lea r4, [r4 + r5]
mova m4, [r0]
mova m5, [r0 + 16]
movu m6, [r1]
movu m7, [r1 + 16]
psadbw m6, m4
psadbw m7, m5
paddd m6, m7
paddd m0, m6
movu m6, [r2]
movu m7, [r2 + 16]
psadbw m6, m4
psadbw m7, m5
paddd m6, m7
paddd m1, m6
movu m6, [r3]
movu m7, [r3 + 16]
psadbw m6, m4
psadbw m7, m5
paddd m6, m7
paddd m2, m6
movu m6, [r4]
movu m7, [r4 + 16]
psadbw m6, m4
psadbw m7, m5
paddd m6, m7
paddd m3, m6
lea r0, [r0 + FENC_STRIDE]
lea r1, [r1 + r5]
lea r2, [r2 + r5]
lea r3, [r3 + r5]
lea r4, [r4 + r5]
mova m4, [r0]
mova m5, [r0 + 16]
movu m6, [r1]
movu m7, [r1 + 16]
psadbw m6, m4
psadbw m7, m5
paddd m6, m7
paddd m0, m6
movu m6, [r2]
movu m7, [r2 + 16]
psadbw m6, m4
psadbw m7, m5
paddd m6, m7
paddd m1, m6
movu m6, [r3]
movu m7, [r3 + 16]
psadbw m6, m4
psadbw m7, m5
paddd m6, m7
paddd m2, m6
movu m6, [r4]
movu m7, [r4 + 16]
psadbw m6, m4
psadbw m7, m5
paddd m6, m7
paddd m3, m6
lea r0, [r0 + FENC_STRIDE]
lea r1, [r1 + r5]
lea r2, [r2 + r5]
lea r3, [r3 + r5]
lea r4, [r4 + r5]
%endmacro
%macro SAD_X3_48x4 0
mova m3, [r0]
mova m4, [r0 + 16]
mova m5, [r0 + 32]
movu m6, [r1]
psadbw m6, m3
paddd m0, m6
movu m6, [r1 + 16]
psadbw m6, m4
paddd m0, m6
movu m6, [r1 + 32]
psadbw m6, m5
paddd m0, m6
movu m6, [r2]
psadbw m6, m3
paddd m1, m6
movu m6, [r2 + 16]
psadbw m6, m4
paddd m1, m6
movu m6, [r2 + 32]
psadbw m6, m5
paddd m1, m6
movu m6, [r3]
psadbw m6, m3
paddd m2, m6
movu m6, [r3 + 16]
psadbw m6, m4
paddd m2, m6
movu m6, [r3 + 32]
psadbw m6, m5
paddd m2, m6
mova m3, [r0 + FENC_STRIDE]
mova m4, [r0 + 16 + FENC_STRIDE]
mova m5, [r0 + 32 + FENC_STRIDE]
movu m6, [r1 + r4]
psadbw m6, m3
paddd m0, m6
movu m6, [r1 + 16 + r4]
psadbw m6, m4
paddd m0, m6
movu m6, [r1 + 32 + r4]
psadbw m6, m5
paddd m0, m6
movu m6, [r2 + r4]
psadbw m6, m3
paddd m1, m6
movu m6, [r2 + 16 + r4]
psadbw m6, m4
paddd m1, m6
movu m6, [r2 + 32 + r4]
psadbw m6, m5
paddd m1, m6
movu m6, [r3 + r4]
psadbw m6, m3
paddd m2, m6
movu m6, [r3 + 16 + r4]
psadbw m6, m4
paddd m2, m6
movu m6, [r3 + 32 + r4]
psadbw m6, m5
paddd m2, m6
mova m3, [r0 + FENC_STRIDE * 2]
mova m4, [r0 + 16 + FENC_STRIDE * 2]
mova m5, [r0 + 32 + FENC_STRIDE * 2]
movu m6, [r1 + r4 * 2]
psadbw m6, m3
paddd m0, m6
movu m6, [r1 + 16 + r4 * 2]
psadbw m6, m4
paddd m0, m6
movu m6, [r1 + 32 + r4 * 2]
psadbw m6, m5
paddd m0, m6
movu m6, [r2 + r4 * 2]
psadbw m6, m3
paddd m1, m6
movu m6, [r2 + 16 + r4 * 2]
psadbw m6, m4
paddd m1, m6
movu m6, [r2 + 32 + r4 * 2]
psadbw m6, m5
paddd m1, m6
movu m6, [r3 + r4 * 2]
psadbw m6, m3
paddd m2, m6
movu m6, [r3 + 16 + r4 * 2]
psadbw m6, m4
paddd m2, m6
movu m6, [r3 + 32 + r4 * 2]
psadbw m6, m5
paddd m2, m6
lea r0, [r0 + FENC_STRIDE * 2]
lea r1, [r1 + r4 * 2]
lea r2, [r2 + r4 * 2]
lea r3, [r3 + r4 * 2]
mova m3, [r0 + FENC_STRIDE]
mova m4, [r0 + 16 + FENC_STRIDE]
mova m5, [r0 + 32 + FENC_STRIDE]
movu m6, [r1 + r4]
psadbw m6, m3
paddd m0, m6
movu m6, [r1 + 16 + r4]
psadbw m6, m4
paddd m0, m6
movu m6, [r1 + 32 + r4]
psadbw m6, m5
paddd m0, m6
movu m6, [r2 + r4]
psadbw m6, m3
paddd m1, m6
movu m6, [r2 + 16 + r4]
psadbw m6, m4
paddd m1, m6
movu m6, [r2 + 32 + r4]
psadbw m6, m5
paddd m1, m6
movu m6, [r3 + r4]
psadbw m6, m3
paddd m2, m6
movu m6, [r3 + 16 + r4]
psadbw m6, m4
paddd m2, m6
movu m6, [r3 + 32 + r4]
psadbw m6, m5
paddd m2, m6
lea r0, [r0 + FENC_STRIDE * 2]
lea r1, [r1 + r4 * 2]
lea r2, [r2 + r4 * 2]
lea r3, [r3 + r4 * 2]
%endmacro
%macro SAD_X4_48x4 0
mova m4, [r0]
mova m5, [r0 + 16]
mova m6, [r0 + 32]
movu m7, [r1]
psadbw m7, m4
paddd m0, m7
movu m7, [r1 + 16]
psadbw m7, m5
paddd m0, m7
movu m7, [r1 + 32]
psadbw m7, m6
paddd m0, m7
movu m7, [r2]
psadbw m7, m4
paddd m1, m7
movu m7, [r2 + 16]
psadbw m7, m5
paddd m1, m7
movu m7, [r2 + 32]
psadbw m7, m6
paddd m1, m7
movu m7, [r3]
psadbw m7, m4
paddd m2, m7
movu m7, [r3 + 16]
psadbw m7, m5
paddd m2, m7
movu m7, [r3 + 32]
psadbw m7, m6
paddd m2, m7
movu m7, [r4]
psadbw m7, m4
paddd m3, m7
movu m7, [r4 + 16]
psadbw m7, m5
paddd m3, m7
movu m7, [r4 + 32]
psadbw m7, m6
paddd m3, m7
mova m4, [r0 + FENC_STRIDE]
mova m5, [r0 + 16 + FENC_STRIDE]
mova m6, [r0 + 32 + FENC_STRIDE]
movu m7, [r1 + r5]
psadbw m7, m4
paddd m0, m7
movu m7, [r1 + 16 + r5]
psadbw m7, m5
paddd m0, m7
movu m7, [r1 + 32 + r5]
psadbw m7, m6
paddd m0, m7
movu m7, [r2 + r5]
psadbw m7, m4
paddd m1, m7
movu m7, [r2 + 16 + r5]
psadbw m7, m5
paddd m1, m7
movu m7, [r2 + 32 + r5]
psadbw m7, m6
paddd m1, m7
movu m7, [r3 + r5]
psadbw m7, m4
paddd m2, m7
movu m7, [r3 + 16 + r5]
psadbw m7, m5
paddd m2, m7
movu m7, [r3 + 32 + r5]
psadbw m7, m6
paddd m2, m7
movu m7, [r4 + r5]
psadbw m7, m4
paddd m3, m7
movu m7, [r4 + 16 + r5]
psadbw m7, m5
paddd m3, m7
movu m7, [r4 + 32 + r5]
psadbw m7, m6
paddd m3, m7
mova m4, [r0 + FENC_STRIDE * 2]
mova m5, [r0 + 16 + FENC_STRIDE * 2]
mova m6, [r0 + 32 + FENC_STRIDE * 2]
movu m7, [r1 + r5 * 2]
psadbw m7, m4
paddd m0, m7
movu m7, [r1 + 16 + r5 * 2]
psadbw m7, m5
paddd m0, m7
movu m7, [r1 + 32 + r5 * 2]
psadbw m7, m6
paddd m0, m7
movu m7, [r2 + r5 * 2]
psadbw m7, m4
paddd m1, m7
movu m7, [r2 + 16 + r5 * 2]
psadbw m7, m5
paddd m1, m7
movu m7, [r2 + 32 + r5 * 2]
psadbw m7, m6
paddd m1, m7
movu m7, [r3 + r5 * 2]
psadbw m7, m4
paddd m2, m7
movu m7, [r3 + 16 + r5 * 2]
psadbw m7, m5
paddd m2, m7
movu m7, [r3 + 32 + r5 * 2]
psadbw m7, m6
paddd m2, m7
movu m7, [r4 + r5 * 2]
psadbw m7, m4
paddd m3, m7
movu m7, [r4 + 16 + r5 * 2]
psadbw m7, m5
paddd m3, m7
movu m7, [r4 + 32 + r5 * 2]
psadbw m7, m6
paddd m3, m7
lea r0, [r0 + FENC_STRIDE * 2]
lea r1, [r1 + r5 * 2]
lea r2, [r2 + r5 * 2]
lea r3, [r3 + r5 * 2]
lea r4, [r4 + r5 * 2]
mova m4, [r0 + FENC_STRIDE]
mova m5, [r0 + 16 + FENC_STRIDE]
mova m6, [r0 + 32 + FENC_STRIDE]
movu m7, [r1 + r5]
psadbw m7, m4
paddd m0, m7
movu m7, [r1 + 16 + r5]
psadbw m7, m5
paddd m0, m7
movu m7, [r1 + 32 + r5]
psadbw m7, m6
paddd m0, m7
movu m7, [r2 + r5]
psadbw m7, m4
paddd m1, m7
movu m7, [r2 + 16 + r5]
psadbw m7, m5
paddd m1, m7
movu m7, [r2 + 32 + r5]
psadbw m7, m6
paddd m1, m7
movu m7, [r3 + r5]
psadbw m7, m4
paddd m2, m7
movu m7, [r3 + 16 + r5]
psadbw m7, m5
paddd m2, m7
movu m7, [r3 + 32 + r5]
psadbw m7, m6
paddd m2, m7
movu m7, [r4 + r5]
psadbw m7, m4
paddd m3, m7
movu m7, [r4 + 16 + r5]
psadbw m7, m5
paddd m3, m7
movu m7, [r4 + 32 + r5]
psadbw m7, m6
paddd m3, m7
lea r0, [r0 + FENC_STRIDE * 2]
lea r1, [r1 + r5 * 2]
lea r2, [r2 + r5 * 2]
lea r3, [r3 + r5 * 2]
lea r4, [r4 + r5 * 2]
%endmacro
%macro SAD_X3_64x4 0
mova m3, [r0]
mova m4, [r0 + 16]
movu m5, [r1]
psadbw m5, m3
paddd m0, m5
movu m5, [r1 + 16]
psadbw m5, m4
paddd m0, m5
movu m5, [r2]
psadbw m5, m3
paddd m1, m5
movu m5, [r2 + 16]
psadbw m5, m4
paddd m1, m5
movu m5, [r3]
psadbw m5, m3
paddd m2, m5
movu m5, [r3 + 16]
psadbw m5, m4
paddd m2, m5
mova m3, [r0 + 32]
mova m4, [r0 + 48]
movu m5, [r1 + 32]
psadbw m5, m3
paddd m0, m5
movu m5, [r1 + 48]
psadbw m5, m4
paddd m0, m5
movu m5, [r2 + 32]
psadbw m5, m3
paddd m1, m5
movu m5, [r2 + 48]
psadbw m5, m4
paddd m1, m5
movu m5, [r3 + 32]
psadbw m5, m3
paddd m2, m5
movu m5, [r3 + 48]
psadbw m5, m4
paddd m2, m5
mova m3, [r0 + FENC_STRIDE]
mova m4, [r0 + 16 + FENC_STRIDE]
movu m5, [r1 + r4]
psadbw m5, m3
paddd m0, m5
movu m5, [r1 + 16 + r4]
psadbw m5, m4
paddd m0, m5
movu m5, [r2 + r4]
psadbw m5, m3
paddd m1, m5
movu m5, [r2 + 16 + r4]
psadbw m5, m4
paddd m1, m5
movu m5, [r3 + r4]
psadbw m5, m3
paddd m2, m5
movu m5, [r3 + 16 + r4]
psadbw m5, m4
paddd m2, m5
mova m3, [r0 + 32 + FENC_STRIDE]
mova m4, [r0 + 48 + FENC_STRIDE]
movu m5, [r1 + 32 + r4]
psadbw m5, m3
paddd m0, m5
movu m5, [r1 + 48 + r4]
psadbw m5, m4
paddd m0, m5
movu m5, [r2 + 32 + r4]
psadbw m5, m3
paddd m1, m5
movu m5, [r2 + 48 + r4]
psadbw m5, m4
paddd m1, m5
movu m5, [r3 + 32 + r4]
psadbw m5, m3
paddd m2, m5
movu m5, [r3 + 48 + r4]
psadbw m5, m4
paddd m2, m5
mova m3, [r0 + FENC_STRIDE * 2]
mova m4, [r0 + 16 + FENC_STRIDE * 2]
movu m5, [r1 + r4 * 2]
psadbw m5, m3
paddd m0, m5
movu m5, [r1 + 16 + r4 * 2]
psadbw m5, m4
paddd m0, m5
movu m5, [r2 + r4 * 2]
psadbw m5, m3
paddd m1, m5
movu m5, [r2 + 16 + r4 * 2]
psadbw m5, m4
paddd m1, m5
movu m5, [r3 + r4 * 2]
psadbw m5, m3
paddd m2, m5
movu m5, [r3 + 16 + r4 * 2]
psadbw m5, m4
paddd m2, m5
mova m3, [r0 + 32 + FENC_STRIDE * 2]
mova m4, [r0 + 48 + FENC_STRIDE * 2]
movu m5, [r1 + 32 + r4 * 2]
psadbw m5, m3
paddd m0, m5
movu m5, [r1 + 48 + r4 * 2]
psadbw m5, m4
paddd m0, m5
movu m5, [r2 + 32 + r4 * 2]
psadbw m5, m3
paddd m1, m5
movu m5, [r2 + 48 + r4 * 2]
psadbw m5, m4
paddd m1, m5
movu m5, [r3 + 32 + r4 * 2]
psadbw m5, m3
paddd m2, m5
movu m5, [r3 + 48 + r4 * 2]
psadbw m5, m4
paddd m2, m5
lea r0, [r0 + FENC_STRIDE * 2]
lea r1, [r1 + r4 * 2]
lea r2, [r2 + r4 * 2]
lea r3, [r3 + r4 * 2]
mova m3, [r0 + FENC_STRIDE]
mova m4, [r0 + 16 + FENC_STRIDE]
movu m5, [r1 + r4]
psadbw m5, m3
paddd m0, m5
movu m5, [r1 + 16 + r4]
psadbw m5, m4
paddd m0, m5
movu m5, [r2 + r4]
psadbw m5, m3
paddd m1, m5
movu m5, [r2 + 16 + r4]
psadbw m5, m4
paddd m1, m5
movu m5, [r3 + r4]
psadbw m5, m3
paddd m2, m5
movu m5, [r3 + 16 + r4]
psadbw m5, m4
paddd m2, m5
mova m3, [r0 + 32 + FENC_STRIDE]
mova m4, [r0 + 48 + FENC_STRIDE]
movu m5, [r1 + 32 + r4]
psadbw m5, m3
paddd m0, m5
movu m5, [r1 + 48 + r4]
psadbw m5, m4
paddd m0, m5
movu m5, [r2 + 32 + r4]
psadbw m5, m3
paddd m1, m5
movu m5, [r2 + 48 + r4]
psadbw m5, m4
paddd m1, m5
movu m5, [r3 + 32 + r4]
psadbw m5, m3
paddd m2, m5
movu m5, [r3 + 48 + r4]
psadbw m5, m4
paddd m2, m5
lea r0, [r0 + FENC_STRIDE * 2]
lea r1, [r1 + r4 * 2]
lea r2, [r2 + r4 * 2]
lea r3, [r3 + r4 * 2]
%endmacro
%macro SAD_X4_64x4 0
mova m4, [r0]
mova m5, [r0 + 16]
movu m6, [r1]
psadbw m6, m4
paddd m0, m6
movu m6, [r1 + 16]
psadbw m6, m5
paddd m0, m6
movu m6, [r2]
psadbw m6, m4
paddd m1, m6
movu m6, [r2 + 16]
psadbw m6, m5
paddd m1, m6
movu m6, [r3]
psadbw m6, m4
paddd m2, m6
movu m6, [r3 + 16]
psadbw m6, m5
paddd m2, m6
movu m6, [r4]
psadbw m6, m4
paddd m3, m6
movu m6, [r4 + 16]
psadbw m6, m5
paddd m3, m6
mova m4, [r0 + 32]
mova m5, [r0 + 48]
movu m6, [r1 + 32]
psadbw m6, m4
paddd m0, m6
movu m6, [r1 + 48]
psadbw m6, m5
paddd m0, m6
movu m6, [r2 + 32]
psadbw m6, m4
paddd m1, m6
movu m6, [r2 + 48]
psadbw m6, m5
paddd m1, m6
movu m6, [r3 + 32]
psadbw m6, m4
paddd m2, m6
movu m6, [r3 + 48]
psadbw m6, m5
paddd m2, m6
movu m6, [r4 + 32]
psadbw m6, m4
paddd m3, m6
movu m6, [r4 + 48]
psadbw m6, m5
paddd m3, m6
mova m4, [r0 + FENC_STRIDE]
mova m5, [r0 + 16 + FENC_STRIDE]
movu m6, [r1 + r5]
psadbw m6, m4
paddd m0, m6
movu m6, [r1 + 16 + r5]
psadbw m6, m5
paddd m0, m6
movu m6, [r2 + r5]
psadbw m6, m4
paddd m1, m6
movu m6, [r2 + 16 + r5]
psadbw m6, m5
paddd m1, m6
movu m6, [r3 + r5]
psadbw m6, m4
paddd m2, m6
movu m6, [r3 + 16 + r5]
psadbw m6, m5
paddd m2, m6
movu m6, [r4 + r5]
psadbw m6, m4
paddd m3, m6
movu m6, [r4 + 16 + r5]
psadbw m6, m5
paddd m3, m6
mova m4, [r0 + 32 + FENC_STRIDE]
mova m5, [r0 + 48 + FENC_STRIDE]
movu m6, [r1 + 32 + r5]
psadbw m6, m4
paddd m0, m6
movu m6, [r1 + 48 + r5]
psadbw m6, m5
paddd m0, m6
movu m6, [r2 + 32 + r5]
psadbw m6, m4
paddd m1, m6
movu m6, [r2 + 48 + r5]
psadbw m6, m5
paddd m1, m6
movu m6, [r3 + 32 + r5]
psadbw m6, m4
paddd m2, m6
movu m6, [r3 + 48 + r5]
psadbw m6, m5
paddd m2, m6
movu m6, [r4 + 32 + r5]
psadbw m6, m4
paddd m3, m6
movu m6, [r4 + 48 + r5]
psadbw m6, m5
paddd m3, m6
mova m4, [r0 + FENC_STRIDE * 2]
mova m5, [r0 + 16 + FENC_STRIDE * 2]
movu m6, [r1 + r5 * 2]
psadbw m6, m4
paddd m0, m6
movu m6, [r1 + 16 + r5 * 2]
psadbw m6, m5
paddd m0, m6
movu m6, [r2 + r5 * 2]
psadbw m6, m4
paddd m1, m6
movu m6, [r2 + 16 + r5 * 2]
psadbw m6, m5
paddd m1, m6
movu m6, [r3 + r5 * 2]
psadbw m6, m4
paddd m2, m6
movu m6, [r3 + 16 + r5 * 2]
psadbw m6, m5
paddd m2, m6
movu m6, [r4 + r5 * 2]
psadbw m6, m4
paddd m3, m6
movu m6, [r4 + 16 + r5 * 2]
psadbw m6, m5
paddd m3, m6
mova m4, [r0 + 32 + FENC_STRIDE * 2]
mova m5, [r0 + 48 + FENC_STRIDE * 2]
movu m6, [r1 + 32 + r5 * 2]
psadbw m6, m4
paddd m0, m6
movu m6, [r1 + 48 + r5 * 2]
psadbw m6, m5
paddd m0, m6
movu m6, [r2 + 32 + r5 * 2]
psadbw m6, m4
paddd m1, m6
movu m6, [r2 + 48 + r5 * 2]
psadbw m6, m5
paddd m1, m6
movu m6, [r3 + 32 + r5 * 2]
psadbw m6, m4
paddd m2, m6
movu m6, [r3 + 48 + r5 * 2]
psadbw m6, m5
paddd m2, m6
movu m6, [r4 + 32 + r5 * 2]
psadbw m6, m4
paddd m3, m6
movu m6, [r4 + 48 + r5 * 2]
psadbw m6, m5
paddd m3, m6
lea r0, [r0 + FENC_STRIDE * 2]
lea r1, [r1 + r5 * 2]
lea r2, [r2 + r5 * 2]
lea r3, [r3 + r5 * 2]
lea r4, [r4 + r5 * 2]
mova m4, [r0 + FENC_STRIDE]
mova m5, [r0 + 16 + FENC_STRIDE]
movu m6, [r1 + r5]
psadbw m6, m4
paddd m0, m6
movu m6, [r1 + 16 + r5]
psadbw m6, m5
paddd m0, m6
movu m6, [r2 + r5]
psadbw m6, m4
paddd m1, m6
movu m6, [r2 + 16 + r5]
psadbw m6, m5
paddd m1, m6
movu m6, [r3 + r5]
psadbw m6, m4
paddd m2, m6
movu m6, [r3 + 16 + r5]
psadbw m6, m5
paddd m2, m6
movu m6, [r4 + r5]
psadbw m6, m4
paddd m3, m6
movu m6, [r4 + 16 + r5]
psadbw m6, m5
paddd m3, m6
mova m4, [r0 + 32 + FENC_STRIDE]
mova m5, [r0 + 48 + FENC_STRIDE]
movu m6, [r1 + 32 + r5]
psadbw m6, m4
paddd m0, m6
movu m6, [r1 + 48 + r5]
psadbw m6, m5
paddd m0, m6
movu m6, [r2 + 32 + r5]
psadbw m6, m4
paddd m1, m6
movu m6, [r2 + 48 + r5]
psadbw m6, m5
paddd m1, m6
movu m6, [r3 + 32 + r5]
psadbw m6, m4
paddd m2, m6
movu m6, [r3 + 48 + r5]
psadbw m6, m5
paddd m2, m6
movu m6, [r4 + 32 + r5]
psadbw m6, m4
paddd m3, m6
movu m6, [r4 + 48 + r5]
psadbw m6, m5
paddd m3, m6
lea r0, [r0 + FENC_STRIDE * 2]
lea r1, [r1 + r5 * 2]
lea r2, [r2 + r5 * 2]
lea r3, [r3 + r5 * 2]
lea r4, [r4 + r5 * 2]
%endmacro
;-----------------------------------------------------------------------------
; void pixel_sad_x3_16x16( uint8_t *fenc, uint8_t *pix0, uint8_t *pix1,
; uint8_t *pix2, intptr_t i_stride, int scores[3] )
;-----------------------------------------------------------------------------
%macro SAD_X 3
cglobal pixel_sad_x%1_%2x%3_mmx2, %1+2, %1+2
SAD_X%1_2x%2P 1
%rep %3/2-1
SAD_X%1_2x%2P 0
%endrep
SAD_X%1_END
%endmacro
INIT_MMX
SAD_X 3, 16, 16
SAD_X 3, 16, 8
SAD_X 3, 8, 16
SAD_X 3, 8, 8
SAD_X 3, 8, 4
SAD_X 3, 4, 16
SAD_X 3, 4, 8
SAD_X 3, 4, 4
SAD_X 4, 16, 16
SAD_X 4, 16, 8
SAD_X 4, 8, 16
SAD_X 4, 8, 8
SAD_X 4, 8, 4
SAD_X 4, 4, 16
SAD_X 4, 4, 8
SAD_X 4, 4, 4
;=============================================================================
; SAD x3/x4 XMM
;=============================================================================
%macro SAD_X3_START_1x16P_SSE2 0
mova m2, [r0]
%if cpuflag(avx)
psadbw m0, m2, [r1]
psadbw m1, m2, [r2]
psadbw m2, [r3]
%else
movu m0, [r1]
movu m1, [r2]
movu m3, [r3]
psadbw m0, m2
psadbw m1, m2
psadbw m2, m3
%endif
%endmacro
%macro SAD_X3_1x16P_SSE2 2
mova m3, [r0+%1]
%if cpuflag(avx)
psadbw m4, m3, [r1+%2]
psadbw m5, m3, [r2+%2]
psadbw m3, [r3+%2]
%else
movu m4, [r1+%2]
movu m5, [r2+%2]
movu m6, [r3+%2]
psadbw m4, m3
psadbw m5, m3
psadbw m3, m6
%endif
paddd m0, m4
paddd m1, m5
paddd m2, m3
%endmacro
%if ARCH_X86_64
DECLARE_REG_TMP 6
%else
DECLARE_REG_TMP 5
%endif
%macro SAD_X3_4x16P_SSE2 2
%if %1==0
lea t0, [r4*3]
SAD_X3_START_1x16P_SSE2
%else
SAD_X3_1x16P_SSE2 FENC_STRIDE*(0+(%1&1)*4), r4*0
%endif
SAD_X3_1x16P_SSE2 FENC_STRIDE*(1+(%1&1)*4), r4*1
SAD_X3_1x16P_SSE2 FENC_STRIDE*(2+(%1&1)*4), r4*2
SAD_X3_1x16P_SSE2 FENC_STRIDE*(3+(%1&1)*4), t0
%if %1 != %2-1
%if (%1&1) != 0
add r0, 8*FENC_STRIDE
%endif
lea r1, [r1+4*r4]
lea r2, [r2+4*r4]
lea r3, [r3+4*r4]
%endif
%endmacro
%macro SAD_X3_START_2x8P_SSE2 0
movq m3, [r0]
movq m0, [r1]
movq m1, [r2]
movq m2, [r3]
movhps m3, [r0+FENC_STRIDE]
movhps m0, [r1+r4]
movhps m1, [r2+r4]
movhps m2, [r3+r4]
psadbw m0, m3
psadbw m1, m3
psadbw m2, m3
%endmacro
%macro SAD_X3_2x8P_SSE2 4
movq m6, [r0+%1]
movq m3, [r1+%2]
movq m4, [r2+%2]
movq m5, [r3+%2]
movhps m6, [r0+%3]
movhps m3, [r1+%4]
movhps m4, [r2+%4]
movhps m5, [r3+%4]
psadbw m3, m6
psadbw m4, m6
psadbw m5, m6
paddd m0, m3
paddd m1, m4
paddd m2, m5
%endmacro
%macro SAD_X4_START_2x8P_SSE2 0
movq m4, [r0]
movq m0, [r1]
movq m1, [r2]
movq m2, [r3]
movq m3, [r4]
movhps m4, [r0+FENC_STRIDE]
movhps m0, [r1+r5]
movhps m1, [r2+r5]
movhps m2, [r3+r5]
movhps m3, [r4+r5]
psadbw m0, m4
psadbw m1, m4
psadbw m2, m4
psadbw m3, m4
%endmacro
%macro SAD_X4_2x8P_SSE2 4
movq m6, [r0+%1]
movq m4, [r1+%2]
movq m5, [r2+%2]
movhps m6, [r0+%3]
movhps m4, [r1+%4]
movhps m5, [r2+%4]
psadbw m4, m6
psadbw m5, m6
paddd m0, m4
paddd m1, m5
movq m4, [r3+%2]
movq m5, [r4+%2]
movhps m4, [r3+%4]
movhps m5, [r4+%4]
psadbw m4, m6
psadbw m5, m6
paddd m2, m4
paddd m3, m5
%endmacro
%macro SAD_X4_START_1x16P_SSE2 0
mova m3, [r0]
%if cpuflag(avx)
psadbw m0, m3, [r1]
psadbw m1, m3, [r2]
psadbw m2, m3, [r3]
psadbw m3, [r4]
%else
movu m0, [r1]
movu m1, [r2]
movu m2, [r3]
movu m4, [r4]
psadbw m0, m3
psadbw m1, m3
psadbw m2, m3
psadbw m3, m4
%endif
%endmacro
%macro SAD_X4_1x16P_SSE2 2
mova m6, [r0+%1]
%if cpuflag(avx)
psadbw m4, m6, [r1+%2]
psadbw m5, m6, [r2+%2]
%else
movu m4, [r1+%2]
movu m5, [r2+%2]
psadbw m4, m6
psadbw m5, m6
%endif
paddd m0, m4
paddd m1, m5
%if cpuflag(avx)
psadbw m4, m6, [r3+%2]
psadbw m5, m6, [r4+%2]
%else
movu m4, [r3+%2]
movu m5, [r4+%2]
psadbw m4, m6
psadbw m5, m6
%endif
paddd m2, m4
paddd m3, m5
%endmacro
%macro SAD_X4_4x16P_SSE2 2
%if %1==0
lea r6, [r5*3]
SAD_X4_START_1x16P_SSE2
%else
SAD_X4_1x16P_SSE2 FENC_STRIDE*(0+(%1&1)*4), r5*0
%endif
SAD_X4_1x16P_SSE2 FENC_STRIDE*(1+(%1&1)*4), r5*1
SAD_X4_1x16P_SSE2 FENC_STRIDE*(2+(%1&1)*4), r5*2
SAD_X4_1x16P_SSE2 FENC_STRIDE*(3+(%1&1)*4), r6
%if %1 != %2-1
%if (%1&1) != 0
add r0, 8*FENC_STRIDE
%endif
lea r1, [r1+4*r5]
lea r2, [r2+4*r5]
lea r3, [r3+4*r5]
lea r4, [r4+4*r5]
%endif
%endmacro
%macro SAD_X3_4x8P_SSE2 2
%if %1==0
lea t0, [r4*3]
SAD_X3_START_2x8P_SSE2
%else
SAD_X3_2x8P_SSE2 FENC_STRIDE*(0+(%1&1)*4), r4*0, FENC_STRIDE*(1+(%1&1)*4), r4*1
%endif
SAD_X3_2x8P_SSE2 FENC_STRIDE*(2+(%1&1)*4), r4*2, FENC_STRIDE*(3+(%1&1)*4), t0
%if %1 != %2-1
%if (%1&1) != 0
add r0, 8*FENC_STRIDE
%endif
lea r1, [r1+4*r4]
lea r2, [r2+4*r4]
lea r3, [r3+4*r4]
%endif
%endmacro
%macro SAD_X4_4x8P_SSE2 2
%if %1==0
lea r6, [r5*3]
SAD_X4_START_2x8P_SSE2
%else
SAD_X4_2x8P_SSE2 FENC_STRIDE*(0+(%1&1)*4), r5*0, FENC_STRIDE*(1+(%1&1)*4), r5*1
%endif
SAD_X4_2x8P_SSE2 FENC_STRIDE*(2+(%1&1)*4), r5*2, FENC_STRIDE*(3+(%1&1)*4), r6
%if %1 != %2-1
%if (%1&1) != 0
add r0, 8*FENC_STRIDE
%endif
lea r1, [r1+4*r5]
lea r2, [r2+4*r5]
lea r3, [r3+4*r5]
lea r4, [r4+4*r5]
%endif
%endmacro
%macro SAD_X3_END_SSE2 1
movifnidn r5, r5mp
movhlps m3, m0
movhlps m4, m1
movhlps m5, m2
paddd m0, m3
paddd m1, m4
paddd m2, m5
movd [r5+0], m0
movd [r5+4], m1
movd [r5+8], m2
RET
%endmacro
%macro SAD_X4_END_SSE2 1
mov r0, r6mp
psllq m1, 32
psllq m3, 32
paddd m0, m1
paddd m2, m3
movhlps m1, m0
movhlps m3, m2
paddd m0, m1
paddd m2, m3
movq [r0+0], m0
movq [r0+8], m2
RET
%endmacro
%macro SAD_X3_START_2x16P_AVX2 0
movu m3, [r0] ; assumes FENC_STRIDE == 16
movu xm0, [r1]
movu xm1, [r2]
movu xm2, [r3]
vinserti128 m0, m0, [r1+r4], 1
vinserti128 m1, m1, [r2+r4], 1
vinserti128 m2, m2, [r3+r4], 1
psadbw m0, m3
psadbw m1, m3
psadbw m2, m3
%endmacro
%macro SAD_X3_2x16P_AVX2 3
movu m3, [r0+%1] ; assumes FENC_STRIDE == 16
movu xm4, [r1+%2]
movu xm5, [r2+%2]
movu xm6, [r3+%2]
vinserti128 m4, m4, [r1+%3], 1
vinserti128 m5, m5, [r2+%3], 1
vinserti128 m6, m6, [r3+%3], 1
psadbw m4, m3
psadbw m5, m3
psadbw m6, m3
paddw m0, m4
paddw m1, m5
paddw m2, m6
%endmacro
%macro SAD_X3_4x16P_AVX2 2
%if %1==0
lea t0, [r4*3]
SAD_X3_START_2x16P_AVX2
%else
SAD_X3_2x16P_AVX2 FENC_STRIDE*(0+(%1&1)*4), r4*0, r4*1
%endif
SAD_X3_2x16P_AVX2 FENC_STRIDE*(2+(%1&1)*4), r4*2, t0
%if %1 != %2-1
%if (%1&1) != 0
add r0, 8*FENC_STRIDE
%endif
lea r1, [r1+4*r4]
lea r2, [r2+4*r4]
lea r3, [r3+4*r4]
%endif
%endmacro
%macro SAD_X4_START_2x16P_AVX2 0
vbroadcasti128 m4, [r0]
vbroadcasti128 m5, [r0+FENC_STRIDE]
movu xm0, [r1]
movu xm1, [r2]
movu xm2, [r1+r5]
movu xm3, [r2+r5]
vinserti128 m0, m0, [r3], 1
vinserti128 m1, m1, [r4], 1
vinserti128 m2, m2, [r3+r5], 1
vinserti128 m3, m3, [r4+r5], 1
psadbw m0, m4
psadbw m1, m4
psadbw m2, m5
psadbw m3, m5
paddw m0, m2
paddw m1, m3
%endmacro
%macro SAD_X4_2x16P_AVX2 4
vbroadcasti128 m6, [r0+%1]
vbroadcasti128 m7, [r0+%3]
movu xm2, [r1+%2]
movu xm3, [r2+%2]
movu xm4, [r1+%4]
movu xm5, [r2+%4]
vinserti128 m2, m2, [r3+%2], 1
vinserti128 m3, m3, [r4+%2], 1
vinserti128 m4, m4, [r3+%4], 1
vinserti128 m5, m5, [r4+%4], 1
psadbw m2, m6
psadbw m3, m6
psadbw m4, m7
psadbw m5, m7
paddd m0, m2
paddd m1, m3
paddd m0, m4
paddd m1, m5
%endmacro
%macro SAD_X4_4x16P_AVX2 2
%if %1==0
lea r6, [r5*3]
SAD_X4_START_2x16P_AVX2
%else
SAD_X4_2x16P_AVX2 FENC_STRIDE*(0+(%1&1)*4), r5*0, FENC_STRIDE*(1+(%1&1)*4), r5*1
%endif
SAD_X4_2x16P_AVX2 FENC_STRIDE*(2+(%1&1)*4), r5*2, FENC_STRIDE*(3+(%1&1)*4), r6
%if %1 != %2-1
%if (%1&1) != 0
add r0, 8*FENC_STRIDE
%endif
lea r1, [r1+4*r5]
lea r2, [r2+4*r5]
lea r3, [r3+4*r5]
lea r4, [r4+4*r5]
%endif
%endmacro
%macro SAD_X4_START_2x32P_AVX2 0
mova m4, [r0]
movu m0, [r1]
movu m2, [r2]
movu m1, [r3]
movu m3, [r4]
psadbw m0, m4
psadbw m2, m4
psadbw m1, m4
psadbw m3, m4
packusdw m0, m2
packusdw m1, m3
mova m6, [r0+FENC_STRIDE]
movu m2, [r1+r5]
movu m4, [r2+r5]
movu m3, [r3+r5]
movu m5, [r4+r5]
psadbw m2, m6
psadbw m4, m6
psadbw m3, m6
psadbw m5, m6
packusdw m2, m4
packusdw m3, m5
paddd m0, m2
paddd m1, m3
%endmacro
%macro SAD_X4_2x32P_AVX2 4
mova m6, [r0+%1]
movu m2, [r1+%2]
movu m4, [r2+%2]
movu m3, [r3+%2]
movu m5, [r4+%2]
psadbw m2, m6
psadbw m4, m6
psadbw m3, m6
psadbw m5, m6
packusdw m2, m4
packusdw m3, m5
paddd m0, m2
paddd m1, m3
mova m6, [r0+%3]
movu m2, [r1+%4]
movu m4, [r2+%4]
movu m3, [r3+%4]
movu m5, [r4+%4]
psadbw m2, m6
psadbw m4, m6
psadbw m3, m6
psadbw m5, m6
packusdw m2, m4
packusdw m3, m5
paddd m0, m2
paddd m1, m3
%endmacro
%macro SAD_X4_4x32P_AVX2 2
%if %1==0
lea r6, [r5*3]
SAD_X4_START_2x32P_AVX2
%else
SAD_X4_2x32P_AVX2 FENC_STRIDE*(0+(%1&1)*4), r5*0, FENC_STRIDE*(1+(%1&1)*4), r5*1
%endif
SAD_X4_2x32P_AVX2 FENC_STRIDE*(2+(%1&1)*4), r5*2, FENC_STRIDE*(3+(%1&1)*4), r6
%if %1 != %2-1
%if (%1&1) != 0
add r0, 8*FENC_STRIDE
%endif
lea r1, [r1+4*r5]
lea r2, [r2+4*r5]
lea r3, [r3+4*r5]
lea r4, [r4+4*r5]
%endif
%endmacro
%macro SAD_X3_END_AVX2 0
movifnidn r5, r5mp
packssdw m0, m1 ; 0 0 1 1 0 0 1 1
packssdw m2, m2 ; 2 2 _ _ 2 2 _ _
phaddd m0, m2 ; 0 1 2 _ 0 1 2 _
vextracti128 xm1, m0, 1
paddd xm0, xm1 ; 0 1 2 _
mova [r5], xm0
RET
%endmacro
%macro SAD_X4_END_AVX2 0
mov r0, r6mp
pshufd m0, m0, 0x8
pshufd m1, m1, 0x8
vextracti128 xm2, m0, 1
vextracti128 xm3, m1, 1
punpcklqdq xm0, xm1
punpcklqdq xm2, xm3
phaddd xm0, xm2 ; 0 1 2 3
mova [r0], xm0
RET
%endmacro
%macro SAD_X4_32P_END_AVX2 0
mov r0, r6mp
vextracti128 xm2, m0, 1
vextracti128 xm3, m1, 1
paddd xm0, xm2
paddd xm1, xm3
phaddd xm0, xm1
mova [r0], xm0
RET
%endmacro
;-----------------------------------------------------------------------------
; void pixel_sad_x3_16x16( uint8_t *fenc, uint8_t *pix0, uint8_t *pix1,
; uint8_t *pix2, intptr_t i_stride, int scores[3] )
;-----------------------------------------------------------------------------
%macro SAD_X_SSE2 4
cglobal pixel_sad_x%1_%2x%3, 2+%1,3+%1,%4
%assign x 0
%rep %3/4
SAD_X%1_4x%2P_SSE2 x, %3/4
%assign x x+1
%endrep
%if %3 == 64
SAD_X%1_END_SSE2 1
%else
SAD_X%1_END_SSE2 0
%endif
%endmacro
%macro SAD_X3_W12 0
cglobal pixel_sad_x3_12x16, 5, 7, 8
mova m4, [MSK]
pxor m0, m0
pxor m1, m1
pxor m2, m2
SAD_X3_12x4
SAD_X3_12x4
SAD_X3_12x4
SAD_X3_12x4
SAD_X3_END_SSE2 1
%endmacro
%macro SAD_X4_W12 0
cglobal pixel_sad_x4_12x16, 6, 8, 8
mova m6, [MSK]
pxor m0, m0
pxor m1, m1
pxor m2, m2
pxor m3, m3
SAD_X4_12x4
SAD_X4_12x4
SAD_X4_12x4
SAD_X4_12x4
SAD_X4_END_SSE2 1
%endmacro
%macro SAD_X3_W24 0
cglobal pixel_sad_x3_24x32, 5, 7, 8
pxor m0, m0
pxor m1, m1
pxor m2, m2
mov r6, 32
.loop:
SAD_X3_24x4
SAD_X3_24x4
SAD_X3_24x4
SAD_X3_24x4
sub r6, 16
cmp r6, 0
jnz .loop
SAD_X3_END_SSE2 1
%endmacro
%macro SAD_X4_W24 0
%if ARCH_X86_64 == 1
cglobal pixel_sad_x4_24x32, 6, 8, 8
%define count r7
%else
cglobal pixel_sad_x4_24x32, 6, 7, 8, 0-4
%define count dword [rsp]
%endif
pxor m0, m0
pxor m1, m1
pxor m2, m2
pxor m3, m3
mov count, 32
.loop:
SAD_X4_24x4
SAD_X4_24x4
SAD_X4_24x4
SAD_X4_24x4
sub count, 16
jnz .loop
SAD_X4_END_SSE2 1
%endmacro
%macro SAD_X3_W32 0
cglobal pixel_sad_x3_32x8, 5, 6, 8
pxor m0, m0
pxor m1, m1
pxor m2, m2
SAD_X3_32x4
SAD_X3_32x4
SAD_X3_END_SSE2 1
cglobal pixel_sad_x3_32x16, 5, 6, 8
pxor m0, m0
pxor m1, m1
pxor m2, m2
SAD_X3_32x4
SAD_X3_32x4
SAD_X3_32x4
SAD_X3_32x4
SAD_X3_END_SSE2 1
cglobal pixel_sad_x3_32x24, 5, 6, 8
pxor m0, m0
pxor m1, m1
pxor m2, m2
SAD_X3_32x4
SAD_X3_32x4
SAD_X3_32x4
SAD_X3_32x4
SAD_X3_32x4
SAD_X3_32x4
SAD_X3_END_SSE2 1
cglobal pixel_sad_x3_32x32, 5, 7, 8
pxor m0, m0
pxor m1, m1
pxor m2, m2
mov r6, 32
.loop:
SAD_X3_32x4
SAD_X3_32x4
SAD_X3_32x4
SAD_X3_32x4
sub r6, 16
cmp r6, 0
jnz .loop
SAD_X3_END_SSE2 1
cglobal pixel_sad_x3_32x64, 5, 7, 8
pxor m0, m0
pxor m1, m1
pxor m2, m2
mov r6, 64
.loop1:
SAD_X3_32x4
SAD_X3_32x4
SAD_X3_32x4
SAD_X3_32x4
sub r6, 16
cmp r6, 0
jnz .loop1
SAD_X3_END_SSE2 1
%endmacro
%macro SAD_X4_W32 0
cglobal pixel_sad_x4_32x8, 6, 7, 8
pxor m0, m0
pxor m1, m1
pxor m2, m2
pxor m3, m3
SAD_X4_32x4
SAD_X4_32x4
SAD_X4_END_SSE2 1
cglobal pixel_sad_x4_32x16, 6, 7, 8
pxor m0, m0
pxor m1, m1
pxor m2, m2
pxor m3, m3
SAD_X4_32x4
SAD_X4_32x4
SAD_X4_32x4
SAD_X4_32x4
SAD_X4_END_SSE2 1
cglobal pixel_sad_x4_32x24, 6, 7, 8
pxor m0, m0
pxor m1, m1
pxor m2, m2
pxor m3, m3
SAD_X4_32x4
SAD_X4_32x4
SAD_X4_32x4
SAD_X4_32x4
SAD_X4_32x4
SAD_X4_32x4
SAD_X4_END_SSE2 1
%if ARCH_X86_64 == 1
cglobal pixel_sad_x4_32x32, 6, 8, 8
%define count r7
%else
cglobal pixel_sad_x4_32x32, 6, 7, 8, 0-4
%define count dword [rsp]
%endif
pxor m0, m0
pxor m1, m1
pxor m2, m2
pxor m3, m3
mov count, 32
.loop:
SAD_X4_32x4
SAD_X4_32x4
SAD_X4_32x4
SAD_X4_32x4
sub count, 16
jnz .loop
SAD_X4_END_SSE2 1
%if ARCH_X86_64 == 1
cglobal pixel_sad_x4_32x64, 6, 8, 8
%define count r7
%else
cglobal pixel_sad_x4_32x64, 6, 7, 8, 0-4
%define count dword [rsp]
%endif
pxor m0, m0
pxor m1, m1
pxor m2, m2
pxor m3, m3
mov count, 64
.loop:
SAD_X4_32x4
SAD_X4_32x4
SAD_X4_32x4
SAD_X4_32x4
sub count, 16
jnz .loop
SAD_X4_END_SSE2 1
%endmacro
%macro SAD_X3_W48 0
cglobal pixel_sad_x3_48x64, 5, 7, 8
pxor m0, m0
pxor m1, m1
pxor m2, m2
mov r6, 64
.loop:
SAD_X3_48x4
SAD_X3_48x4
SAD_X3_48x4
SAD_X3_48x4
sub r6, 16
jnz .loop
SAD_X3_END_SSE2 1
%endmacro
%macro SAD_X4_W48 0
%if ARCH_X86_64 == 1
cglobal pixel_sad_x4_48x64, 6, 8, 8
%define count r7
%else
cglobal pixel_sad_x4_48x64, 6, 7, 8, 0-4
%define count dword [rsp]
%endif
pxor m0, m0
pxor m1, m1
pxor m2, m2
pxor m3, m3
mov count, 64
.loop:
SAD_X4_48x4
SAD_X4_48x4
SAD_X4_48x4
SAD_X4_48x4
sub count, 16
jnz .loop
SAD_X4_END_SSE2 1
%endmacro
%macro SAD_X3_W64 0
cglobal pixel_sad_x3_64x16, 5, 7, 7
pxor m0, m0
pxor m1, m1
pxor m2, m2
mov r6, 16
.loop:
SAD_X3_64x4
SAD_X3_64x4
sub r6, 8
jnz .loop
SAD_X3_END_SSE2 1
cglobal pixel_sad_x3_64x32, 5, 7, 7
pxor m0, m0
pxor m1, m1
pxor m2, m2
mov r6, 32
.loop:
SAD_X3_64x4
SAD_X3_64x4
sub r6, 8
jnz .loop
SAD_X3_END_SSE2 1
cglobal pixel_sad_x3_64x48, 5, 7, 7
pxor m0, m0
pxor m1, m1
pxor m2, m2
mov r6, 48
.loop:
SAD_X3_64x4
SAD_X3_64x4
sub r6, 8
jnz .loop
SAD_X3_END_SSE2 1
cglobal pixel_sad_x3_64x64, 5, 7, 7
pxor m0, m0
pxor m1, m1
pxor m2, m2
mov r6, 64
.loop:
SAD_X3_64x4
SAD_X3_64x4
sub r6, 8
jnz .loop
SAD_X3_END_SSE2 1
%endmacro
%macro SAD_X4_W64 0
%if ARCH_X86_64 == 1
cglobal pixel_sad_x4_64x16, 6, 8, 8
%define count r7
%else
cglobal pixel_sad_x4_64x16, 6, 7, 8, 0-4
%define count dword [rsp]
%endif
pxor m0, m0
pxor m1, m1
pxor m2, m2
pxor m3, m3
mov count, 16
.loop:
SAD_X4_64x4
SAD_X4_64x4
sub count, 8
jnz .loop
SAD_X4_END_SSE2 1
%if ARCH_X86_64 == 1
cglobal pixel_sad_x4_64x32, 6, 8, 8
%define count r7
%else
cglobal pixel_sad_x4_64x32, 6, 7, 8, 0-4
%define count dword [rsp]
%endif
pxor m0, m0
pxor m1, m1
pxor m2, m2
pxor m3, m3
mov count, 32
.loop:
SAD_X4_64x4
SAD_X4_64x4
sub count, 8
jnz .loop
SAD_X4_END_SSE2 1
%if ARCH_X86_64 == 1
cglobal pixel_sad_x4_64x48, 6, 8, 8
%define count r7
%else
cglobal pixel_sad_x4_64x48, 6, 7, 8, 0-4
%define count dword [rsp]
%endif
pxor m0, m0
pxor m1, m1
pxor m2, m2
pxor m3, m3
mov count, 48
.loop:
SAD_X4_64x4
SAD_X4_64x4
sub count, 8
jnz .loop
SAD_X4_END_SSE2 1
%if ARCH_X86_64 == 1
cglobal pixel_sad_x4_64x64, 6, 8, 8
%define count r7
%else
cglobal pixel_sad_x4_64x64, 6, 7, 8, 0-4
%define count dword [rsp]
%endif
pxor m0, m0
pxor m1, m1
pxor m2, m2
pxor m3, m3
mov count, 64
.loop:
SAD_X4_64x4
SAD_X4_64x4
sub count, 8
jnz .loop
SAD_X4_END_SSE2 1
%endmacro
INIT_XMM sse2
SAD_X_SSE2 3, 16, 16, 7
SAD_X_SSE2 3, 16, 8, 7
SAD_X_SSE2 3, 8, 16, 7
SAD_X_SSE2 3, 8, 8, 7
SAD_X_SSE2 3, 8, 4, 7
SAD_X_SSE2 4, 16, 16, 7
SAD_X_SSE2 4, 16, 8, 7
SAD_X_SSE2 4, 8, 16, 7
SAD_X_SSE2 4, 8, 8, 7
SAD_X_SSE2 4, 8, 4, 7
INIT_XMM sse3
SAD_X_SSE2 3, 16, 16, 7
SAD_X_SSE2 3, 16, 8, 7
SAD_X_SSE2 3, 16, 4, 7
SAD_X_SSE2 4, 16, 16, 7
SAD_X_SSE2 4, 16, 8, 7
SAD_X_SSE2 4, 16, 4, 7
INIT_XMM ssse3
SAD_X3_W12
SAD_X3_W32
SAD_X3_W24
SAD_X3_W48
SAD_X3_W64
SAD_X_SSE2 3, 16, 64, 7
SAD_X_SSE2 3, 16, 32, 7
SAD_X_SSE2 3, 16, 16, 7
SAD_X_SSE2 3, 16, 12, 7
SAD_X_SSE2 3, 16, 8, 7
SAD_X_SSE2 3, 8, 32, 7
SAD_X_SSE2 3, 8, 16, 7
SAD_X4_W12
SAD_X4_W24
SAD_X4_W32
SAD_X4_W48
SAD_X4_W64
SAD_X_SSE2 4, 16, 64, 7
SAD_X_SSE2 4, 16, 32, 7
SAD_X_SSE2 4, 16, 16, 7
SAD_X_SSE2 4, 16, 12, 7
SAD_X_SSE2 4, 16, 8, 7
SAD_X_SSE2 4, 8, 32, 7
SAD_X_SSE2 4, 8, 16, 7
SAD_X_SSE2 4, 8, 8, 7
SAD_X_SSE2 4, 8, 4, 7
INIT_XMM avx
SAD_X3_W12
SAD_X3_W32
SAD_X3_W24
SAD_X3_W48
SAD_X3_W64
SAD_X_SSE2 3, 16, 64, 7
SAD_X_SSE2 3, 16, 32, 6
SAD_X_SSE2 3, 16, 16, 6
SAD_X_SSE2 3, 16, 12, 6
SAD_X_SSE2 3, 16, 8, 6
SAD_X_SSE2 3, 16, 4, 6
SAD_X4_W12
SAD_X4_W24
SAD_X4_W32
SAD_X4_W48
SAD_X4_W64
SAD_X_SSE2 4, 16, 64, 7
SAD_X_SSE2 4, 16, 32, 7
SAD_X_SSE2 4, 16, 16, 7
SAD_X_SSE2 4, 16, 12, 7
SAD_X_SSE2 4, 16, 8, 7
SAD_X_SSE2 4, 16, 4, 7
%macro SAD_X_AVX2 4
cglobal pixel_sad_x%1_%2x%3, 2+%1,3+%1,%4
%assign x 0
%rep %3/4
SAD_X%1_4x%2P_AVX2 x, %3/4
%assign x x+1
%endrep
%if (%1==4) && (%2==32)
SAD_X%1_32P_END_AVX2
%else
SAD_X%1_END_AVX2
%endif
%endmacro
INIT_YMM avx2
SAD_X_AVX2 3, 16, 32, 7
SAD_X_AVX2 3, 16, 16, 7
SAD_X_AVX2 3, 16, 12, 7
SAD_X_AVX2 3, 16, 8, 7
SAD_X_AVX2 4, 16, 32, 8
SAD_X_AVX2 4, 16, 16, 8
SAD_X_AVX2 4, 16, 12, 8
SAD_X_AVX2 4, 16, 8, 8
SAD_X_AVX2 4, 32, 8, 8
SAD_X_AVX2 4, 32, 16, 8
SAD_X_AVX2 4, 32, 24, 8
SAD_X_AVX2 4, 32, 32, 8
SAD_X_AVX2 4, 32, 64, 8
;=============================================================================
; SAD cacheline split
;=============================================================================
; Core2 (Conroe) can load unaligned data just as quickly as aligned data...
; unless the unaligned data spans the border between 2 cachelines, in which
; case it's really slow. The exact numbers may differ, but all Intel cpus prior
; to Nehalem have a large penalty for cacheline splits.
; (8-byte alignment exactly half way between two cachelines is ok though.)
; LDDQU was supposed to fix this, but it only works on Pentium 4.
; So in the split case we load aligned data and explicitly perform the
; alignment between registers. Like on archs that have only aligned loads,
; except complicated by the fact that PALIGNR takes only an immediate, not
; a variable alignment.
; It is also possible to hoist the realignment to the macroblock level (keep
; 2 copies of the reference frame, offset by 32 bytes), but the extra memory
; needed for that method makes it often slower.
; sad 16x16 costs on Core2:
; good offsets: 49 cycles (50/64 of all mvs)
; cacheline split: 234 cycles (14/64 of all mvs. ammortized: +40 cycles)
; page split: 3600 cycles (14/4096 of all mvs. ammortized: +11.5 cycles)
; cache or page split with palignr: 57 cycles (ammortized: +2 cycles)
; computed jump assumes this loop is exactly 80 bytes
%macro SAD16_CACHELINE_LOOP_SSE2 1 ; alignment
ALIGN 16
sad_w16_align%1_sse2:
movdqa xmm1, [r2+16]
movdqa xmm2, [r2+r3+16]
movdqa xmm3, [r2]
movdqa xmm4, [r2+r3]
pslldq xmm1, 16-%1
pslldq xmm2, 16-%1
psrldq xmm3, %1
psrldq xmm4, %1
por xmm1, xmm3
por xmm2, xmm4
psadbw xmm1, [r0]
psadbw xmm2, [r0+r1]
paddw xmm0, xmm1
paddw xmm0, xmm2
lea r0, [r0+2*r1]
lea r2, [r2+2*r3]
dec r4
jg sad_w16_align%1_sse2
ret
%endmacro
; computed jump assumes this loop is exactly 64 bytes
%macro SAD16_CACHELINE_LOOP_SSSE3 1 ; alignment
ALIGN 16
sad_w16_align%1_ssse3:
movdqa xmm1, [r2+16]
movdqa xmm2, [r2+r3+16]
palignr xmm1, [r2], %1
palignr xmm2, [r2+r3], %1
psadbw xmm1, [r0]
psadbw xmm2, [r0+r1]
paddw xmm0, xmm1
paddw xmm0, xmm2
lea r0, [r0+2*r1]
lea r2, [r2+2*r3]
dec r4
jg sad_w16_align%1_ssse3
ret
%endmacro
%macro SAD16_CACHELINE_FUNC 2 ; cpu, height
cglobal pixel_sad_16x%2_cache64_%1
mov eax, r2m
and eax, 0x37
cmp eax, 0x30
jle pixel_sad_16x%2_sse2
PROLOGUE 4,6
mov r4d, r2d
and r4d, 15
%ifidn %1, ssse3
shl r4d, 6 ; code size = 64
%else
lea r4, [r4*5]
shl r4d, 4 ; code size = 80
%endif
%define sad_w16_addr (sad_w16_align1_%1 + (sad_w16_align1_%1 - sad_w16_align2_%1))
%ifdef PIC
lea r5, [sad_w16_addr]
add r5, r4
%else
lea r5, [sad_w16_addr + r4]
%endif
and r2, ~15
mov r4d, %2/2
pxor xmm0, xmm0
call r5
movhlps xmm1, xmm0
paddw xmm0, xmm1
movd eax, xmm0
RET
%endmacro
%macro SAD_CACHELINE_START_MMX2 4 ; width, height, iterations, cacheline
mov eax, r2m
and eax, 0x17|%1|(%4>>1)
cmp eax, 0x10|%1|(%4>>1)
jle pixel_sad_%1x%2_mmx2
and eax, 7
shl eax, 3
movd mm6, [pd_64]
movd mm7, eax
psubw mm6, mm7
PROLOGUE 4,5
and r2, ~7
mov r4d, %3
pxor mm0, mm0
%endmacro
%macro SAD16_CACHELINE_FUNC_MMX2 2 ; height, cacheline
cglobal pixel_sad_16x%1_cache%2_mmx2
SAD_CACHELINE_START_MMX2 16, %1, %1, %2
.loop:
movq mm1, [r2]
movq mm2, [r2+8]
movq mm3, [r2+16]
movq mm4, mm2
psrlq mm1, mm7
psllq mm2, mm6
psllq mm3, mm6
psrlq mm4, mm7
por mm1, mm2
por mm3, mm4
psadbw mm1, [r0]
psadbw mm3, [r0+8]
paddw mm0, mm1
paddw mm0, mm3
add r2, r3
add r0, r1
dec r4
jg .loop
movd eax, mm0
RET
%endmacro
%macro SAD8_CACHELINE_FUNC_MMX2 2 ; height, cacheline
cglobal pixel_sad_8x%1_cache%2_mmx2
SAD_CACHELINE_START_MMX2 8, %1, %1/2, %2
.loop:
movq mm1, [r2+8]
movq mm2, [r2+r3+8]
movq mm3, [r2]
movq mm4, [r2+r3]
psllq mm1, mm6
psllq mm2, mm6
psrlq mm3, mm7
psrlq mm4, mm7
por mm1, mm3
por mm2, mm4
psadbw mm1, [r0]
psadbw mm2, [r0+r1]
paddw mm0, mm1
paddw mm0, mm2
lea r2, [r2+2*r3]
lea r0, [r0+2*r1]
dec r4
jg .loop
movd eax, mm0
RET
%endmacro
; sad_x3/x4_cache64: check each mv.
; if they're all within a cacheline, use normal sad_x3/x4.
; otherwise, send them individually to sad_cache64.
%macro CHECK_SPLIT 3 ; pix, width, cacheline
mov eax, %1
and eax, 0x17|%2|(%3>>1)
cmp eax, 0x10|%2|(%3>>1)
jg .split
%endmacro
%macro SADX3_CACHELINE_FUNC 6 ; width, height, cacheline, normal_ver, split_ver, name
cglobal pixel_sad_x3_%1x%2_cache%3_%6
CHECK_SPLIT r1m, %1, %3
CHECK_SPLIT r2m, %1, %3
CHECK_SPLIT r3m, %1, %3
jmp pixel_sad_x3_%1x%2_%4
.split:
%if ARCH_X86_64
PROLOGUE 6,9
push r3
push r2
%if WIN64
movsxd r4, r4d
sub rsp, 40 ; shadow space and alignment
%endif
mov r2, r1
mov r1, FENC_STRIDE
mov r3, r4
mov r7, r0
mov r8, r5
call pixel_sad_%1x%2_cache%3_%5
mov [r8], eax
%if WIN64
mov r2, [rsp+40+0*8]
%else
pop r2
%endif
mov r0, r7
call pixel_sad_%1x%2_cache%3_%5
mov [r8+4], eax
%if WIN64
mov r2, [rsp+40+1*8]
%else
pop r2
%endif
mov r0, r7
call pixel_sad_%1x%2_cache%3_%5
mov [r8+8], eax
%if WIN64
add rsp, 40+2*8
%endif
RET
%else
push edi
mov edi, [esp+28]
push dword [esp+24]
push dword [esp+16]
push dword 16
push dword [esp+20]
call pixel_sad_%1x%2_cache%3_%5
mov ecx, [esp+32]
mov [edi], eax
mov [esp+8], ecx
call pixel_sad_%1x%2_cache%3_%5
mov ecx, [esp+36]
mov [edi+4], eax
mov [esp+8], ecx
call pixel_sad_%1x%2_cache%3_%5
mov [edi+8], eax
add esp, 16
pop edi
ret
%endif
%endmacro
%macro SADX4_CACHELINE_FUNC 6 ; width, height, cacheline, normal_ver, split_ver, name
cglobal pixel_sad_x4_%1x%2_cache%3_%6
CHECK_SPLIT r1m, %1, %3
CHECK_SPLIT r2m, %1, %3
CHECK_SPLIT r3m, %1, %3
CHECK_SPLIT r4m, %1, %3
jmp pixel_sad_x4_%1x%2_%4
.split:
%if ARCH_X86_64
PROLOGUE 6,9
mov r8, r6mp
push r4
push r3
push r2
%if WIN64
sub rsp, 32 ; shadow space
%endif
mov r2, r1
mov r1, FENC_STRIDE
mov r3, r5
mov r7, r0
call pixel_sad_%1x%2_cache%3_%5
mov [r8], eax
%if WIN64
mov r2, [rsp+32+0*8]
%else
pop r2
%endif
mov r0, r7
call pixel_sad_%1x%2_cache%3_%5
mov [r8+4], eax
%if WIN64
mov r2, [rsp+32+1*8]
%else
pop r2
%endif
mov r0, r7
call pixel_sad_%1x%2_cache%3_%5
mov [r8+8], eax
%if WIN64
mov r2, [rsp+32+2*8]
%else
pop r2
%endif
mov r0, r7
call pixel_sad_%1x%2_cache%3_%5
mov [r8+12], eax
%if WIN64
add rsp, 32+3*8
%endif
RET
%else
push edi
mov edi, [esp+32]
push dword [esp+28]
push dword [esp+16]
push dword 16
push dword [esp+20]
call pixel_sad_%1x%2_cache%3_%5
mov ecx, [esp+32]
mov [edi], eax
mov [esp+8], ecx
call pixel_sad_%1x%2_cache%3_%5
mov ecx, [esp+36]
mov [edi+4], eax
mov [esp+8], ecx
call pixel_sad_%1x%2_cache%3_%5
mov ecx, [esp+40]
mov [edi+8], eax
mov [esp+8], ecx
call pixel_sad_%1x%2_cache%3_%5
mov [edi+12], eax
add esp, 16
pop edi
ret
%endif
%endmacro
%macro SADX34_CACHELINE_FUNC 1+
SADX3_CACHELINE_FUNC %1
SADX4_CACHELINE_FUNC %1
%endmacro
; instantiate the aligned sads
INIT_MMX
%if ARCH_X86_64 == 0
SAD16_CACHELINE_FUNC_MMX2 8, 32
SAD16_CACHELINE_FUNC_MMX2 16, 32
SAD8_CACHELINE_FUNC_MMX2 4, 32
SAD8_CACHELINE_FUNC_MMX2 8, 32
SAD8_CACHELINE_FUNC_MMX2 16, 32
SAD16_CACHELINE_FUNC_MMX2 8, 64
SAD16_CACHELINE_FUNC_MMX2 16, 64
%endif ; !ARCH_X86_64
SAD8_CACHELINE_FUNC_MMX2 4, 64
SAD8_CACHELINE_FUNC_MMX2 8, 64
SAD8_CACHELINE_FUNC_MMX2 16, 64
%if ARCH_X86_64 == 0
SADX34_CACHELINE_FUNC 16, 16, 32, mmx2, mmx2, mmx2
SADX34_CACHELINE_FUNC 16, 8, 32, mmx2, mmx2, mmx2
SADX34_CACHELINE_FUNC 8, 16, 32, mmx2, mmx2, mmx2
SADX34_CACHELINE_FUNC 8, 8, 32, mmx2, mmx2, mmx2
SADX34_CACHELINE_FUNC 16, 16, 64, mmx2, mmx2, mmx2
SADX34_CACHELINE_FUNC 16, 8, 64, mmx2, mmx2, mmx2
%endif ; !ARCH_X86_64
SADX34_CACHELINE_FUNC 8, 16, 64, mmx2, mmx2, mmx2
SADX34_CACHELINE_FUNC 8, 8, 64, mmx2, mmx2, mmx2
%if ARCH_X86_64 == 0
SAD16_CACHELINE_FUNC sse2, 8
SAD16_CACHELINE_FUNC sse2, 16
%assign i 1
%rep 15
SAD16_CACHELINE_LOOP_SSE2 i
%assign i i+1
%endrep
SADX34_CACHELINE_FUNC 16, 16, 64, sse2, sse2, sse2
SADX34_CACHELINE_FUNC 16, 8, 64, sse2, sse2, sse2
%endif ; !ARCH_X86_64
SADX34_CACHELINE_FUNC 8, 16, 64, sse2, mmx2, sse2
SAD16_CACHELINE_FUNC ssse3, 8
SAD16_CACHELINE_FUNC ssse3, 16
%assign i 1
%rep 15
SAD16_CACHELINE_LOOP_SSSE3 i
%assign i i+1
%endrep
SADX34_CACHELINE_FUNC 16, 16, 64, sse2, ssse3, ssse3
SADX34_CACHELINE_FUNC 16, 8, 64, sse2, ssse3, ssse3
%if HIGH_BIT_DEPTH==0
INIT_YMM avx2
cglobal pixel_sad_x3_8x4, 6,6,5
xorps m0, m0
xorps m1, m1
sub r2, r1 ; rebase on pointer r1
sub r3, r1
; row 0
vpbroadcastq xm2, [r0 + 0 * FENC_STRIDE]
movq xm3, [r1]
movhps xm3, [r1 + r2]
movq xm4, [r1 + r3]
psadbw xm3, xm2
psadbw xm4, xm2
paddd xm0, xm3
paddd xm1, xm4
add r1, r4
; row 1
vpbroadcastq xm2, [r0 + 1 * FENC_STRIDE]
movq xm3, [r1]
movhps xm3, [r1 + r2]
movq xm4, [r1 + r3]
psadbw xm3, xm2
psadbw xm4, xm2
paddd xm0, xm3
paddd xm1, xm4
add r1, r4
; row 2
vpbroadcastq xm2, [r0 + 2 * FENC_STRIDE]
movq xm3, [r1]
movhps xm3, [r1 + r2]
movq xm4, [r1 + r3]
psadbw xm3, xm2
psadbw xm4, xm2
paddd xm0, xm3
paddd xm1, xm4
add r1, r4
; row 3
vpbroadcastq xm2, [r0 + 3 * FENC_STRIDE]
movq xm3, [r1]
movhps xm3, [r1 + r2]
movq xm4, [r1 + r3]
psadbw xm3, xm2
psadbw xm4, xm2
paddd xm0, xm3
paddd xm1, xm4
pshufd xm0, xm0, q0020
movq [r5 + 0], xm0
movd [r5 + 8], xm1
RET
INIT_YMM avx2
cglobal pixel_sad_x3_8x8, 6,6,5
xorps m0, m0
xorps m1, m1
sub r2, r1 ; rebase on pointer r1
sub r3, r1
%assign x 0
%rep 4
; row 0
vpbroadcastq xm2, [r0 + 0 * FENC_STRIDE]
movq xm3, [r1]
movhps xm3, [r1 + r2]
movq xm4, [r1 + r3]
psadbw xm3, xm2
psadbw xm4, xm2
paddd xm0, xm3
paddd xm1, xm4
add r1, r4
; row 1
vpbroadcastq xm2, [r0 + 1 * FENC_STRIDE]
movq xm3, [r1]
movhps xm3, [r1 + r2]
movq xm4, [r1 + r3]
psadbw xm3, xm2
psadbw xm4, xm2
paddd xm0, xm3
paddd xm1, xm4
%assign x x+1
%if x < 4
add r1, r4
add r0, 2 * FENC_STRIDE
%endif
%endrep
pshufd xm0, xm0, q0020
movq [r5 + 0], xm0
movd [r5 + 8], xm1
RET
INIT_YMM avx2
cglobal pixel_sad_x3_8x16, 6,6,5
xorps m0, m0
xorps m1, m1
sub r2, r1 ; rebase on pointer r1
sub r3, r1
%assign x 0
%rep 8
; row 0
vpbroadcastq xm2, [r0 + 0 * FENC_STRIDE]
movq xm3, [r1]
movhps xm3, [r1 + r2]
movq xm4, [r1 + r3]
psadbw xm3, xm2
psadbw xm4, xm2
paddd xm0, xm3
paddd xm1, xm4
add r1, r4
; row 1
vpbroadcastq xm2, [r0 + 1 * FENC_STRIDE]
movq xm3, [r1]
movhps xm3, [r1 + r2]
movq xm4, [r1 + r3]
psadbw xm3, xm2
psadbw xm4, xm2
paddd xm0, xm3
paddd xm1, xm4
%assign x x+1
%if x < 8
add r1, r4
add r0, 2 * FENC_STRIDE
%endif
%endrep
pshufd xm0, xm0, q0020
movq [r5 + 0], xm0
movd [r5 + 8], xm1
RET
INIT_YMM avx2
cglobal pixel_sad_x4_8x8, 7,7,5
xorps m0, m0
xorps m1, m1
sub r2, r1 ; rebase on pointer r1
sub r3, r1
sub r4, r1
%assign x 0
%rep 4
; row 0
vpbroadcastq xm2, [r0 + 0 * FENC_STRIDE]
movq xm3, [r1]
movhps xm3, [r1 + r2]
movq xm4, [r1 + r3]
movhps xm4, [r1 + r4]
psadbw xm3, xm2
psadbw xm4, xm2
paddd xm0, xm3
paddd xm1, xm4
add r1, r5
; row 1
vpbroadcastq xm2, [r0 + 1 * FENC_STRIDE]
movq xm3, [r1]
movhps xm3, [r1 + r2]
movq xm4, [r1 + r3]
movhps xm4, [r1 + r4]
psadbw xm3, xm2
psadbw xm4, xm2
paddd xm0, xm3
paddd xm1, xm4
%assign x x+1
%if x < 4
add r1, r5
add r0, 2 * FENC_STRIDE
%endif
%endrep
pshufd xm0, xm0, q0020
pshufd xm1, xm1, q0020
movq [r6 + 0], xm0
movq [r6 + 8], xm1
RET
INIT_YMM avx2
cglobal pixel_sad_32x8, 4,4,6
xorps m0, m0
xorps m5, m5
movu m1, [r0] ; row 0 of pix0
movu m2, [r2] ; row 0 of pix1
movu m3, [r0 + r1] ; row 1 of pix0
movu m4, [r2 + r3] ; row 1 of pix1
psadbw m1, m2
psadbw m3, m4
paddd m0, m1
paddd m5, m3
lea r2, [r2 + 2 * r3]
lea r0, [r0 + 2 * r1]
movu m1, [r0] ; row 2 of pix0
movu m2, [r2] ; row 2 of pix1
movu m3, [r0 + r1] ; row 3 of pix0
movu m4, [r2 + r3] ; row 3 of pix1
psadbw m1, m2
psadbw m3, m4
paddd m0, m1
paddd m5, m3
lea r2, [r2 + 2 * r3]
lea r0, [r0 + 2 * r1]
movu m1, [r0] ; row 4 of pix0
movu m2, [r2] ; row 4 of pix1
movu m3, [r0 + r1] ; row 5 of pix0
movu m4, [r2 + r3] ; row 5 of pix1
psadbw m1, m2
psadbw m3, m4
paddd m0, m1
paddd m5, m3
lea r2, [r2 + 2 * r3]
lea r0, [r0 + 2 * r1]
movu m1, [r0] ; row 6 of pix0
movu m2, [r2] ; row 6 of pix1
movu m3, [r0 + r1] ; row 7 of pix0
movu m4, [r2 + r3] ; row 7 of pix1
psadbw m1, m2
psadbw m3, m4
paddd m0, m1
paddd m5, m3
paddd m0, m5
vextracti128 xm1, m0, 1
paddd xm0, xm1
pshufd xm1, xm0, 2
paddd xm0,xm1
movd eax, xm0
RET
INIT_YMM avx2
cglobal pixel_sad_32x16, 4,5,6
xorps m0, m0
xorps m5, m5
mov r4d, 4
.loop
movu m1, [r0] ; row 0 of pix0
movu m2, [r2] ; row 0 of pix1
movu m3, [r0 + r1] ; row 1 of pix0
movu m4, [r2 + r3] ; row 1 of pix1
psadbw m1, m2
psadbw m3, m4
paddd m0, m1
paddd m5, m3
lea r2, [r2 + 2 * r3]
lea r0, [r0 + 2 * r1]
movu m1, [r0] ; row 2 of pix0
movu m2, [r2] ; row 2 of pix1
movu m3, [r0 + r1] ; row 3 of pix0
movu m4, [r2 + r3] ; row 3 of pix1
psadbw m1, m2
psadbw m3, m4
paddd m0, m1
paddd m5, m3
lea r2, [r2 + 2 * r3]
lea r0, [r0 + 2 * r1]
dec r4d
jnz .loop
paddd m0, m5
vextracti128 xm1, m0, 1
paddd xm0, xm1
pshufd xm1, xm0, 2
paddd xm0,xm1
movd eax, xm0
RET
INIT_YMM avx2
cglobal pixel_sad_32x24, 4,7,6
xorps m0, m0
xorps m5, m5
mov r4d, 6
lea r5, [r1 * 3]
lea r6, [r3 * 3]
.loop
movu m1, [r0] ; row 0 of pix0
movu m2, [r2] ; row 0 of pix1
movu m3, [r0 + r1] ; row 1 of pix0
movu m4, [r2 + r3] ; row 1 of pix1
psadbw m1, m2
psadbw m3, m4
paddd m0, m1
paddd m5, m3
movu m1, [r0 + 2 * r1] ; row 2 of pix0
movu m2, [r2 + 2 * r3] ; row 2 of pix1
movu m3, [r0 + r5] ; row 3 of pix0
movu m4, [r2 + r6] ; row 3 of pix1
psadbw m1, m2
psadbw m3, m4
paddd m0, m1
paddd m5, m3
lea r2, [r2 + 4 * r3]
lea r0, [r0 + 4 * r1]
dec r4d
jnz .loop
paddd m0, m5
vextracti128 xm1, m0, 1
paddd xm0, xm1
pshufd xm1, xm0, 2
paddd xm0,xm1
movd eax, xm0
RET
INIT_YMM avx2
cglobal pixel_sad_32x32, 4,7,5
xorps m0, m0
mov r4d, 32/4
lea r5, [r1 * 3]
lea r6, [r3 * 3]
.loop
movu m1, [r0] ; row 0 of pix0
movu m2, [r2] ; row 0 of pix1
movu m3, [r0 + r1] ; row 1 of pix0
movu m4, [r2 + r3] ; row 1 of pix1
psadbw m1, m2
psadbw m3, m4
paddd m0, m1
paddd m0, m3
movu m1, [r0 + 2 * r1] ; row 2 of pix0
movu m2, [r2 + 2 * r3] ; row 2 of pix1
movu m3, [r0 + r5] ; row 3 of pix0
movu m4, [r2 + r6] ; row 3 of pix1
psadbw m1, m2
psadbw m3, m4
paddd m0, m1
paddd m0, m3
lea r2, [r2 + 4 * r3]
lea r0, [r0 + 4 * r1]
dec r4d
jnz .loop
vextracti128 xm1, m0, 1
paddd xm0, xm1
pshufd xm1, xm0, 2
paddd xm0,xm1
movd eax, xm0
RET
INIT_YMM avx2
cglobal pixel_sad_32x64, 4,7,5
xorps m0, m0
mov r4d, 64/8
lea r5, [r1 * 3]
lea r6, [r3 * 3]
.loop
movu m1, [r0] ; row 0 of pix0
movu m2, [r2] ; row 0 of pix1
movu m3, [r0 + r1] ; row 1 of pix0
movu m4, [r2 + r3] ; row 1 of pix1
psadbw m1, m2
psadbw m3, m4
paddd m0, m1
paddd m0, m3
movu m1, [r0 + 2 * r1] ; row 2 of pix0
movu m2, [r2 + 2 * r3] ; row 2 of pix1
movu m3, [r0 + r5] ; row 3 of pix0
movu m4, [r2 + r6] ; row 3 of pix1
psadbw m1, m2
psadbw m3, m4
paddd m0, m1
paddd m0, m3
lea r2, [r2 + 4 * r3]
lea r0, [r0 + 4 * r1]
movu m1, [r0] ; row 4 of pix0
movu m2, [r2] ; row 4 of pix1
movu m3, [r0 + r1] ; row 5 of pix0
movu m4, [r2 + r3] ; row 5 of pix1
psadbw m1, m2
psadbw m3, m4
paddd m0, m1
paddd m0, m3
movu m1, [r0 + 2 * r1] ; row 6 of pix0
movu m2, [r2 + 2 * r3] ; row 6 of pix1
movu m3, [r0 + r5] ; row 7 of pix0
movu m4, [r2 + r6] ; row 7 of pix1
psadbw m1, m2
psadbw m3, m4
paddd m0, m1
paddd m0, m3
lea r2, [r2 + 4 * r3]
lea r0, [r0 + 4 * r1]
dec r4d
jnz .loop
vextracti128 xm1, m0, 1
paddd xm0, xm1
pshufd xm1, xm0, 2
paddd xm0,xm1
movd eax, xm0
RET
INIT_YMM avx2
cglobal pixel_sad_48x64, 4,7,7
xorps m0, m0
mov r4d, 64/4
lea r5, [r1 * 3]
lea r6, [r3 * 3]
.loop
movu m1, [r0] ; row 0 of pix0
movu m2, [r2] ; row 0 of pix1
movu m3, [r0 + r1] ; row 1 of pix0
movu m4, [r2 + r3] ; row 1 of pix1
movu xm5, [r0 +32] ; last 16 of row 0 of pix0
vinserti128 m5, m5, [r0 + r1 + 32], 1
movu xm6, [r2 +32] ; last 16 of row 0 of pix1
vinserti128 m6, m6, [r2 + r3 + 32], 1
psadbw m1, m2
psadbw m3, m4
psadbw m5, m6
paddd m0, m1
paddd m0, m3
paddd m0, m5
movu m1, [r0 + 2 * r1] ; row 2 of pix0
movu m2, [r2 + 2 * r3] ; row 2 of pix1
movu m3, [r0 + r5] ; row 3 of pix0
movu m4, [r2 + r6] ; row 3 of pix1
movu xm5, [r0 +32 + 2 * r1]
vinserti128 m5, m5, [r0 + r5 + 32], 1
movu xm6, [r2 +32 + 2 * r3]
vinserti128 m6, m6, [r2 + r6 + 32], 1
psadbw m1, m2
psadbw m3, m4
psadbw m5, m6
paddd m0, m1
paddd m0, m3
paddd m0, m5
lea r2, [r2 + 4 * r3]
lea r0, [r0 + 4 * r1]
dec r4d
jnz .loop
vextracti128 xm1, m0, 1
paddd xm0, xm1
pshufd xm1, xm0, 2
paddd xm0,xm1
movd eax, xm0
RET
INIT_YMM avx2
cglobal pixel_sad_64x16, 4,5,6
xorps m0, m0
xorps m5, m5
mov r4d, 4
.loop
movu m1, [r0] ; first 32 of row 0 of pix0
movu m2, [r2] ; first 32 of row 0 of pix1
movu m3, [r0 + 32] ; second 32 of row 0 of pix0
movu m4, [r2 + 32] ; second 32 of row 0 of pix1
psadbw m1, m2
psadbw m3, m4
paddd m0, m1
paddd m5, m3
movu m1, [r0 + r1] ; first 32 of row 1 of pix0
movu m2, [r2 + r3] ; first 32 of row 1 of pix1
movu m3, [r0 + 32 + r1] ; second 32 of row 1 of pix0
movu m4, [r2 + 32 + r3] ; second 32 of row 1 of pix1
psadbw m1, m2
psadbw m3, m4
paddd m0, m1
paddd m5, m3
lea r2, [r2 + 2 * r3]
lea r0, [r0 + 2 * r1]
movu m1, [r0] ; first 32 of row 2 of pix0
movu m2, [r2] ; first 32 of row 2 of pix1
movu m3, [r0 + 32] ; second 32 of row 2 of pix0
movu m4, [r2 + 32] ; second 32 of row 2 of pix1
psadbw m1, m2
psadbw m3, m4
paddd m0, m1
paddd m5, m3
movu m1, [r0 + r1] ; first 32 of row 3 of pix0
movu m2, [r2 + r3] ; first 32 of row 3 of pix1
movu m3, [r0 + 32 + r1] ; second 32 of row 3 of pix0
movu m4, [r2 + 32 + r3] ; second 32 of row 3 of pix1
psadbw m1, m2
psadbw m3, m4
paddd m0, m1
paddd m5, m3
lea r2, [r2 + 2 * r3]
lea r0, [r0 + 2 * r1]
dec r4d
jnz .loop
paddd m0, m5
vextracti128 xm1, m0, 1
paddd xm0, xm1
pshufd xm1, xm0, 2
paddd xm0,xm1
movd eax, xm0
RET
INIT_YMM avx2
cglobal pixel_sad_64x32, 4,5,6
xorps m0, m0
xorps m5, m5
mov r4d, 16
.loop
movu m1, [r0] ; first 32 of row 0 of pix0
movu m2, [r2] ; first 32 of row 0 of pix1
movu m3, [r0 + 32] ; second 32 of row 0 of pix0
movu m4, [r2 + 32] ; second 32 of row 0 of pix1
psadbw m1, m2
psadbw m3, m4
paddd m0, m1
paddd m5, m3
movu m1, [r0 + r1] ; first 32 of row 1 of pix0
movu m2, [r2 + r3] ; first 32 of row 1 of pix1
movu m3, [r0 + 32 + r1] ; second 32 of row 1 of pix0
movu m4, [r2 + 32 + r3] ; second 32 of row 1 of pix1
psadbw m1, m2
psadbw m3, m4
paddd m0, m1
paddd m5, m3
lea r2, [r2 + 2 * r3]
lea r0, [r0 + 2 * r1]
dec r4d
jnz .loop
paddd m0, m5
vextracti128 xm1, m0, 1
paddd xm0, xm1
pshufd xm1, xm0, 2
paddd xm0,xm1
movd eax, xm0
RET
INIT_YMM avx2
cglobal pixel_sad_64x48, 4,7,6
xorps m0, m0
xorps m5, m5
mov r4d, 12
lea r5, [r1 * 3]
lea r6, [r3 * 3]
.loop
movu m1, [r0] ; first 32 of row 0 of pix0
movu m2, [r2] ; first 32 of row 0 of pix1
movu m3, [r0 + 32] ; second 32 of row 0 of pix0
movu m4, [r2 + 32] ; second 32 of row 0 of pix1
psadbw m1, m2
psadbw m3, m4
paddd m0, m1
paddd m5, m3
movu m1, [r0 + r1] ; first 32 of row 1 of pix0
movu m2, [r2 + r3] ; first 32 of row 1 of pix1
movu m3, [r0 + 32 + r1] ; second 32 of row 1 of pix0
movu m4, [r2 + 32 + r3] ; second 32 of row 1 of pix1
psadbw m1, m2
psadbw m3, m4
paddd m0, m1
paddd m5, m3
movu m1, [r0 + 2 * r1] ; first 32 of row 0 of pix0
movu m2, [r2 + 2 * r3] ; first 32 of row 0 of pix1
movu m3, [r0 + 2 * r1 + 32] ; second 32 of row 0 of pix0
movu m4, [r2 + 2 * r3 + 32] ; second 32 of row 0 of pix1
psadbw m1, m2
psadbw m3, m4
paddd m0, m1
paddd m5, m3
movu m1, [r0 + r5] ; first 32 of row 1 of pix0
movu m2, [r2 + r6] ; first 32 of row 1 of pix1
movu m3, [r0 + 32 + r5] ; second 32 of row 1 of pix0
movu m4, [r2 + 32 + r6] ; second 32 of row 1 of pix1
psadbw m1, m2
psadbw m3, m4
paddd m0, m1
paddd m5, m3
lea r2, [r2 + 4 * r3]
lea r0, [r0 + 4 * r1]
dec r4d
jnz .loop
paddd m0, m5
vextracti128 xm1, m0, 1
paddd xm0, xm1
pshufd xm1, xm0, 2
paddd xm0,xm1
movd eax, xm0
RET
INIT_YMM avx2
cglobal pixel_sad_64x64, 4,7,6
xorps m0, m0
xorps m5, m5
mov r4d, 8
lea r5, [r1 * 3]
lea r6, [r3 * 3]
.loop
movu m1, [r0] ; first 32 of row 0 of pix0
movu m2, [r2] ; first 32 of row 0 of pix1
movu m3, [r0 + 32] ; second 32 of row 0 of pix0
movu m4, [r2 + 32] ; second 32 of row 0 of pix1
psadbw m1, m2
psadbw m3, m4
paddd m0, m1
paddd m5, m3
movu m1, [r0 + r1] ; first 32 of row 1 of pix0
movu m2, [r2 + r3] ; first 32 of row 1 of pix1
movu m3, [r0 + 32 + r1] ; second 32 of row 1 of pix0
movu m4, [r2 + 32 + r3] ; second 32 of row 1 of pix1
psadbw m1, m2
psadbw m3, m4
paddd m0, m1
paddd m5, m3
movu m1, [r0 + 2 * r1] ; first 32 of row 2 of pix0
movu m2, [r2 + 2 * r3] ; first 32 of row 2 of pix1
movu m3, [r0 + 2 * r1 + 32] ; second 32 of row 2 of pix0
movu m4, [r2 + 2 * r3 + 32] ; second 32 of row 2 of pix1
psadbw m1, m2
psadbw m3, m4
paddd m0, m1
paddd m5, m3
movu m1, [r0 + r5] ; first 32 of row 3 of pix0
movu m2, [r2 + r6] ; first 32 of row 3 of pix1
movu m3, [r0 + 32 + r5] ; second 32 of row 3 of pix0
movu m4, [r2 + 32 + r6] ; second 32 of row 3 of pix1
psadbw m1, m2
psadbw m3, m4
paddd m0, m1
paddd m5, m3
lea r2, [r2 + 4 * r3]
lea r0, [r0 + 4 * r1]
movu m1, [r0] ; first 32 of row 4 of pix0
movu m2, [r2] ; first 32 of row 4 of pix1
movu m3, [r0 + 32] ; second 32 of row 4 of pix0
movu m4, [r2 + 32] ; second 32 of row 4 of pix1
psadbw m1, m2
psadbw m3, m4
paddd m0, m1
paddd m5, m3
movu m1, [r0 + r1] ; first 32 of row 5 of pix0
movu m2, [r2 + r3] ; first 32 of row 5 of pix1
movu m3, [r0 + 32 + r1] ; second 32 of row 5 of pix0
movu m4, [r2 + 32 + r3] ; second 32 of row 5 of pix1
psadbw m1, m2
psadbw m3, m4
paddd m0, m1
paddd m5, m3
movu m1, [r0 + 2 * r1] ; first 32 of row 6 of pix0
movu m2, [r2 + 2 * r3] ; first 32 of row 6 of pix1
movu m3, [r0 + 2 * r1 + 32] ; second 32 of row 6 of pix0
movu m4, [r2 + 2 * r3 + 32] ; second 32 of row 6 of pix1
psadbw m1, m2
psadbw m3, m4
paddd m0, m1
paddd m5, m3
movu m1, [r0 + r5] ; first 32 of row 7 of pix0
movu m2, [r2 + r6] ; first 32 of row 7 of pix1
movu m3, [r0 + 32 + r5] ; second 32 of row 7 of pix0
movu m4, [r2 + 32 + r6] ; second 32 of row 7 of pix1
psadbw m1, m2
psadbw m3, m4
paddd m0, m1
paddd m5, m3
lea r2, [r2 + 4 * r3]
lea r0, [r0 + 4 * r1]
dec r4d
jnz .loop
paddd m0, m5
vextracti128 xm1, m0, 1
paddd xm0, xm1
pshufd xm1, xm0, 2
paddd xm0,xm1
movd eax, xm0
RET
%endif
Reference in a new issue View git blame Copy permalink