/* poly1305-avx2-amd64.S - AMD64/AVX2 implementation of Poly1305
*
* Copyright (C) 2014 Jussi Kivilinna <jussi.kivilinna@iki.fi>
*
* This file is part of Libgcrypt.
*
* Libgcrypt is free software; you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as
* published by the Free Software Foundation; either version 2.1 of
* the License, or (at your option) any later version.
*
* Libgcrypt is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this program; if not, see <http://www.gnu.org/licenses/>.
*/
/*
* Based on public domain implementation by Andrew Moon at
* https://github.com/floodyberry/poly1305-opt
*/
#include <config.h>
#if defined(__x86_64__) && (defined(HAVE_COMPATIBLE_GCC_AMD64_PLATFORM_AS) || \
defined(HAVE_COMPATIBLE_GCC_WIN64_PLATFORM_AS)) && \
defined(ENABLE_AVX2_SUPPORT)
#ifdef HAVE_COMPATIBLE_GCC_AMD64_PLATFORM_AS
# define ELF(...) __VA_ARGS__
#else
# define ELF(...) /*_*/
#endif
.text
.align 8
.globl _gcry_poly1305_amd64_avx2_init_ext
ELF(.type _gcry_poly1305_amd64_avx2_init_ext,@function;)
_gcry_poly1305_amd64_avx2_init_ext:
.Lpoly1305_init_ext_avx2_local:
xor %edx, %edx
vzeroupper
pushq %r12
pushq %r13
pushq %r14
pushq %r15
pushq %rbx
movq %rdx, %rcx
vpxor %ymm0, %ymm0, %ymm0
movq $-1, %r8
testq %rcx, %rcx
vmovdqu %ymm0, (%rdi)
vmovdqu %ymm0, 32(%rdi)
vmovdqu %ymm0, 64(%rdi)
vmovdqu %ymm0, 96(%rdi)
vmovdqu %ymm0, 128(%rdi)
movq 8(%rsi), %r9
cmove %r8, %rcx
movq $0xffc0fffffff, %r8
movq %r9, %r13
movq (%rsi), %r10
andq %r10, %r8
shrq $44, %r10
movq %r8, %r14
shlq $20, %r13
orq %r13, %r10
movq $0xfffffc0ffff, %r13
shrq $24, %r9
andq %r13, %r10
movq $0xffffffc0f, %r13
andq %r13, %r9
movl %r8d, %r13d
andl $67108863, %r13d
movl %r13d, 164(%rdi)
movq %r10, %r13
shrq $26, %r14
shlq $18, %r13
orq %r13, %r14
movq %r10, %r13
shrq $8, %r13
andl $67108863, %r14d
andl $67108863, %r13d
movl %r14d, 172(%rdi)
movq %r10, %r14
movl %r13d, 180(%rdi)
movq %r9, %r13
shrq $34, %r14
shlq $10, %r13
orq %r13, %r14
movq %r9, %r13
shrq $16, %r13
andl $67108863, %r14d
movl %r14d, 188(%rdi)
movl %r13d, 196(%rdi)
cmpq $16, %rcx
jbe .Lpoly1305_init_ext_avx2_continue
lea (%r9,%r9,4), %r11
shlq $2, %r11
lea (%r10,%r10), %rax
mulq %r11
movq %rax, %r13
movq %r8, %rax
movq %rdx, %r14
mulq %r8
addq %rax, %r13
lea (%r8,%r8), %rax
movq %r13, %r12
adcq %rdx, %r14
mulq %r10
shlq $20, %r14
movq %rax, %r15
shrq $44, %r12
movq %r11, %rax
orq %r12, %r14
movq %rdx, %r12
mulq %r9
addq %rax, %r15
movq %r8, %rax
adcq %rdx, %r12
addq %r15, %r14
lea (%r9,%r9), %r15
movq %r14, %rbx
adcq $0, %r12
mulq %r15
shlq $20, %r12
movq %rdx, %r11
shrq $44, %rbx
orq %rbx, %r12
movq %rax, %rbx
movq %r10, %rax
mulq %r10
addq %rax, %rbx
adcq %rdx, %r11
addq %rbx, %r12
movq $0xfffffffffff, %rbx
movq %r12, %r15
adcq $0, %r11
andq %rbx, %r13
shlq $22, %r11
andq %rbx, %r14
shrq $42, %r15
orq %r15, %r11
lea (%r11,%r11,4), %r11
addq %r11, %r13
movq %rbx, %r11
andq %r13, %r11
shrq $44, %r13
movq %r11, %r15
addq %r13, %r14
movq $0x3ffffffffff, %r13
andq %r14, %rbx
andq %r13, %r12
movq %rbx, %r13
shrq $26, %r15
shlq $18, %r13
orq %r13, %r15
movq %rbx, %r13
shrq $44, %r14
shrq $8, %r13
addq %r14, %r12
movl %r11d, %r14d
andl $67108863, %r15d
andl $67108863, %r14d
andl $67108863, %r13d
movl %r14d, 204(%rdi)
movq %rbx, %r14
movl %r13d, 220(%rdi)
movq %r12, %r13
shrq $34, %r14
shlq $10, %r13
orq %r13, %r14
movq %r12, %r13
shrq $16, %r13
andl $67108863, %r14d
movl %r15d, 212(%rdi)
movl %r14d, 228(%rdi)
movl %r13d, 236(%rdi)
cmpq $32, %rcx
jbe .Lpoly1305_init_ext_avx2_continue
movq %r9, %rax
lea (%rbx,%rbx,4), %r14
shlq $2, %r14
mulq %r14
movq %rdi, -32(%rsp)
lea (%r12,%r12,4), %rdi
shlq $2, %rdi
movq %rax, %r14
movq %r10, %rax
movq %rdx, %r15
mulq %rdi
movq %rax, %r13
movq %r11, %rax
movq %rcx, -16(%rsp)
movq %rdx, %rcx
mulq %r8
addq %rax, %r13
movq %rdi, %rax
movq %rsi, -24(%rsp)
adcq %rdx, %rcx
addq %r13, %r14
adcq %rcx, %r15
movq %r14, %rcx
mulq %r9
shlq $20, %r15
movq %rax, %r13
shrq $44, %rcx
movq %r11, %rax
orq %rcx, %r15
movq %rdx, %rcx
mulq %r10
movq %rax, %rsi
movq %rbx, %rax
movq %rdx, %rdi
mulq %r8
addq %rax, %rsi
movq %r11, %rax
adcq %rdx, %rdi
addq %rsi, %r13
adcq %rdi, %rcx
addq %r13, %r15
movq %r15, %rdi
adcq $0, %rcx
mulq %r9
shlq $20, %rcx
movq %rdx, %rsi
shrq $44, %rdi
orq %rdi, %rcx
movq %rax, %rdi
movq %rbx, %rax
mulq %r10
movq %rax, %r9
movq %r8, %rax
movq %rdx, %r10
movq $0xfffffffffff, %r8
mulq %r12
addq %rax, %r9
adcq %rdx, %r10
andq %r8, %r14
addq %r9, %rdi
adcq %r10, %rsi
andq %r8, %r15
addq %rdi, %rcx
movq $0x3ffffffffff, %rdi
movq %rcx, %r10
adcq $0, %rsi
andq %rdi, %rcx
shlq $22, %rsi
shrq $42, %r10
orq %r10, %rsi
movq -32(%rsp), %rdi
lea (%rsi,%rsi,4), %r9
movq %r8, %rsi
addq %r9, %r14
andq %r14, %rsi
shrq $44, %r14
addq %r14, %r15
andq %r15, %r8
shrq $44, %r15
movq %r8, %r14
addq %r15, %rcx
movl %esi, %r15d
movq %rcx, %r10
movq %r8, %r9
shrq $26, %rsi
andl $67108863, %r15d
shlq $18, %r14
shrq $34, %r8
orq %r14, %rsi
shlq $10, %r10
shrq $8, %r9
orq %r10, %r8
shrq $16, %rcx
andl $67108863, %esi
movl %esi, 252(%rdi)
andl $67108863, %r9d
movl %ecx, 276(%rdi)
andl $67108863, %r8d
movl %r15d, 244(%rdi)
movl %r9d, 260(%rdi)
movl %r8d, 268(%rdi)
movq -16(%rsp), %rcx
movq -24(%rsp), %rsi
.Lpoly1305_init_ext_avx2_continue:
movl 16(%rsi), %r8d
movl %r8d, 284(%rdi)
movl 20(%rsi), %r9d
movl %r9d, 292(%rdi)
movl 24(%rsi), %r10d
movl %r10d, 300(%rdi)
movl 28(%rsi), %esi
movl %esi, 308(%rdi)
cmpq $48, %rcx
jbe .Lpoly1305_init_ext_avx2_done
lea (%r12,%r12,4), %r9
shlq $2, %r9
lea (%rbx,%rbx), %rax
mulq %r9
movq %rax, %rsi
movq %r11, %rax
movq %rdx, %r8
mulq %r11
addq %rax, %rsi
lea (%r11,%r11), %rax
movq %rsi, %r10
adcq %rdx, %r8
mulq %rbx
movq %rax, %r13
movq %r12, %rax
movq %rdx, %rcx
addq %r12, %r12
mulq %r9
addq %rax, %r13
movq %r11, %rax
movq $0xfffffffffff, %r9
adcq %rdx, %rcx
andq %r9, %rsi
mulq %r12
shlq $20, %r8
movq %rax, %r11
shrq $44, %r10
movq %rbx, %rax
orq %r10, %r8
movq %rdx, %r12
mulq %rbx
addq %r13, %r8
movq %r8, %r14
adcq $0, %rcx
andq %r9, %r8
addq %rax, %r11
adcq %rdx, %r12
shlq $20, %rcx
shrq $44, %r14
orq %r14, %rcx
addq %r11, %rcx
movq %rcx, %rbx
adcq $0, %r12
shlq $22, %r12
shrq $42, %rbx
orq %rbx, %r12
movq %r9, %rbx
lea (%r12,%r12,4), %r15
addq %r15, %rsi
andq %rsi, %rbx
shrq $44, %rsi
movl %ebx, %r11d
addq %rsi, %r8
movq $0x3ffffffffff, %rsi
andq %r8, %r9
andq %rsi, %rcx
shrq $44, %r8
movq %r9, %rax
addq %r8, %rcx
movq %r9, %r8
movq %rcx, %r10
andl $67108863, %r11d
shrq $26, %rbx
shlq $18, %r8
shrq $34, %r9
orq %r8, %rbx
shlq $10, %r10
shrq $8, %rax
orq %r10, %r9
shrq $16, %rcx
andl $67108863, %ebx
andl $67108863, %eax
andl $67108863, %r9d
movl %r11d, 184(%rdi)
movl %r11d, 176(%rdi)
movl %r11d, 168(%rdi)
movl %r11d, 160(%rdi)
movl %ebx, 216(%rdi)
movl %ebx, 208(%rdi)
movl %ebx, 200(%rdi)
movl %ebx, 192(%rdi)
movl %eax, 248(%rdi)
movl %eax, 240(%rdi)
movl %eax, 232(%rdi)
movl %eax, 224(%rdi)
movl %r9d, 280(%rdi)
movl %r9d, 272(%rdi)
movl %r9d, 264(%rdi)
movl %r9d, 256(%rdi)
movl %ecx, 312(%rdi)
movl %ecx, 304(%rdi)
movl %ecx, 296(%rdi)
movl %ecx, 288(%rdi)
.Lpoly1305_init_ext_avx2_done:
movq $0, 320(%rdi)
vzeroall
popq %rbx
popq %r15
popq %r14
popq %r13
popq %r12
ret
ELF(.size _gcry_poly1305_amd64_avx2_init_ext,.-_gcry_poly1305_amd64_avx2_init_ext;)
.align 8
.globl _gcry_poly1305_amd64_avx2_blocks
ELF(.type _gcry_poly1305_amd64_avx2_blocks,@function;)
_gcry_poly1305_amd64_avx2_blocks:
.Lpoly1305_blocks_avx2_local:
vzeroupper
pushq %rbp
movq %rsp, %rbp
pushq %rbx
andq $-64, %rsp
subq $200, %rsp
movl $((1<<26)-1), %r8d
movl $(5), %r9d
movl $((1<<24)), %r10d
vmovd %r8d, %xmm0
vmovd %r9d, %xmm8
vmovd %r10d, %xmm7
vpbroadcastq %xmm0, %ymm0
vpbroadcastq %xmm8, %ymm8
vpbroadcastq %xmm7, %ymm7
vmovdqa %ymm7, 168(%rsp)
movq 320(%rdi), %rax
testb $60, %al
je .Lpoly1305_blocks_avx2_9
vmovdqa 168(%rsp), %ymm7
vpsrldq $8, %ymm7, %ymm1
vmovdqa %ymm1, 168(%rsp)
testb $4, %al
je .Lpoly1305_blocks_avx2_10
vpermq $192, %ymm1, %ymm7
vmovdqa %ymm7, 168(%rsp)
.Lpoly1305_blocks_avx2_10:
testb $8, %al
je .Lpoly1305_blocks_avx2_11
vpermq $240, 168(%rsp), %ymm7
vmovdqa %ymm7, 168(%rsp)
.Lpoly1305_blocks_avx2_11:
testb $16, %al
je .Lpoly1305_blocks_avx2_12
vpermq $252, 168(%rsp), %ymm6
vmovdqa %ymm6, 168(%rsp)
.Lpoly1305_blocks_avx2_12:
testb $32, %al
je .Lpoly1305_blocks_avx2_9
vpxor %xmm6, %xmm6, %xmm6
vmovdqa %ymm6, 168(%rsp)
.Lpoly1305_blocks_avx2_9:
testb $1, %al
jne .Lpoly1305_blocks_avx2_13
vmovdqu (%rsi), %ymm3
vmovdqu 32(%rsi), %ymm1
vpunpcklqdq %ymm1, %ymm3, %ymm2
vpunpckhqdq %ymm1, %ymm3, %ymm1
vpermq $216, %ymm2, %ymm2
vpermq $216, %ymm1, %ymm1
vpand %ymm2, %ymm0, %ymm5
vpsrlq $26, %ymm2, %ymm4
vpand %ymm4, %ymm0, %ymm4
vpsllq $12, %ymm1, %ymm3
vpsrlq $52, %ymm2, %ymm2
vpor %ymm3, %ymm2, %ymm2
vpand %ymm2, %ymm0, %ymm3
vpsrlq $26, %ymm2, %ymm2
vpand %ymm2, %ymm0, %ymm2
vpsrlq $40, %ymm1, %ymm1
vpor 168(%rsp), %ymm1, %ymm1
addq $64, %rsi
subq $64, %rdx
orq $1, 320(%rdi)
jmp .Lpoly1305_blocks_avx2_14
.Lpoly1305_blocks_avx2_13:
vmovdqa (%rdi), %ymm5
vmovdqa 32(%rdi), %ymm4
vmovdqa 64(%rdi), %ymm3
vmovdqa 96(%rdi), %ymm2
vmovdqa 128(%rdi), %ymm1
.Lpoly1305_blocks_avx2_14:
cmpq $63, %rdx
jbe .Lpoly1305_blocks_avx2_15
vmovdqa 160(%rdi), %ymm6
vmovdqa %ymm8, 136(%rsp)
vmovdqa 192(%rdi), %ymm7
vpmuludq %ymm8, %ymm7, %ymm11
vmovdqa %ymm11, 104(%rsp)
vmovdqa 224(%rdi), %ymm11
vmovdqa %ymm11, 72(%rsp)
vpmuludq %ymm11, %ymm8, %ymm11
vmovdqa %ymm11, 40(%rsp)
vmovdqa 256(%rdi), %ymm11
vmovdqa %ymm11, 8(%rsp)
vpmuludq %ymm11, %ymm8, %ymm11
vmovdqa %ymm11, -24(%rsp)
vmovdqa 288(%rdi), %ymm13
vmovdqa %ymm13, -56(%rsp)
vpmuludq %ymm13, %ymm8, %ymm13
vmovdqa %ymm13, -88(%rsp)
.Lpoly1305_blocks_avx2_16:
vpmuludq 104(%rsp), %ymm1, %ymm14
vmovdqa 40(%rsp), %ymm13
vpmuludq %ymm13, %ymm2, %ymm8
vpmuludq %ymm13, %ymm1, %ymm13
vmovdqa -24(%rsp), %ymm9
vpmuludq %ymm9, %ymm2, %ymm10
vpmuludq %ymm9, %ymm1, %ymm11
vpaddq %ymm8, %ymm14, %ymm14
vpmuludq %ymm9, %ymm3, %ymm8
vmovdqa -88(%rsp), %ymm12
vpmuludq %ymm12, %ymm1, %ymm9
vpaddq %ymm10, %ymm13, %ymm13
vpmuludq %ymm12, %ymm4, %ymm15
vmovdqa %ymm12, %ymm10
vpmuludq %ymm12, %ymm3, %ymm12
vpaddq %ymm8, %ymm14, %ymm14
vpmuludq %ymm10, %ymm2, %ymm10
vpmuludq %ymm6, %ymm2, %ymm8
vpaddq %ymm15, %ymm14, %ymm14
vpmuludq %ymm6, %ymm1, %ymm1
vpaddq %ymm12, %ymm13, %ymm13
vpmuludq %ymm6, %ymm5, %ymm15
vpaddq %ymm10, %ymm11, %ymm11
vpmuludq %ymm6, %ymm4, %ymm12
vpaddq %ymm8, %ymm9, %ymm9
vpmuludq %ymm6, %ymm3, %ymm10
vpmuludq %ymm7, %ymm3, %ymm8
vpaddq %ymm15, %ymm14, %ymm14
vpmuludq %ymm7, %ymm2, %ymm2
vpaddq %ymm12, %ymm13, %ymm12
vpmuludq %ymm7, %ymm5, %ymm15
vpaddq %ymm10, %ymm11, %ymm10
vpmuludq %ymm7, %ymm4, %ymm13
vpaddq %ymm8, %ymm9, %ymm8
vmovdqa 72(%rsp), %ymm9
vpmuludq %ymm9, %ymm4, %ymm11
vpaddq %ymm2, %ymm1, %ymm1
vpmuludq %ymm9, %ymm3, %ymm3
vpaddq %ymm15, %ymm12, %ymm12
vpmuludq %ymm9, %ymm5, %ymm15
vpaddq %ymm13, %ymm10, %ymm10
vmovdqa 8(%rsp), %ymm2
vpmuludq %ymm2, %ymm5, %ymm9
vpaddq %ymm11, %ymm8, %ymm8
vpmuludq %ymm2, %ymm4, %ymm4
vpaddq %ymm3, %ymm1, %ymm1
vpmuludq -56(%rsp), %ymm5, %ymm5
vpaddq %ymm15, %ymm10, %ymm10
vpaddq %ymm9, %ymm8, %ymm8
vpaddq %ymm4, %ymm1, %ymm1
vpaddq %ymm5, %ymm1, %ymm5
vmovdqu (%rsi), %ymm3
vmovdqu 32(%rsi), %ymm2
vperm2i128 $32, %ymm2, %ymm3, %ymm1
vperm2i128 $49, %ymm2, %ymm3, %ymm2
vpunpckldq %ymm2, %ymm1, %ymm15
vpunpckhdq %ymm2, %ymm1, %ymm2
vpxor %xmm4, %xmm4, %xmm4
vpunpckldq %ymm4, %ymm15, %ymm1
vpunpckhdq %ymm4, %ymm15, %ymm15
vpunpckldq %ymm4, %ymm2, %ymm3
vpunpckhdq %ymm4, %ymm2, %ymm2
vpsllq $6, %ymm15, %ymm15
vpsllq $12, %ymm3, %ymm3
vpsllq $18, %ymm2, %ymm2
vpaddq %ymm1, %ymm14, %ymm14
vpaddq %ymm15, %ymm12, %ymm12
vpaddq %ymm3, %ymm10, %ymm10
vpaddq %ymm2, %ymm8, %ymm8
vpaddq 168(%rsp), %ymm5, %ymm5
addq $64, %rsi
vpsrlq $26, %ymm14, %ymm4
vpsrlq $26, %ymm8, %ymm2
vpand %ymm0, %ymm14, %ymm14
vpand %ymm0, %ymm8, %ymm8
vpaddq %ymm4, %ymm12, %ymm12
vpaddq %ymm2, %ymm5, %ymm5
vpsrlq $26, %ymm12, %ymm3
vpsrlq $26, %ymm5, %ymm9
vpand %ymm0, %ymm12, %ymm12
vpand %ymm0, %ymm5, %ymm11
vpaddq %ymm3, %ymm10, %ymm3
vpmuludq 136(%rsp), %ymm9, %ymm9
vpaddq %ymm9, %ymm14, %ymm14
vpsrlq $26, %ymm3, %ymm2
vpsrlq $26, %ymm14, %ymm4
vpand %ymm0, %ymm3, %ymm3
vpand %ymm0, %ymm14, %ymm5
vpaddq %ymm2, %ymm8, %ymm2
vpaddq %ymm4, %ymm12, %ymm4
vpsrlq $26, %ymm2, %ymm1
vpand %ymm0, %ymm2, %ymm2
vpaddq %ymm1, %ymm11, %ymm1
subq $64, %rdx
cmpq $63, %rdx
ja .Lpoly1305_blocks_avx2_16
.Lpoly1305_blocks_avx2_15:
testb $64, 320(%rdi)
jne .Lpoly1305_blocks_avx2_17
vmovdqa %ymm5, (%rdi)
vmovdqa %ymm4, 32(%rdi)
vmovdqa %ymm3, 64(%rdi)
vmovdqa %ymm2, 96(%rdi)
vmovdqa %ymm1, 128(%rdi)
jmp .Lpoly1305_blocks_avx2_8
.Lpoly1305_blocks_avx2_17:
vpermq $245, %ymm5, %ymm0
vpaddq %ymm0, %ymm5, %ymm5
vpermq $245, %ymm4, %ymm0
vpaddq %ymm0, %ymm4, %ymm4
vpermq $245, %ymm3, %ymm0
vpaddq %ymm0, %ymm3, %ymm3
vpermq $245, %ymm2, %ymm0
vpaddq %ymm0, %ymm2, %ymm2
vpermq $245, %ymm1, %ymm0
vpaddq %ymm0, %ymm1, %ymm1
vpermq $170, %ymm5, %ymm0
vpaddq %ymm0, %ymm5, %ymm5
vpermq $170, %ymm4, %ymm0
vpaddq %ymm0, %ymm4, %ymm4
vpermq $170, %ymm3, %ymm0
vpaddq %ymm0, %ymm3, %ymm3
vpermq $170, %ymm2, %ymm0
vpaddq %ymm0, %ymm2, %ymm2
vpermq $170, %ymm1, %ymm0
vpaddq %ymm0, %ymm1, %ymm1
vmovd %xmm5, %eax
vmovd %xmm4, %edx
movl %eax, %ecx
shrl $26, %ecx
addl %edx, %ecx
movl %ecx, %edx
andl $67108863, %edx
vmovd %xmm3, %esi
shrl $26, %ecx
movl %ecx, %r11d
addl %esi, %r11d
vmovd %xmm2, %ecx
movl %r11d, %r10d
shrl $26, %r10d
addl %ecx, %r10d
movl %r10d, %r9d
andl $67108863, %r9d
vmovd %xmm1, %r8d
movl %edx, %esi
salq $26, %rsi
andl $67108863, %eax
orq %rax, %rsi
movabsq $17592186044415, %rax
andq %rax, %rsi
andl $67108863, %r11d
salq $8, %r11
shrl $18, %edx
movl %edx, %edx
orq %r11, %rdx
movq %r9, %rcx
salq $34, %rcx
orq %rcx, %rdx
andq %rax, %rdx
shrl $26, %r10d
addl %r10d, %r8d
salq $16, %r8
shrl $10, %r9d
movl %r9d, %r9d
orq %r9, %r8
movabsq $4398046511103, %r10
movq %r8, %r9
andq %r10, %r9
shrq $42, %r8
leaq (%r8,%r8,4), %rcx
addq %rcx, %rsi
movq %rsi, %r8
andq %rax, %r8
movq %rsi, %rcx
shrq $44, %rcx
addq %rdx, %rcx
movq %rcx, %rsi
andq %rax, %rsi
shrq $44, %rcx
movq %rcx, %rdx
addq %r9, %rdx
andq %rdx, %r10
shrq $42, %rdx
leaq (%r8,%rdx,4), %rcx
leaq (%rcx,%rdx), %rdx
movq %rdx, %rbx
andq %rax, %rbx
shrq $44, %rdx
movq %rdx, %r11
addq %rsi, %r11
leaq 5(%rbx), %r9
movq %r9, %r8
shrq $44, %r8
addq %r11, %r8
movabsq $-4398046511104, %rsi
addq %r10, %rsi
movq %r8, %rdx
shrq $44, %rdx
addq %rdx, %rsi
movq %rsi, %rdx
shrq $63, %rdx
subq $1, %rdx
movq %rdx, %rcx
notq %rcx
andq %rcx, %rbx
andq %rcx, %r11
andq %r10, %rcx
andq %rax, %r9
andq %rdx, %r9
orq %r9, %rbx
movq %rbx, (%rdi)
andq %r8, %rax
andq %rdx, %rax
orq %rax, %r11
movq %r11, 8(%rdi)
andq %rsi, %rdx
orq %rcx, %rdx
movq %rdx, 16(%rdi)
.Lpoly1305_blocks_avx2_8:
movq -8(%rbp), %rbx
vzeroall
movq %rbp, %rax
subq %rsp, %rax
leave
addq $8, %rax
ret
ELF(.size _gcry_poly1305_amd64_avx2_blocks,.-_gcry_poly1305_amd64_avx2_blocks;)
.align 8
.globl _gcry_poly1305_amd64_avx2_finish_ext
ELF(.type _gcry_poly1305_amd64_avx2_finish_ext,@function;)
_gcry_poly1305_amd64_avx2_finish_ext:
.Lpoly1305_finish_ext_avx2_local:
vzeroupper
pushq %rbp
movq %rsp, %rbp
pushq %r13
pushq %r12
pushq %rbx
andq $-64, %rsp
subq $64, %rsp
movq %rdi, %rbx
movq %rdx, %r13
movq %rcx, %r12
testq %rdx, %rdx
je .Lpoly1305_finish_ext_avx2_22
vpxor %xmm0, %xmm0, %xmm0
vmovdqa %ymm0, (%rsp)
vmovdqa %ymm0, 32(%rsp)
movq %rsp, %rax
subq %rsp, %rsi
testb $32, %dl
je .Lpoly1305_finish_ext_avx2_23
vmovdqu (%rsp,%rsi), %ymm0
vmovdqa %ymm0, (%rsp)
leaq 32(%rsp), %rax
.Lpoly1305_finish_ext_avx2_23:
testb $16, %r13b
je .Lpoly1305_finish_ext_avx2_24
vmovdqu (%rax,%rsi), %xmm0
vmovdqa %xmm0, (%rax)
addq $16, %rax
.Lpoly1305_finish_ext_avx2_24:
testb $8, %r13b
je .Lpoly1305_finish_ext_avx2_25
movq (%rax,%rsi), %rdx
movq %rdx, (%rax)
addq $8, %rax
.Lpoly1305_finish_ext_avx2_25:
testb $4, %r13b
je .Lpoly1305_finish_ext_avx2_26
movl (%rax,%rsi), %edx
movl %edx, (%rax)
addq $4, %rax
.Lpoly1305_finish_ext_avx2_26:
testb $2, %r13b
je .Lpoly1305_finish_ext_avx2_27
movzwl (%rax,%rsi), %edx
movw %dx, (%rax)
addq $2, %rax
.Lpoly1305_finish_ext_avx2_27:
testb $1, %r13b
je .Lpoly1305_finish_ext_avx2_28
movzbl (%rax,%rsi), %edx
movb %dl, (%rax)
.Lpoly1305_finish_ext_avx2_28:
testb $15, %r13b
je .Lpoly1305_finish_ext_avx2_29
movb $1, (%rsp,%r13)
.Lpoly1305_finish_ext_avx2_29:
cmpq $47, %r13
jbe .Lpoly1305_finish_ext_avx2_30
orq $4, 320(%rbx)
jmp .Lpoly1305_finish_ext_avx2_31
.Lpoly1305_finish_ext_avx2_30:
cmpq $31, %r13
jbe .Lpoly1305_finish_ext_avx2_32
orq $8, 320(%rbx)
jmp .Lpoly1305_finish_ext_avx2_31
.Lpoly1305_finish_ext_avx2_32:
cmpq $15, %r13
jbe .Lpoly1305_finish_ext_avx2_33
orq $16, 320(%rbx)
jmp .Lpoly1305_finish_ext_avx2_31
.Lpoly1305_finish_ext_avx2_33:
orq $32, 320(%rbx)
.Lpoly1305_finish_ext_avx2_31:
testb $1, 320(%rbx)
je .Lpoly1305_finish_ext_avx2_34
cmpq $32, %r13
ja .Lpoly1305_finish_ext_avx2_34
cmpq $17, %r13
sbbq %rsi, %rsi
notq %rsi
addq $2, %rsi
cmpq $17, %r13
sbbq %rax, %rax
movq %rbx, %rdx
addq $23, %rax
leaq (%rbx,%rax,8), %rax
movl $0, %ecx
.Lpoly1305_finish_ext_avx2_37:
movl 244(%rdx), %edi
movl %edi, (%rax)
movl 252(%rdx), %edi
movl %edi, 32(%rax)
movl 260(%rdx), %edi
movl %edi, 64(%rax)
movl 268(%rdx), %edi
movl %edi, 96(%rax)
movl 276(%rdx), %edi
movl %edi, 128(%rax)
addq $1, %rcx
subq $40, %rdx
addq $8, %rax
cmpq %rcx, %rsi
ja .Lpoly1305_finish_ext_avx2_37
.Lpoly1305_finish_ext_avx2_34:
movl $64, %edx
movq %rsp, %rsi
movq %rbx, %rdi
call .Lpoly1305_blocks_avx2_local
.Lpoly1305_finish_ext_avx2_22:
movq 320(%rbx), %r8
testb $1, %r8b
je .Lpoly1305_finish_ext_avx2_38
leaq -1(%r13), %rax
cmpq $47, %rax
ja .Lpoly1305_finish_ext_avx2_46
cmpq $32, %r13
ja .Lpoly1305_finish_ext_avx2_47
cmpq $17, %r13
sbbq %r9, %r9
addq $2, %r9
movl $0, %edi
cmpq $17, %r13
sbbq %rax, %rax
notq %rax
andl $5, %eax
jmp .Lpoly1305_finish_ext_avx2_39
.Lpoly1305_finish_ext_avx2_41:
movl (%rdx), %esi
movl %esi, (%rax)
movl 8(%rdx), %esi
movl %esi, 32(%rax)
movl 16(%rdx), %esi
movl %esi, 64(%rax)
movl 24(%rdx), %esi
movl %esi, 96(%rax)
movl 32(%rdx), %esi
movl %esi, 128(%rax)
addq $1, %rcx
subq $40, %rdx
addq $8, %rax
movq %rcx, %rsi
subq %rdi, %rsi
cmpq %rsi, %r9
ja .Lpoly1305_finish_ext_avx2_41
cmpq $3, %rcx
ja .Lpoly1305_finish_ext_avx2_42
leaq 160(%rbx,%rcx,8), %rax
.Lpoly1305_finish_ext_avx2_43:
movl $1, (%rax)
movl $0, 32(%rax)
movl $0, 64(%rax)
movl $0, 96(%rax)
movl $0, 128(%rax)
addq $1, %rcx
addq $8, %rax
cmpq $4, %rcx
jne .Lpoly1305_finish_ext_avx2_43
.Lpoly1305_finish_ext_avx2_42:
orq $96, %r8
movq %r8, 320(%rbx)
vpxor %ymm0, %ymm0, %ymm0
vmovdqa %ymm0, (%rsp)
vmovdqa %ymm0, 32(%rsp)
movl $64, %edx
movq %rsp, %rsi
movq %rbx, %rdi
call .Lpoly1305_blocks_avx2_local
.Lpoly1305_finish_ext_avx2_38:
movq 8(%rbx), %rax
movq %rax, %rdx
salq $44, %rdx
orq (%rbx), %rdx
shrq $20, %rax
movl $24, %edi
shlx %rdi, 16(%rbx), %rcx
orq %rcx, %rax
movl 292(%rbx), %ecx
salq $32, %rcx
movl 284(%rbx), %esi
orq %rsi, %rcx
movl 308(%rbx), %esi
salq $32, %rsi
movl 300(%rbx), %edi
orq %rdi, %rsi
addq %rcx, %rdx
adcq %rsi, %rax
movq %rdx, (%r12)
movq %rax, 8(%r12)
vpxor %xmm0, %xmm0, %xmm0
vmovdqu %ymm0, (%rbx)
vmovdqu %ymm0, 32(%rbx)
vmovdqu %ymm0, 64(%rbx)
vmovdqu %ymm0, 96(%rbx)
vmovdqu %ymm0, 128(%rbx)
vmovdqu %ymm0, 160(%rbx)
vmovdqu %ymm0, 192(%rbx)
vmovdqu %ymm0, 224(%rbx)
jmp .Lpoly1305_finish_ext_avx2_49
.Lpoly1305_finish_ext_avx2_46:
movl $3, %r9d
movl $1, %edi
movl $10, %eax
jmp .Lpoly1305_finish_ext_avx2_39
.Lpoly1305_finish_ext_avx2_47:
movl $3, %r9d
movl $0, %edi
movl $10, %eax
.Lpoly1305_finish_ext_avx2_39:
leaq 164(%rbx,%rax,8), %rdx
leaq 160(%rbx,%rdi,8), %rax
movq %rdi, %rcx
jmp .Lpoly1305_finish_ext_avx2_41
.Lpoly1305_finish_ext_avx2_49:
movq %rbp, %rax
subq %rsp, %rax
leaq -24(%rbp), %rsp
vzeroall
popq %rbx
popq %r12
popq %r13
popq %rbp
addq $(8*5), %rax
ret
ELF(.size _gcry_poly1305_amd64_avx2_finish_ext,.-_gcry_poly1305_amd64_avx2_finish_ext;)
#endif
