| #! /usr/bin/env perl |
| # Copyright 2013-2020 The OpenSSL Project Authors. All Rights Reserved. |
| # Copyright (c) 2012, Intel Corporation. All Rights Reserved. |
| # |
| # Licensed under the Apache License 2.0 (the "License"). You may not use |
| # this file except in compliance with the License. You can obtain a copy |
| # in the file LICENSE in the source distribution or at |
| # https://www.openssl.org/source/license.html |
| # |
| # Originally written by Shay Gueron (1, 2), and Vlad Krasnov (1) |
| # (1) Intel Corporation, Israel Development Center, Haifa, Israel |
| # (2) University of Haifa, Israel |
| # |
| # References: |
| # [1] S. Gueron, "Efficient Software Implementations of Modular |
| # Exponentiation", http://eprint.iacr.org/2011/239 |
| # [2] S. Gueron, V. Krasnov. "Speeding up Big-Numbers Squaring". |
| # IEEE Proceedings of 9th International Conference on Information |
| # Technology: New Generations (ITNG 2012), 821-823 (2012). |
| # [3] S. Gueron, Efficient Software Implementations of Modular Exponentiation |
| # Journal of Cryptographic Engineering 2:31-43 (2012). |
| # [4] S. Gueron, V. Krasnov: "[PATCH] Efficient and side channel analysis |
| # resistant 512-bit and 1024-bit modular exponentiation for optimizing |
| # RSA1024 and RSA2048 on x86_64 platforms", |
| # http://rt.openssl.org/Ticket/Display.html?id=2582&user=guest&pass=guest |
| # |
| # While original submission covers 512- and 1024-bit exponentiation, |
| # this module is limited to 512-bit version only (and as such |
| # accelerates RSA1024 sign). This is because improvement for longer |
| # keys is not high enough to justify the effort, highest measured |
| # was ~5% on Westmere. [This is relative to OpenSSL 1.0.2, upcoming |
| # for the moment of this writing!] Nor does this module implement |
| # "monolithic" complete exponentiation jumbo-subroutine, but adheres |
| # to more modular mixture of C and assembly. And it's optimized even |
| # for processors other than Intel Core family (see table below for |
| # improvement coefficients). |
| # <appro@openssl.org> |
| # |
| # RSA1024 sign/sec this/original |this/rsax(*) this/fips(*) |
| # ----------------+--------------------------- |
| # Opteron +13% |+5% +20% |
| # Bulldozer -0% |-1% +10% |
| # P4 +11% |+7% +8% |
| # Westmere +5% |+14% +17% |
| # Sandy Bridge +2% |+12% +29% |
| # Ivy Bridge +1% |+11% +35% |
| # Haswell(**) -0% |+12% +39% |
| # Atom +13% |+11% +4% |
| # VIA Nano +70% |+9% +25% |
| # |
| # (*) rsax engine and fips numbers are presented for reference |
| # purposes; |
| # (**) MULX was attempted, but found to give only marginal improvement; |
| |
| # $output is the last argument if it looks like a file (it has an extension) |
| # $flavour is the first argument if it doesn't look like a file |
| $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; |
| $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef; |
| |
| $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); |
| |
| $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; |
| ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or |
| ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or |
| die "can't locate x86_64-xlate.pl"; |
| |
| open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"" |
| or die "can't call $xlate: $!"; |
| *STDOUT=*OUT; |
| |
| if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` |
| =~ /GNU assembler version ([2-9]\.[0-9]+)/) { |
| $addx = ($1>=2.23); |
| } |
| |
| if (!$addx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && |
| `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) { |
| $addx = ($1>=2.10); |
| } |
| |
| if (!$addx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) && |
| `ml64 2>&1` =~ /Version ([0-9]+)\./) { |
| $addx = ($1>=12); |
| } |
| |
| if (!$addx && `$ENV{CC} -v 2>&1` =~ /((?:clang|LLVM) version|.*based on LLVM) ([0-9]+)\.([0-9]+)/) { |
| my $ver = $2 + $3/100.0; # 3.1->3.01, 3.10->3.10 |
| $addx = ($ver>=3.03); |
| } |
| |
| ($out, $inp, $mod) = ("%rdi", "%rsi", "%rbp"); # common internal API |
| { |
| my ($out,$inp,$mod,$n0,$times) = ("%rdi","%rsi","%rdx","%rcx","%r8d"); |
| |
| $code.=<<___; |
| .text |
| |
| .extern OPENSSL_ia32cap_P |
| |
| .globl rsaz_512_sqr |
| .type rsaz_512_sqr,\@function,5 |
| .align 32 |
| rsaz_512_sqr: # 25-29% faster than rsaz_512_mul |
| .cfi_startproc |
| push %rbx |
| .cfi_push %rbx |
| push %rbp |
| .cfi_push %rbp |
| push %r12 |
| .cfi_push %r12 |
| push %r13 |
| .cfi_push %r13 |
| push %r14 |
| .cfi_push %r14 |
| push %r15 |
| .cfi_push %r15 |
| |
| subq \$128+24, %rsp |
| .cfi_adjust_cfa_offset 128+24 |
| .Lsqr_body: |
| movq $mod, %xmm1 # common off-load |
| movq ($inp), %rdx |
| movq 8($inp), %rax |
| movq $n0, 128(%rsp) |
| ___ |
| $code.=<<___ if ($addx); |
| movl \$0x80100,%r11d |
| andl OPENSSL_ia32cap_P+8(%rip),%r11d |
| cmpl \$0x80100,%r11d # check for MULX and ADO/CX |
| je .Loop_sqrx |
| ___ |
| $code.=<<___; |
| jmp .Loop_sqr |
| |
| .align 32 |
| .Loop_sqr: |
| movl $times,128+8(%rsp) |
| #first iteration |
| movq %rdx, %rbx # 0($inp) |
| mov %rax, %rbp # 8($inp) |
| mulq %rdx |
| movq %rax, %r8 |
| movq 16($inp), %rax |
| movq %rdx, %r9 |
| |
| mulq %rbx |
| addq %rax, %r9 |
| movq 24($inp), %rax |
| movq %rdx, %r10 |
| adcq \$0, %r10 |
| |
| mulq %rbx |
| addq %rax, %r10 |
| movq 32($inp), %rax |
| movq %rdx, %r11 |
| adcq \$0, %r11 |
| |
| mulq %rbx |
| addq %rax, %r11 |
| movq 40($inp), %rax |
| movq %rdx, %r12 |
| adcq \$0, %r12 |
| |
| mulq %rbx |
| addq %rax, %r12 |
| movq 48($inp), %rax |
| movq %rdx, %r13 |
| adcq \$0, %r13 |
| |
| mulq %rbx |
| addq %rax, %r13 |
| movq 56($inp), %rax |
| movq %rdx, %r14 |
| adcq \$0, %r14 |
| |
| mulq %rbx |
| addq %rax, %r14 |
| movq %rbx, %rax |
| adcq \$0, %rdx |
| |
| xorq %rcx,%rcx # rcx:r8 = r8 << 1 |
| addq %r8, %r8 |
| movq %rdx, %r15 |
| adcq \$0, %rcx |
| |
| mulq %rax |
| addq %r8, %rdx |
| adcq \$0, %rcx |
| |
| movq %rax, (%rsp) |
| movq %rdx, 8(%rsp) |
| |
| #second iteration |
| movq 16($inp), %rax |
| mulq %rbp |
| addq %rax, %r10 |
| movq 24($inp), %rax |
| movq %rdx, %rbx |
| adcq \$0, %rbx |
| |
| mulq %rbp |
| addq %rax, %r11 |
| movq 32($inp), %rax |
| adcq \$0, %rdx |
| addq %rbx, %r11 |
| movq %rdx, %rbx |
| adcq \$0, %rbx |
| |
| mulq %rbp |
| addq %rax, %r12 |
| movq 40($inp), %rax |
| adcq \$0, %rdx |
| addq %rbx, %r12 |
| movq %rdx, %rbx |
| adcq \$0, %rbx |
| |
| mulq %rbp |
| addq %rax, %r13 |
| movq 48($inp), %rax |
| adcq \$0, %rdx |
| addq %rbx, %r13 |
| movq %rdx, %rbx |
| adcq \$0, %rbx |
| |
| mulq %rbp |
| addq %rax, %r14 |
| movq 56($inp), %rax |
| adcq \$0, %rdx |
| addq %rbx, %r14 |
| movq %rdx, %rbx |
| adcq \$0, %rbx |
| |
| mulq %rbp |
| addq %rax, %r15 |
| movq %rbp, %rax |
| adcq \$0, %rdx |
| addq %rbx, %r15 |
| adcq \$0, %rdx |
| |
| xorq %rbx, %rbx # rbx:r10:r9 = r10:r9 << 1 |
| addq %r9, %r9 |
| movq %rdx, %r8 |
| adcq %r10, %r10 |
| adcq \$0, %rbx |
| |
| mulq %rax |
| # rcx <= 2 and rax <= 0xFFFF..F9, so carry must be zero here |
| addq %rcx, %rax |
| movq 16($inp), %rbp |
| addq %rax, %r9 |
| movq 24($inp), %rax |
| adcq %rdx, %r10 |
| adcq \$0, %rbx |
| |
| movq %r9, 16(%rsp) |
| movq %r10, 24(%rsp) |
| |
| #third iteration |
| mulq %rbp |
| addq %rax, %r12 |
| movq 32($inp), %rax |
| movq %rdx, %rcx |
| adcq \$0, %rcx |
| |
| mulq %rbp |
| addq %rax, %r13 |
| movq 40($inp), %rax |
| adcq \$0, %rdx |
| addq %rcx, %r13 |
| movq %rdx, %rcx |
| adcq \$0, %rcx |
| |
| mulq %rbp |
| addq %rax, %r14 |
| movq 48($inp), %rax |
| adcq \$0, %rdx |
| addq %rcx, %r14 |
| movq %rdx, %rcx |
| adcq \$0, %rcx |
| |
| mulq %rbp |
| addq %rax, %r15 |
| movq 56($inp), %rax |
| adcq \$0, %rdx |
| addq %rcx, %r15 |
| movq %rdx, %rcx |
| adcq \$0, %rcx |
| |
| mulq %rbp |
| addq %rax, %r8 |
| movq %rbp, %rax |
| adcq \$0, %rdx |
| addq %rcx, %r8 |
| adcq \$0, %rdx |
| |
| xorq %rcx, %rcx # rcx:r12:r11 = r12:r11 << 1 |
| addq %r11, %r11 |
| movq %rdx, %r9 |
| adcq %r12, %r12 |
| adcq \$0, %rcx |
| |
| mulq %rax |
| # rbx <= 2 and rax <= 0xFFFF..F9, so carry must be zero here |
| addq %rbx, %rax |
| movq 24($inp), %r10 |
| addq %rax, %r11 |
| movq 32($inp), %rax |
| adcq %rdx, %r12 |
| adcq \$0, %rcx |
| |
| movq %r11, 32(%rsp) |
| movq %r12, 40(%rsp) |
| |
| #fourth iteration |
| mov %rax, %r11 # 32($inp) |
| mulq %r10 |
| addq %rax, %r14 |
| movq 40($inp), %rax |
| movq %rdx, %rbx |
| adcq \$0, %rbx |
| |
| mov %rax, %r12 # 40($inp) |
| mulq %r10 |
| addq %rax, %r15 |
| movq 48($inp), %rax |
| adcq \$0, %rdx |
| addq %rbx, %r15 |
| movq %rdx, %rbx |
| adcq \$0, %rbx |
| |
| mov %rax, %rbp # 48($inp) |
| mulq %r10 |
| addq %rax, %r8 |
| movq 56($inp), %rax |
| adcq \$0, %rdx |
| addq %rbx, %r8 |
| movq %rdx, %rbx |
| adcq \$0, %rbx |
| |
| mulq %r10 |
| addq %rax, %r9 |
| movq %r10, %rax |
| adcq \$0, %rdx |
| addq %rbx, %r9 |
| adcq \$0, %rdx |
| |
| xorq %rbx, %rbx # rbx:r13:r14 = r13:r14 << 1 |
| addq %r13, %r13 |
| movq %rdx, %r10 |
| adcq %r14, %r14 |
| adcq \$0, %rbx |
| |
| mulq %rax |
| # rcx <= 2 and rax <= 0xFFFF..F9, so carry must be zero here |
| addq %rcx, %rax |
| addq %rax, %r13 |
| movq %r12, %rax # 40($inp) |
| adcq %rdx, %r14 |
| adcq \$0, %rbx |
| |
| movq %r13, 48(%rsp) |
| movq %r14, 56(%rsp) |
| |
| #fifth iteration |
| mulq %r11 |
| addq %rax, %r8 |
| movq %rbp, %rax # 48($inp) |
| movq %rdx, %rcx |
| adcq \$0, %rcx |
| |
| mulq %r11 |
| addq %rax, %r9 |
| movq 56($inp), %rax |
| adcq \$0, %rdx |
| addq %rcx, %r9 |
| movq %rdx, %rcx |
| adcq \$0, %rcx |
| |
| mov %rax, %r14 # 56($inp) |
| mulq %r11 |
| addq %rax, %r10 |
| movq %r11, %rax |
| adcq \$0, %rdx |
| addq %rcx, %r10 |
| adcq \$0, %rdx |
| |
| xorq %rcx, %rcx # rcx:r8:r15 = r8:r15 << 1 |
| addq %r15, %r15 |
| movq %rdx, %r11 |
| adcq %r8, %r8 |
| adcq \$0, %rcx |
| |
| mulq %rax |
| # rbx <= 2 and rax <= 0xFFFF..F9, so carry must be zero here |
| addq %rbx, %rax |
| addq %rax, %r15 |
| movq %rbp, %rax # 48($inp) |
| adcq %rdx, %r8 |
| adcq \$0, %rcx |
| |
| movq %r15, 64(%rsp) |
| movq %r8, 72(%rsp) |
| |
| #sixth iteration |
| mulq %r12 |
| addq %rax, %r10 |
| movq %r14, %rax # 56($inp) |
| movq %rdx, %rbx |
| adcq \$0, %rbx |
| |
| mulq %r12 |
| addq %rax, %r11 |
| movq %r12, %rax |
| adcq \$0, %rdx |
| addq %rbx, %r11 |
| adcq \$0, %rdx |
| |
| xorq %rbx, %rbx # rbx:r10:r9 = r10:r9 << 1 |
| addq %r9, %r9 |
| movq %rdx, %r12 |
| adcq %r10, %r10 |
| adcq \$0, %rbx |
| |
| mulq %rax |
| # rcx <= 2 and rax <= 0xFFFF..F9, so carry must be zero here |
| addq %rcx, %rax |
| addq %rax, %r9 |
| movq %r14, %rax # 56($inp) |
| adcq %rdx, %r10 |
| adcq \$0, %rbx |
| |
| movq %r9, 80(%rsp) |
| movq %r10, 88(%rsp) |
| |
| #seventh iteration |
| mulq %rbp |
| addq %rax, %r12 |
| movq %rbp, %rax |
| adcq \$0, %rdx |
| |
| xorq %rcx, %rcx # rcx:r12:r11 = r12:r11 << 1 |
| addq %r11, %r11 |
| movq %rdx, %r13 |
| adcq %r12, %r12 |
| adcq \$0, %rcx |
| |
| mulq %rax |
| # rbx <= 2 and rax <= 0xFFFF..F9, so carry must be zero here |
| addq %rbx, %rax |
| addq %rax, %r11 |
| movq %r14, %rax # 56($inp) |
| adcq %rdx, %r12 |
| adcq \$0, %rcx |
| |
| movq %r11, 96(%rsp) |
| movq %r12, 104(%rsp) |
| |
| #eighth iteration |
| xorq %rbx, %rbx # rbx:r13 = r13 << 1 |
| addq %r13, %r13 |
| adcq \$0, %rbx |
| |
| mulq %rax |
| # rcx <= 2 and rax <= 0xFFFF..F9, so carry must be zero here |
| addq %rcx, %rax |
| addq %r13, %rax |
| adcq %rbx, %rdx |
| |
| movq (%rsp), %r8 |
| movq 8(%rsp), %r9 |
| movq 16(%rsp), %r10 |
| movq 24(%rsp), %r11 |
| movq 32(%rsp), %r12 |
| movq 40(%rsp), %r13 |
| movq 48(%rsp), %r14 |
| movq 56(%rsp), %r15 |
| movq %xmm1, %rbp |
| |
| movq %rax, 112(%rsp) |
| movq %rdx, 120(%rsp) |
| |
| call __rsaz_512_reduce |
| |
| addq 64(%rsp), %r8 |
| adcq 72(%rsp), %r9 |
| adcq 80(%rsp), %r10 |
| adcq 88(%rsp), %r11 |
| adcq 96(%rsp), %r12 |
| adcq 104(%rsp), %r13 |
| adcq 112(%rsp), %r14 |
| adcq 120(%rsp), %r15 |
| sbbq %rcx, %rcx |
| |
| call __rsaz_512_subtract |
| |
| movq %r8, %rdx |
| movq %r9, %rax |
| movl 128+8(%rsp), $times |
| movq $out, $inp |
| |
| decl $times |
| jnz .Loop_sqr |
| ___ |
| if ($addx) { |
| $code.=<<___; |
| jmp .Lsqr_tail |
| |
| .align 32 |
| .Loop_sqrx: |
| movl $times,128+8(%rsp) |
| movq $out, %xmm0 # off-load |
| #first iteration |
| mulx %rax, %r8, %r9 |
| mov %rax, %rbx |
| |
| mulx 16($inp), %rcx, %r10 |
| xor %rbp, %rbp # cf=0, of=0 |
| |
| mulx 24($inp), %rax, %r11 |
| adcx %rcx, %r9 |
| |
| .byte 0xc4,0x62,0xf3,0xf6,0xa6,0x20,0x00,0x00,0x00 # mulx 32($inp), %rcx, %r12 |
| adcx %rax, %r10 |
| |
| .byte 0xc4,0x62,0xfb,0xf6,0xae,0x28,0x00,0x00,0x00 # mulx 40($inp), %rax, %r13 |
| adcx %rcx, %r11 |
| |
| mulx 48($inp), %rcx, %r14 |
| adcx %rax, %r12 |
| adcx %rcx, %r13 |
| |
| mulx 56($inp), %rax, %r15 |
| adcx %rax, %r14 |
| adcx %rbp, %r15 # %rbp is 0 |
| |
| mulx %rdx, %rax, $out |
| mov %rbx, %rdx # 8($inp) |
| xor %rcx, %rcx |
| adox %r8, %r8 |
| adcx $out, %r8 |
| adox %rbp, %rcx |
| adcx %rbp, %rcx |
| |
| mov %rax, (%rsp) |
| mov %r8, 8(%rsp) |
| |
| #second iteration |
| .byte 0xc4,0xe2,0xfb,0xf6,0x9e,0x10,0x00,0x00,0x00 # mulx 16($inp), %rax, %rbx |
| adox %rax, %r10 |
| adcx %rbx, %r11 |
| |
| mulx 24($inp), $out, %r8 |
| adox $out, %r11 |
| .byte 0x66 |
| adcx %r8, %r12 |
| |
| mulx 32($inp), %rax, %rbx |
| adox %rax, %r12 |
| adcx %rbx, %r13 |
| |
| mulx 40($inp), $out, %r8 |
| adox $out, %r13 |
| adcx %r8, %r14 |
| |
| .byte 0xc4,0xe2,0xfb,0xf6,0x9e,0x30,0x00,0x00,0x00 # mulx 48($inp), %rax, %rbx |
| adox %rax, %r14 |
| adcx %rbx, %r15 |
| |
| .byte 0xc4,0x62,0xc3,0xf6,0x86,0x38,0x00,0x00,0x00 # mulx 56($inp), $out, %r8 |
| adox $out, %r15 |
| adcx %rbp, %r8 |
| mulx %rdx, %rax, $out |
| adox %rbp, %r8 |
| .byte 0x48,0x8b,0x96,0x10,0x00,0x00,0x00 # mov 16($inp), %rdx |
| |
| xor %rbx, %rbx |
| adox %r9, %r9 |
| # rcx <= 2 and rax <= 0xFFFF..F9, so carry must be zero here |
| adcx %rcx, %rax |
| adox %r10, %r10 |
| adcx %rax, %r9 |
| adox %rbp, %rbx |
| adcx $out, %r10 |
| adcx %rbp, %rbx |
| |
| mov %r9, 16(%rsp) |
| .byte 0x4c,0x89,0x94,0x24,0x18,0x00,0x00,0x00 # mov %r10, 24(%rsp) |
| |
| #third iteration |
| mulx 24($inp), $out, %r9 |
| adox $out, %r12 |
| adcx %r9, %r13 |
| |
| mulx 32($inp), %rax, %rcx |
| adox %rax, %r13 |
| adcx %rcx, %r14 |
| |
| .byte 0xc4,0x62,0xc3,0xf6,0x8e,0x28,0x00,0x00,0x00 # mulx 40($inp), $out, %r9 |
| adox $out, %r14 |
| adcx %r9, %r15 |
| |
| .byte 0xc4,0xe2,0xfb,0xf6,0x8e,0x30,0x00,0x00,0x00 # mulx 48($inp), %rax, %rcx |
| adox %rax, %r15 |
| adcx %rcx, %r8 |
| |
| mulx 56($inp), $out, %r9 |
| adox $out, %r8 |
| adcx %rbp, %r9 |
| mulx %rdx, %rax, $out |
| adox %rbp, %r9 |
| mov 24($inp), %rdx |
| |
| xor %rcx, %rcx |
| adox %r11, %r11 |
| # rbx <= 2 and rax <= 0xFFFF..F9, so carry must be zero here |
| adcx %rbx, %rax |
| adox %r12, %r12 |
| adcx %rax, %r11 |
| adox %rbp, %rcx |
| adcx $out, %r12 |
| adcx %rbp, %rcx |
| |
| mov %r11, 32(%rsp) |
| mov %r12, 40(%rsp) |
| |
| #fourth iteration |
| mulx 32($inp), %rax, %rbx |
| adox %rax, %r14 |
| adcx %rbx, %r15 |
| |
| mulx 40($inp), $out, %r10 |
| adox $out, %r15 |
| adcx %r10, %r8 |
| |
| mulx 48($inp), %rax, %rbx |
| adox %rax, %r8 |
| adcx %rbx, %r9 |
| |
| mulx 56($inp), $out, %r10 |
| adox $out, %r9 |
| adcx %rbp, %r10 |
| mulx %rdx, %rax, $out |
| adox %rbp, %r10 |
| mov 32($inp), %rdx |
| |
| xor %rbx, %rbx |
| adox %r13, %r13 |
| # rcx <= 2 and rax <= 0xFFFF..F9, so carry must be zero here |
| adcx %rcx, %rax |
| adox %r14, %r14 |
| adcx %rax, %r13 |
| adox %rbp, %rbx |
| adcx $out, %r14 |
| adcx %rbp, %rbx |
| |
| mov %r13, 48(%rsp) |
| mov %r14, 56(%rsp) |
| |
| #fifth iteration |
| mulx 40($inp), $out, %r11 |
| adox $out, %r8 |
| adcx %r11, %r9 |
| |
| mulx 48($inp), %rax, %rcx |
| adox %rax, %r9 |
| adcx %rcx, %r10 |
| |
| mulx 56($inp), $out, %r11 |
| adox $out, %r10 |
| adcx %rbp, %r11 |
| mulx %rdx, %rax, $out |
| mov 40($inp), %rdx |
| adox %rbp, %r11 |
| |
| xor %rcx, %rcx |
| adox %r15, %r15 |
| # rbx <= 2 and rax <= 0xFFFF..F9, so carry must be zero here |
| adcx %rbx, %rax |
| adox %r8, %r8 |
| adcx %rax, %r15 |
| adox %rbp, %rcx |
| adcx $out, %r8 |
| adcx %rbp, %rcx |
| |
| mov %r15, 64(%rsp) |
| mov %r8, 72(%rsp) |
| |
| #sixth iteration |
| .byte 0xc4,0xe2,0xfb,0xf6,0x9e,0x30,0x00,0x00,0x00 # mulx 48($inp), %rax, %rbx |
| adox %rax, %r10 |
| adcx %rbx, %r11 |
| |
| .byte 0xc4,0x62,0xc3,0xf6,0xa6,0x38,0x00,0x00,0x00 # mulx 56($inp), $out, %r12 |
| adox $out, %r11 |
| adcx %rbp, %r12 |
| mulx %rdx, %rax, $out |
| adox %rbp, %r12 |
| mov 48($inp), %rdx |
| |
| xor %rbx, %rbx |
| adox %r9, %r9 |
| # rcx <= 2 and rax <= 0xFFFF..F9, so carry must be zero here |
| adcx %rcx, %rax |
| adox %r10, %r10 |
| adcx %rax, %r9 |
| adcx $out, %r10 |
| adox %rbp, %rbx |
| adcx %rbp, %rbx |
| |
| mov %r9, 80(%rsp) |
| mov %r10, 88(%rsp) |
| |
| #seventh iteration |
| .byte 0xc4,0x62,0xfb,0xf6,0xae,0x38,0x00,0x00,0x00 # mulx 56($inp), %rax, %r13 |
| adox %rax, %r12 |
| adox %rbp, %r13 |
| |
| mulx %rdx, %rax, $out |
| xor %rcx, %rcx |
| mov 56($inp), %rdx |
| adox %r11, %r11 |
| # rbx <= 2 and rax <= 0xFFFF..F9, so carry must be zero here |
| adcx %rbx, %rax |
| adox %r12, %r12 |
| adcx %rax, %r11 |
| adox %rbp, %rcx |
| adcx $out, %r12 |
| adcx %rbp, %rcx |
| |
| .byte 0x4c,0x89,0x9c,0x24,0x60,0x00,0x00,0x00 # mov %r11, 96(%rsp) |
| .byte 0x4c,0x89,0xa4,0x24,0x68,0x00,0x00,0x00 # mov %r12, 104(%rsp) |
| |
| #eighth iteration |
| mulx %rdx, %rax, %rdx |
| xor %rbx, %rbx |
| adox %r13, %r13 |
| # rcx <= 2 and rax <= 0xFFFF..F9, so carry must be zero here |
| adcx %rcx, %rax |
| adox %rbp, %rbx |
| adcx %r13, %rax |
| adcx %rdx, %rbx |
| |
| movq %xmm0, $out |
| movq %xmm1, %rbp |
| |
| movq 128(%rsp), %rdx # pull $n0 |
| movq (%rsp), %r8 |
| movq 8(%rsp), %r9 |
| movq 16(%rsp), %r10 |
| movq 24(%rsp), %r11 |
| movq 32(%rsp), %r12 |
| movq 40(%rsp), %r13 |
| movq 48(%rsp), %r14 |
| movq 56(%rsp), %r15 |
| |
| movq %rax, 112(%rsp) |
| movq %rbx, 120(%rsp) |
| |
| call __rsaz_512_reducex |
| |
| addq 64(%rsp), %r8 |
| adcq 72(%rsp), %r9 |
| adcq 80(%rsp), %r10 |
| adcq 88(%rsp), %r11 |
| adcq 96(%rsp), %r12 |
| adcq 104(%rsp), %r13 |
| adcq 112(%rsp), %r14 |
| adcq 120(%rsp), %r15 |
| sbbq %rcx, %rcx |
| |
| call __rsaz_512_subtract |
| |
| movq %r8, %rdx |
| movq %r9, %rax |
| movl 128+8(%rsp), $times |
| movq $out, $inp |
| |
| decl $times |
| jnz .Loop_sqrx |
| |
| .Lsqr_tail: |
| ___ |
| } |
| $code.=<<___; |
| |
| leaq 128+24+48(%rsp), %rax |
| .cfi_def_cfa %rax,8 |
| movq -48(%rax), %r15 |
| .cfi_restore %r15 |
| movq -40(%rax), %r14 |
| .cfi_restore %r14 |
| movq -32(%rax), %r13 |
| .cfi_restore %r13 |
| movq -24(%rax), %r12 |
| .cfi_restore %r12 |
| movq -16(%rax), %rbp |
| .cfi_restore %rbp |
| movq -8(%rax), %rbx |
| .cfi_restore %rbx |
| leaq (%rax), %rsp |
| .cfi_def_cfa_register %rsp |
| .Lsqr_epilogue: |
| ret |
| .cfi_endproc |
| .size rsaz_512_sqr,.-rsaz_512_sqr |
| ___ |
| } |
| { |
| my ($out,$ap,$bp,$mod,$n0) = ("%rdi","%rsi","%rdx","%rcx","%r8"); |
| $code.=<<___; |
| .globl rsaz_512_mul |
| .type rsaz_512_mul,\@function,5 |
| .align 32 |
| rsaz_512_mul: |
| .cfi_startproc |
| push %rbx |
| .cfi_push %rbx |
| push %rbp |
| .cfi_push %rbp |
| push %r12 |
| .cfi_push %r12 |
| push %r13 |
| .cfi_push %r13 |
| push %r14 |
| .cfi_push %r14 |
| push %r15 |
| .cfi_push %r15 |
| |
| subq \$128+24, %rsp |
| .cfi_adjust_cfa_offset 128+24 |
| .Lmul_body: |
| movq $out, %xmm0 # off-load arguments |
| movq $mod, %xmm1 |
| movq $n0, 128(%rsp) |
| ___ |
| $code.=<<___ if ($addx); |
| movl \$0x80100,%r11d |
| andl OPENSSL_ia32cap_P+8(%rip),%r11d |
| cmpl \$0x80100,%r11d # check for MULX and ADO/CX |
| je .Lmulx |
| ___ |
| $code.=<<___; |
| movq ($bp), %rbx # pass b[0] |
| movq $bp, %rbp # pass argument |
| call __rsaz_512_mul |
| |
| movq %xmm0, $out |
| movq %xmm1, %rbp |
| |
| movq (%rsp), %r8 |
| movq 8(%rsp), %r9 |
| movq 16(%rsp), %r10 |
| movq 24(%rsp), %r11 |
| movq 32(%rsp), %r12 |
| movq 40(%rsp), %r13 |
| movq 48(%rsp), %r14 |
| movq 56(%rsp), %r15 |
| |
| call __rsaz_512_reduce |
| ___ |
| $code.=<<___ if ($addx); |
| jmp .Lmul_tail |
| |
| .align 32 |
| .Lmulx: |
| movq $bp, %rbp # pass argument |
| movq ($bp), %rdx # pass b[0] |
| call __rsaz_512_mulx |
| |
| movq %xmm0, $out |
| movq %xmm1, %rbp |
| |
| movq 128(%rsp), %rdx # pull $n0 |
| movq (%rsp), %r8 |
| movq 8(%rsp), %r9 |
| movq 16(%rsp), %r10 |
| movq 24(%rsp), %r11 |
| movq 32(%rsp), %r12 |
| movq 40(%rsp), %r13 |
| movq 48(%rsp), %r14 |
| movq 56(%rsp), %r15 |
| |
| call __rsaz_512_reducex |
| .Lmul_tail: |
| ___ |
| $code.=<<___; |
| addq 64(%rsp), %r8 |
| adcq 72(%rsp), %r9 |
| adcq 80(%rsp), %r10 |
| adcq 88(%rsp), %r11 |
| adcq 96(%rsp), %r12 |
| adcq 104(%rsp), %r13 |
| adcq 112(%rsp), %r14 |
| adcq 120(%rsp), %r15 |
| sbbq %rcx, %rcx |
| |
| call __rsaz_512_subtract |
| |
| leaq 128+24+48(%rsp), %rax |
| .cfi_def_cfa %rax,8 |
| movq -48(%rax), %r15 |
| .cfi_restore %r15 |
| movq -40(%rax), %r14 |
| .cfi_restore %r14 |
| movq -32(%rax), %r13 |
| .cfi_restore %r13 |
| movq -24(%rax), %r12 |
| .cfi_restore %r12 |
| movq -16(%rax), %rbp |
| .cfi_restore %rbp |
| movq -8(%rax), %rbx |
| .cfi_restore %rbx |
| leaq (%rax), %rsp |
| .cfi_def_cfa_register %rsp |
| .Lmul_epilogue: |
| ret |
| .cfi_endproc |
| .size rsaz_512_mul,.-rsaz_512_mul |
| ___ |
| } |
| { |
| my ($out,$ap,$bp,$mod,$n0,$pwr) = ("%rdi","%rsi","%rdx","%rcx","%r8","%r9d"); |
| $code.=<<___; |
| .globl rsaz_512_mul_gather4 |
| .type rsaz_512_mul_gather4,\@function,6 |
| .align 32 |
| rsaz_512_mul_gather4: |
| .cfi_startproc |
| push %rbx |
| .cfi_push %rbx |
| push %rbp |
| .cfi_push %rbp |
| push %r12 |
| .cfi_push %r12 |
| push %r13 |
| .cfi_push %r13 |
| push %r14 |
| .cfi_push %r14 |
| push %r15 |
| .cfi_push %r15 |
| |
| subq \$`128+24+($win64?0xb0:0)`, %rsp |
| .cfi_adjust_cfa_offset `128+24+($win64?0xb0:0)` |
| ___ |
| $code.=<<___ if ($win64); |
| movaps %xmm6,0xa0(%rsp) |
| movaps %xmm7,0xb0(%rsp) |
| movaps %xmm8,0xc0(%rsp) |
| movaps %xmm9,0xd0(%rsp) |
| movaps %xmm10,0xe0(%rsp) |
| movaps %xmm11,0xf0(%rsp) |
| movaps %xmm12,0x100(%rsp) |
| movaps %xmm13,0x110(%rsp) |
| movaps %xmm14,0x120(%rsp) |
| movaps %xmm15,0x130(%rsp) |
| ___ |
| $code.=<<___; |
| .Lmul_gather4_body: |
| movd $pwr,%xmm8 |
| movdqa .Linc+16(%rip),%xmm1 # 00000002000000020000000200000002 |
| movdqa .Linc(%rip),%xmm0 # 00000001000000010000000000000000 |
| |
| pshufd \$0,%xmm8,%xmm8 # broadcast $power |
| movdqa %xmm1,%xmm7 |
| movdqa %xmm1,%xmm2 |
| ___ |
| ######################################################################## |
| # calculate mask by comparing 0..15 to $power |
| # |
| for($i=0;$i<4;$i++) { |
| $code.=<<___; |
| paddd %xmm`$i`,%xmm`$i+1` |
| pcmpeqd %xmm8,%xmm`$i` |
| movdqa %xmm7,%xmm`$i+3` |
| ___ |
| } |
| for(;$i<7;$i++) { |
| $code.=<<___; |
| paddd %xmm`$i`,%xmm`$i+1` |
| pcmpeqd %xmm8,%xmm`$i` |
| ___ |
| } |
| $code.=<<___; |
| pcmpeqd %xmm8,%xmm7 |
| |
| movdqa 16*0($bp),%xmm8 |
| movdqa 16*1($bp),%xmm9 |
| movdqa 16*2($bp),%xmm10 |
| movdqa 16*3($bp),%xmm11 |
| pand %xmm0,%xmm8 |
| movdqa 16*4($bp),%xmm12 |
| pand %xmm1,%xmm9 |
| movdqa 16*5($bp),%xmm13 |
| pand %xmm2,%xmm10 |
| movdqa 16*6($bp),%xmm14 |
| pand %xmm3,%xmm11 |
| movdqa 16*7($bp),%xmm15 |
| leaq 128($bp), %rbp |
| pand %xmm4,%xmm12 |
| pand %xmm5,%xmm13 |
| pand %xmm6,%xmm14 |
| pand %xmm7,%xmm15 |
| por %xmm10,%xmm8 |
| por %xmm11,%xmm9 |
| por %xmm12,%xmm8 |
| por %xmm13,%xmm9 |
| por %xmm14,%xmm8 |
| por %xmm15,%xmm9 |
| |
| por %xmm9,%xmm8 |
| pshufd \$0x4e,%xmm8,%xmm9 |
| por %xmm9,%xmm8 |
| ___ |
| $code.=<<___ if ($addx); |
| movl \$0x80100,%r11d |
| andl OPENSSL_ia32cap_P+8(%rip),%r11d |
| cmpl \$0x80100,%r11d # check for MULX and ADO/CX |
| je .Lmulx_gather |
| ___ |
| $code.=<<___; |
| movq %xmm8,%rbx |
| |
| movq $n0, 128(%rsp) # off-load arguments |
| movq $out, 128+8(%rsp) |
| movq $mod, 128+16(%rsp) |
| |
| movq ($ap), %rax |
| movq 8($ap), %rcx |
| mulq %rbx # 0 iteration |
| movq %rax, (%rsp) |
| movq %rcx, %rax |
| movq %rdx, %r8 |
| |
| mulq %rbx |
| addq %rax, %r8 |
| movq 16($ap), %rax |
| movq %rdx, %r9 |
| adcq \$0, %r9 |
| |
| mulq %rbx |
| addq %rax, %r9 |
| movq 24($ap), %rax |
| movq %rdx, %r10 |
| adcq \$0, %r10 |
| |
| mulq %rbx |
| addq %rax, %r10 |
| movq 32($ap), %rax |
| movq %rdx, %r11 |
| adcq \$0, %r11 |
| |
| mulq %rbx |
| addq %rax, %r11 |
| movq 40($ap), %rax |
| movq %rdx, %r12 |
| adcq \$0, %r12 |
| |
| mulq %rbx |
| addq %rax, %r12 |
| movq 48($ap), %rax |
| movq %rdx, %r13 |
| adcq \$0, %r13 |
| |
| mulq %rbx |
| addq %rax, %r13 |
| movq 56($ap), %rax |
| movq %rdx, %r14 |
| adcq \$0, %r14 |
| |
| mulq %rbx |
| addq %rax, %r14 |
| movq ($ap), %rax |
| movq %rdx, %r15 |
| adcq \$0, %r15 |
| |
| leaq 8(%rsp), %rdi |
| movl \$7, %ecx |
| jmp .Loop_mul_gather |
| |
| .align 32 |
| .Loop_mul_gather: |
| movdqa 16*0(%rbp),%xmm8 |
| movdqa 16*1(%rbp),%xmm9 |
| movdqa 16*2(%rbp),%xmm10 |
| movdqa 16*3(%rbp),%xmm11 |
| pand %xmm0,%xmm8 |
| movdqa 16*4(%rbp),%xmm12 |
| pand %xmm1,%xmm9 |
| movdqa 16*5(%rbp),%xmm13 |
| pand %xmm2,%xmm10 |
| movdqa 16*6(%rbp),%xmm14 |
| pand %xmm3,%xmm11 |
| movdqa 16*7(%rbp),%xmm15 |
| leaq 128(%rbp), %rbp |
| pand %xmm4,%xmm12 |
| pand %xmm5,%xmm13 |
| pand %xmm6,%xmm14 |
| pand %xmm7,%xmm15 |
| por %xmm10,%xmm8 |
| por %xmm11,%xmm9 |
| por %xmm12,%xmm8 |
| por %xmm13,%xmm9 |
| por %xmm14,%xmm8 |
| por %xmm15,%xmm9 |
| |
| por %xmm9,%xmm8 |
| pshufd \$0x4e,%xmm8,%xmm9 |
| por %xmm9,%xmm8 |
| movq %xmm8,%rbx |
| |
| mulq %rbx |
| addq %rax, %r8 |
| movq 8($ap), %rax |
| movq %r8, (%rdi) |
| movq %rdx, %r8 |
| adcq \$0, %r8 |
| |
| mulq %rbx |
| addq %rax, %r9 |
| movq 16($ap), %rax |
| adcq \$0, %rdx |
| addq %r9, %r8 |
| movq %rdx, %r9 |
| adcq \$0, %r9 |
| |
| mulq %rbx |
| addq %rax, %r10 |
| movq 24($ap), %rax |
| adcq \$0, %rdx |
| addq %r10, %r9 |
| movq %rdx, %r10 |
| adcq \$0, %r10 |
| |
| mulq %rbx |
| addq %rax, %r11 |
| movq 32($ap), %rax |
| adcq \$0, %rdx |
| addq %r11, %r10 |
| movq %rdx, %r11 |
| adcq \$0, %r11 |
| |
| mulq %rbx |
| addq %rax, %r12 |
| movq 40($ap), %rax |
| adcq \$0, %rdx |
| addq %r12, %r11 |
| movq %rdx, %r12 |
| adcq \$0, %r12 |
| |
| mulq %rbx |
| addq %rax, %r13 |
| movq 48($ap), %rax |
| adcq \$0, %rdx |
| addq %r13, %r12 |
| movq %rdx, %r13 |
| adcq \$0, %r13 |
| |
| mulq %rbx |
| addq %rax, %r14 |
| movq 56($ap), %rax |
| adcq \$0, %rdx |
| addq %r14, %r13 |
| movq %rdx, %r14 |
| adcq \$0, %r14 |
| |
| mulq %rbx |
| addq %rax, %r15 |
| movq ($ap), %rax |
| adcq \$0, %rdx |
| addq %r15, %r14 |
| movq %rdx, %r15 |
| adcq \$0, %r15 |
| |
| leaq 8(%rdi), %rdi |
| |
| decl %ecx |
| jnz .Loop_mul_gather |
| |
| movq %r8, (%rdi) |
| movq %r9, 8(%rdi) |
| movq %r10, 16(%rdi) |
| movq %r11, 24(%rdi) |
| movq %r12, 32(%rdi) |
| movq %r13, 40(%rdi) |
| movq %r14, 48(%rdi) |
| movq %r15, 56(%rdi) |
| |
| movq 128+8(%rsp), $out |
| movq 128+16(%rsp), %rbp |
| |
| movq (%rsp), %r8 |
| movq 8(%rsp), %r9 |
| movq 16(%rsp), %r10 |
| movq 24(%rsp), %r11 |
| movq 32(%rsp), %r12 |
| movq 40(%rsp), %r13 |
| movq 48(%rsp), %r14 |
| movq 56(%rsp), %r15 |
| |
| call __rsaz_512_reduce |
| ___ |
| $code.=<<___ if ($addx); |
| jmp .Lmul_gather_tail |
| |
| .align 32 |
| .Lmulx_gather: |
| movq %xmm8,%rdx |
| |
| mov $n0, 128(%rsp) # off-load arguments |
| mov $out, 128+8(%rsp) |
| mov $mod, 128+16(%rsp) |
| |
| mulx ($ap), %rbx, %r8 # 0 iteration |
| mov %rbx, (%rsp) |
| xor %edi, %edi # cf=0, of=0 |
| |
| mulx 8($ap), %rax, %r9 |
| |
| mulx 16($ap), %rbx, %r10 |
| adcx %rax, %r8 |
| |
| mulx 24($ap), %rax, %r11 |
| adcx %rbx, %r9 |
| |
| mulx 32($ap), %rbx, %r12 |
| adcx %rax, %r10 |
| |
| mulx 40($ap), %rax, %r13 |
| adcx %rbx, %r11 |
| |
| mulx 48($ap), %rbx, %r14 |
| adcx %rax, %r12 |
| |
| mulx 56($ap), %rax, %r15 |
| adcx %rbx, %r13 |
| adcx %rax, %r14 |
| .byte 0x67 |
| mov %r8, %rbx |
| adcx %rdi, %r15 # %rdi is 0 |
| |
| mov \$-7, %rcx |
| jmp .Loop_mulx_gather |
| |
| .align 32 |
| .Loop_mulx_gather: |
| movdqa 16*0(%rbp),%xmm8 |
| movdqa 16*1(%rbp),%xmm9 |
| movdqa 16*2(%rbp),%xmm10 |
| movdqa 16*3(%rbp),%xmm11 |
| pand %xmm0,%xmm8 |
| movdqa 16*4(%rbp),%xmm12 |
| pand %xmm1,%xmm9 |
| movdqa 16*5(%rbp),%xmm13 |
| pand %xmm2,%xmm10 |
| movdqa 16*6(%rbp),%xmm14 |
| pand %xmm3,%xmm11 |
| movdqa 16*7(%rbp),%xmm15 |
| leaq 128(%rbp), %rbp |
| pand %xmm4,%xmm12 |
| pand %xmm5,%xmm13 |
| pand %xmm6,%xmm14 |
| pand %xmm7,%xmm15 |
| por %xmm10,%xmm8 |
| por %xmm11,%xmm9 |
| por %xmm12,%xmm8 |
| por %xmm13,%xmm9 |
| por %xmm14,%xmm8 |
| por %xmm15,%xmm9 |
| |
| por %xmm9,%xmm8 |
| pshufd \$0x4e,%xmm8,%xmm9 |
| por %xmm9,%xmm8 |
| movq %xmm8,%rdx |
| |
| .byte 0xc4,0x62,0xfb,0xf6,0x86,0x00,0x00,0x00,0x00 # mulx ($ap), %rax, %r8 |
| adcx %rax, %rbx |
| adox %r9, %r8 |
| |
| mulx 8($ap), %rax, %r9 |
| adcx %rax, %r8 |
| adox %r10, %r9 |
| |
| mulx 16($ap), %rax, %r10 |
| adcx %rax, %r9 |
| adox %r11, %r10 |
| |
| .byte 0xc4,0x62,0xfb,0xf6,0x9e,0x18,0x00,0x00,0x00 # mulx 24($ap), %rax, %r11 |
| adcx %rax, %r10 |
| adox %r12, %r11 |
| |
| mulx 32($ap), %rax, %r12 |
| adcx %rax, %r11 |
| adox %r13, %r12 |
| |
| mulx 40($ap), %rax, %r13 |
| adcx %rax, %r12 |
| adox %r14, %r13 |
| |
| .byte 0xc4,0x62,0xfb,0xf6,0xb6,0x30,0x00,0x00,0x00 # mulx 48($ap), %rax, %r14 |
| adcx %rax, %r13 |
| .byte 0x67 |
| adox %r15, %r14 |
| |
| mulx 56($ap), %rax, %r15 |
| mov %rbx, 64(%rsp,%rcx,8) |
| adcx %rax, %r14 |
| adox %rdi, %r15 |
| mov %r8, %rbx |
| adcx %rdi, %r15 # cf=0 |
| |
| inc %rcx # of=0 |
| jnz .Loop_mulx_gather |
| |
| mov %r8, 64(%rsp) |
| mov %r9, 64+8(%rsp) |
| mov %r10, 64+16(%rsp) |
| mov %r11, 64+24(%rsp) |
| mov %r12, 64+32(%rsp) |
| mov %r13, 64+40(%rsp) |
| mov %r14, 64+48(%rsp) |
| mov %r15, 64+56(%rsp) |
| |
| mov 128(%rsp), %rdx # pull arguments |
| mov 128+8(%rsp), $out |
| mov 128+16(%rsp), %rbp |
| |
| mov (%rsp), %r8 |
| mov 8(%rsp), %r9 |
| mov 16(%rsp), %r10 |
| mov 24(%rsp), %r11 |
| mov 32(%rsp), %r12 |
| mov 40(%rsp), %r13 |
| mov 48(%rsp), %r14 |
| mov 56(%rsp), %r15 |
| |
| call __rsaz_512_reducex |
| |
| .Lmul_gather_tail: |
| ___ |
| $code.=<<___; |
| addq 64(%rsp), %r8 |
| adcq 72(%rsp), %r9 |
| adcq 80(%rsp), %r10 |
| adcq 88(%rsp), %r11 |
| adcq 96(%rsp), %r12 |
| adcq 104(%rsp), %r13 |
| adcq 112(%rsp), %r14 |
| adcq 120(%rsp), %r15 |
| sbbq %rcx, %rcx |
| |
| call __rsaz_512_subtract |
| |
| leaq 128+24+48(%rsp), %rax |
| ___ |
| $code.=<<___ if ($win64); |
| movaps 0xa0-0xc8(%rax),%xmm6 |
| movaps 0xb0-0xc8(%rax),%xmm7 |
| movaps 0xc0-0xc8(%rax),%xmm8 |
| movaps 0xd0-0xc8(%rax),%xmm9 |
| movaps 0xe0-0xc8(%rax),%xmm10 |
| movaps 0xf0-0xc8(%rax),%xmm11 |
| movaps 0x100-0xc8(%rax),%xmm12 |
| movaps 0x110-0xc8(%rax),%xmm13 |
| movaps 0x120-0xc8(%rax),%xmm14 |
| movaps 0x130-0xc8(%rax),%xmm15 |
| lea 0xb0(%rax),%rax |
| ___ |
| $code.=<<___; |
| .cfi_def_cfa %rax,8 |
| movq -48(%rax), %r15 |
| .cfi_restore %r15 |
| movq -40(%rax), %r14 |
| .cfi_restore %r14 |
| movq -32(%rax), %r13 |
| .cfi_restore %r13 |
| movq -24(%rax), %r12 |
| .cfi_restore %r12 |
| movq -16(%rax), %rbp |
| .cfi_restore %rbp |
| movq -8(%rax), %rbx |
| .cfi_restore %rbx |
| leaq (%rax), %rsp |
| .cfi_def_cfa_register %rsp |
| .Lmul_gather4_epilogue: |
| ret |
| .cfi_endproc |
| .size rsaz_512_mul_gather4,.-rsaz_512_mul_gather4 |
| ___ |
| } |
| { |
| my ($out,$ap,$mod,$n0,$tbl,$pwr) = ("%rdi","%rsi","%rdx","%rcx","%r8","%r9d"); |
| $code.=<<___; |
| .globl rsaz_512_mul_scatter4 |
| .type rsaz_512_mul_scatter4,\@function,6 |
| .align 32 |
| rsaz_512_mul_scatter4: |
| .cfi_startproc |
| push %rbx |
| .cfi_push %rbx |
| push %rbp |
| .cfi_push %rbp |
| push %r12 |
| .cfi_push %r12 |
| push %r13 |
| .cfi_push %r13 |
| push %r14 |
| .cfi_push %r14 |
| push %r15 |
| .cfi_push %r15 |
| |
| mov $pwr, $pwr |
| subq \$128+24, %rsp |
| .cfi_adjust_cfa_offset 128+24 |
| .Lmul_scatter4_body: |
| leaq ($tbl,$pwr,8), $tbl |
| movq $out, %xmm0 # off-load arguments |
| movq $mod, %xmm1 |
| movq $tbl, %xmm2 |
| movq $n0, 128(%rsp) |
| |
| movq $out, %rbp |
| ___ |
| $code.=<<___ if ($addx); |
| movl \$0x80100,%r11d |
| andl OPENSSL_ia32cap_P+8(%rip),%r11d |
| cmpl \$0x80100,%r11d # check for MULX and ADO/CX |
| je .Lmulx_scatter |
| ___ |
| $code.=<<___; |
| movq ($out),%rbx # pass b[0] |
| call __rsaz_512_mul |
| |
| movq %xmm0, $out |
| movq %xmm1, %rbp |
| |
| movq (%rsp), %r8 |
| movq 8(%rsp), %r9 |
| movq 16(%rsp), %r10 |
| movq 24(%rsp), %r11 |
| movq 32(%rsp), %r12 |
| movq 40(%rsp), %r13 |
| movq 48(%rsp), %r14 |
| movq 56(%rsp), %r15 |
| |
| call __rsaz_512_reduce |
| ___ |
| $code.=<<___ if ($addx); |
| jmp .Lmul_scatter_tail |
| |
| .align 32 |
| .Lmulx_scatter: |
| movq ($out), %rdx # pass b[0] |
| call __rsaz_512_mulx |
| |
| movq %xmm0, $out |
| movq %xmm1, %rbp |
| |
| movq 128(%rsp), %rdx # pull $n0 |
| movq (%rsp), %r8 |
| movq 8(%rsp), %r9 |
| movq 16(%rsp), %r10 |
| movq 24(%rsp), %r11 |
| movq 32(%rsp), %r12 |
| movq 40(%rsp), %r13 |
| movq 48(%rsp), %r14 |
| movq 56(%rsp), %r15 |
| |
| call __rsaz_512_reducex |
| |
| .Lmul_scatter_tail: |
| ___ |
| $code.=<<___; |
| addq 64(%rsp), %r8 |
| adcq 72(%rsp), %r9 |
| adcq 80(%rsp), %r10 |
| adcq 88(%rsp), %r11 |
| adcq 96(%rsp), %r12 |
| adcq 104(%rsp), %r13 |
| adcq 112(%rsp), %r14 |
| adcq 120(%rsp), %r15 |
| movq %xmm2, $inp |
| sbbq %rcx, %rcx |
| |
| call __rsaz_512_subtract |
| |
| movq %r8, 128*0($inp) # scatter |
| movq %r9, 128*1($inp) |
| movq %r10, 128*2($inp) |
| movq %r11, 128*3($inp) |
| movq %r12, 128*4($inp) |
| movq %r13, 128*5($inp) |
| movq %r14, 128*6($inp) |
| movq %r15, 128*7($inp) |
| |
| leaq 128+24+48(%rsp), %rax |
| .cfi_def_cfa %rax,8 |
| movq -48(%rax), %r15 |
| .cfi_restore %r15 |
| movq -40(%rax), %r14 |
| .cfi_restore %r14 |
| movq -32(%rax), %r13 |
| .cfi_restore %r13 |
| movq -24(%rax), %r12 |
| .cfi_restore %r12 |
| movq -16(%rax), %rbp |
| .cfi_restore %rbp |
| movq -8(%rax), %rbx |
| .cfi_restore %rbx |
| leaq (%rax), %rsp |
| .cfi_def_cfa_register %rsp |
| .Lmul_scatter4_epilogue: |
| ret |
| .cfi_endproc |
| .size rsaz_512_mul_scatter4,.-rsaz_512_mul_scatter4 |
| ___ |
| } |
| { |
| my ($out,$inp,$mod,$n0) = ("%rdi","%rsi","%rdx","%rcx"); |
| $code.=<<___; |
| .globl rsaz_512_mul_by_one |
| .type rsaz_512_mul_by_one,\@function,4 |
| .align 32 |
| rsaz_512_mul_by_one: |
| .cfi_startproc |
| push %rbx |
| .cfi_push %rbx |
| push %rbp |
| .cfi_push %rbp |
| push %r12 |
| .cfi_push %r12 |
| push %r13 |
| .cfi_push %r13 |
| push %r14 |
| .cfi_push %r14 |
| push %r15 |
| .cfi_push %r15 |
| |
| subq \$128+24, %rsp |
| .cfi_adjust_cfa_offset 128+24 |
| .Lmul_by_one_body: |
| ___ |
| $code.=<<___ if ($addx); |
| movl OPENSSL_ia32cap_P+8(%rip),%eax |
| ___ |
| $code.=<<___; |
| movq $mod, %rbp # reassign argument |
| movq $n0, 128(%rsp) |
| |
| movq ($inp), %r8 |
| pxor %xmm0, %xmm0 |
| movq 8($inp), %r9 |
| movq 16($inp), %r10 |
| movq 24($inp), %r11 |
| movq 32($inp), %r12 |
| movq 40($inp), %r13 |
| movq 48($inp), %r14 |
| movq 56($inp), %r15 |
| |
| movdqa %xmm0, (%rsp) |
| movdqa %xmm0, 16(%rsp) |
| movdqa %xmm0, 32(%rsp) |
| movdqa %xmm0, 48(%rsp) |
| movdqa %xmm0, 64(%rsp) |
| movdqa %xmm0, 80(%rsp) |
| movdqa %xmm0, 96(%rsp) |
| ___ |
| $code.=<<___ if ($addx); |
| andl \$0x80100,%eax |
| cmpl \$0x80100,%eax # check for MULX and ADO/CX |
| je .Lby_one_callx |
| ___ |
| $code.=<<___; |
| call __rsaz_512_reduce |
| ___ |
| $code.=<<___ if ($addx); |
| jmp .Lby_one_tail |
| .align 32 |
| .Lby_one_callx: |
| movq 128(%rsp), %rdx # pull $n0 |
| call __rsaz_512_reducex |
| .Lby_one_tail: |
| ___ |
| $code.=<<___; |
| movq %r8, ($out) |
| movq %r9, 8($out) |
| movq %r10, 16($out) |
| movq %r11, 24($out) |
| movq %r12, 32($out) |
| movq %r13, 40($out) |
| movq %r14, 48($out) |
| movq %r15, 56($out) |
| |
| leaq 128+24+48(%rsp), %rax |
| .cfi_def_cfa %rax,8 |
| movq -48(%rax), %r15 |
| .cfi_restore %r15 |
| movq -40(%rax), %r14 |
| .cfi_restore %r14 |
| movq -32(%rax), %r13 |
| .cfi_restore %r13 |
| movq -24(%rax), %r12 |
| .cfi_restore %r12 |
| movq -16(%rax), %rbp |
| .cfi_restore %rbp |
| movq -8(%rax), %rbx |
| .cfi_restore %rbx |
| leaq (%rax), %rsp |
| .cfi_def_cfa_register %rsp |
| .Lmul_by_one_epilogue: |
| ret |
| .cfi_endproc |
| .size rsaz_512_mul_by_one,.-rsaz_512_mul_by_one |
| ___ |
| } |
| { # __rsaz_512_reduce |
| # |
| # input: %r8-%r15, %rbp - mod, 128(%rsp) - n0 |
| # output: %r8-%r15 |
| # clobbers: everything except %rbp and %rdi |
| $code.=<<___; |
| .type __rsaz_512_reduce,\@abi-omnipotent |
| .align 32 |
| __rsaz_512_reduce: |
| .cfi_startproc |
| movq %r8, %rbx |
| imulq 128+8(%rsp), %rbx |
| movq 0(%rbp), %rax |
| movl \$8, %ecx |
| jmp .Lreduction_loop |
| |
| .align 32 |
| .Lreduction_loop: |
| mulq %rbx |
| movq 8(%rbp), %rax |
| negq %r8 |
| movq %rdx, %r8 |
| adcq \$0, %r8 |
| |
| mulq %rbx |
| addq %rax, %r9 |
| movq 16(%rbp), %rax |
| adcq \$0, %rdx |
| addq %r9, %r8 |
| movq %rdx, %r9 |
| adcq \$0, %r9 |
| |
| mulq %rbx |
| addq %rax, %r10 |
| movq 24(%rbp), %rax |
| adcq \$0, %rdx |
| addq %r10, %r9 |
| movq %rdx, %r10 |
| adcq \$0, %r10 |
| |
| mulq %rbx |
| addq %rax, %r11 |
| movq 32(%rbp), %rax |
| adcq \$0, %rdx |
| addq %r11, %r10 |
| movq 128+8(%rsp), %rsi |
| #movq %rdx, %r11 |
| #adcq \$0, %r11 |
| adcq \$0, %rdx |
| movq %rdx, %r11 |
| |
| mulq %rbx |
| addq %rax, %r12 |
| movq 40(%rbp), %rax |
| adcq \$0, %rdx |
| imulq %r8, %rsi |
| addq %r12, %r11 |
| movq %rdx, %r12 |
| adcq \$0, %r12 |
| |
| mulq %rbx |
| addq %rax, %r13 |
| movq 48(%rbp), %rax |
| adcq \$0, %rdx |
| addq %r13, %r12 |
| movq %rdx, %r13 |
| adcq \$0, %r13 |
| |
| mulq %rbx |
| addq %rax, %r14 |
| movq 56(%rbp), %rax |
| adcq \$0, %rdx |
| addq %r14, %r13 |
| movq %rdx, %r14 |
| adcq \$0, %r14 |
| |
| mulq %rbx |
| movq %rsi, %rbx |
| addq %rax, %r15 |
| movq 0(%rbp), %rax |
| adcq \$0, %rdx |
| addq %r15, %r14 |
| movq %rdx, %r15 |
| adcq \$0, %r15 |
| |
| decl %ecx |
| jne .Lreduction_loop |
| |
| ret |
| .cfi_endproc |
| .size __rsaz_512_reduce,.-__rsaz_512_reduce |
| ___ |
| } |
| if ($addx) { |
| # __rsaz_512_reducex |
| # |
| # input: %r8-%r15, %rbp - mod, 128(%rsp) - n0 |
| # output: %r8-%r15 |
| # clobbers: everything except %rbp and %rdi |
| $code.=<<___; |
| .type __rsaz_512_reducex,\@abi-omnipotent |
| .align 32 |
| __rsaz_512_reducex: |
| .cfi_startproc |
| #movq 128+8(%rsp), %rdx # pull $n0 |
| imulq %r8, %rdx |
| xorq %rsi, %rsi # cf=0,of=0 |
| movl \$8, %ecx |
| jmp .Lreduction_loopx |
| |
| .align 32 |
| .Lreduction_loopx: |
| mov %r8, %rbx |
| mulx 0(%rbp), %rax, %r8 |
| adcx %rbx, %rax |
| adox %r9, %r8 |
| |
| mulx 8(%rbp), %rax, %r9 |
| adcx %rax, %r8 |
| adox %r10, %r9 |
| |
| mulx 16(%rbp), %rbx, %r10 |
| adcx %rbx, %r9 |
| adox %r11, %r10 |
| |
| mulx 24(%rbp), %rbx, %r11 |
| adcx %rbx, %r10 |
| adox %r12, %r11 |
| |
| .byte 0xc4,0x62,0xe3,0xf6,0xa5,0x20,0x00,0x00,0x00 # mulx 32(%rbp), %rbx, %r12 |
| mov %rdx, %rax |
| mov %r8, %rdx |
| adcx %rbx, %r11 |
| adox %r13, %r12 |
| |
| mulx 128+8(%rsp), %rbx, %rdx |
| mov %rax, %rdx |
| |
| mulx 40(%rbp), %rax, %r13 |
| adcx %rax, %r12 |
| adox %r14, %r13 |
| |
| .byte 0xc4,0x62,0xfb,0xf6,0xb5,0x30,0x00,0x00,0x00 # mulx 48(%rbp), %rax, %r14 |
| adcx %rax, %r13 |
| adox %r15, %r14 |
| |
| mulx 56(%rbp), %rax, %r15 |
| mov %rbx, %rdx |
| adcx %rax, %r14 |
| adox %rsi, %r15 # %rsi is 0 |
| adcx %rsi, %r15 # cf=0 |
| |
| decl %ecx # of=0 |
| jne .Lreduction_loopx |
| |
| ret |
| .cfi_endproc |
| .size __rsaz_512_reducex,.-__rsaz_512_reducex |
| ___ |
| } |
| { # __rsaz_512_subtract |
| # input: %r8-%r15, %rdi - $out, %rbp - $mod, %rcx - mask |
| # output: |
| # clobbers: everything but %rdi, %rsi and %rbp |
| $code.=<<___; |
| .type __rsaz_512_subtract,\@abi-omnipotent |
| .align 32 |
| __rsaz_512_subtract: |
| .cfi_startproc |
| movq %r8, ($out) |
| movq %r9, 8($out) |
| movq %r10, 16($out) |
| movq %r11, 24($out) |
| movq %r12, 32($out) |
| movq %r13, 40($out) |
| movq %r14, 48($out) |
| movq %r15, 56($out) |
| |
| movq 0($mod), %r8 |
| movq 8($mod), %r9 |
| negq %r8 |
| notq %r9 |
| andq %rcx, %r8 |
| movq 16($mod), %r10 |
| andq %rcx, %r9 |
| notq %r10 |
| movq 24($mod), %r11 |
| andq %rcx, %r10 |
| notq %r11 |
| movq 32($mod), %r12 |
| andq %rcx, %r11 |
| notq %r12 |
| movq 40($mod), %r13 |
| andq %rcx, %r12 |
| notq %r13 |
| movq 48($mod), %r14 |
| andq %rcx, %r13 |
| notq %r14 |
| movq 56($mod), %r15 |
| andq %rcx, %r14 |
| notq %r15 |
| andq %rcx, %r15 |
| |
| addq ($out), %r8 |
| adcq 8($out), %r9 |
| adcq 16($out), %r10 |
| adcq 24($out), %r11 |
| adcq 32($out), %r12 |
| adcq 40($out), %r13 |
| adcq 48($out), %r14 |
| adcq 56($out), %r15 |
| |
| movq %r8, ($out) |
| movq %r9, 8($out) |
| movq %r10, 16($out) |
| movq %r11, 24($out) |
| movq %r12, 32($out) |
| movq %r13, 40($out) |
| movq %r14, 48($out) |
| movq %r15, 56($out) |
| |
| ret |
| .cfi_endproc |
| .size __rsaz_512_subtract,.-__rsaz_512_subtract |
| ___ |
| } |
| { # __rsaz_512_mul |
| # |
| # input: %rsi - ap, %rbp - bp |
| # output: |
| # clobbers: everything |
| my ($ap,$bp) = ("%rsi","%rbp"); |
| $code.=<<___; |
| .type __rsaz_512_mul,\@abi-omnipotent |
| .align 32 |
| __rsaz_512_mul: |
| .cfi_startproc |
| leaq 8(%rsp), %rdi |
| |
| movq ($ap), %rax |
| mulq %rbx |
| movq %rax, (%rdi) |
| movq 8($ap), %rax |
| movq %rdx, %r8 |
| |
| mulq %rbx |
| addq %rax, %r8 |
| movq 16($ap), %rax |
| movq %rdx, %r9 |
| adcq \$0, %r9 |
| |
| mulq %rbx |
| addq %rax, %r9 |
| movq 24($ap), %rax |
| movq %rdx, %r10 |
| adcq \$0, %r10 |
| |
| mulq %rbx |
| addq %rax, %r10 |
| movq 32($ap), %rax |
| movq %rdx, %r11 |
| adcq \$0, %r11 |
| |
| mulq %rbx |
| addq %rax, %r11 |
| movq 40($ap), %rax |
| movq %rdx, %r12 |
| adcq \$0, %r12 |
| |
| mulq %rbx |
| addq %rax, %r12 |
| movq 48($ap), %rax |
| movq %rdx, %r13 |
| adcq \$0, %r13 |
| |
| mulq %rbx |
| addq %rax, %r13 |
| movq 56($ap), %rax |
| movq %rdx, %r14 |
| adcq \$0, %r14 |
| |
| mulq %rbx |
| addq %rax, %r14 |
| movq ($ap), %rax |
| movq %rdx, %r15 |
| adcq \$0, %r15 |
| |
| leaq 8($bp), $bp |
| leaq 8(%rdi), %rdi |
| |
| movl \$7, %ecx |
| jmp .Loop_mul |
| |
| .align 32 |
| .Loop_mul: |
| movq ($bp), %rbx |
| mulq %rbx |
| addq %rax, %r8 |
| movq 8($ap), %rax |
| movq %r8, (%rdi) |
| movq %rdx, %r8 |
| adcq \$0, %r8 |
| |
| mulq %rbx |
| addq %rax, %r9 |
| movq 16($ap), %rax |
| adcq \$0, %rdx |
| addq %r9, %r8 |
| movq %rdx, %r9 |
| adcq \$0, %r9 |
| |
| mulq %rbx |
| addq %rax, %r10 |
| movq 24($ap), %rax |
| adcq \$0, %rdx |
| addq %r10, %r9 |
| movq %rdx, %r10 |
| adcq \$0, %r10 |
| |
| mulq %rbx |
| addq %rax, %r11 |
| movq 32($ap), %rax |
| adcq \$0, %rdx |
| addq %r11, %r10 |
| movq %rdx, %r11 |
| adcq \$0, %r11 |
| |
| mulq %rbx |
| addq %rax, %r12 |
| movq 40($ap), %rax |
| adcq \$0, %rdx |
| addq %r12, %r11 |
| movq %rdx, %r12 |
| adcq \$0, %r12 |
| |
| mulq %rbx |
| addq %rax, %r13 |
| movq 48($ap), %rax |
| adcq \$0, %rdx |
| addq %r13, %r12 |
| movq %rdx, %r13 |
| adcq \$0, %r13 |
| |
| mulq %rbx |
| addq %rax, %r14 |
| movq 56($ap), %rax |
| adcq \$0, %rdx |
| addq %r14, %r13 |
| movq %rdx, %r14 |
| leaq 8($bp), $bp |
| adcq \$0, %r14 |
| |
| mulq %rbx |
| addq %rax, %r15 |
| movq ($ap), %rax |
| adcq \$0, %rdx |
| addq %r15, %r14 |
| movq %rdx, %r15 |
| adcq \$0, %r15 |
| |
| leaq 8(%rdi), %rdi |
| |
| decl %ecx |
| jnz .Loop_mul |
| |
| movq %r8, (%rdi) |
| movq %r9, 8(%rdi) |
| movq %r10, 16(%rdi) |
| movq %r11, 24(%rdi) |
| movq %r12, 32(%rdi) |
| movq %r13, 40(%rdi) |
| movq %r14, 48(%rdi) |
| movq %r15, 56(%rdi) |
| |
| ret |
| .cfi_endproc |
| .size __rsaz_512_mul,.-__rsaz_512_mul |
| ___ |
| } |
| if ($addx) { |
| # __rsaz_512_mulx |
| # |
| # input: %rsi - ap, %rbp - bp |
| # output: |
| # clobbers: everything |
| my ($ap,$bp,$zero) = ("%rsi","%rbp","%rdi"); |
| $code.=<<___; |
| .type __rsaz_512_mulx,\@abi-omnipotent |
| .align 32 |
| __rsaz_512_mulx: |
| .cfi_startproc |
| mulx ($ap), %rbx, %r8 # initial %rdx preloaded by caller |
| mov \$-6, %rcx |
| |
| mulx 8($ap), %rax, %r9 |
| movq %rbx, 8(%rsp) |
| |
| mulx 16($ap), %rbx, %r10 |
| adc %rax, %r8 |
| |
| mulx 24($ap), %rax, %r11 |
| adc %rbx, %r9 |
| |
| mulx 32($ap), %rbx, %r12 |
| adc %rax, %r10 |
| |
| mulx 40($ap), %rax, %r13 |
| adc %rbx, %r11 |
| |
| mulx 48($ap), %rbx, %r14 |
| adc %rax, %r12 |
| |
| mulx 56($ap), %rax, %r15 |
| mov 8($bp), %rdx |
| adc %rbx, %r13 |
| adc %rax, %r14 |
| adc \$0, %r15 |
| |
| xor $zero, $zero # cf=0,of=0 |
| jmp .Loop_mulx |
| |
| .align 32 |
| .Loop_mulx: |
| movq %r8, %rbx |
| mulx ($ap), %rax, %r8 |
| adcx %rax, %rbx |
| adox %r9, %r8 |
| |
| mulx 8($ap), %rax, %r9 |
| adcx %rax, %r8 |
| adox %r10, %r9 |
| |
| mulx 16($ap), %rax, %r10 |
| adcx %rax, %r9 |
| adox %r11, %r10 |
| |
| mulx 24($ap), %rax, %r11 |
| adcx %rax, %r10 |
| adox %r12, %r11 |
| |
| .byte 0x3e,0xc4,0x62,0xfb,0xf6,0xa6,0x20,0x00,0x00,0x00 # mulx 32($ap), %rax, %r12 |
| adcx %rax, %r11 |
| adox %r13, %r12 |
| |
| mulx 40($ap), %rax, %r13 |
| adcx %rax, %r12 |
| adox %r14, %r13 |
| |
| mulx 48($ap), %rax, %r14 |
| adcx %rax, %r13 |
| adox %r15, %r14 |
| |
| mulx 56($ap), %rax, %r15 |
| movq 64($bp,%rcx,8), %rdx |
| movq %rbx, 8+64-8(%rsp,%rcx,8) |
| adcx %rax, %r14 |
| adox $zero, %r15 |
| adcx $zero, %r15 # cf=0 |
| |
| inc %rcx # of=0 |
| jnz .Loop_mulx |
| |
| movq %r8, %rbx |
| mulx ($ap), %rax, %r8 |
| adcx %rax, %rbx |
| adox %r9, %r8 |
| |
| .byte 0xc4,0x62,0xfb,0xf6,0x8e,0x08,0x00,0x00,0x00 # mulx 8($ap), %rax, %r9 |
| adcx %rax, %r8 |
| adox %r10, %r9 |
| |
| .byte 0xc4,0x62,0xfb,0xf6,0x96,0x10,0x00,0x00,0x00 # mulx 16($ap), %rax, %r10 |
| adcx %rax, %r9 |
| adox %r11, %r10 |
| |
| mulx 24($ap), %rax, %r11 |
| adcx %rax, %r10 |
| adox %r12, %r11 |
| |
| mulx 32($ap), %rax, %r12 |
| adcx %rax, %r11 |
| adox %r13, %r12 |
| |
| mulx 40($ap), %rax, %r13 |
| adcx %rax, %r12 |
| adox %r14, %r13 |
| |
| .byte 0xc4,0x62,0xfb,0xf6,0xb6,0x30,0x00,0x00,0x00 # mulx 48($ap), %rax, %r14 |
| adcx %rax, %r13 |
| adox %r15, %r14 |
| |
| .byte 0xc4,0x62,0xfb,0xf6,0xbe,0x38,0x00,0x00,0x00 # mulx 56($ap), %rax, %r15 |
| adcx %rax, %r14 |
| adox $zero, %r15 |
| adcx $zero, %r15 |
| |
| mov %rbx, 8+64-8(%rsp) |
| mov %r8, 8+64(%rsp) |
| mov %r9, 8+64+8(%rsp) |
| mov %r10, 8+64+16(%rsp) |
| mov %r11, 8+64+24(%rsp) |
| mov %r12, 8+64+32(%rsp) |
| mov %r13, 8+64+40(%rsp) |
| mov %r14, 8+64+48(%rsp) |
| mov %r15, 8+64+56(%rsp) |
| |
| ret |
| .cfi_endproc |
| .size __rsaz_512_mulx,.-__rsaz_512_mulx |
| ___ |
| } |
| { |
| my ($out,$inp,$power)= $win64 ? ("%rcx","%rdx","%r8d") : ("%rdi","%rsi","%edx"); |
| $code.=<<___; |
| .globl rsaz_512_scatter4 |
| .type rsaz_512_scatter4,\@abi-omnipotent |
| .align 16 |
| rsaz_512_scatter4: |
| .cfi_startproc |
| leaq ($out,$power,8), $out |
| movl \$8, %r9d |
| jmp .Loop_scatter |
| .align 16 |
| .Loop_scatter: |
| movq ($inp), %rax |
| leaq 8($inp), $inp |
| movq %rax, ($out) |
| leaq 128($out), $out |
| decl %r9d |
| jnz .Loop_scatter |
| ret |
| .cfi_endproc |
| .size rsaz_512_scatter4,.-rsaz_512_scatter4 |
| |
| .globl rsaz_512_gather4 |
| .type rsaz_512_gather4,\@abi-omnipotent |
| .align 16 |
| rsaz_512_gather4: |
| .cfi_startproc |
| ___ |
| $code.=<<___ if ($win64); |
| .LSEH_begin_rsaz_512_gather4: |
| .byte 0x48,0x81,0xec,0xa8,0x00,0x00,0x00 # sub $0xa8,%rsp |
| .byte 0x0f,0x29,0x34,0x24 # movaps %xmm6,(%rsp) |
| .byte 0x0f,0x29,0x7c,0x24,0x10 # movaps %xmm7,0x10(%rsp) |
| .byte 0x44,0x0f,0x29,0x44,0x24,0x20 # movaps %xmm8,0x20(%rsp) |
| .byte 0x44,0x0f,0x29,0x4c,0x24,0x30 # movaps %xmm9,0x30(%rsp) |
| .byte 0x44,0x0f,0x29,0x54,0x24,0x40 # movaps %xmm10,0x40(%rsp) |
| .byte 0x44,0x0f,0x29,0x5c,0x24,0x50 # movaps %xmm11,0x50(%rsp) |
| .byte 0x44,0x0f,0x29,0x64,0x24,0x60 # movaps %xmm12,0x60(%rsp) |
| .byte 0x44,0x0f,0x29,0x6c,0x24,0x70 # movaps %xmm13,0x70(%rsp) |
| .byte 0x44,0x0f,0x29,0xb4,0x24,0x80,0,0,0 # movaps %xmm14,0x80(%rsp) |
| .byte 0x44,0x0f,0x29,0xbc,0x24,0x90,0,0,0 # movaps %xmm15,0x90(%rsp) |
| ___ |
| $code.=<<___; |
| movd $power,%xmm8 |
| movdqa .Linc+16(%rip),%xmm1 # 00000002000000020000000200000002 |
| movdqa .Linc(%rip),%xmm0 # 00000001000000010000000000000000 |
| |
| pshufd \$0,%xmm8,%xmm8 # broadcast $power |
| movdqa %xmm1,%xmm7 |
| movdqa %xmm1,%xmm2 |
| ___ |
| ######################################################################## |
| # calculate mask by comparing 0..15 to $power |
| # |
| for($i=0;$i<4;$i++) { |
| $code.=<<___; |
| paddd %xmm`$i`,%xmm`$i+1` |
| pcmpeqd %xmm8,%xmm`$i` |
| movdqa %xmm7,%xmm`$i+3` |
| ___ |
| } |
| for(;$i<7;$i++) { |
| $code.=<<___; |
| paddd %xmm`$i`,%xmm`$i+1` |
| pcmpeqd %xmm8,%xmm`$i` |
| ___ |
| } |
| $code.=<<___; |
| pcmpeqd %xmm8,%xmm7 |
| movl \$8, %r9d |
| jmp .Loop_gather |
| .align 16 |
| .Loop_gather: |
| movdqa 16*0($inp),%xmm8 |
| movdqa 16*1($inp),%xmm9 |
| movdqa 16*2($inp),%xmm10 |
| movdqa 16*3($inp),%xmm11 |
| pand %xmm0,%xmm8 |
| movdqa 16*4($inp),%xmm12 |
| pand %xmm1,%xmm9 |
| movdqa 16*5($inp),%xmm13 |
| pand %xmm2,%xmm10 |
| movdqa 16*6($inp),%xmm14 |
| pand %xmm3,%xmm11 |
| movdqa 16*7($inp),%xmm15 |
| leaq 128($inp), $inp |
| pand %xmm4,%xmm12 |
| pand %xmm5,%xmm13 |
| pand %xmm6,%xmm14 |
| pand %xmm7,%xmm15 |
| por %xmm10,%xmm8 |
| por %xmm11,%xmm9 |
| por %xmm12,%xmm8 |
| por %xmm13,%xmm9 |
| por %xmm14,%xmm8 |
| por %xmm15,%xmm9 |
| |
| por %xmm9,%xmm8 |
| pshufd \$0x4e,%xmm8,%xmm9 |
| por %xmm9,%xmm8 |
| movq %xmm8,($out) |
| leaq 8($out), $out |
| decl %r9d |
| jnz .Loop_gather |
| ___ |
| $code.=<<___ if ($win64); |
| movaps 0x00(%rsp),%xmm6 |
| movaps 0x10(%rsp),%xmm7 |
| movaps 0x20(%rsp),%xmm8 |
| movaps 0x30(%rsp),%xmm9 |
| movaps 0x40(%rsp),%xmm10 |
| movaps 0x50(%rsp),%xmm11 |
| movaps 0x60(%rsp),%xmm12 |
| movaps 0x70(%rsp),%xmm13 |
| movaps 0x80(%rsp),%xmm14 |
| movaps 0x90(%rsp),%xmm15 |
| add \$0xa8,%rsp |
| ___ |
| $code.=<<___; |
| ret |
| .LSEH_end_rsaz_512_gather4: |
| .cfi_endproc |
| .size rsaz_512_gather4,.-rsaz_512_gather4 |
| |
| .align 64 |
| .Linc: |
| .long 0,0, 1,1 |
| .long 2,2, 2,2 |
| ___ |
| } |
| |
| # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, |
| # CONTEXT *context,DISPATCHER_CONTEXT *disp) |
| if ($win64) { |
| $rec="%rcx"; |
| $frame="%rdx"; |
| $context="%r8"; |
| $disp="%r9"; |
| |
| $code.=<<___; |
| .extern __imp_RtlVirtualUnwind |
| .type se_handler,\@abi-omnipotent |
| .align 16 |
| se_handler: |
| push %rsi |
| push %rdi |
| push %rbx |
| push %rbp |
| push %r12 |
| push %r13 |
| push %r14 |
| push %r15 |
| pushfq |
| sub \$64,%rsp |
| |
| mov 120($context),%rax # pull context->Rax |
| mov 248($context),%rbx # pull context->Rip |
| |
| mov 8($disp),%rsi # disp->ImageBase |
| mov 56($disp),%r11 # disp->HandlerData |
| |
| mov 0(%r11),%r10d # HandlerData[0] |
| lea (%rsi,%r10),%r10 # end of prologue label |
| cmp %r10,%rbx # context->Rip<end of prologue label |
| jb .Lcommon_seh_tail |
| |
| mov 152($context),%rax # pull context->Rsp |
| |
| mov 4(%r11),%r10d # HandlerData[1] |
| lea (%rsi,%r10),%r10 # epilogue label |
| cmp %r10,%rbx # context->Rip>=epilogue label |
| jae .Lcommon_seh_tail |
| |
| lea 128+24+48(%rax),%rax |
| |
| lea .Lmul_gather4_epilogue(%rip),%rbx |
| cmp %r10,%rbx |
| jne .Lse_not_in_mul_gather4 |
| |
| lea 0xb0(%rax),%rax |
| |
| lea -48-0xa8(%rax),%rsi |
| lea 512($context),%rdi |
| mov \$20,%ecx |
| .long 0xa548f3fc # cld; rep movsq |
| |
| .Lse_not_in_mul_gather4: |
| mov -8(%rax),%rbx |
| mov -16(%rax),%rbp |
| mov -24(%rax),%r12 |
| mov -32(%rax),%r13 |
| mov -40(%rax),%r14 |
| mov -48(%rax),%r15 |
| mov %rbx,144($context) # restore context->Rbx |
| mov %rbp,160($context) # restore context->Rbp |
| mov %r12,216($context) # restore context->R12 |
| mov %r13,224($context) # restore context->R13 |
| mov %r14,232($context) # restore context->R14 |
| mov %r15,240($context) # restore context->R15 |
| |
| .Lcommon_seh_tail: |
| mov 8(%rax),%rdi |
| mov 16(%rax),%rsi |
| mov %rax,152($context) # restore context->Rsp |
| mov %rsi,168($context) # restore context->Rsi |
| mov %rdi,176($context) # restore context->Rdi |
| |
| mov 40($disp),%rdi # disp->ContextRecord |
| mov $context,%rsi # context |
| mov \$154,%ecx # sizeof(CONTEXT) |
| .long 0xa548f3fc # cld; rep movsq |
| |
| mov $disp,%rsi |
| xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER |
| mov 8(%rsi),%rdx # arg2, disp->ImageBase |
| mov 0(%rsi),%r8 # arg3, disp->ControlPc |
| mov 16(%rsi),%r9 # arg4, disp->FunctionEntry |
| mov 40(%rsi),%r10 # disp->ContextRecord |
| lea 56(%rsi),%r11 # &disp->HandlerData |
| lea 24(%rsi),%r12 # &disp->EstablisherFrame |
| mov %r10,32(%rsp) # arg5 |
| mov %r11,40(%rsp) # arg6 |
| mov %r12,48(%rsp) # arg7 |
| mov %rcx,56(%rsp) # arg8, (NULL) |
| call *__imp_RtlVirtualUnwind(%rip) |
| |
| mov \$1,%eax # ExceptionContinueSearch |
| add \$64,%rsp |
| popfq |
| pop %r15 |
| pop %r14 |
| pop %r13 |
| pop %r12 |
| pop %rbp |
| pop %rbx |
| pop %rdi |
| pop %rsi |
| ret |
| .size se_handler,.-se_handler |
| |
| .section .pdata |
| .align 4 |
| .rva .LSEH_begin_rsaz_512_sqr |
| .rva .LSEH_end_rsaz_512_sqr |
| .rva .LSEH_info_rsaz_512_sqr |
| |
| .rva .LSEH_begin_rsaz_512_mul |
| .rva .LSEH_end_rsaz_512_mul |
| .rva .LSEH_info_rsaz_512_mul |
| |
| .rva .LSEH_begin_rsaz_512_mul_gather4 |
| .rva .LSEH_end_rsaz_512_mul_gather4 |
| .rva .LSEH_info_rsaz_512_mul_gather4 |
| |
| .rva .LSEH_begin_rsaz_512_mul_scatter4 |
| .rva .LSEH_end_rsaz_512_mul_scatter4 |
| .rva .LSEH_info_rsaz_512_mul_scatter4 |
| |
| .rva .LSEH_begin_rsaz_512_mul_by_one |
| .rva .LSEH_end_rsaz_512_mul_by_one |
| .rva .LSEH_info_rsaz_512_mul_by_one |
| |
| .rva .LSEH_begin_rsaz_512_gather4 |
| .rva .LSEH_end_rsaz_512_gather4 |
| .rva .LSEH_info_rsaz_512_gather4 |
| |
| .section .xdata |
| .align 8 |
| .LSEH_info_rsaz_512_sqr: |
| .byte 9,0,0,0 |
| .rva se_handler |
| .rva .Lsqr_body,.Lsqr_epilogue # HandlerData[] |
| .LSEH_info_rsaz_512_mul: |
| .byte 9,0,0,0 |
| .rva se_handler |
| .rva .Lmul_body,.Lmul_epilogue # HandlerData[] |
| .LSEH_info_rsaz_512_mul_gather4: |
| .byte 9,0,0,0 |
| .rva se_handler |
| .rva .Lmul_gather4_body,.Lmul_gather4_epilogue # HandlerData[] |
| .LSEH_info_rsaz_512_mul_scatter4: |
| .byte 9,0,0,0 |
| .rva se_handler |
| .rva .Lmul_scatter4_body,.Lmul_scatter4_epilogue # HandlerData[] |
| .LSEH_info_rsaz_512_mul_by_one: |
| .byte 9,0,0,0 |
| .rva se_handler |
| .rva .Lmul_by_one_body,.Lmul_by_one_epilogue # HandlerData[] |
| .LSEH_info_rsaz_512_gather4: |
| .byte 0x01,0x46,0x16,0x00 |
| .byte 0x46,0xf8,0x09,0x00 # vmovaps 0x90(rsp),xmm15 |
| .byte 0x3d,0xe8,0x08,0x00 # vmovaps 0x80(rsp),xmm14 |
| .byte 0x34,0xd8,0x07,0x00 # vmovaps 0x70(rsp),xmm13 |
| .byte 0x2e,0xc8,0x06,0x00 # vmovaps 0x60(rsp),xmm12 |
| .byte 0x28,0xb8,0x05,0x00 # vmovaps 0x50(rsp),xmm11 |
| .byte 0x22,0xa8,0x04,0x00 # vmovaps 0x40(rsp),xmm10 |
| .byte 0x1c,0x98,0x03,0x00 # vmovaps 0x30(rsp),xmm9 |
| .byte 0x16,0x88,0x02,0x00 # vmovaps 0x20(rsp),xmm8 |
| .byte 0x10,0x78,0x01,0x00 # vmovaps 0x10(rsp),xmm7 |
| .byte 0x0b,0x68,0x00,0x00 # vmovaps 0x00(rsp),xmm6 |
| .byte 0x07,0x01,0x15,0x00 # sub rsp,0xa8 |
| ___ |
| } |
| |
| $code =~ s/\`([^\`]*)\`/eval $1/gem; |
| print $code; |
| close STDOUT or die "error closing STDOUT: $!"; |
