| #!/usr/bin/env perl |
| # Copyright 2018-2020 The OpenSSL Project Authors. All Rights Reserved. |
| # |
| # Licensed under the Apache License 2.0 (the "License"). You may not use |
| # this file except in compliance with the License. You can obtain a copy |
| # in the file LICENSE in the source distribution or at |
| # https://www.openssl.org/source/license.html |
| # |
| # ==================================================================== |
| # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL |
| # project. The module is, however, dual licensed under OpenSSL and |
| # CRYPTOGAMS licenses depending on where you obtain it. For further |
| # details see http://www.openssl.org/~appro/cryptogams/. |
| # ==================================================================== |
| # |
| # X25519 lower-level primitives for x86_64. |
| # |
| # February 2018. |
| # |
| # This module implements radix 2^51 multiplication and squaring, and |
| # radix 2^64 multiplication, squaring, addition, subtraction and final |
| # reduction. Latter radix is used on ADCX/ADOX-capable processors such |
| # as Broadwell. On related note one should mention that there are |
| # vector implementations that provide significantly better performance |
| # on some processors(*), but they are large and overly complex. Which |
| # in combination with them being effectively processor-specific makes |
| # the undertaking hard to justify. The goal for this implementation |
| # is rather versatility and simplicity [and ultimately formal |
| # verification]. |
| # |
| # (*) For example sandy2x should provide ~30% improvement on Sandy |
| # Bridge, but only nominal ~5% on Haswell [and big loss on |
| # Broadwell and successors]. |
| # |
| ###################################################################### |
| # Improvement coefficients: |
| # |
| # amd64-51(*) gcc-5.x(**) |
| # |
| # P4 +22% +40% |
| # Sandy Bridge -3% +11% |
| # Haswell -1% +13% |
| # Broadwell(***) +30% +35% |
| # Skylake(***) +33% +47% |
| # Silvermont +20% +26% |
| # Goldmont +40% +50% |
| # Bulldozer +20% +9% |
| # Ryzen(***) +43% +40% |
| # VIA +170% +120% |
| # |
| # (*) amd64-51 is popular assembly implementation with 2^51 radix, |
| # only multiplication and squaring subroutines were linked |
| # for comparison, but not complete ladder step; gain on most |
| # processors is because this module refrains from shld, and |
| # minor regression on others is because this does result in |
| # higher instruction count; |
| # (**) compiler is free to inline functions, in assembly one would |
| # need to implement ladder step to do that, and it will improve |
| # performance by several percent; |
| # (***) ADCX/ADOX result for 2^64 radix, there is no corresponding |
| # C implementation, so that comparison is always against |
| # 2^51 radix; |
| |
| # $output is the last argument if it looks like a file (it has an extension) |
| # $flavour is the first argument if it doesn't look like a file |
| $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; |
| $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef; |
| |
| $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); |
| |
| $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; |
| ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or |
| ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or |
| die "can't locate x86_64-xlate.pl"; |
| |
| open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"" |
| or die "can't call $xlate: $!"; |
| *STDOUT=*OUT; |
| |
| if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` |
| =~ /GNU assembler version ([2-9]\.[0-9]+)/) { |
| $addx = ($1>=2.23); |
| } |
| |
| if (!$addx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && |
| `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) { |
| $addx = ($1>=2.10); |
| } |
| |
| if (!$addx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) && |
| `ml64 2>&1` =~ /Version ([0-9]+)\./) { |
| $addx = ($1>=12); |
| } |
| |
| if (!$addx && `$ENV{CC} -v 2>&1` =~ /((?:clang|LLVM) version|.*based on LLVM) ([0-9]+)\.([0-9]+)/) { |
| my $ver = $2 + $3/100.0; # 3.1->3.01, 3.10->3.10 |
| $addx = ($ver>=3.03); |
| } |
| |
| $code.=<<___; |
| .text |
| |
| .globl x25519_fe51_mul |
| .type x25519_fe51_mul,\@function,3 |
| .align 32 |
| x25519_fe51_mul: |
| .cfi_startproc |
| push %rbp |
| .cfi_push %rbp |
| push %rbx |
| .cfi_push %rbx |
| push %r12 |
| .cfi_push %r12 |
| push %r13 |
| .cfi_push %r13 |
| push %r14 |
| .cfi_push %r14 |
| push %r15 |
| .cfi_push %r15 |
| lea -8*5(%rsp),%rsp |
| .cfi_adjust_cfa_offset 40 |
| .Lfe51_mul_body: |
| |
| mov 8*0(%rsi),%rax # f[0] |
| mov 8*0(%rdx),%r11 # load g[0-4] |
| mov 8*1(%rdx),%r12 |
| mov 8*2(%rdx),%r13 |
| mov 8*3(%rdx),%rbp |
| mov 8*4(%rdx),%r14 |
| |
| mov %rdi,8*4(%rsp) # offload 1st argument |
| mov %rax,%rdi |
| mulq %r11 # f[0]*g[0] |
| mov %r11,8*0(%rsp) # offload g[0] |
| mov %rax,%rbx # %rbx:%rcx = h0 |
| mov %rdi,%rax |
| mov %rdx,%rcx |
| mulq %r12 # f[0]*g[1] |
| mov %r12,8*1(%rsp) # offload g[1] |
| mov %rax,%r8 # %r8:%r9 = h1 |
| mov %rdi,%rax |
| lea (%r14,%r14,8),%r15 |
| mov %rdx,%r9 |
| mulq %r13 # f[0]*g[2] |
| mov %r13,8*2(%rsp) # offload g[2] |
| mov %rax,%r10 # %r10:%r11 = h2 |
| mov %rdi,%rax |
| lea (%r14,%r15,2),%rdi # g[4]*19 |
| mov %rdx,%r11 |
| mulq %rbp # f[0]*g[3] |
| mov %rax,%r12 # %r12:%r13 = h3 |
| mov 8*0(%rsi),%rax # f[0] |
| mov %rdx,%r13 |
| mulq %r14 # f[0]*g[4] |
| mov %rax,%r14 # %r14:%r15 = h4 |
| mov 8*1(%rsi),%rax # f[1] |
| mov %rdx,%r15 |
| |
| mulq %rdi # f[1]*g[4]*19 |
| add %rax,%rbx |
| mov 8*2(%rsi),%rax # f[2] |
| adc %rdx,%rcx |
| mulq %rdi # f[2]*g[4]*19 |
| add %rax,%r8 |
| mov 8*3(%rsi),%rax # f[3] |
| adc %rdx,%r9 |
| mulq %rdi # f[3]*g[4]*19 |
| add %rax,%r10 |
| mov 8*4(%rsi),%rax # f[4] |
| adc %rdx,%r11 |
| mulq %rdi # f[4]*g[4]*19 |
| imulq \$19,%rbp,%rdi # g[3]*19 |
| add %rax,%r12 |
| mov 8*1(%rsi),%rax # f[1] |
| adc %rdx,%r13 |
| mulq %rbp # f[1]*g[3] |
| mov 8*2(%rsp),%rbp # g[2] |
| add %rax,%r14 |
| mov 8*2(%rsi),%rax # f[2] |
| adc %rdx,%r15 |
| |
| mulq %rdi # f[2]*g[3]*19 |
| add %rax,%rbx |
| mov 8*3(%rsi),%rax # f[3] |
| adc %rdx,%rcx |
| mulq %rdi # f[3]*g[3]*19 |
| add %rax,%r8 |
| mov 8*4(%rsi),%rax # f[4] |
| adc %rdx,%r9 |
| mulq %rdi # f[4]*g[3]*19 |
| imulq \$19,%rbp,%rdi # g[2]*19 |
| add %rax,%r10 |
| mov 8*1(%rsi),%rax # f[1] |
| adc %rdx,%r11 |
| mulq %rbp # f[1]*g[2] |
| add %rax,%r12 |
| mov 8*2(%rsi),%rax # f[2] |
| adc %rdx,%r13 |
| mulq %rbp # f[2]*g[2] |
| mov 8*1(%rsp),%rbp # g[1] |
| add %rax,%r14 |
| mov 8*3(%rsi),%rax # f[3] |
| adc %rdx,%r15 |
| |
| mulq %rdi # f[3]*g[2]*19 |
| add %rax,%rbx |
| mov 8*4(%rsi),%rax # f[3] |
| adc %rdx,%rcx |
| mulq %rdi # f[4]*g[2]*19 |
| add %rax,%r8 |
| mov 8*1(%rsi),%rax # f[1] |
| adc %rdx,%r9 |
| mulq %rbp # f[1]*g[1] |
| imulq \$19,%rbp,%rdi |
| add %rax,%r10 |
| mov 8*2(%rsi),%rax # f[2] |
| adc %rdx,%r11 |
| mulq %rbp # f[2]*g[1] |
| add %rax,%r12 |
| mov 8*3(%rsi),%rax # f[3] |
| adc %rdx,%r13 |
| mulq %rbp # f[3]*g[1] |
| mov 8*0(%rsp),%rbp # g[0] |
| add %rax,%r14 |
| mov 8*4(%rsi),%rax # f[4] |
| adc %rdx,%r15 |
| |
| mulq %rdi # f[4]*g[1]*19 |
| add %rax,%rbx |
| mov 8*1(%rsi),%rax # f[1] |
| adc %rdx,%rcx |
| mul %rbp # f[1]*g[0] |
| add %rax,%r8 |
| mov 8*2(%rsi),%rax # f[2] |
| adc %rdx,%r9 |
| mul %rbp # f[2]*g[0] |
| add %rax,%r10 |
| mov 8*3(%rsi),%rax # f[3] |
| adc %rdx,%r11 |
| mul %rbp # f[3]*g[0] |
| add %rax,%r12 |
| mov 8*4(%rsi),%rax # f[4] |
| adc %rdx,%r13 |
| mulq %rbp # f[4]*g[0] |
| add %rax,%r14 |
| adc %rdx,%r15 |
| |
| mov 8*4(%rsp),%rdi # restore 1st argument |
| jmp .Lreduce51 |
| .Lfe51_mul_epilogue: |
| .cfi_endproc |
| .size x25519_fe51_mul,.-x25519_fe51_mul |
| |
| .globl x25519_fe51_sqr |
| .type x25519_fe51_sqr,\@function,2 |
| .align 32 |
| x25519_fe51_sqr: |
| .cfi_startproc |
| push %rbp |
| .cfi_push %rbp |
| push %rbx |
| .cfi_push %rbx |
| push %r12 |
| .cfi_push %r12 |
| push %r13 |
| .cfi_push %r13 |
| push %r14 |
| .cfi_push %r14 |
| push %r15 |
| .cfi_push %r15 |
| lea -8*5(%rsp),%rsp |
| .cfi_adjust_cfa_offset 40 |
| .Lfe51_sqr_body: |
| |
| mov 8*0(%rsi),%rax # g[0] |
| mov 8*2(%rsi),%r15 # g[2] |
| mov 8*4(%rsi),%rbp # g[4] |
| |
| mov %rdi,8*4(%rsp) # offload 1st argument |
| lea (%rax,%rax),%r14 |
| mulq %rax # g[0]*g[0] |
| mov %rax,%rbx |
| mov 8*1(%rsi),%rax # g[1] |
| mov %rdx,%rcx |
| mulq %r14 # 2*g[0]*g[1] |
| mov %rax,%r8 |
| mov %r15,%rax |
| mov %r15,8*0(%rsp) # offload g[2] |
| mov %rdx,%r9 |
| mulq %r14 # 2*g[0]*g[2] |
| mov %rax,%r10 |
| mov 8*3(%rsi),%rax |
| mov %rdx,%r11 |
| imulq \$19,%rbp,%rdi # g[4]*19 |
| mulq %r14 # 2*g[0]*g[3] |
| mov %rax,%r12 |
| mov %rbp,%rax |
| mov %rdx,%r13 |
| mulq %r14 # 2*g[0]*g[4] |
| mov %rax,%r14 |
| mov %rbp,%rax |
| mov %rdx,%r15 |
| |
| mulq %rdi # g[4]*g[4]*19 |
| add %rax,%r12 |
| mov 8*1(%rsi),%rax # g[1] |
| adc %rdx,%r13 |
| |
| mov 8*3(%rsi),%rsi # g[3] |
| lea (%rax,%rax),%rbp |
| mulq %rax # g[1]*g[1] |
| add %rax,%r10 |
| mov 8*0(%rsp),%rax # g[2] |
| adc %rdx,%r11 |
| mulq %rbp # 2*g[1]*g[2] |
| add %rax,%r12 |
| mov %rbp,%rax |
| adc %rdx,%r13 |
| mulq %rsi # 2*g[1]*g[3] |
| add %rax,%r14 |
| mov %rbp,%rax |
| adc %rdx,%r15 |
| imulq \$19,%rsi,%rbp # g[3]*19 |
| mulq %rdi # 2*g[1]*g[4]*19 |
| add %rax,%rbx |
| lea (%rsi,%rsi),%rax |
| adc %rdx,%rcx |
| |
| mulq %rdi # 2*g[3]*g[4]*19 |
| add %rax,%r10 |
| mov %rsi,%rax |
| adc %rdx,%r11 |
| mulq %rbp # g[3]*g[3]*19 |
| add %rax,%r8 |
| mov 8*0(%rsp),%rax # g[2] |
| adc %rdx,%r9 |
| |
| lea (%rax,%rax),%rsi |
| mulq %rax # g[2]*g[2] |
| add %rax,%r14 |
| mov %rbp,%rax |
| adc %rdx,%r15 |
| mulq %rsi # 2*g[2]*g[3]*19 |
| add %rax,%rbx |
| mov %rsi,%rax |
| adc %rdx,%rcx |
| mulq %rdi # 2*g[2]*g[4]*19 |
| add %rax,%r8 |
| adc %rdx,%r9 |
| |
| mov 8*4(%rsp),%rdi # restore 1st argument |
| jmp .Lreduce51 |
| |
| .align 32 |
| .Lreduce51: |
| mov \$0x7ffffffffffff,%rbp |
| |
| mov %r10,%rdx |
| shr \$51,%r10 |
| shl \$13,%r11 |
| and %rbp,%rdx # %rdx = g2 = h2 & mask |
| or %r10,%r11 # h2>>51 |
| add %r11,%r12 |
| adc \$0,%r13 # h3 += h2>>51 |
| |
| mov %rbx,%rax |
| shr \$51,%rbx |
| shl \$13,%rcx |
| and %rbp,%rax # %rax = g0 = h0 & mask |
| or %rbx,%rcx # h0>>51 |
| add %rcx,%r8 # h1 += h0>>51 |
| adc \$0,%r9 |
| |
| mov %r12,%rbx |
| shr \$51,%r12 |
| shl \$13,%r13 |
| and %rbp,%rbx # %rbx = g3 = h3 & mask |
| or %r12,%r13 # h3>>51 |
| add %r13,%r14 # h4 += h3>>51 |
| adc \$0,%r15 |
| |
| mov %r8,%rcx |
| shr \$51,%r8 |
| shl \$13,%r9 |
| and %rbp,%rcx # %rcx = g1 = h1 & mask |
| or %r8,%r9 |
| add %r9,%rdx # g2 += h1>>51 |
| |
| mov %r14,%r10 |
| shr \$51,%r14 |
| shl \$13,%r15 |
| and %rbp,%r10 # %r10 = g4 = h0 & mask |
| or %r14,%r15 # h0>>51 |
| |
| lea (%r15,%r15,8),%r14 |
| lea (%r15,%r14,2),%r15 |
| add %r15,%rax # g0 += (h0>>51)*19 |
| |
| mov %rdx,%r8 |
| and %rbp,%rdx # g2 &= mask |
| shr \$51,%r8 |
| add %r8,%rbx # g3 += g2>>51 |
| |
| mov %rax,%r9 |
| and %rbp,%rax # g0 &= mask |
| shr \$51,%r9 |
| add %r9,%rcx # g1 += g0>>51 |
| |
| mov %rax,8*0(%rdi) # save the result |
| mov %rcx,8*1(%rdi) |
| mov %rdx,8*2(%rdi) |
| mov %rbx,8*3(%rdi) |
| mov %r10,8*4(%rdi) |
| |
| mov 8*5(%rsp),%r15 |
| .cfi_restore %r15 |
| mov 8*6(%rsp),%r14 |
| .cfi_restore %r14 |
| mov 8*7(%rsp),%r13 |
| .cfi_restore %r13 |
| mov 8*8(%rsp),%r12 |
| .cfi_restore %r12 |
| mov 8*9(%rsp),%rbx |
| .cfi_restore %rbx |
| mov 8*10(%rsp),%rbp |
| .cfi_restore %rbp |
| lea 8*11(%rsp),%rsp |
| .cfi_adjust_cfa_offset 88 |
| .Lfe51_sqr_epilogue: |
| ret |
| .cfi_endproc |
| .size x25519_fe51_sqr,.-x25519_fe51_sqr |
| |
| .globl x25519_fe51_mul121666 |
| .type x25519_fe51_mul121666,\@function,2 |
| .align 32 |
| x25519_fe51_mul121666: |
| .cfi_startproc |
| push %rbp |
| .cfi_push %rbp |
| push %rbx |
| .cfi_push %rbx |
| push %r12 |
| .cfi_push %r12 |
| push %r13 |
| .cfi_push %r13 |
| push %r14 |
| .cfi_push %r14 |
| push %r15 |
| .cfi_push %r15 |
| lea -8*5(%rsp),%rsp |
| .cfi_adjust_cfa_offset 40 |
| .Lfe51_mul121666_body: |
| mov \$121666,%eax |
| |
| mulq 8*0(%rsi) |
| mov %rax,%rbx # %rbx:%rcx = h0 |
| mov \$121666,%eax |
| mov %rdx,%rcx |
| mulq 8*1(%rsi) |
| mov %rax,%r8 # %r8:%r9 = h1 |
| mov \$121666,%eax |
| mov %rdx,%r9 |
| mulq 8*2(%rsi) |
| mov %rax,%r10 # %r10:%r11 = h2 |
| mov \$121666,%eax |
| mov %rdx,%r11 |
| mulq 8*3(%rsi) |
| mov %rax,%r12 # %r12:%r13 = h3 |
| mov \$121666,%eax # f[0] |
| mov %rdx,%r13 |
| mulq 8*4(%rsi) |
| mov %rax,%r14 # %r14:%r15 = h4 |
| mov %rdx,%r15 |
| |
| jmp .Lreduce51 |
| .Lfe51_mul121666_epilogue: |
| .cfi_endproc |
| .size x25519_fe51_mul121666,.-x25519_fe51_mul121666 |
| ___ |
| ######################################################################## |
| # Base 2^64 subroutines modulo 2*(2^255-19) |
| # |
| if ($addx) { |
| my ($acc0,$acc1,$acc2,$acc3,$acc4,$acc5,$acc6,$acc7) = map("%r$_",(8..15)); |
| |
| $code.=<<___; |
| .extern OPENSSL_ia32cap_P |
| .globl x25519_fe64_eligible |
| .type x25519_fe64_eligible,\@abi-omnipotent |
| .align 32 |
| x25519_fe64_eligible: |
| .cfi_startproc |
| mov OPENSSL_ia32cap_P+8(%rip),%ecx |
| xor %eax,%eax |
| and \$0x80100,%ecx |
| cmp \$0x80100,%ecx |
| cmove %ecx,%eax |
| ret |
| .cfi_endproc |
| .size x25519_fe64_eligible,.-x25519_fe64_eligible |
| |
| .globl x25519_fe64_mul |
| .type x25519_fe64_mul,\@function,3 |
| .align 32 |
| x25519_fe64_mul: |
| .cfi_startproc |
| push %rbp |
| .cfi_push %rbp |
| push %rbx |
| .cfi_push %rbx |
| push %r12 |
| .cfi_push %r12 |
| push %r13 |
| .cfi_push %r13 |
| push %r14 |
| .cfi_push %r14 |
| push %r15 |
| .cfi_push %r15 |
| push %rdi # offload dst |
| .cfi_push %rdi |
| lea -8*2(%rsp),%rsp |
| .cfi_adjust_cfa_offset 16 |
| .Lfe64_mul_body: |
| |
| mov %rdx,%rax |
| mov 8*0(%rdx),%rbp # b[0] |
| mov 8*0(%rsi),%rdx # a[0] |
| mov 8*1(%rax),%rcx # b[1] |
| mov 8*2(%rax),$acc6 # b[2] |
| mov 8*3(%rax),$acc7 # b[3] |
| |
| mulx %rbp,$acc0,%rax # a[0]*b[0] |
| xor %edi,%edi # cf=0,of=0 |
| mulx %rcx,$acc1,%rbx # a[0]*b[1] |
| adcx %rax,$acc1 |
| mulx $acc6,$acc2,%rax # a[0]*b[2] |
| adcx %rbx,$acc2 |
| mulx $acc7,$acc3,$acc4 # a[0]*b[3] |
| mov 8*1(%rsi),%rdx # a[1] |
| adcx %rax,$acc3 |
| mov $acc6,(%rsp) # offload b[2] |
| adcx %rdi,$acc4 # cf=0 |
| |
| mulx %rbp,%rax,%rbx # a[1]*b[0] |
| adox %rax,$acc1 |
| adcx %rbx,$acc2 |
| mulx %rcx,%rax,%rbx # a[1]*b[1] |
| adox %rax,$acc2 |
| adcx %rbx,$acc3 |
| mulx $acc6,%rax,%rbx # a[1]*b[2] |
| adox %rax,$acc3 |
| adcx %rbx,$acc4 |
| mulx $acc7,%rax,$acc5 # a[1]*b[3] |
| mov 8*2(%rsi),%rdx # a[2] |
| adox %rax,$acc4 |
| adcx %rdi,$acc5 # cf=0 |
| adox %rdi,$acc5 # of=0 |
| |
| mulx %rbp,%rax,%rbx # a[2]*b[0] |
| adcx %rax,$acc2 |
| adox %rbx,$acc3 |
| mulx %rcx,%rax,%rbx # a[2]*b[1] |
| adcx %rax,$acc3 |
| adox %rbx,$acc4 |
| mulx $acc6,%rax,%rbx # a[2]*b[2] |
| adcx %rax,$acc4 |
| adox %rbx,$acc5 |
| mulx $acc7,%rax,$acc6 # a[2]*b[3] |
| mov 8*3(%rsi),%rdx # a[3] |
| adcx %rax,$acc5 |
| adox %rdi,$acc6 # of=0 |
| adcx %rdi,$acc6 # cf=0 |
| |
| mulx %rbp,%rax,%rbx # a[3]*b[0] |
| adox %rax,$acc3 |
| adcx %rbx,$acc4 |
| mulx %rcx,%rax,%rbx # a[3]*b[1] |
| adox %rax,$acc4 |
| adcx %rbx,$acc5 |
| mulx (%rsp),%rax,%rbx # a[3]*b[2] |
| adox %rax,$acc5 |
| adcx %rbx,$acc6 |
| mulx $acc7,%rax,$acc7 # a[3]*b[3] |
| mov \$38,%edx |
| adox %rax,$acc6 |
| adcx %rdi,$acc7 # cf=0 |
| adox %rdi,$acc7 # of=0 |
| |
| jmp .Lreduce64 |
| .Lfe64_mul_epilogue: |
| .cfi_endproc |
| .size x25519_fe64_mul,.-x25519_fe64_mul |
| |
| .globl x25519_fe64_sqr |
| .type x25519_fe64_sqr,\@function,2 |
| .align 32 |
| x25519_fe64_sqr: |
| .cfi_startproc |
| push %rbp |
| .cfi_push %rbp |
| push %rbx |
| .cfi_push %rbx |
| push %r12 |
| .cfi_push %r12 |
| push %r13 |
| .cfi_push %r13 |
| push %r14 |
| .cfi_push %r14 |
| push %r15 |
| .cfi_push %r15 |
| push %rdi # offload dst |
| .cfi_push %rdi |
| lea -8*2(%rsp),%rsp |
| .cfi_adjust_cfa_offset 16 |
| .Lfe64_sqr_body: |
| |
| mov 8*0(%rsi),%rdx # a[0] |
| mov 8*1(%rsi),%rcx # a[1] |
| mov 8*2(%rsi),%rbp # a[2] |
| mov 8*3(%rsi),%rsi # a[3] |
| |
| ################################################################ |
| mulx %rdx,$acc0,$acc7 # a[0]*a[0] |
| mulx %rcx,$acc1,%rax # a[0]*a[1] |
| xor %edi,%edi # cf=0,of=0 |
| mulx %rbp,$acc2,%rbx # a[0]*a[2] |
| adcx %rax,$acc2 |
| mulx %rsi,$acc3,$acc4 # a[0]*a[3] |
| mov %rcx,%rdx # a[1] |
| adcx %rbx,$acc3 |
| adcx %rdi,$acc4 # cf=0 |
| |
| ################################################################ |
| mulx %rbp,%rax,%rbx # a[1]*a[2] |
| adox %rax,$acc3 |
| adcx %rbx,$acc4 |
| mulx %rsi,%rax,$acc5 # a[1]*a[3] |
| mov %rbp,%rdx # a[2] |
| adox %rax,$acc4 |
| adcx %rdi,$acc5 |
| |
| ################################################################ |
| mulx %rsi,%rax,$acc6 # a[2]*a[3] |
| mov %rcx,%rdx # a[1] |
| adox %rax,$acc5 |
| adcx %rdi,$acc6 # cf=0 |
| adox %rdi,$acc6 # of=0 |
| |
| adcx $acc1,$acc1 # acc1:6<<1 |
| adox $acc7,$acc1 |
| adcx $acc2,$acc2 |
| mulx %rdx,%rax,%rbx # a[1]*a[1] |
| mov %rbp,%rdx # a[2] |
| adcx $acc3,$acc3 |
| adox %rax,$acc2 |
| adcx $acc4,$acc4 |
| adox %rbx,$acc3 |
| mulx %rdx,%rax,%rbx # a[2]*a[2] |
| mov %rsi,%rdx # a[3] |
| adcx $acc5,$acc5 |
| adox %rax,$acc4 |
| adcx $acc6,$acc6 |
| adox %rbx,$acc5 |
| mulx %rdx,%rax,$acc7 # a[3]*a[3] |
| mov \$38,%edx |
| adox %rax,$acc6 |
| adcx %rdi,$acc7 # cf=0 |
| adox %rdi,$acc7 # of=0 |
| jmp .Lreduce64 |
| |
| .align 32 |
| .Lreduce64: |
| mulx $acc4,%rax,%rbx |
| adcx %rax,$acc0 |
| adox %rbx,$acc1 |
| mulx $acc5,%rax,%rbx |
| adcx %rax,$acc1 |
| adox %rbx,$acc2 |
| mulx $acc6,%rax,%rbx |
| adcx %rax,$acc2 |
| adox %rbx,$acc3 |
| mulx $acc7,%rax,$acc4 |
| adcx %rax,$acc3 |
| adox %rdi,$acc4 |
| adcx %rdi,$acc4 |
| |
| mov 8*2(%rsp),%rdi # restore dst |
| imulq %rdx,$acc4 |
| |
| add $acc4,$acc0 |
| adc \$0,$acc1 |
| adc \$0,$acc2 |
| adc \$0,$acc3 |
| |
| sbb %rax,%rax # cf -> mask |
| and \$38,%rax |
| |
| add %rax,$acc0 |
| mov $acc1,8*1(%rdi) |
| mov $acc2,8*2(%rdi) |
| mov $acc3,8*3(%rdi) |
| mov $acc0,8*0(%rdi) |
| |
| mov 8*3(%rsp),%r15 |
| .cfi_restore %r15 |
| mov 8*4(%rsp),%r14 |
| .cfi_restore %r14 |
| mov 8*5(%rsp),%r13 |
| .cfi_restore %r13 |
| mov 8*6(%rsp),%r12 |
| .cfi_restore %r12 |
| mov 8*7(%rsp),%rbx |
| .cfi_restore %rbx |
| mov 8*8(%rsp),%rbp |
| .cfi_restore %rbp |
| lea 8*9(%rsp),%rsp |
| .cfi_adjust_cfa_offset 88 |
| .Lfe64_sqr_epilogue: |
| ret |
| .cfi_endproc |
| .size x25519_fe64_sqr,.-x25519_fe64_sqr |
| |
| .globl x25519_fe64_mul121666 |
| .type x25519_fe64_mul121666,\@function,2 |
| .align 32 |
| x25519_fe64_mul121666: |
| .Lfe64_mul121666_body: |
| .cfi_startproc |
| mov \$121666,%edx |
| mulx 8*0(%rsi),$acc0,%rcx |
| mulx 8*1(%rsi),$acc1,%rax |
| add %rcx,$acc1 |
| mulx 8*2(%rsi),$acc2,%rcx |
| adc %rax,$acc2 |
| mulx 8*3(%rsi),$acc3,%rax |
| adc %rcx,$acc3 |
| adc \$0,%rax |
| |
| imulq \$38,%rax,%rax |
| |
| add %rax,$acc0 |
| adc \$0,$acc1 |
| adc \$0,$acc2 |
| adc \$0,$acc3 |
| |
| sbb %rax,%rax # cf -> mask |
| and \$38,%rax |
| |
| add %rax,$acc0 |
| mov $acc1,8*1(%rdi) |
| mov $acc2,8*2(%rdi) |
| mov $acc3,8*3(%rdi) |
| mov $acc0,8*0(%rdi) |
| |
| .Lfe64_mul121666_epilogue: |
| ret |
| .cfi_endproc |
| .size x25519_fe64_mul121666,.-x25519_fe64_mul121666 |
| |
| .globl x25519_fe64_add |
| .type x25519_fe64_add,\@function,3 |
| .align 32 |
| x25519_fe64_add: |
| .Lfe64_add_body: |
| .cfi_startproc |
| mov 8*0(%rsi),$acc0 |
| mov 8*1(%rsi),$acc1 |
| mov 8*2(%rsi),$acc2 |
| mov 8*3(%rsi),$acc3 |
| |
| add 8*0(%rdx),$acc0 |
| adc 8*1(%rdx),$acc1 |
| adc 8*2(%rdx),$acc2 |
| adc 8*3(%rdx),$acc3 |
| |
| sbb %rax,%rax # cf -> mask |
| and \$38,%rax |
| |
| add %rax,$acc0 |
| adc \$0,$acc1 |
| adc \$0,$acc2 |
| mov $acc1,8*1(%rdi) |
| adc \$0,$acc3 |
| mov $acc2,8*2(%rdi) |
| sbb %rax,%rax # cf -> mask |
| mov $acc3,8*3(%rdi) |
| and \$38,%rax |
| |
| add %rax,$acc0 |
| mov $acc0,8*0(%rdi) |
| |
| .Lfe64_add_epilogue: |
| ret |
| .cfi_endproc |
| .size x25519_fe64_add,.-x25519_fe64_add |
| |
| .globl x25519_fe64_sub |
| .type x25519_fe64_sub,\@function,3 |
| .align 32 |
| x25519_fe64_sub: |
| .Lfe64_sub_body: |
| .cfi_startproc |
| mov 8*0(%rsi),$acc0 |
| mov 8*1(%rsi),$acc1 |
| mov 8*2(%rsi),$acc2 |
| mov 8*3(%rsi),$acc3 |
| |
| sub 8*0(%rdx),$acc0 |
| sbb 8*1(%rdx),$acc1 |
| sbb 8*2(%rdx),$acc2 |
| sbb 8*3(%rdx),$acc3 |
| |
| sbb %rax,%rax # cf -> mask |
| and \$38,%rax |
| |
| sub %rax,$acc0 |
| sbb \$0,$acc1 |
| sbb \$0,$acc2 |
| mov $acc1,8*1(%rdi) |
| sbb \$0,$acc3 |
| mov $acc2,8*2(%rdi) |
| sbb %rax,%rax # cf -> mask |
| mov $acc3,8*3(%rdi) |
| and \$38,%rax |
| |
| sub %rax,$acc0 |
| mov $acc0,8*0(%rdi) |
| |
| .Lfe64_sub_epilogue: |
| ret |
| .cfi_endproc |
| .size x25519_fe64_sub,.-x25519_fe64_sub |
| |
| .globl x25519_fe64_tobytes |
| .type x25519_fe64_tobytes,\@function,2 |
| .align 32 |
| x25519_fe64_tobytes: |
| .Lfe64_to_body: |
| .cfi_startproc |
| mov 8*0(%rsi),$acc0 |
| mov 8*1(%rsi),$acc1 |
| mov 8*2(%rsi),$acc2 |
| mov 8*3(%rsi),$acc3 |
| |
| ################################# reduction modulo 2^255-19 |
| lea ($acc3,$acc3),%rax |
| sar \$63,$acc3 # most significant bit -> mask |
| shr \$1,%rax # most significant bit cleared |
| and \$19,$acc3 |
| add \$19,$acc3 # compare to modulus in the same go |
| |
| add $acc3,$acc0 |
| adc \$0,$acc1 |
| adc \$0,$acc2 |
| adc \$0,%rax |
| |
| lea (%rax,%rax),$acc3 |
| sar \$63,%rax # most significant bit -> mask |
| shr \$1,$acc3 # most significant bit cleared |
| not %rax |
| and \$19,%rax |
| |
| sub %rax,$acc0 |
| sbb \$0,$acc1 |
| sbb \$0,$acc2 |
| sbb \$0,$acc3 |
| |
| mov $acc0,8*0(%rdi) |
| mov $acc1,8*1(%rdi) |
| mov $acc2,8*2(%rdi) |
| mov $acc3,8*3(%rdi) |
| |
| .Lfe64_to_epilogue: |
| ret |
| .cfi_endproc |
| .size x25519_fe64_tobytes,.-x25519_fe64_tobytes |
| ___ |
| } else { |
| $code.=<<___; |
| .globl x25519_fe64_eligible |
| .type x25519_fe64_eligible,\@abi-omnipotent |
| .align 32 |
| x25519_fe64_eligible: |
| .cfi_startproc |
| xor %eax,%eax |
| ret |
| .cfi_endproc |
| .size x25519_fe64_eligible,.-x25519_fe64_eligible |
| |
| .globl x25519_fe64_mul |
| .type x25519_fe64_mul,\@abi-omnipotent |
| .globl x25519_fe64_sqr |
| .globl x25519_fe64_mul121666 |
| .globl x25519_fe64_add |
| .globl x25519_fe64_sub |
| .globl x25519_fe64_tobytes |
| x25519_fe64_mul: |
| x25519_fe64_sqr: |
| x25519_fe64_mul121666: |
| x25519_fe64_add: |
| x25519_fe64_sub: |
| x25519_fe64_tobytes: |
| .cfi_startproc |
| .byte 0x0f,0x0b # ud2 |
| ret |
| .cfi_endproc |
| .size x25519_fe64_mul,.-x25519_fe64_mul |
| ___ |
| } |
| $code.=<<___; |
| .asciz "X25519 primitives for x86_64, CRYPTOGAMS by <appro\@openssl.org>" |
| ___ |
| |
| # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, |
| # CONTEXT *context,DISPATCHER_CONTEXT *disp) |
| if ($win64) { |
| $rec="%rcx"; |
| $frame="%rdx"; |
| $context="%r8"; |
| $disp="%r9"; |
| |
| $code.=<<___; |
| .extern __imp_RtlVirtualUnwind |
| |
| .type short_handler,\@abi-omnipotent |
| .align 16 |
| short_handler: |
| push %rsi |
| push %rdi |
| push %rbx |
| push %rbp |
| push %r12 |
| push %r13 |
| push %r14 |
| push %r15 |
| pushfq |
| sub \$64,%rsp |
| |
| mov 120($context),%rax # pull context->Rax |
| mov 248($context),%rbx # pull context->Rip |
| |
| mov 8($disp),%rsi # disp->ImageBase |
| mov 56($disp),%r11 # disp->HandlerData |
| |
| mov 0(%r11),%r10d # HandlerData[0] |
| lea (%rsi,%r10),%r10 # end of prologue label |
| cmp %r10,%rbx # context->Rip<end of prologue label |
| jb .Lcommon_seh_tail |
| |
| mov 152($context),%rax # pull context->Rsp |
| jmp .Lcommon_seh_tail |
| .size short_handler,.-short_handler |
| |
| .type full_handler,\@abi-omnipotent |
| .align 16 |
| full_handler: |
| push %rsi |
| push %rdi |
| push %rbx |
| push %rbp |
| push %r12 |
| push %r13 |
| push %r14 |
| push %r15 |
| pushfq |
| sub \$64,%rsp |
| |
| mov 120($context),%rax # pull context->Rax |
| mov 248($context),%rbx # pull context->Rip |
| |
| mov 8($disp),%rsi # disp->ImageBase |
| mov 56($disp),%r11 # disp->HandlerData |
| |
| mov 0(%r11),%r10d # HandlerData[0] |
| lea (%rsi,%r10),%r10 # end of prologue label |
| cmp %r10,%rbx # context->Rip<end of prologue label |
| jb .Lcommon_seh_tail |
| |
| mov 152($context),%rax # pull context->Rsp |
| |
| mov 4(%r11),%r10d # HandlerData[1] |
| lea (%rsi,%r10),%r10 # epilogue label |
| cmp %r10,%rbx # context->Rip>=epilogue label |
| jae .Lcommon_seh_tail |
| |
| mov 8(%r11),%r10d # HandlerData[2] |
| lea (%rax,%r10),%rax |
| |
| mov -8(%rax),%rbp |
| mov -16(%rax),%rbx |
| mov -24(%rax),%r12 |
| mov -32(%rax),%r13 |
| mov -40(%rax),%r14 |
| mov -48(%rax),%r15 |
| mov %rbx,144($context) # restore context->Rbx |
| mov %rbp,160($context) # restore context->Rbp |
| mov %r12,216($context) # restore context->R12 |
| mov %r13,224($context) # restore context->R13 |
| mov %r14,232($context) # restore context->R14 |
| mov %r15,240($context) # restore context->R15 |
| |
| .Lcommon_seh_tail: |
| mov 8(%rax),%rdi |
| mov 16(%rax),%rsi |
| mov %rax,152($context) # restore context->Rsp |
| mov %rsi,168($context) # restore context->Rsi |
| mov %rdi,176($context) # restore context->Rdi |
| |
| mov 40($disp),%rdi # disp->ContextRecord |
| mov $context,%rsi # context |
| mov \$154,%ecx # sizeof(CONTEXT) |
| .long 0xa548f3fc # cld; rep movsq |
| |
| mov $disp,%rsi |
| xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER |
| mov 8(%rsi),%rdx # arg2, disp->ImageBase |
| mov 0(%rsi),%r8 # arg3, disp->ControlPc |
| mov 16(%rsi),%r9 # arg4, disp->FunctionEntry |
| mov 40(%rsi),%r10 # disp->ContextRecord |
| lea 56(%rsi),%r11 # &disp->HandlerData |
| lea 24(%rsi),%r12 # &disp->EstablisherFrame |
| mov %r10,32(%rsp) # arg5 |
| mov %r11,40(%rsp) # arg6 |
| mov %r12,48(%rsp) # arg7 |
| mov %rcx,56(%rsp) # arg8, (NULL) |
| call *__imp_RtlVirtualUnwind(%rip) |
| |
| mov \$1,%eax # ExceptionContinueSearch |
| add \$64,%rsp |
| popfq |
| pop %r15 |
| pop %r14 |
| pop %r13 |
| pop %r12 |
| pop %rbp |
| pop %rbx |
| pop %rdi |
| pop %rsi |
| ret |
| .size full_handler,.-full_handler |
| |
| .section .pdata |
| .align 4 |
| .rva .LSEH_begin_x25519_fe51_mul |
| .rva .LSEH_end_x25519_fe51_mul |
| .rva .LSEH_info_x25519_fe51_mul |
| |
| .rva .LSEH_begin_x25519_fe51_sqr |
| .rva .LSEH_end_x25519_fe51_sqr |
| .rva .LSEH_info_x25519_fe51_sqr |
| |
| .rva .LSEH_begin_x25519_fe51_mul121666 |
| .rva .LSEH_end_x25519_fe51_mul121666 |
| .rva .LSEH_info_x25519_fe51_mul121666 |
| ___ |
| $code.=<<___ if ($addx); |
| .rva .LSEH_begin_x25519_fe64_mul |
| .rva .LSEH_end_x25519_fe64_mul |
| .rva .LSEH_info_x25519_fe64_mul |
| |
| .rva .LSEH_begin_x25519_fe64_sqr |
| .rva .LSEH_end_x25519_fe64_sqr |
| .rva .LSEH_info_x25519_fe64_sqr |
| |
| .rva .LSEH_begin_x25519_fe64_mul121666 |
| .rva .LSEH_end_x25519_fe64_mul121666 |
| .rva .LSEH_info_x25519_fe64_mul121666 |
| |
| .rva .LSEH_begin_x25519_fe64_add |
| .rva .LSEH_end_x25519_fe64_add |
| .rva .LSEH_info_x25519_fe64_add |
| |
| .rva .LSEH_begin_x25519_fe64_sub |
| .rva .LSEH_end_x25519_fe64_sub |
| .rva .LSEH_info_x25519_fe64_sub |
| |
| .rva .LSEH_begin_x25519_fe64_tobytes |
| .rva .LSEH_end_x25519_fe64_tobytes |
| .rva .LSEH_info_x25519_fe64_tobytes |
| ___ |
| $code.=<<___; |
| .section .xdata |
| .align 8 |
| .LSEH_info_x25519_fe51_mul: |
| .byte 9,0,0,0 |
| .rva full_handler |
| .rva .Lfe51_mul_body,.Lfe51_mul_epilogue # HandlerData[] |
| .long 88,0 |
| .LSEH_info_x25519_fe51_sqr: |
| .byte 9,0,0,0 |
| .rva full_handler |
| .rva .Lfe51_sqr_body,.Lfe51_sqr_epilogue # HandlerData[] |
| .long 88,0 |
| .LSEH_info_x25519_fe51_mul121666: |
| .byte 9,0,0,0 |
| .rva full_handler |
| .rva .Lfe51_mul121666_body,.Lfe51_mul121666_epilogue # HandlerData[] |
| .long 88,0 |
| ___ |
| $code.=<<___ if ($addx); |
| .LSEH_info_x25519_fe64_mul: |
| .byte 9,0,0,0 |
| .rva full_handler |
| .rva .Lfe64_mul_body,.Lfe64_mul_epilogue # HandlerData[] |
| .long 72,0 |
| .LSEH_info_x25519_fe64_sqr: |
| .byte 9,0,0,0 |
| .rva full_handler |
| .rva .Lfe64_sqr_body,.Lfe64_sqr_epilogue # HandlerData[] |
| .long 72,0 |
| .LSEH_info_x25519_fe64_mul121666: |
| .byte 9,0,0,0 |
| .rva short_handler |
| .rva .Lfe64_mul121666_body,.Lfe64_mul121666_epilogue # HandlerData[] |
| .LSEH_info_x25519_fe64_add: |
| .byte 9,0,0,0 |
| .rva short_handler |
| .rva .Lfe64_add_body,.Lfe64_add_epilogue # HandlerData[] |
| .LSEH_info_x25519_fe64_sub: |
| .byte 9,0,0,0 |
| .rva short_handler |
| .rva .Lfe64_sub_body,.Lfe64_sub_epilogue # HandlerData[] |
| .LSEH_info_x25519_fe64_tobytes: |
| .byte 9,0,0,0 |
| .rva short_handler |
| .rva .Lfe64_to_body,.Lfe64_to_epilogue # HandlerData[] |
| ___ |
| } |
| |
| $code =~ s/\`([^\`]*)\`/eval $1/gem; |
| print $code; |
| close STDOUT or die "error closing STDOUT: $!"; |