| # Copyright 2021-2022 The OpenSSL Project Authors. All Rights Reserved. |
| # Copyright (c) 2021, Intel Corporation. All Rights Reserved. |
| # |
| # Licensed under the Apache License 2.0 (the "License"). You may not use |
| # this file except in compliance with the License. You can obtain a copy |
| # in the file LICENSE in the source distribution or at |
| # https://www.openssl.org/source/license.html |
| # |
| # |
| # Originally written by Sergey Kirillov and Andrey Matyukov |
| # Intel Corporation |
| # |
| # March 2021 |
| # |
| # Initial release. |
| # |
| # Implementation utilizes 256-bit (ymm) registers to avoid frequency scaling issues. |
| # |
| # IceLake-Client @ 1.3GHz |
| # |---------+-----------------------+---------------+-------------| |
| # | | OpenSSL 3.0.0-alpha15 | this | Unit | |
| # |---------+-----------------------+---------------+-------------| |
| # | rsa4096 | 14 301 4300 | 5 813 953 | cycles/sign | |
| # | | 90.9 | 223.6 / +146% | sign/s | |
| # |---------+-----------------------+---------------+-------------| |
| # |
| |
| # $output is the last argument if it looks like a file (it has an extension) |
| # $flavour is the first argument if it doesn't look like a file |
| $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; |
| $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef; |
| |
| $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); |
| $avx512ifma=0; |
| |
| $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; |
| ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or |
| ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or |
| die "can't locate x86_64-xlate.pl"; |
| |
| if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` |
| =~ /GNU assembler version ([2-9]\.[0-9]+)/) { |
| $avx512ifma = ($1>=2.26); |
| } |
| |
| if (!$avx512 && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && |
| `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)(?:\.([0-9]+))?/) { |
| $avx512ifma = ($1==2.11 && $2>=8) + ($1>=2.12); |
| } |
| |
| if (!$avx512 && `$ENV{CC} -v 2>&1` =~ /((?:clang|LLVM) version|.*based on LLVM) ([0-9]+\.[0-9]+)/) { |
| $avx512ifma = ($2>=7.0); |
| } |
| |
| open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"" |
| or die "can't call $xlate: $!"; |
| *STDOUT=*OUT; |
| |
| if ($avx512ifma>0) {{{ |
| @_6_args_universal_ABI = ("%rdi","%rsi","%rdx","%rcx","%r8","%r9"); |
| |
| ############################################################################### |
| # Almost Montgomery Multiplication (AMM) for 40-digit number in radix 2^52. |
| # |
| # AMM is defined as presented in the paper [1]. |
| # |
| # The input and output are presented in 2^52 radix domain, i.e. |
| # |res|, |a|, |b|, |m| are arrays of 40 64-bit qwords with 12 high bits zeroed. |
| # |k0| is a Montgomery coefficient, which is here k0 = -1/m mod 2^64 |
| # |
| # NB: the AMM implementation does not perform "conditional" subtraction step |
| # specified in the original algorithm as according to the Lemma 1 from the paper |
| # [2], the result will be always < 2*m and can be used as a direct input to |
| # the next AMM iteration. This post-condition is true, provided the correct |
| # parameter |s| (notion of the Lemma 1 from [2]) is chosen, i.e. s >= n + 2 * k, |
| # which matches our case: 2080 > 2048 + 2 * 1. |
| # |
| # [1] Gueron, S. Efficient software implementations of modular exponentiation. |
| # DOI: 10.1007/s13389-012-0031-5 |
| # [2] Gueron, S. Enhanced Montgomery Multiplication. |
| # DOI: 10.1007/3-540-36400-5_5 |
| # |
| # void ossl_rsaz_amm52x40_x1_ifma256(BN_ULONG *res, |
| # const BN_ULONG *a, |
| # const BN_ULONG *b, |
| # const BN_ULONG *m, |
| # BN_ULONG k0); |
| ############################################################################### |
| { |
| # input parameters ("%rdi","%rsi","%rdx","%rcx","%r8") |
| my ($res,$a,$b,$m,$k0) = @_6_args_universal_ABI; |
| |
| my $mask52 = "%rax"; |
| my $acc0_0 = "%r9"; |
| my $acc0_0_low = "%r9d"; |
| my $acc0_1 = "%r15"; |
| my $acc0_1_low = "%r15d"; |
| my $b_ptr = "%r11"; |
| |
| my $iter = "%ebx"; |
| |
| my $zero = "%ymm0"; |
| my $Bi = "%ymm1"; |
| my $Yi = "%ymm2"; |
| my ($R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$R2_0h,$R3_0,$R3_0h,$R4_0,$R4_0h) = map("%ymm$_",(3..12)); |
| my ($R0_1,$R0_1h,$R1_1,$R1_1h,$R2_1,$R2_1h,$R3_1,$R3_1h,$R4_1,$R4_1h) = map("%ymm$_",(13..22)); |
| |
| # Registers mapping for normalization |
| my ($T0,$T0h,$T1,$T1h,$T2,$T2h,$T3,$T3h,$T4,$T4h) = ("$zero", "$Bi", "$Yi", map("%ymm$_", (23..29))); |
| |
| sub amm52x40_x1() { |
| # _data_offset - offset in the |a| or |m| arrays pointing to the beginning |
| # of data for corresponding AMM operation; |
| # _b_offset - offset in the |b| array pointing to the next qword digit; |
| my ($_data_offset,$_b_offset,$_acc,$_R0,$_R0h,$_R1,$_R1h,$_R2,$_R2h,$_R3,$_R3h,$_R4,$_R4h,$_k0) = @_; |
| my $_R0_xmm = $_R0; |
| $_R0_xmm =~ s/%y/%x/; |
| $code.=<<___; |
| movq $_b_offset($b_ptr), %r13 # b[i] |
| |
| vpbroadcastq %r13, $Bi # broadcast b[i] |
| movq $_data_offset($a), %rdx |
| mulx %r13, %r13, %r12 # a[0]*b[i] = (t0,t2) |
| addq %r13, $_acc # acc += t0 |
| movq %r12, %r10 |
| adcq \$0, %r10 # t2 += CF |
| |
| movq $_k0, %r13 |
| imulq $_acc, %r13 # acc * k0 |
| andq $mask52, %r13 # yi = (acc * k0) & mask52 |
| |
| vpbroadcastq %r13, $Yi # broadcast y[i] |
| movq $_data_offset($m), %rdx |
| mulx %r13, %r13, %r12 # yi * m[0] = (t0,t1) |
| addq %r13, $_acc # acc += t0 |
| adcq %r12, %r10 # t2 += (t1 + CF) |
| |
| shrq \$52, $_acc |
| salq \$12, %r10 |
| or %r10, $_acc # acc = ((acc >> 52) | (t2 << 12)) |
| |
| vpmadd52luq `$_data_offset+64*0`($a), $Bi, $_R0 |
| vpmadd52luq `$_data_offset+64*0+32`($a), $Bi, $_R0h |
| vpmadd52luq `$_data_offset+64*1`($a), $Bi, $_R1 |
| vpmadd52luq `$_data_offset+64*1+32`($a), $Bi, $_R1h |
| vpmadd52luq `$_data_offset+64*2`($a), $Bi, $_R2 |
| vpmadd52luq `$_data_offset+64*2+32`($a), $Bi, $_R2h |
| vpmadd52luq `$_data_offset+64*3`($a), $Bi, $_R3 |
| vpmadd52luq `$_data_offset+64*3+32`($a), $Bi, $_R3h |
| vpmadd52luq `$_data_offset+64*4`($a), $Bi, $_R4 |
| vpmadd52luq `$_data_offset+64*4+32`($a), $Bi, $_R4h |
| |
| vpmadd52luq `$_data_offset+64*0`($m), $Yi, $_R0 |
| vpmadd52luq `$_data_offset+64*0+32`($m), $Yi, $_R0h |
| vpmadd52luq `$_data_offset+64*1`($m), $Yi, $_R1 |
| vpmadd52luq `$_data_offset+64*1+32`($m), $Yi, $_R1h |
| vpmadd52luq `$_data_offset+64*2`($m), $Yi, $_R2 |
| vpmadd52luq `$_data_offset+64*2+32`($m), $Yi, $_R2h |
| vpmadd52luq `$_data_offset+64*3`($m), $Yi, $_R3 |
| vpmadd52luq `$_data_offset+64*3+32`($m), $Yi, $_R3h |
| vpmadd52luq `$_data_offset+64*4`($m), $Yi, $_R4 |
| vpmadd52luq `$_data_offset+64*4+32`($m), $Yi, $_R4h |
| |
| # Shift accumulators right by 1 qword, zero extending the highest one |
| valignq \$1, $_R0, $_R0h, $_R0 |
| valignq \$1, $_R0h, $_R1, $_R0h |
| valignq \$1, $_R1, $_R1h, $_R1 |
| valignq \$1, $_R1h, $_R2, $_R1h |
| valignq \$1, $_R2, $_R2h, $_R2 |
| valignq \$1, $_R2h, $_R3, $_R2h |
| valignq \$1, $_R3, $_R3h, $_R3 |
| valignq \$1, $_R3h, $_R4, $_R3h |
| valignq \$1, $_R4, $_R4h, $_R4 |
| valignq \$1, $_R4h, $zero, $_R4h |
| |
| vmovq $_R0_xmm, %r13 |
| addq %r13, $_acc # acc += R0[0] |
| |
| vpmadd52huq `$_data_offset+64*0`($a), $Bi, $_R0 |
| vpmadd52huq `$_data_offset+64*0+32`($a), $Bi, $_R0h |
| vpmadd52huq `$_data_offset+64*1`($a), $Bi, $_R1 |
| vpmadd52huq `$_data_offset+64*1+32`($a), $Bi, $_R1h |
| vpmadd52huq `$_data_offset+64*2`($a), $Bi, $_R2 |
| vpmadd52huq `$_data_offset+64*2+32`($a), $Bi, $_R2h |
| vpmadd52huq `$_data_offset+64*3`($a), $Bi, $_R3 |
| vpmadd52huq `$_data_offset+64*3+32`($a), $Bi, $_R3h |
| vpmadd52huq `$_data_offset+64*4`($a), $Bi, $_R4 |
| vpmadd52huq `$_data_offset+64*4+32`($a), $Bi, $_R4h |
| |
| vpmadd52huq `$_data_offset+64*0`($m), $Yi, $_R0 |
| vpmadd52huq `$_data_offset+64*0+32`($m), $Yi, $_R0h |
| vpmadd52huq `$_data_offset+64*1`($m), $Yi, $_R1 |
| vpmadd52huq `$_data_offset+64*1+32`($m), $Yi, $_R1h |
| vpmadd52huq `$_data_offset+64*2`($m), $Yi, $_R2 |
| vpmadd52huq `$_data_offset+64*2+32`($m), $Yi, $_R2h |
| vpmadd52huq `$_data_offset+64*3`($m), $Yi, $_R3 |
| vpmadd52huq `$_data_offset+64*3+32`($m), $Yi, $_R3h |
| vpmadd52huq `$_data_offset+64*4`($m), $Yi, $_R4 |
| vpmadd52huq `$_data_offset+64*4+32`($m), $Yi, $_R4h |
| ___ |
| } |
| |
| # Normalization routine: handles carry bits and gets bignum qwords to normalized |
| # 2^52 representation. |
| # |
| # Uses %r8-14,%e[abcd]x |
| sub amm52x40_x1_norm { |
| my ($_acc,$_R0,$_R0h,$_R1,$_R1h,$_R2,$_R2h,$_R3,$_R3h,$_R4,$_R4h) = @_; |
| $code.=<<___; |
| # Put accumulator to low qword in R0 |
| vpbroadcastq $_acc, $T0 |
| vpblendd \$3, $T0, $_R0, $_R0 |
| |
| # Extract "carries" (12 high bits) from each QW of the bignum |
| # Save them to LSB of QWs in T0..Tn |
| vpsrlq \$52, $_R0, $T0 |
| vpsrlq \$52, $_R0h, $T0h |
| vpsrlq \$52, $_R1, $T1 |
| vpsrlq \$52, $_R1h, $T1h |
| vpsrlq \$52, $_R2, $T2 |
| vpsrlq \$52, $_R2h, $T2h |
| vpsrlq \$52, $_R3, $T3 |
| vpsrlq \$52, $_R3h, $T3h |
| vpsrlq \$52, $_R4, $T4 |
| vpsrlq \$52, $_R4h, $T4h |
| |
| # "Shift left" T0..Tn by 1 QW |
| valignq \$3, $T4, $T4h, $T4h |
| valignq \$3, $T3h, $T4, $T4 |
| valignq \$3, $T3, $T3h, $T3h |
| valignq \$3, $T2h, $T3, $T3 |
| valignq \$3, $T2, $T2h, $T2h |
| valignq \$3, $T1h, $T2, $T2 |
| valignq \$3, $T1, $T1h, $T1h |
| valignq \$3, $T0h, $T1, $T1 |
| valignq \$3, $T0, $T0h, $T0h |
| valignq \$3, .Lzeros(%rip), $T0, $T0 |
| |
| # Drop "carries" from R0..Rn QWs |
| vpandq .Lmask52x4(%rip), $_R0, $_R0 |
| vpandq .Lmask52x4(%rip), $_R0h, $_R0h |
| vpandq .Lmask52x4(%rip), $_R1, $_R1 |
| vpandq .Lmask52x4(%rip), $_R1h, $_R1h |
| vpandq .Lmask52x4(%rip), $_R2, $_R2 |
| vpandq .Lmask52x4(%rip), $_R2h, $_R2h |
| vpandq .Lmask52x4(%rip), $_R3, $_R3 |
| vpandq .Lmask52x4(%rip), $_R3h, $_R3h |
| vpandq .Lmask52x4(%rip), $_R4, $_R4 |
| vpandq .Lmask52x4(%rip), $_R4h, $_R4h |
| |
| # Sum R0..Rn with corresponding adjusted carries |
| vpaddq $T0, $_R0, $_R0 |
| vpaddq $T0h, $_R0h, $_R0h |
| vpaddq $T1, $_R1, $_R1 |
| vpaddq $T1h, $_R1h, $_R1h |
| vpaddq $T2, $_R2, $_R2 |
| vpaddq $T2h, $_R2h, $_R2h |
| vpaddq $T3, $_R3, $_R3 |
| vpaddq $T3h, $_R3h, $_R3h |
| vpaddq $T4, $_R4, $_R4 |
| vpaddq $T4h, $_R4h, $_R4h |
| |
| # Now handle carry bits from this addition |
| # Get mask of QWs whose 52-bit parts overflow |
| vpcmpuq \$6,.Lmask52x4(%rip),${_R0},%k1 # OP=nle (i.e. gt) |
| vpcmpuq \$6,.Lmask52x4(%rip),${_R0h},%k2 |
| kmovb %k1,%r14d |
| kmovb %k2,%r13d |
| shl \$4,%r13b |
| or %r13b,%r14b |
| |
| vpcmpuq \$6,.Lmask52x4(%rip),${_R1},%k1 |
| vpcmpuq \$6,.Lmask52x4(%rip),${_R1h},%k2 |
| kmovb %k1,%r13d |
| kmovb %k2,%r12d |
| shl \$4,%r12b |
| or %r12b,%r13b |
| |
| vpcmpuq \$6,.Lmask52x4(%rip),${_R2},%k1 |
| vpcmpuq \$6,.Lmask52x4(%rip),${_R2h},%k2 |
| kmovb %k1,%r12d |
| kmovb %k2,%r11d |
| shl \$4,%r11b |
| or %r11b,%r12b |
| |
| vpcmpuq \$6,.Lmask52x4(%rip),${_R3},%k1 |
| vpcmpuq \$6,.Lmask52x4(%rip),${_R3h},%k2 |
| kmovb %k1,%r11d |
| kmovb %k2,%r10d |
| shl \$4,%r10b |
| or %r10b,%r11b |
| |
| vpcmpuq \$6,.Lmask52x4(%rip),${_R4},%k1 |
| vpcmpuq \$6,.Lmask52x4(%rip),${_R4h},%k2 |
| kmovb %k1,%r10d |
| kmovb %k2,%r9d |
| shl \$4,%r9b |
| or %r9b,%r10b |
| |
| addb %r14b,%r14b |
| adcb %r13b,%r13b |
| adcb %r12b,%r12b |
| adcb %r11b,%r11b |
| adcb %r10b,%r10b |
| |
| # Get mask of QWs whose 52-bit parts saturated |
| vpcmpuq \$0,.Lmask52x4(%rip),${_R0},%k1 # OP=eq |
| vpcmpuq \$0,.Lmask52x4(%rip),${_R0h},%k2 |
| kmovb %k1,%r9d |
| kmovb %k2,%r8d |
| shl \$4,%r8b |
| or %r8b,%r9b |
| |
| vpcmpuq \$0,.Lmask52x4(%rip),${_R1},%k1 |
| vpcmpuq \$0,.Lmask52x4(%rip),${_R1h},%k2 |
| kmovb %k1,%r8d |
| kmovb %k2,%edx |
| shl \$4,%dl |
| or %dl,%r8b |
| |
| vpcmpuq \$0,.Lmask52x4(%rip),${_R2},%k1 |
| vpcmpuq \$0,.Lmask52x4(%rip),${_R2h},%k2 |
| kmovb %k1,%edx |
| kmovb %k2,%ecx |
| shl \$4,%cl |
| or %cl,%dl |
| |
| vpcmpuq \$0,.Lmask52x4(%rip),${_R3},%k1 |
| vpcmpuq \$0,.Lmask52x4(%rip),${_R3h},%k2 |
| kmovb %k1,%ecx |
| kmovb %k2,%ebx |
| shl \$4,%bl |
| or %bl,%cl |
| |
| vpcmpuq \$0,.Lmask52x4(%rip),${_R4},%k1 |
| vpcmpuq \$0,.Lmask52x4(%rip),${_R4h},%k2 |
| kmovb %k1,%ebx |
| kmovb %k2,%eax |
| shl \$4,%al |
| or %al,%bl |
| |
| addb %r9b,%r14b |
| adcb %r8b,%r13b |
| adcb %dl,%r12b |
| adcb %cl,%r11b |
| adcb %bl,%r10b |
| |
| xor %r9b,%r14b |
| xor %r8b,%r13b |
| xor %dl,%r12b |
| xor %cl,%r11b |
| xor %bl,%r10b |
| |
| kmovb %r14d,%k1 |
| shr \$4,%r14b |
| kmovb %r14d,%k2 |
| kmovb %r13d,%k3 |
| shr \$4,%r13b |
| kmovb %r13d,%k4 |
| kmovb %r12d,%k5 |
| shr \$4,%r12b |
| kmovb %r12d,%k6 |
| kmovb %r11d,%k7 |
| |
| vpsubq .Lmask52x4(%rip), $_R0, ${_R0}{%k1} |
| vpsubq .Lmask52x4(%rip), $_R0h, ${_R0h}{%k2} |
| vpsubq .Lmask52x4(%rip), $_R1, ${_R1}{%k3} |
| vpsubq .Lmask52x4(%rip), $_R1h, ${_R1h}{%k4} |
| vpsubq .Lmask52x4(%rip), $_R2, ${_R2}{%k5} |
| vpsubq .Lmask52x4(%rip), $_R2h, ${_R2h}{%k6} |
| vpsubq .Lmask52x4(%rip), $_R3, ${_R3}{%k7} |
| |
| vpandq .Lmask52x4(%rip), $_R0, $_R0 |
| vpandq .Lmask52x4(%rip), $_R0h, $_R0h |
| vpandq .Lmask52x4(%rip), $_R1, $_R1 |
| vpandq .Lmask52x4(%rip), $_R1h, $_R1h |
| vpandq .Lmask52x4(%rip), $_R2, $_R2 |
| vpandq .Lmask52x4(%rip), $_R2h, $_R2h |
| vpandq .Lmask52x4(%rip), $_R3, $_R3 |
| |
| shr \$4,%r11b |
| kmovb %r11d,%k1 |
| kmovb %r10d,%k2 |
| shr \$4,%r10b |
| kmovb %r10d,%k3 |
| |
| vpsubq .Lmask52x4(%rip), $_R3h, ${_R3h}{%k1} |
| vpsubq .Lmask52x4(%rip), $_R4, ${_R4}{%k2} |
| vpsubq .Lmask52x4(%rip), $_R4h, ${_R4h}{%k3} |
| |
| vpandq .Lmask52x4(%rip), $_R3h, $_R3h |
| vpandq .Lmask52x4(%rip), $_R4, $_R4 |
| vpandq .Lmask52x4(%rip), $_R4h, $_R4h |
| ___ |
| } |
| |
| $code.=<<___; |
| .text |
| |
| .globl ossl_rsaz_amm52x40_x1_ifma256 |
| .type ossl_rsaz_amm52x40_x1_ifma256,\@function,5 |
| .align 32 |
| ossl_rsaz_amm52x40_x1_ifma256: |
| .cfi_startproc |
| endbranch |
| push %rbx |
| .cfi_push %rbx |
| push %rbp |
| .cfi_push %rbp |
| push %r12 |
| .cfi_push %r12 |
| push %r13 |
| .cfi_push %r13 |
| push %r14 |
| .cfi_push %r14 |
| push %r15 |
| .cfi_push %r15 |
| ___ |
| $code.=<<___ if ($win64); |
| lea -168(%rsp),%rsp # 16*10 + (8 bytes to get correct 16-byte SIMD alignment) |
| vmovdqa64 %xmm6, `0*16`(%rsp) # save non-volatile registers |
| vmovdqa64 %xmm7, `1*16`(%rsp) |
| vmovdqa64 %xmm8, `2*16`(%rsp) |
| vmovdqa64 %xmm9, `3*16`(%rsp) |
| vmovdqa64 %xmm10,`4*16`(%rsp) |
| vmovdqa64 %xmm11,`5*16`(%rsp) |
| vmovdqa64 %xmm12,`6*16`(%rsp) |
| vmovdqa64 %xmm13,`7*16`(%rsp) |
| vmovdqa64 %xmm14,`8*16`(%rsp) |
| vmovdqa64 %xmm15,`9*16`(%rsp) |
| .Lossl_rsaz_amm52x40_x1_ifma256_body: |
| ___ |
| $code.=<<___; |
| # Zeroing accumulators |
| vpxord $zero, $zero, $zero |
| vmovdqa64 $zero, $R0_0 |
| vmovdqa64 $zero, $R0_0h |
| vmovdqa64 $zero, $R1_0 |
| vmovdqa64 $zero, $R1_0h |
| vmovdqa64 $zero, $R2_0 |
| vmovdqa64 $zero, $R2_0h |
| vmovdqa64 $zero, $R3_0 |
| vmovdqa64 $zero, $R3_0h |
| vmovdqa64 $zero, $R4_0 |
| vmovdqa64 $zero, $R4_0h |
| |
| xorl $acc0_0_low, $acc0_0_low |
| |
| movq $b, $b_ptr # backup address of b |
| movq \$0xfffffffffffff, $mask52 # 52-bit mask |
| |
| # Loop over 40 digits unrolled by 4 |
| mov \$10, $iter |
| |
| .align 32 |
| .Lloop10: |
| ___ |
| foreach my $idx (0..3) { |
| &amm52x40_x1(0,8*$idx,$acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$R2_0h,$R3_0,$R3_0h,$R4_0,$R4_0h,$k0); |
| } |
| $code.=<<___; |
| lea `4*8`($b_ptr), $b_ptr |
| dec $iter |
| jne .Lloop10 |
| ___ |
| &amm52x40_x1_norm($acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$R2_0h,$R3_0,$R3_0h,$R4_0,$R4_0h); |
| $code.=<<___; |
| |
| vmovdqu64 $R0_0, `0*32`($res) |
| vmovdqu64 $R0_0h, `1*32`($res) |
| vmovdqu64 $R1_0, `2*32`($res) |
| vmovdqu64 $R1_0h, `3*32`($res) |
| vmovdqu64 $R2_0, `4*32`($res) |
| vmovdqu64 $R2_0h, `5*32`($res) |
| vmovdqu64 $R3_0, `6*32`($res) |
| vmovdqu64 $R3_0h, `7*32`($res) |
| vmovdqu64 $R4_0, `8*32`($res) |
| vmovdqu64 $R4_0h, `9*32`($res) |
| |
| vzeroupper |
| lea (%rsp),%rax |
| .cfi_def_cfa_register %rax |
| ___ |
| $code.=<<___ if ($win64); |
| vmovdqa64 `0*16`(%rax),%xmm6 |
| vmovdqa64 `1*16`(%rax),%xmm7 |
| vmovdqa64 `2*16`(%rax),%xmm8 |
| vmovdqa64 `3*16`(%rax),%xmm9 |
| vmovdqa64 `4*16`(%rax),%xmm10 |
| vmovdqa64 `5*16`(%rax),%xmm11 |
| vmovdqa64 `6*16`(%rax),%xmm12 |
| vmovdqa64 `7*16`(%rax),%xmm13 |
| vmovdqa64 `8*16`(%rax),%xmm14 |
| vmovdqa64 `9*16`(%rax),%xmm15 |
| lea 168(%rsp),%rax |
| ___ |
| $code.=<<___; |
| mov 0(%rax),%r15 |
| .cfi_restore %r15 |
| mov 8(%rax),%r14 |
| .cfi_restore %r14 |
| mov 16(%rax),%r13 |
| .cfi_restore %r13 |
| mov 24(%rax),%r12 |
| .cfi_restore %r12 |
| mov 32(%rax),%rbp |
| .cfi_restore %rbp |
| mov 40(%rax),%rbx |
| .cfi_restore %rbx |
| lea 48(%rax),%rsp # restore rsp |
| .cfi_def_cfa %rsp,8 |
| .Lossl_rsaz_amm52x40_x1_ifma256_epilogue: |
| |
| ret |
| .cfi_endproc |
| .size ossl_rsaz_amm52x40_x1_ifma256, .-ossl_rsaz_amm52x40_x1_ifma256 |
| ___ |
| |
| $code.=<<___; |
| .data |
| .align 32 |
| .Lmask52x4: |
| .quad 0xfffffffffffff |
| .quad 0xfffffffffffff |
| .quad 0xfffffffffffff |
| .quad 0xfffffffffffff |
| ___ |
| |
| ############################################################################### |
| # Dual Almost Montgomery Multiplication for 40-digit number in radix 2^52 |
| # |
| # See description of ossl_rsaz_amm52x40_x1_ifma256() above for details about Almost |
| # Montgomery Multiplication algorithm and function input parameters description. |
| # |
| # This function does two AMMs for two independent inputs, hence dual. |
| # |
| # void ossl_rsaz_amm52x40_x2_ifma256(BN_ULONG out[2][40], |
| # const BN_ULONG a[2][40], |
| # const BN_ULONG b[2][40], |
| # const BN_ULONG m[2][40], |
| # const BN_ULONG k0[2]); |
| ############################################################################### |
| |
| $code.=<<___; |
| .text |
| |
| .globl ossl_rsaz_amm52x40_x2_ifma256 |
| .type ossl_rsaz_amm52x40_x2_ifma256,\@function,5 |
| .align 32 |
| ossl_rsaz_amm52x40_x2_ifma256: |
| .cfi_startproc |
| endbranch |
| push %rbx |
| .cfi_push %rbx |
| push %rbp |
| .cfi_push %rbp |
| push %r12 |
| .cfi_push %r12 |
| push %r13 |
| .cfi_push %r13 |
| push %r14 |
| .cfi_push %r14 |
| push %r15 |
| .cfi_push %r15 |
| ___ |
| $code.=<<___ if ($win64); |
| lea -168(%rsp),%rsp |
| vmovdqa64 %xmm6, `0*16`(%rsp) # save non-volatile registers |
| vmovdqa64 %xmm7, `1*16`(%rsp) |
| vmovdqa64 %xmm8, `2*16`(%rsp) |
| vmovdqa64 %xmm9, `3*16`(%rsp) |
| vmovdqa64 %xmm10,`4*16`(%rsp) |
| vmovdqa64 %xmm11,`5*16`(%rsp) |
| vmovdqa64 %xmm12,`6*16`(%rsp) |
| vmovdqa64 %xmm13,`7*16`(%rsp) |
| vmovdqa64 %xmm14,`8*16`(%rsp) |
| vmovdqa64 %xmm15,`9*16`(%rsp) |
| .Lossl_rsaz_amm52x40_x2_ifma256_body: |
| ___ |
| $code.=<<___; |
| # Zeroing accumulators |
| vpxord $zero, $zero, $zero |
| vmovdqa64 $zero, $R0_0 |
| vmovdqa64 $zero, $R0_0h |
| vmovdqa64 $zero, $R1_0 |
| vmovdqa64 $zero, $R1_0h |
| vmovdqa64 $zero, $R2_0 |
| vmovdqa64 $zero, $R2_0h |
| vmovdqa64 $zero, $R3_0 |
| vmovdqa64 $zero, $R3_0h |
| vmovdqa64 $zero, $R4_0 |
| vmovdqa64 $zero, $R4_0h |
| |
| vmovdqa64 $zero, $R0_1 |
| vmovdqa64 $zero, $R0_1h |
| vmovdqa64 $zero, $R1_1 |
| vmovdqa64 $zero, $R1_1h |
| vmovdqa64 $zero, $R2_1 |
| vmovdqa64 $zero, $R2_1h |
| vmovdqa64 $zero, $R3_1 |
| vmovdqa64 $zero, $R3_1h |
| vmovdqa64 $zero, $R4_1 |
| vmovdqa64 $zero, $R4_1h |
| |
| |
| xorl $acc0_0_low, $acc0_0_low |
| xorl $acc0_1_low, $acc0_1_low |
| |
| movq $b, $b_ptr # backup address of b |
| movq \$0xfffffffffffff, $mask52 # 52-bit mask |
| |
| mov \$40, $iter |
| |
| .align 32 |
| .Lloop40: |
| ___ |
| &amm52x40_x1( 0, 0,$acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$R2_0h,$R3_0,$R3_0h,$R4_0,$R4_0h,"($k0)"); |
| # 40*8 = offset of the next dimension in two-dimension array |
| &amm52x40_x1(40*8,40*8,$acc0_1,$R0_1,$R0_1h,$R1_1,$R1_1h,$R2_1,$R2_1h,$R3_1,$R3_1h,$R4_1,$R4_1h,"8($k0)"); |
| $code.=<<___; |
| lea 8($b_ptr), $b_ptr |
| dec $iter |
| jne .Lloop40 |
| ___ |
| &amm52x40_x1_norm($acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$R2_0h,$R3_0,$R3_0h,$R4_0,$R4_0h); |
| &amm52x40_x1_norm($acc0_1,$R0_1,$R0_1h,$R1_1,$R1_1h,$R2_1,$R2_1h,$R3_1,$R3_1h,$R4_1,$R4_1h); |
| $code.=<<___; |
| |
| vmovdqu64 $R0_0, `0*32`($res) |
| vmovdqu64 $R0_0h, `1*32`($res) |
| vmovdqu64 $R1_0, `2*32`($res) |
| vmovdqu64 $R1_0h, `3*32`($res) |
| vmovdqu64 $R2_0, `4*32`($res) |
| vmovdqu64 $R2_0h, `5*32`($res) |
| vmovdqu64 $R3_0, `6*32`($res) |
| vmovdqu64 $R3_0h, `7*32`($res) |
| vmovdqu64 $R4_0, `8*32`($res) |
| vmovdqu64 $R4_0h, `9*32`($res) |
| |
| vmovdqu64 $R0_1, `10*32`($res) |
| vmovdqu64 $R0_1h, `11*32`($res) |
| vmovdqu64 $R1_1, `12*32`($res) |
| vmovdqu64 $R1_1h, `13*32`($res) |
| vmovdqu64 $R2_1, `14*32`($res) |
| vmovdqu64 $R2_1h, `15*32`($res) |
| vmovdqu64 $R3_1, `16*32`($res) |
| vmovdqu64 $R3_1h, `17*32`($res) |
| vmovdqu64 $R4_1, `18*32`($res) |
| vmovdqu64 $R4_1h, `19*32`($res) |
| |
| vzeroupper |
| lea (%rsp),%rax |
| .cfi_def_cfa_register %rax |
| ___ |
| $code.=<<___ if ($win64); |
| vmovdqa64 `0*16`(%rax),%xmm6 |
| vmovdqa64 `1*16`(%rax),%xmm7 |
| vmovdqa64 `2*16`(%rax),%xmm8 |
| vmovdqa64 `3*16`(%rax),%xmm9 |
| vmovdqa64 `4*16`(%rax),%xmm10 |
| vmovdqa64 `5*16`(%rax),%xmm11 |
| vmovdqa64 `6*16`(%rax),%xmm12 |
| vmovdqa64 `7*16`(%rax),%xmm13 |
| vmovdqa64 `8*16`(%rax),%xmm14 |
| vmovdqa64 `9*16`(%rax),%xmm15 |
| lea 168(%rsp),%rax |
| ___ |
| $code.=<<___; |
| mov 0(%rax),%r15 |
| .cfi_restore %r15 |
| mov 8(%rax),%r14 |
| .cfi_restore %r14 |
| mov 16(%rax),%r13 |
| .cfi_restore %r13 |
| mov 24(%rax),%r12 |
| .cfi_restore %r12 |
| mov 32(%rax),%rbp |
| .cfi_restore %rbp |
| mov 40(%rax),%rbx |
| .cfi_restore %rbx |
| lea 48(%rax),%rsp |
| .cfi_def_cfa %rsp,8 |
| .Lossl_rsaz_amm52x40_x2_ifma256_epilogue: |
| ret |
| .cfi_endproc |
| .size ossl_rsaz_amm52x40_x2_ifma256, .-ossl_rsaz_amm52x40_x2_ifma256 |
| ___ |
| } |
| |
| ############################################################################### |
| # Constant time extraction from the precomputed table of powers base^i, where |
| # i = 0..2^EXP_WIN_SIZE-1 |
| # |
| # The input |red_table| contains precomputations for two independent base values. |
| # |red_table_idx1| and |red_table_idx2| are corresponding power indexes. |
| # |
| # Extracted value (output) is 2 40 digits numbers in 2^52 radix. |
| # |
| # void ossl_extract_multiplier_2x40_win5(BN_ULONG *red_Y, |
| # const BN_ULONG red_table[1 << EXP_WIN_SIZE][2][40], |
| # int red_table_idx1, int red_table_idx2); |
| # |
| # EXP_WIN_SIZE = 5 |
| ############################################################################### |
| { |
| # input parameters |
| my ($out,$red_tbl,$red_tbl_idx1,$red_tbl_idx2)=$win64 ? ("%rcx","%rdx","%r8", "%r9") : # Win64 order |
| ("%rdi","%rsi","%rdx","%rcx"); # Unix order |
| |
| my ($t0,$t1,$t2,$t3,$t4,$t5) = map("%ymm$_", (0..5)); |
| my ($t6,$t7,$t8,$t9) = map("%ymm$_", (16..19)); |
| my ($tmp,$cur_idx,$idx1,$idx2,$ones) = map("%ymm$_", (20..24)); |
| |
| my @t = ($t0,$t1,$t2,$t3,$t4,$t5,$t6,$t7,$t8,$t9); |
| my $t0xmm = $t0; |
| $t0xmm =~ s/%y/%x/; |
| |
| sub get_table_value_consttime() { |
| my ($_idx,$_offset) = @_; |
| $code.=<<___; |
| vpxorq $cur_idx, $cur_idx, $cur_idx |
| .align 32 |
| .Lloop_$_offset: |
| vpcmpq \$0, $cur_idx, $_idx, %k1 # mask of (idx == cur_idx) |
| ___ |
| foreach (0..9) { |
| $code.=<<___; |
| vmovdqu64 `$_offset+${_}*32`($red_tbl), $tmp # load data from red_tbl |
| vpblendmq $tmp, $t[$_], ${t[$_]}{%k1} # extract data when mask is not zero |
| ___ |
| } |
| $code.=<<___; |
| vpaddq $ones, $cur_idx, $cur_idx # increment cur_idx |
| addq \$`2*40*8`, $red_tbl |
| cmpq $red_tbl, %rax |
| jne .Lloop_$_offset |
| ___ |
| } |
| |
| $code.=<<___; |
| .text |
| |
| .align 32 |
| .globl ossl_extract_multiplier_2x40_win5 |
| .type ossl_extract_multiplier_2x40_win5,\@abi-omnipotent |
| ossl_extract_multiplier_2x40_win5: |
| .cfi_startproc |
| endbranch |
| vmovdqa64 .Lones(%rip), $ones # broadcast ones |
| vpbroadcastq $red_tbl_idx1, $idx1 |
| vpbroadcastq $red_tbl_idx2, $idx2 |
| leaq `(1<<5)*2*40*8`($red_tbl), %rax # holds end of the tbl |
| |
| # backup red_tbl address |
| movq $red_tbl, %r10 |
| |
| # zeroing t0..n, cur_idx |
| vpxor $t0xmm, $t0xmm, $t0xmm |
| ___ |
| foreach (1..9) { |
| $code.="vmovdqa64 $t0, $t[$_] \n"; |
| } |
| |
| &get_table_value_consttime($idx1, 0); |
| foreach (0..9) { |
| $code.="vmovdqu64 $t[$_], `(0+$_)*32`($out) \n"; |
| } |
| $code.="movq %r10, $red_tbl \n"; |
| &get_table_value_consttime($idx2, 40*8); |
| foreach (0..9) { |
| $code.="vmovdqu64 $t[$_], `(10+$_)*32`($out) \n"; |
| } |
| $code.=<<___; |
| |
| ret |
| .cfi_endproc |
| .size ossl_extract_multiplier_2x40_win5, .-ossl_extract_multiplier_2x40_win5 |
| ___ |
| $code.=<<___; |
| .data |
| .align 32 |
| .Lones: |
| .quad 1,1,1,1 |
| .Lzeros: |
| .quad 0,0,0,0 |
| ___ |
| } |
| |
| if ($win64) { |
| $rec="%rcx"; |
| $frame="%rdx"; |
| $context="%r8"; |
| $disp="%r9"; |
| |
| $code.=<<___; |
| .extern __imp_RtlVirtualUnwind |
| .type rsaz_avx_handler,\@abi-omnipotent |
| .align 16 |
| rsaz_avx_handler: |
| push %rsi |
| push %rdi |
| push %rbx |
| push %rbp |
| push %r12 |
| push %r13 |
| push %r14 |
| push %r15 |
| pushfq |
| sub \$64,%rsp |
| |
| mov 120($context),%rax # pull context->Rax |
| mov 248($context),%rbx # pull context->Rip |
| |
| mov 8($disp),%rsi # disp->ImageBase |
| mov 56($disp),%r11 # disp->HandlerData |
| |
| mov 0(%r11),%r10d # HandlerData[0] |
| lea (%rsi,%r10),%r10 # prologue label |
| cmp %r10,%rbx # context->Rip<.Lprologue |
| jb .Lcommon_seh_tail |
| |
| mov 4(%r11),%r10d # HandlerData[1] |
| lea (%rsi,%r10),%r10 # epilogue label |
| cmp %r10,%rbx # context->Rip>=.Lepilogue |
| jae .Lcommon_seh_tail |
| |
| mov 152($context),%rax # pull context->Rsp |
| |
| lea (%rax),%rsi # %xmm save area |
| lea 512($context),%rdi # & context.Xmm6 |
| mov \$20,%ecx # 10*sizeof(%xmm0)/sizeof(%rax) |
| .long 0xa548f3fc # cld; rep movsq |
| |
| lea `48+168`(%rax),%rax |
| |
| mov -8(%rax),%rbx |
| mov -16(%rax),%rbp |
| mov -24(%rax),%r12 |
| mov -32(%rax),%r13 |
| mov -40(%rax),%r14 |
| mov -48(%rax),%r15 |
| mov %rbx,144($context) # restore context->Rbx |
| mov %rbp,160($context) # restore context->Rbp |
| mov %r12,216($context) # restore context->R12 |
| mov %r13,224($context) # restore context->R13 |
| mov %r14,232($context) # restore context->R14 |
| mov %r15,240($context) # restore context->R14 |
| |
| .Lcommon_seh_tail: |
| mov 8(%rax),%rdi |
| mov 16(%rax),%rsi |
| mov %rax,152($context) # restore context->Rsp |
| mov %rsi,168($context) # restore context->Rsi |
| mov %rdi,176($context) # restore context->Rdi |
| |
| mov 40($disp),%rdi # disp->ContextRecord |
| mov $context,%rsi # context |
| mov \$154,%ecx # sizeof(CONTEXT) |
| .long 0xa548f3fc # cld; rep movsq |
| |
| mov $disp,%rsi |
| xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER |
| mov 8(%rsi),%rdx # arg2, disp->ImageBase |
| mov 0(%rsi),%r8 # arg3, disp->ControlPc |
| mov 16(%rsi),%r9 # arg4, disp->FunctionEntry |
| mov 40(%rsi),%r10 # disp->ContextRecord |
| lea 56(%rsi),%r11 # &disp->HandlerData |
| lea 24(%rsi),%r12 # &disp->EstablisherFrame |
| mov %r10,32(%rsp) # arg5 |
| mov %r11,40(%rsp) # arg6 |
| mov %r12,48(%rsp) # arg7 |
| mov %rcx,56(%rsp) # arg8, (NULL) |
| call *__imp_RtlVirtualUnwind(%rip) |
| |
| mov \$1,%eax # ExceptionContinueSearch |
| add \$64,%rsp |
| popfq |
| pop %r15 |
| pop %r14 |
| pop %r13 |
| pop %r12 |
| pop %rbp |
| pop %rbx |
| pop %rdi |
| pop %rsi |
| ret |
| .size rsaz_avx_handler,.-rsaz_avx_handler |
| |
| .section .pdata |
| .align 4 |
| .rva .LSEH_begin_ossl_rsaz_amm52x40_x1_ifma256 |
| .rva .LSEH_end_ossl_rsaz_amm52x40_x1_ifma256 |
| .rva .LSEH_info_ossl_rsaz_amm52x40_x1_ifma256 |
| |
| .rva .LSEH_begin_ossl_rsaz_amm52x40_x2_ifma256 |
| .rva .LSEH_end_ossl_rsaz_amm52x40_x2_ifma256 |
| .rva .LSEH_info_ossl_rsaz_amm52x40_x2_ifma256 |
| |
| .section .xdata |
| .align 8 |
| .LSEH_info_ossl_rsaz_amm52x40_x1_ifma256: |
| .byte 9,0,0,0 |
| .rva rsaz_avx_handler |
| .rva .Lossl_rsaz_amm52x40_x1_ifma256_body,.Lossl_rsaz_amm52x40_x1_ifma256_epilogue |
| .LSEH_info_ossl_rsaz_amm52x40_x2_ifma256: |
| .byte 9,0,0,0 |
| .rva rsaz_avx_handler |
| .rva .Lossl_rsaz_amm52x40_x2_ifma256_body,.Lossl_rsaz_amm52x40_x2_ifma256_epilogue |
| ___ |
| } |
| }}} else {{{ # fallback for old assembler |
| $code.=<<___; |
| .text |
| |
| .globl ossl_rsaz_amm52x40_x1_ifma256 |
| .globl ossl_rsaz_amm52x40_x2_ifma256 |
| .globl ossl_extract_multiplier_2x40_win5 |
| .type ossl_rsaz_amm52x40_x1_ifma256,\@abi-omnipotent |
| ossl_rsaz_amm52x40_x1_ifma256: |
| ossl_rsaz_amm52x40_x2_ifma256: |
| ossl_extract_multiplier_2x40_win5: |
| .byte 0x0f,0x0b # ud2 |
| ret |
| .size ossl_rsaz_amm52x40_x1_ifma256, .-ossl_rsaz_amm52x40_x1_ifma256 |
| ___ |
| }}} |
| |
| $code =~ s/\`([^\`]*)\`/eval $1/gem; |
| print $code; |
| close STDOUT or die "error closing STDOUT: $!"; |
