| #! /usr/bin/env perl |
| # Copyright 2013-2020 The OpenSSL Project Authors. All Rights Reserved. |
| # |
| # Licensed under the Apache License 2.0 (the "License"). You may not use |
| # this file except in compliance with the License. You can obtain a copy |
| # in the file LICENSE in the source distribution or at |
| # https://www.openssl.org/source/license.html |
| |
| |
| # ==================================================================== |
| # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL |
| # project. The module is, however, dual licensed under OpenSSL and |
| # CRYPTOGAMS licenses depending on where you obtain it. For further |
| # details see http://www.openssl.org/~appro/cryptogams/. |
| # ==================================================================== |
| |
| # Multi-buffer SHA1 procedure processes n buffers in parallel by |
| # placing buffer data to designated lane of SIMD register. n is |
| # naturally limited to 4 on pre-AVX2 processors and to 8 on |
| # AVX2-capable processors such as Haswell. |
| # |
| # this +aesni(i) sha1 aesni-sha1 gain(iv) |
| # ------------------------------------------------------------------- |
| # Westmere(ii) 10.7/n +1.28=3.96(n=4) 5.30 6.66 +68% |
| # Atom(ii) 18.1/n +3.93=8.46(n=4) 9.37 12.8 +51% |
| # Sandy Bridge (8.16 +5.15=13.3)/n 4.99 5.98 +80% |
| # Ivy Bridge (8.08 +5.14=13.2)/n 4.60 5.54 +68% |
| # Haswell(iii) (8.96 +5.00=14.0)/n 3.57 4.55 +160% |
| # Skylake (8.70 +5.00=13.7)/n 3.64 4.20 +145% |
| # Bulldozer (9.76 +5.76=15.5)/n 5.95 6.37 +64% |
| # |
| # (i) multi-block CBC encrypt with 128-bit key; |
| # (ii) (HASH+AES)/n does not apply to Westmere for n>3 and Atom, |
| # because of lower AES-NI instruction throughput; |
| # (iii) "this" is for n=8, when we gather twice as much data, result |
| # for n=4 is 8.00+4.44=12.4; |
| # (iv) presented improvement coefficients are asymptotic limits and |
| # in real-life application are somewhat lower, e.g. for 2KB |
| # fragments they range from 30% to 100% (on Haswell); |
| |
| # $output is the last argument if it looks like a file (it has an extension) |
| # $flavour is the first argument if it doesn't look like a file |
| $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; |
| $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef; |
| |
| $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); |
| |
| $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; |
| ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or |
| ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or |
| die "can't locate x86_64-xlate.pl"; |
| |
| push(@INC,"${dir}","${dir}../../perlasm"); |
| require "x86_64-support.pl"; |
| |
| $ptr_size=&pointer_size($flavour); |
| |
| $avx=0; |
| |
| if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` |
| =~ /GNU assembler version ([2-9]\.[0-9]+)/) { |
| $avx = ($1>=2.19) + ($1>=2.22); |
| } |
| |
| if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && |
| `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) { |
| $avx = ($1>=2.09) + ($1>=2.10); |
| } |
| |
| if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) && |
| `ml64 2>&1` =~ /Version ([0-9]+)\./) { |
| $avx = ($1>=10) + ($1>=11); |
| } |
| |
| if (!$avx && `$ENV{CC} -v 2>&1` =~ /((?:clang|LLVM) version|.*based on LLVM) ([0-9]+\.[0-9]+)/) { |
| $avx = ($2>=3.0) + ($2>3.0); |
| } |
| |
| open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"" |
| or die "can't call $xlate: $!"; |
| *STDOUT=*OUT; |
| |
| # void sha1_multi_block ( |
| # struct { unsigned int A[8]; |
| # unsigned int B[8]; |
| # unsigned int C[8]; |
| # unsigned int D[8]; |
| # unsigned int E[8]; } *ctx, |
| # struct { void *ptr; int blocks; } inp[8], |
| # int num); /* 1 or 2 */ |
| # |
| $ctx="%rdi"; # 1st arg |
| $inp="%rsi"; # 2nd arg |
| $num="%edx"; |
| @ptr=map("%r$_",(8..11)); |
| $Tbl="%rbp"; |
| $inp_elm_size=2*$ptr_size; |
| |
| @V=($A,$B,$C,$D,$E)=map("%xmm$_",(0..4)); |
| ($t0,$t1,$t2,$t3,$tx)=map("%xmm$_",(5..9)); |
| @Xi=map("%xmm$_",(10..14)); |
| $K="%xmm15"; |
| |
| if (1) { |
| # Atom-specific optimization aiming to eliminate pshufb with high |
| # registers [and thus get rid of 48 cycles accumulated penalty] |
| @Xi=map("%xmm$_",(0..4)); |
| ($tx,$t0,$t1,$t2,$t3)=map("%xmm$_",(5..9)); |
| @V=($A,$B,$C,$D,$E)=map("%xmm$_",(10..14)); |
| } |
| |
| $REG_SZ=16; |
| |
| sub Xi_off { |
| my $off = shift; |
| |
| $off %= 16; $off *= $REG_SZ; |
| $off<256 ? "$off-128(%rax)" : "$off-256-128(%rbx)"; |
| } |
| |
| sub BODY_00_19 { |
| my ($i,$a,$b,$c,$d,$e)=@_; |
| my $j=$i+1; |
| my $k=$i+2; |
| |
| # Loads are performed 2+3/4 iterations in advance. 3/4 means that out |
| # of 4 words you would expect to be loaded per given iteration one is |
| # spilled to next iteration. In other words indices in four input |
| # streams are distributed as following: |
| # |
| # $i==0: 0,0,0,0,1,1,1,1,2,2,2, |
| # $i==1: 2,3,3,3, |
| # $i==2: 3,4,4,4, |
| # ... |
| # $i==13: 14,15,15,15, |
| # $i==14: 15 |
| # |
| # Then at $i==15 Xupdate is applied one iteration in advance... |
| $code.=<<___ if ($i==0); |
| movd (@ptr[0]),@Xi[0] |
| lea `16*4`(@ptr[0]),@ptr[0] |
| movd (@ptr[1]),@Xi[2] # borrow @Xi[2] |
| lea `16*4`(@ptr[1]),@ptr[1] |
| movd (@ptr[2]),@Xi[3] # borrow @Xi[3] |
| lea `16*4`(@ptr[2]),@ptr[2] |
| movd (@ptr[3]),@Xi[4] # borrow @Xi[4] |
| lea `16*4`(@ptr[3]),@ptr[3] |
| punpckldq @Xi[3],@Xi[0] |
| movd `4*$j-16*4`(@ptr[0]),@Xi[1] |
| punpckldq @Xi[4],@Xi[2] |
| movd `4*$j-16*4`(@ptr[1]),$t3 |
| punpckldq @Xi[2],@Xi[0] |
| movd `4*$j-16*4`(@ptr[2]),$t2 |
| pshufb $tx,@Xi[0] |
| ___ |
| $code.=<<___ if ($i<14); # just load input |
| movd `4*$j-16*4`(@ptr[3]),$t1 |
| punpckldq $t2,@Xi[1] |
| movdqa $a,$t2 |
| paddd $K,$e # e+=K_00_19 |
| punpckldq $t1,$t3 |
| movdqa $b,$t1 |
| movdqa $b,$t0 |
| pslld \$5,$t2 |
| pandn $d,$t1 |
| pand $c,$t0 |
| punpckldq $t3,@Xi[1] |
| movdqa $a,$t3 |
| |
| movdqa @Xi[0],`&Xi_off($i)` |
| paddd @Xi[0],$e # e+=X[i] |
| movd `4*$k-16*4`(@ptr[0]),@Xi[2] |
| psrld \$27,$t3 |
| pxor $t1,$t0 # Ch(b,c,d) |
| movdqa $b,$t1 |
| |
| por $t3,$t2 # rol(a,5) |
| movd `4*$k-16*4`(@ptr[1]),$t3 |
| pslld \$30,$t1 |
| paddd $t0,$e # e+=Ch(b,c,d) |
| |
| psrld \$2,$b |
| paddd $t2,$e # e+=rol(a,5) |
| pshufb $tx,@Xi[1] |
| movd `4*$k-16*4`(@ptr[2]),$t2 |
| por $t1,$b # b=rol(b,30) |
| ___ |
| $code.=<<___ if ($i==14); # just load input |
| movd `4*$j-16*4`(@ptr[3]),$t1 |
| punpckldq $t2,@Xi[1] |
| movdqa $a,$t2 |
| paddd $K,$e # e+=K_00_19 |
| punpckldq $t1,$t3 |
| movdqa $b,$t1 |
| movdqa $b,$t0 |
| pslld \$5,$t2 |
| prefetcht0 63(@ptr[0]) |
| pandn $d,$t1 |
| pand $c,$t0 |
| punpckldq $t3,@Xi[1] |
| movdqa $a,$t3 |
| |
| movdqa @Xi[0],`&Xi_off($i)` |
| paddd @Xi[0],$e # e+=X[i] |
| psrld \$27,$t3 |
| pxor $t1,$t0 # Ch(b,c,d) |
| movdqa $b,$t1 |
| prefetcht0 63(@ptr[1]) |
| |
| por $t3,$t2 # rol(a,5) |
| pslld \$30,$t1 |
| paddd $t0,$e # e+=Ch(b,c,d) |
| prefetcht0 63(@ptr[2]) |
| |
| psrld \$2,$b |
| paddd $t2,$e # e+=rol(a,5) |
| pshufb $tx,@Xi[1] |
| prefetcht0 63(@ptr[3]) |
| por $t1,$b # b=rol(b,30) |
| ___ |
| $code.=<<___ if ($i>=13 && $i<15); |
| movdqa `&Xi_off($j+2)`,@Xi[3] # preload "X[2]" |
| ___ |
| $code.=<<___ if ($i>=15); # apply Xupdate |
| pxor @Xi[-2],@Xi[1] # "X[13]" |
| movdqa `&Xi_off($j+2)`,@Xi[3] # "X[2]" |
| |
| movdqa $a,$t2 |
| pxor `&Xi_off($j+8)`,@Xi[1] |
| paddd $K,$e # e+=K_00_19 |
| movdqa $b,$t1 |
| pslld \$5,$t2 |
| pxor @Xi[3],@Xi[1] |
| movdqa $b,$t0 |
| pandn $d,$t1 |
| movdqa @Xi[1],$tx |
| pand $c,$t0 |
| movdqa $a,$t3 |
| psrld \$31,$tx |
| paddd @Xi[1],@Xi[1] |
| |
| movdqa @Xi[0],`&Xi_off($i)` |
| paddd @Xi[0],$e # e+=X[i] |
| psrld \$27,$t3 |
| pxor $t1,$t0 # Ch(b,c,d) |
| |
| movdqa $b,$t1 |
| por $t3,$t2 # rol(a,5) |
| pslld \$30,$t1 |
| paddd $t0,$e # e+=Ch(b,c,d) |
| |
| psrld \$2,$b |
| paddd $t2,$e # e+=rol(a,5) |
| por $tx,@Xi[1] # rol \$1,@Xi[1] |
| por $t1,$b # b=rol(b,30) |
| ___ |
| push(@Xi,shift(@Xi)); |
| } |
| |
| sub BODY_20_39 { |
| my ($i,$a,$b,$c,$d,$e)=@_; |
| my $j=$i+1; |
| |
| $code.=<<___ if ($i<79); |
| pxor @Xi[-2],@Xi[1] # "X[13]" |
| movdqa `&Xi_off($j+2)`,@Xi[3] # "X[2]" |
| |
| movdqa $a,$t2 |
| movdqa $d,$t0 |
| pxor `&Xi_off($j+8)`,@Xi[1] |
| paddd $K,$e # e+=K_20_39 |
| pslld \$5,$t2 |
| pxor $b,$t0 |
| |
| movdqa $a,$t3 |
| ___ |
| $code.=<<___ if ($i<72); |
| movdqa @Xi[0],`&Xi_off($i)` |
| ___ |
| $code.=<<___ if ($i<79); |
| paddd @Xi[0],$e # e+=X[i] |
| pxor @Xi[3],@Xi[1] |
| psrld \$27,$t3 |
| pxor $c,$t0 # Parity(b,c,d) |
| movdqa $b,$t1 |
| |
| pslld \$30,$t1 |
| movdqa @Xi[1],$tx |
| por $t3,$t2 # rol(a,5) |
| psrld \$31,$tx |
| paddd $t0,$e # e+=Parity(b,c,d) |
| paddd @Xi[1],@Xi[1] |
| |
| psrld \$2,$b |
| paddd $t2,$e # e+=rol(a,5) |
| por $tx,@Xi[1] # rol(@Xi[1],1) |
| por $t1,$b # b=rol(b,30) |
| ___ |
| $code.=<<___ if ($i==79); |
| movdqa $a,$t2 |
| paddd $K,$e # e+=K_20_39 |
| movdqa $d,$t0 |
| pslld \$5,$t2 |
| pxor $b,$t0 |
| |
| movdqa $a,$t3 |
| paddd @Xi[0],$e # e+=X[i] |
| psrld \$27,$t3 |
| movdqa $b,$t1 |
| pxor $c,$t0 # Parity(b,c,d) |
| |
| pslld \$30,$t1 |
| por $t3,$t2 # rol(a,5) |
| paddd $t0,$e # e+=Parity(b,c,d) |
| |
| psrld \$2,$b |
| paddd $t2,$e # e+=rol(a,5) |
| por $t1,$b # b=rol(b,30) |
| ___ |
| push(@Xi,shift(@Xi)); |
| } |
| |
| sub BODY_40_59 { |
| my ($i,$a,$b,$c,$d,$e)=@_; |
| my $j=$i+1; |
| |
| $code.=<<___; |
| pxor @Xi[-2],@Xi[1] # "X[13]" |
| movdqa `&Xi_off($j+2)`,@Xi[3] # "X[2]" |
| |
| movdqa $a,$t2 |
| movdqa $d,$t1 |
| pxor `&Xi_off($j+8)`,@Xi[1] |
| pxor @Xi[3],@Xi[1] |
| paddd $K,$e # e+=K_40_59 |
| pslld \$5,$t2 |
| movdqa $a,$t3 |
| pand $c,$t1 |
| |
| movdqa $d,$t0 |
| movdqa @Xi[1],$tx |
| psrld \$27,$t3 |
| paddd $t1,$e |
| pxor $c,$t0 |
| |
| movdqa @Xi[0],`&Xi_off($i)` |
| paddd @Xi[0],$e # e+=X[i] |
| por $t3,$t2 # rol(a,5) |
| psrld \$31,$tx |
| pand $b,$t0 |
| movdqa $b,$t1 |
| |
| pslld \$30,$t1 |
| paddd @Xi[1],@Xi[1] |
| paddd $t0,$e # e+=Maj(b,d,c) |
| |
| psrld \$2,$b |
| paddd $t2,$e # e+=rol(a,5) |
| por $tx,@Xi[1] # rol(@X[1],1) |
| por $t1,$b # b=rol(b,30) |
| ___ |
| push(@Xi,shift(@Xi)); |
| } |
| |
| $code.=<<___; |
| .text |
| |
| .extern OPENSSL_ia32cap_P |
| |
| .globl sha1_multi_block |
| .type sha1_multi_block,\@function,3 |
| .align 32 |
| sha1_multi_block: |
| .cfi_startproc |
| mov OPENSSL_ia32cap_P+4(%rip),%rcx |
| bt \$61,%rcx # check SHA bit |
| jc _shaext_shortcut |
| ___ |
| $code.=<<___ if ($avx); |
| test \$`1<<28`,%ecx |
| jnz _avx_shortcut |
| ___ |
| $code.=<<___; |
| mov %rsp,%rax |
| .cfi_def_cfa_register %rax |
| push %rbx |
| .cfi_push %rbx |
| push %rbp |
| .cfi_push %rbx |
| ___ |
| $code.=<<___ if ($win64); |
| lea -0xa8(%rsp),%rsp |
| movaps %xmm6,(%rsp) |
| movaps %xmm7,0x10(%rsp) |
| movaps %xmm8,0x20(%rsp) |
| movaps %xmm9,0x30(%rsp) |
| movaps %xmm10,-0x78(%rax) |
| movaps %xmm11,-0x68(%rax) |
| movaps %xmm12,-0x58(%rax) |
| movaps %xmm13,-0x48(%rax) |
| movaps %xmm14,-0x38(%rax) |
| movaps %xmm15,-0x28(%rax) |
| ___ |
| $code.=<<___; |
| sub \$`$REG_SZ*18`,%rsp |
| and \$-256,%rsp |
| mov %rax,`$REG_SZ*17`(%rsp) # original %rsp |
| .cfi_cfa_expression %rsp+`$REG_SZ*17`,deref,+8 |
| .Lbody: |
| lea K_XX_XX(%rip),$Tbl |
| lea `$REG_SZ*16`(%rsp),%rbx |
| |
| .Loop_grande: |
| mov $num,`$REG_SZ*17+8`(%rsp) # original $num |
| xor $num,$num |
| ___ |
| for($i=0;$i<4;$i++) { |
| $ptr_reg=&pointer_register($flavour,@ptr[$i]); |
| $code.=<<___; |
| # input pointer |
| mov `$inp_elm_size*$i+0`($inp),$ptr_reg |
| # number of blocks |
| mov `$inp_elm_size*$i+$ptr_size`($inp),%ecx |
| cmp $num,%ecx |
| cmovg %ecx,$num # find maximum |
| test %ecx,%ecx |
| mov %ecx,`4*$i`(%rbx) # initialize counters |
| cmovle $Tbl,@ptr[$i] # cancel input |
| ___ |
| } |
| $code.=<<___; |
| test $num,$num |
| jz .Ldone |
| |
| movdqu 0x00($ctx),$A # load context |
| lea 128(%rsp),%rax |
| movdqu 0x20($ctx),$B |
| movdqu 0x40($ctx),$C |
| movdqu 0x60($ctx),$D |
| movdqu 0x80($ctx),$E |
| movdqa 0x60($Tbl),$tx # pbswap_mask |
| movdqa -0x20($Tbl),$K # K_00_19 |
| jmp .Loop |
| |
| .align 32 |
| .Loop: |
| ___ |
| for($i=0;$i<20;$i++) { &BODY_00_19($i,@V); unshift(@V,pop(@V)); } |
| $code.=" movdqa 0x00($Tbl),$K\n"; # K_20_39 |
| for(;$i<40;$i++) { &BODY_20_39($i,@V); unshift(@V,pop(@V)); } |
| $code.=" movdqa 0x20($Tbl),$K\n"; # K_40_59 |
| for(;$i<60;$i++) { &BODY_40_59($i,@V); unshift(@V,pop(@V)); } |
| $code.=" movdqa 0x40($Tbl),$K\n"; # K_60_79 |
| for(;$i<80;$i++) { &BODY_20_39($i,@V); unshift(@V,pop(@V)); } |
| $code.=<<___; |
| movdqa (%rbx),@Xi[0] # pull counters |
| mov \$1,%ecx |
| cmp 4*0(%rbx),%ecx # examine counters |
| pxor $t2,$t2 |
| cmovge $Tbl,@ptr[0] # cancel input |
| cmp 4*1(%rbx),%ecx |
| movdqa @Xi[0],@Xi[1] |
| cmovge $Tbl,@ptr[1] |
| cmp 4*2(%rbx),%ecx |
| pcmpgtd $t2,@Xi[1] # mask value |
| cmovge $Tbl,@ptr[2] |
| cmp 4*3(%rbx),%ecx |
| paddd @Xi[1],@Xi[0] # counters-- |
| cmovge $Tbl,@ptr[3] |
| |
| movdqu 0x00($ctx),$t0 |
| pand @Xi[1],$A |
| movdqu 0x20($ctx),$t1 |
| pand @Xi[1],$B |
| paddd $t0,$A |
| movdqu 0x40($ctx),$t2 |
| pand @Xi[1],$C |
| paddd $t1,$B |
| movdqu 0x60($ctx),$t3 |
| pand @Xi[1],$D |
| paddd $t2,$C |
| movdqu 0x80($ctx),$tx |
| pand @Xi[1],$E |
| movdqu $A,0x00($ctx) |
| paddd $t3,$D |
| movdqu $B,0x20($ctx) |
| paddd $tx,$E |
| movdqu $C,0x40($ctx) |
| movdqu $D,0x60($ctx) |
| movdqu $E,0x80($ctx) |
| |
| movdqa @Xi[0],(%rbx) # save counters |
| movdqa 0x60($Tbl),$tx # pbswap_mask |
| movdqa -0x20($Tbl),$K # K_00_19 |
| dec $num |
| jnz .Loop |
| |
| mov `$REG_SZ*17+8`(%rsp),$num |
| lea $REG_SZ($ctx),$ctx |
| lea `$inp_elm_size*$REG_SZ/4`($inp),$inp |
| dec $num |
| jnz .Loop_grande |
| |
| .Ldone: |
| mov `$REG_SZ*17`(%rsp),%rax # original %rsp |
| .cfi_def_cfa %rax,8 |
| ___ |
| $code.=<<___ if ($win64); |
| movaps -0xb8(%rax),%xmm6 |
| movaps -0xa8(%rax),%xmm7 |
| movaps -0x98(%rax),%xmm8 |
| movaps -0x88(%rax),%xmm9 |
| movaps -0x78(%rax),%xmm10 |
| movaps -0x68(%rax),%xmm11 |
| movaps -0x58(%rax),%xmm12 |
| movaps -0x48(%rax),%xmm13 |
| movaps -0x38(%rax),%xmm14 |
| movaps -0x28(%rax),%xmm15 |
| ___ |
| $code.=<<___; |
| mov -16(%rax),%rbp |
| .cfi_restore %rbp |
| mov -8(%rax),%rbx |
| .cfi_restore %rbx |
| lea (%rax),%rsp |
| .cfi_def_cfa_register %rsp |
| .Lepilogue: |
| ret |
| .cfi_endproc |
| .size sha1_multi_block,.-sha1_multi_block |
| ___ |
| {{{ |
| my ($ABCD0,$E0,$E0_,$BSWAP,$ABCD1,$E1,$E1_)=map("%xmm$_",(0..3,8..10)); |
| my @MSG0=map("%xmm$_",(4..7)); |
| my @MSG1=map("%xmm$_",(11..14)); |
| |
| $code.=<<___; |
| .type sha1_multi_block_shaext,\@function,3 |
| .align 32 |
| sha1_multi_block_shaext: |
| .cfi_startproc |
| _shaext_shortcut: |
| mov %rsp,%rax |
| .cfi_def_cfa_register %rax |
| push %rbx |
| .cfi_push %rbx |
| push %rbp |
| .cfi_push %rbp |
| ___ |
| $code.=<<___ if ($win64); |
| lea -0xa8(%rsp),%rsp |
| movaps %xmm6,(%rsp) |
| movaps %xmm7,0x10(%rsp) |
| movaps %xmm8,0x20(%rsp) |
| movaps %xmm9,0x30(%rsp) |
| movaps %xmm10,-0x78(%rax) |
| movaps %xmm11,-0x68(%rax) |
| movaps %xmm12,-0x58(%rax) |
| movaps %xmm13,-0x48(%rax) |
| movaps %xmm14,-0x38(%rax) |
| movaps %xmm15,-0x28(%rax) |
| ___ |
| $code.=<<___; |
| sub \$`$REG_SZ*18`,%rsp |
| shl \$1,$num # we process pair at a time |
| and \$-256,%rsp |
| lea 0x40($ctx),$ctx # size optimization |
| mov %rax,`$REG_SZ*17`(%rsp) # original %rsp |
| .Lbody_shaext: |
| lea `$REG_SZ*16`(%rsp),%rbx |
| movdqa K_XX_XX+0x80(%rip),$BSWAP # byte-n-word swap |
| |
| .Loop_grande_shaext: |
| mov $num,`$REG_SZ*17+8`(%rsp) # original $num |
| xor $num,$num |
| ___ |
| for($i=0;$i<2;$i++) { |
| $ptr_reg=&pointer_register($flavour,@ptr[$i]); |
| $code.=<<___; |
| # input pointer |
| mov `$inp_elm_size*$i+0`($inp),$ptr_reg |
| # number of blocks |
| mov `$inp_elm_size*$i+$ptr_size`($inp),%ecx |
| cmp $num,%ecx |
| cmovg %ecx,$num # find maximum |
| test %ecx,%ecx |
| mov %ecx,`4*$i`(%rbx) # initialize counters |
| cmovle %rsp,@ptr[$i] # cancel input |
| ___ |
| } |
| $code.=<<___; |
| test $num,$num |
| jz .Ldone_shaext |
| |
| movq 0x00-0x40($ctx),$ABCD0 # a1.a0 |
| movq 0x20-0x40($ctx),@MSG0[0]# b1.b0 |
| movq 0x40-0x40($ctx),@MSG0[1]# c1.c0 |
| movq 0x60-0x40($ctx),@MSG0[2]# d1.d0 |
| movq 0x80-0x40($ctx),@MSG0[3]# e1.e0 |
| |
| punpckldq @MSG0[0],$ABCD0 # b1.a1.b0.a0 |
| punpckldq @MSG0[2],@MSG0[1] # d1.c1.d0.c0 |
| |
| movdqa $ABCD0,$ABCD1 |
| punpcklqdq @MSG0[1],$ABCD0 # d0.c0.b0.a0 |
| punpckhqdq @MSG0[1],$ABCD1 # d1.c1.b1.a1 |
| |
| pshufd \$0b00111111,@MSG0[3],$E0 |
| pshufd \$0b01111111,@MSG0[3],$E1 |
| pshufd \$0b00011011,$ABCD0,$ABCD0 |
| pshufd \$0b00011011,$ABCD1,$ABCD1 |
| jmp .Loop_shaext |
| |
| .align 32 |
| .Loop_shaext: |
| movdqu 0x00(@ptr[0]),@MSG0[0] |
| movdqu 0x00(@ptr[1]),@MSG1[0] |
| movdqu 0x10(@ptr[0]),@MSG0[1] |
| movdqu 0x10(@ptr[1]),@MSG1[1] |
| movdqu 0x20(@ptr[0]),@MSG0[2] |
| pshufb $BSWAP,@MSG0[0] |
| movdqu 0x20(@ptr[1]),@MSG1[2] |
| pshufb $BSWAP,@MSG1[0] |
| movdqu 0x30(@ptr[0]),@MSG0[3] |
| lea 0x40(@ptr[0]),@ptr[0] |
| pshufb $BSWAP,@MSG0[1] |
| movdqu 0x30(@ptr[1]),@MSG1[3] |
| lea 0x40(@ptr[1]),@ptr[1] |
| pshufb $BSWAP,@MSG1[1] |
| |
| movdqa $E0,0x50(%rsp) # offload |
| paddd @MSG0[0],$E0 |
| movdqa $E1,0x70(%rsp) |
| paddd @MSG1[0],$E1 |
| movdqa $ABCD0,0x40(%rsp) # offload |
| movdqa $ABCD0,$E0_ |
| movdqa $ABCD1,0x60(%rsp) |
| movdqa $ABCD1,$E1_ |
| sha1rnds4 \$0,$E0,$ABCD0 # 0-3 |
| sha1nexte @MSG0[1],$E0_ |
| sha1rnds4 \$0,$E1,$ABCD1 # 0-3 |
| sha1nexte @MSG1[1],$E1_ |
| pshufb $BSWAP,@MSG0[2] |
| prefetcht0 127(@ptr[0]) |
| sha1msg1 @MSG0[1],@MSG0[0] |
| pshufb $BSWAP,@MSG1[2] |
| prefetcht0 127(@ptr[1]) |
| sha1msg1 @MSG1[1],@MSG1[0] |
| |
| pshufb $BSWAP,@MSG0[3] |
| movdqa $ABCD0,$E0 |
| pshufb $BSWAP,@MSG1[3] |
| movdqa $ABCD1,$E1 |
| sha1rnds4 \$0,$E0_,$ABCD0 # 4-7 |
| sha1nexte @MSG0[2],$E0 |
| sha1rnds4 \$0,$E1_,$ABCD1 # 4-7 |
| sha1nexte @MSG1[2],$E1 |
| pxor @MSG0[2],@MSG0[0] |
| sha1msg1 @MSG0[2],@MSG0[1] |
| pxor @MSG1[2],@MSG1[0] |
| sha1msg1 @MSG1[2],@MSG1[1] |
| ___ |
| for($i=2;$i<20-4;$i++) { |
| $code.=<<___; |
| movdqa $ABCD0,$E0_ |
| movdqa $ABCD1,$E1_ |
| sha1rnds4 \$`int($i/5)`,$E0,$ABCD0 # 8-11 |
| sha1nexte @MSG0[3],$E0_ |
| sha1rnds4 \$`int($i/5)`,$E1,$ABCD1 # 8-11 |
| sha1nexte @MSG1[3],$E1_ |
| sha1msg2 @MSG0[3],@MSG0[0] |
| sha1msg2 @MSG1[3],@MSG1[0] |
| pxor @MSG0[3],@MSG0[1] |
| sha1msg1 @MSG0[3],@MSG0[2] |
| pxor @MSG1[3],@MSG1[1] |
| sha1msg1 @MSG1[3],@MSG1[2] |
| ___ |
| ($E0,$E0_)=($E0_,$E0); ($E1,$E1_)=($E1_,$E1); |
| push(@MSG0,shift(@MSG0)); push(@MSG1,shift(@MSG1)); |
| } |
| $code.=<<___; |
| movdqa $ABCD0,$E0_ |
| movdqa $ABCD1,$E1_ |
| sha1rnds4 \$3,$E0,$ABCD0 # 64-67 |
| sha1nexte @MSG0[3],$E0_ |
| sha1rnds4 \$3,$E1,$ABCD1 # 64-67 |
| sha1nexte @MSG1[3],$E1_ |
| sha1msg2 @MSG0[3],@MSG0[0] |
| sha1msg2 @MSG1[3],@MSG1[0] |
| pxor @MSG0[3],@MSG0[1] |
| pxor @MSG1[3],@MSG1[1] |
| |
| mov \$1,%ecx |
| pxor @MSG0[2],@MSG0[2] # zero |
| cmp 4*0(%rbx),%ecx # examine counters |
| cmovge %rsp,@ptr[0] # cancel input |
| |
| movdqa $ABCD0,$E0 |
| movdqa $ABCD1,$E1 |
| sha1rnds4 \$3,$E0_,$ABCD0 # 68-71 |
| sha1nexte @MSG0[0],$E0 |
| sha1rnds4 \$3,$E1_,$ABCD1 # 68-71 |
| sha1nexte @MSG1[0],$E1 |
| sha1msg2 @MSG0[0],@MSG0[1] |
| sha1msg2 @MSG1[0],@MSG1[1] |
| |
| cmp 4*1(%rbx),%ecx |
| cmovge %rsp,@ptr[1] |
| movq (%rbx),@MSG0[0] # pull counters |
| |
| movdqa $ABCD0,$E0_ |
| movdqa $ABCD1,$E1_ |
| sha1rnds4 \$3,$E0,$ABCD0 # 72-75 |
| sha1nexte @MSG0[1],$E0_ |
| sha1rnds4 \$3,$E1,$ABCD1 # 72-75 |
| sha1nexte @MSG1[1],$E1_ |
| |
| pshufd \$0x00,@MSG0[0],@MSG1[2] |
| pshufd \$0x55,@MSG0[0],@MSG1[3] |
| movdqa @MSG0[0],@MSG0[1] |
| pcmpgtd @MSG0[2],@MSG1[2] |
| pcmpgtd @MSG0[2],@MSG1[3] |
| |
| movdqa $ABCD0,$E0 |
| movdqa $ABCD1,$E1 |
| sha1rnds4 \$3,$E0_,$ABCD0 # 76-79 |
| sha1nexte $MSG0[2],$E0 |
| sha1rnds4 \$3,$E1_,$ABCD1 # 76-79 |
| sha1nexte $MSG0[2],$E1 |
| |
| pcmpgtd @MSG0[2],@MSG0[1] # counter mask |
| pand @MSG1[2],$ABCD0 |
| pand @MSG1[2],$E0 |
| pand @MSG1[3],$ABCD1 |
| pand @MSG1[3],$E1 |
| paddd @MSG0[1],@MSG0[0] # counters-- |
| |
| paddd 0x40(%rsp),$ABCD0 |
| paddd 0x50(%rsp),$E0 |
| paddd 0x60(%rsp),$ABCD1 |
| paddd 0x70(%rsp),$E1 |
| |
| movq @MSG0[0],(%rbx) # save counters |
| dec $num |
| jnz .Loop_shaext |
| |
| mov `$REG_SZ*17+8`(%rsp),$num |
| |
| pshufd \$0b00011011,$ABCD0,$ABCD0 |
| pshufd \$0b00011011,$ABCD1,$ABCD1 |
| |
| movdqa $ABCD0,@MSG0[0] |
| punpckldq $ABCD1,$ABCD0 # b1.b0.a1.a0 |
| punpckhdq $ABCD1,@MSG0[0] # d1.d0.c1.c0 |
| punpckhdq $E1,$E0 # e1.e0.xx.xx |
| movq $ABCD0,0x00-0x40($ctx) # a1.a0 |
| psrldq \$8,$ABCD0 |
| movq @MSG0[0],0x40-0x40($ctx)# c1.c0 |
| psrldq \$8,@MSG0[0] |
| movq $ABCD0,0x20-0x40($ctx) # b1.b0 |
| psrldq \$8,$E0 |
| movq @MSG0[0],0x60-0x40($ctx)# d1.d0 |
| movq $E0,0x80-0x40($ctx) # e1.e0 |
| |
| lea `$REG_SZ/2`($ctx),$ctx |
| lea `$inp_elm_size*2`($inp),$inp |
| dec $num |
| jnz .Loop_grande_shaext |
| |
| .Ldone_shaext: |
| #mov `$REG_SZ*17`(%rsp),%rax # original %rsp |
| ___ |
| $code.=<<___ if ($win64); |
| movaps -0xb8(%rax),%xmm6 |
| movaps -0xa8(%rax),%xmm7 |
| movaps -0x98(%rax),%xmm8 |
| movaps -0x88(%rax),%xmm9 |
| movaps -0x78(%rax),%xmm10 |
| movaps -0x68(%rax),%xmm11 |
| movaps -0x58(%rax),%xmm12 |
| movaps -0x48(%rax),%xmm13 |
| movaps -0x38(%rax),%xmm14 |
| movaps -0x28(%rax),%xmm15 |
| ___ |
| $code.=<<___; |
| mov -16(%rax),%rbp |
| .cfi_restore %rbp |
| mov -8(%rax),%rbx |
| .cfi_restore %rbx |
| lea (%rax),%rsp |
| .cfi_def_cfa_register %rsp |
| .Lepilogue_shaext: |
| ret |
| .cfi_endproc |
| .size sha1_multi_block_shaext,.-sha1_multi_block_shaext |
| ___ |
| }}} |
| |
| if ($avx) {{{ |
| sub BODY_00_19_avx { |
| my ($i,$a,$b,$c,$d,$e)=@_; |
| my $j=$i+1; |
| my $k=$i+2; |
| my $vpack = $REG_SZ==16 ? "vpunpckldq" : "vinserti128"; |
| my $ptr_n = $REG_SZ==16 ? @ptr[1] : @ptr[4]; |
| |
| $code.=<<___ if ($i==0 && $REG_SZ==16); |
| vmovd (@ptr[0]),@Xi[0] |
| lea `16*4`(@ptr[0]),@ptr[0] |
| vmovd (@ptr[1]),@Xi[2] # borrow Xi[2] |
| lea `16*4`(@ptr[1]),@ptr[1] |
| vpinsrd \$1,(@ptr[2]),@Xi[0],@Xi[0] |
| lea `16*4`(@ptr[2]),@ptr[2] |
| vpinsrd \$1,(@ptr[3]),@Xi[2],@Xi[2] |
| lea `16*4`(@ptr[3]),@ptr[3] |
| vmovd `4*$j-16*4`(@ptr[0]),@Xi[1] |
| vpunpckldq @Xi[2],@Xi[0],@Xi[0] |
| vmovd `4*$j-16*4`($ptr_n),$t3 |
| vpshufb $tx,@Xi[0],@Xi[0] |
| ___ |
| $code.=<<___ if ($i<15 && $REG_SZ==16); # just load input |
| vpinsrd \$1,`4*$j-16*4`(@ptr[2]),@Xi[1],@Xi[1] |
| vpinsrd \$1,`4*$j-16*4`(@ptr[3]),$t3,$t3 |
| ___ |
| $code.=<<___ if ($i==0 && $REG_SZ==32); |
| vmovd (@ptr[0]),@Xi[0] |
| lea `16*4`(@ptr[0]),@ptr[0] |
| vmovd (@ptr[4]),@Xi[2] # borrow Xi[2] |
| lea `16*4`(@ptr[4]),@ptr[4] |
| vmovd (@ptr[1]),$t2 |
| lea `16*4`(@ptr[1]),@ptr[1] |
| vmovd (@ptr[5]),$t1 |
| lea `16*4`(@ptr[5]),@ptr[5] |
| vpinsrd \$1,(@ptr[2]),@Xi[0],@Xi[0] |
| lea `16*4`(@ptr[2]),@ptr[2] |
| vpinsrd \$1,(@ptr[6]),@Xi[2],@Xi[2] |
| lea `16*4`(@ptr[6]),@ptr[6] |
| vpinsrd \$1,(@ptr[3]),$t2,$t2 |
| lea `16*4`(@ptr[3]),@ptr[3] |
| vpunpckldq $t2,@Xi[0],@Xi[0] |
| vpinsrd \$1,(@ptr[7]),$t1,$t1 |
| lea `16*4`(@ptr[7]),@ptr[7] |
| vpunpckldq $t1,@Xi[2],@Xi[2] |
| vmovd `4*$j-16*4`(@ptr[0]),@Xi[1] |
| vinserti128 @Xi[2],@Xi[0],@Xi[0] |
| vmovd `4*$j-16*4`($ptr_n),$t3 |
| vpshufb $tx,@Xi[0],@Xi[0] |
| ___ |
| $code.=<<___ if ($i<15 && $REG_SZ==32); # just load input |
| vmovd `4*$j-16*4`(@ptr[1]),$t2 |
| vmovd `4*$j-16*4`(@ptr[5]),$t1 |
| vpinsrd \$1,`4*$j-16*4`(@ptr[2]),@Xi[1],@Xi[1] |
| vpinsrd \$1,`4*$j-16*4`(@ptr[6]),$t3,$t3 |
| vpinsrd \$1,`4*$j-16*4`(@ptr[3]),$t2,$t2 |
| vpunpckldq $t2,@Xi[1],@Xi[1] |
| vpinsrd \$1,`4*$j-16*4`(@ptr[7]),$t1,$t1 |
| vpunpckldq $t1,$t3,$t3 |
| ___ |
| $code.=<<___ if ($i<14); |
| vpaddd $K,$e,$e # e+=K_00_19 |
| vpslld \$5,$a,$t2 |
| vpandn $d,$b,$t1 |
| vpand $c,$b,$t0 |
| |
| vmovdqa @Xi[0],`&Xi_off($i)` |
| vpaddd @Xi[0],$e,$e # e+=X[i] |
| $vpack $t3,@Xi[1],@Xi[1] |
| vpsrld \$27,$a,$t3 |
| vpxor $t1,$t0,$t0 # Ch(b,c,d) |
| vmovd `4*$k-16*4`(@ptr[0]),@Xi[2] |
| |
| vpslld \$30,$b,$t1 |
| vpor $t3,$t2,$t2 # rol(a,5) |
| vmovd `4*$k-16*4`($ptr_n),$t3 |
| vpaddd $t0,$e,$e # e+=Ch(b,c,d) |
| |
| vpsrld \$2,$b,$b |
| vpaddd $t2,$e,$e # e+=rol(a,5) |
| vpshufb $tx,@Xi[1],@Xi[1] |
| vpor $t1,$b,$b # b=rol(b,30) |
| ___ |
| $code.=<<___ if ($i==14); |
| vpaddd $K,$e,$e # e+=K_00_19 |
| prefetcht0 63(@ptr[0]) |
| vpslld \$5,$a,$t2 |
| vpandn $d,$b,$t1 |
| vpand $c,$b,$t0 |
| |
| vmovdqa @Xi[0],`&Xi_off($i)` |
| vpaddd @Xi[0],$e,$e # e+=X[i] |
| $vpack $t3,@Xi[1],@Xi[1] |
| vpsrld \$27,$a,$t3 |
| prefetcht0 63(@ptr[1]) |
| vpxor $t1,$t0,$t0 # Ch(b,c,d) |
| |
| vpslld \$30,$b,$t1 |
| vpor $t3,$t2,$t2 # rol(a,5) |
| prefetcht0 63(@ptr[2]) |
| vpaddd $t0,$e,$e # e+=Ch(b,c,d) |
| |
| vpsrld \$2,$b,$b |
| vpaddd $t2,$e,$e # e+=rol(a,5) |
| prefetcht0 63(@ptr[3]) |
| vpshufb $tx,@Xi[1],@Xi[1] |
| vpor $t1,$b,$b # b=rol(b,30) |
| ___ |
| $code.=<<___ if ($i>=13 && $i<15); |
| vmovdqa `&Xi_off($j+2)`,@Xi[3] # preload "X[2]" |
| ___ |
| $code.=<<___ if ($i>=15); # apply Xupdate |
| vpxor @Xi[-2],@Xi[1],@Xi[1] # "X[13]" |
| vmovdqa `&Xi_off($j+2)`,@Xi[3] # "X[2]" |
| |
| vpaddd $K,$e,$e # e+=K_00_19 |
| vpslld \$5,$a,$t2 |
| vpandn $d,$b,$t1 |
| `"prefetcht0 63(@ptr[4])" if ($i==15 && $REG_SZ==32)` |
| vpand $c,$b,$t0 |
| |
| vmovdqa @Xi[0],`&Xi_off($i)` |
| vpaddd @Xi[0],$e,$e # e+=X[i] |
| vpxor `&Xi_off($j+8)`,@Xi[1],@Xi[1] |
| vpsrld \$27,$a,$t3 |
| vpxor $t1,$t0,$t0 # Ch(b,c,d) |
| vpxor @Xi[3],@Xi[1],@Xi[1] |
| `"prefetcht0 63(@ptr[5])" if ($i==15 && $REG_SZ==32)` |
| |
| vpslld \$30,$b,$t1 |
| vpor $t3,$t2,$t2 # rol(a,5) |
| vpaddd $t0,$e,$e # e+=Ch(b,c,d) |
| `"prefetcht0 63(@ptr[6])" if ($i==15 && $REG_SZ==32)` |
| vpsrld \$31,@Xi[1],$tx |
| vpaddd @Xi[1],@Xi[1],@Xi[1] |
| |
| vpsrld \$2,$b,$b |
| `"prefetcht0 63(@ptr[7])" if ($i==15 && $REG_SZ==32)` |
| vpaddd $t2,$e,$e # e+=rol(a,5) |
| vpor $tx,@Xi[1],@Xi[1] # rol \$1,@Xi[1] |
| vpor $t1,$b,$b # b=rol(b,30) |
| ___ |
| push(@Xi,shift(@Xi)); |
| } |
| |
| sub BODY_20_39_avx { |
| my ($i,$a,$b,$c,$d,$e)=@_; |
| my $j=$i+1; |
| |
| $code.=<<___ if ($i<79); |
| vpxor @Xi[-2],@Xi[1],@Xi[1] # "X[13]" |
| vmovdqa `&Xi_off($j+2)`,@Xi[3] # "X[2]" |
| |
| vpslld \$5,$a,$t2 |
| vpaddd $K,$e,$e # e+=K_20_39 |
| vpxor $b,$d,$t0 |
| ___ |
| $code.=<<___ if ($i<72); |
| vmovdqa @Xi[0],`&Xi_off($i)` |
| ___ |
| $code.=<<___ if ($i<79); |
| vpaddd @Xi[0],$e,$e # e+=X[i] |
| vpxor `&Xi_off($j+8)`,@Xi[1],@Xi[1] |
| vpsrld \$27,$a,$t3 |
| vpxor $c,$t0,$t0 # Parity(b,c,d) |
| vpxor @Xi[3],@Xi[1],@Xi[1] |
| |
| vpslld \$30,$b,$t1 |
| vpor $t3,$t2,$t2 # rol(a,5) |
| vpaddd $t0,$e,$e # e+=Parity(b,c,d) |
| vpsrld \$31,@Xi[1],$tx |
| vpaddd @Xi[1],@Xi[1],@Xi[1] |
| |
| vpsrld \$2,$b,$b |
| vpaddd $t2,$e,$e # e+=rol(a,5) |
| vpor $tx,@Xi[1],@Xi[1] # rol(@Xi[1],1) |
| vpor $t1,$b,$b # b=rol(b,30) |
| ___ |
| $code.=<<___ if ($i==79); |
| vpslld \$5,$a,$t2 |
| vpaddd $K,$e,$e # e+=K_20_39 |
| vpxor $b,$d,$t0 |
| |
| vpsrld \$27,$a,$t3 |
| vpaddd @Xi[0],$e,$e # e+=X[i] |
| vpxor $c,$t0,$t0 # Parity(b,c,d) |
| |
| vpslld \$30,$b,$t1 |
| vpor $t3,$t2,$t2 # rol(a,5) |
| vpaddd $t0,$e,$e # e+=Parity(b,c,d) |
| |
| vpsrld \$2,$b,$b |
| vpaddd $t2,$e,$e # e+=rol(a,5) |
| vpor $t1,$b,$b # b=rol(b,30) |
| ___ |
| push(@Xi,shift(@Xi)); |
| } |
| |
| sub BODY_40_59_avx { |
| my ($i,$a,$b,$c,$d,$e)=@_; |
| my $j=$i+1; |
| |
| $code.=<<___; |
| vpxor @Xi[-2],@Xi[1],@Xi[1] # "X[13]" |
| vmovdqa `&Xi_off($j+2)`,@Xi[3] # "X[2]" |
| |
| vpaddd $K,$e,$e # e+=K_40_59 |
| vpslld \$5,$a,$t2 |
| vpand $c,$d,$t1 |
| vpxor `&Xi_off($j+8)`,@Xi[1],@Xi[1] |
| |
| vpaddd $t1,$e,$e |
| vpsrld \$27,$a,$t3 |
| vpxor $c,$d,$t0 |
| vpxor @Xi[3],@Xi[1],@Xi[1] |
| |
| vmovdqu @Xi[0],`&Xi_off($i)` |
| vpaddd @Xi[0],$e,$e # e+=X[i] |
| vpor $t3,$t2,$t2 # rol(a,5) |
| vpsrld \$31,@Xi[1],$tx |
| vpand $b,$t0,$t0 |
| vpaddd @Xi[1],@Xi[1],@Xi[1] |
| |
| vpslld \$30,$b,$t1 |
| vpaddd $t0,$e,$e # e+=Maj(b,d,c) |
| |
| vpsrld \$2,$b,$b |
| vpaddd $t2,$e,$e # e+=rol(a,5) |
| vpor $tx,@Xi[1],@Xi[1] # rol(@X[1],1) |
| vpor $t1,$b,$b # b=rol(b,30) |
| ___ |
| push(@Xi,shift(@Xi)); |
| } |
| |
| $code.=<<___; |
| .type sha1_multi_block_avx,\@function,3 |
| .align 32 |
| sha1_multi_block_avx: |
| .cfi_startproc |
| _avx_shortcut: |
| ___ |
| $code.=<<___ if ($avx>1); |
| shr \$32,%rcx |
| cmp \$2,$num |
| jb .Lavx |
| test \$`1<<5`,%ecx |
| jnz _avx2_shortcut |
| jmp .Lavx |
| .align 32 |
| .Lavx: |
| ___ |
| $code.=<<___; |
| mov %rsp,%rax |
| .cfi_def_cfa_register %rax |
| push %rbx |
| .cfi_push %rbx |
| push %rbp |
| .cfi_push %rbp |
| ___ |
| $code.=<<___ if ($win64); |
| lea -0xa8(%rsp),%rsp |
| movaps %xmm6,(%rsp) |
| movaps %xmm7,0x10(%rsp) |
| movaps %xmm8,0x20(%rsp) |
| movaps %xmm9,0x30(%rsp) |
| movaps %xmm10,-0x78(%rax) |
| movaps %xmm11,-0x68(%rax) |
| movaps %xmm12,-0x58(%rax) |
| movaps %xmm13,-0x48(%rax) |
| movaps %xmm14,-0x38(%rax) |
| movaps %xmm15,-0x28(%rax) |
| ___ |
| $code.=<<___; |
| sub \$`$REG_SZ*18`, %rsp |
| and \$-256,%rsp |
| mov %rax,`$REG_SZ*17`(%rsp) # original %rsp |
| .cfi_cfa_expression %rsp+`$REG_SZ*17`,deref,+8 |
| .Lbody_avx: |
| lea K_XX_XX(%rip),$Tbl |
| lea `$REG_SZ*16`(%rsp),%rbx |
| |
| vzeroupper |
| .Loop_grande_avx: |
| mov $num,`$REG_SZ*17+8`(%rsp) # original $num |
| xor $num,$num |
| ___ |
| for($i=0;$i<4;$i++) { |
| $ptr_reg=&pointer_register($flavour,@ptr[$i]); |
| $code.=<<___; |
| # input pointer |
| mov `$inp_elm_size*$i+0`($inp),$ptr_reg |
| # number of blocks |
| mov `$inp_elm_size*$i+$ptr_size`($inp),%ecx |
| cmp $num,%ecx |
| cmovg %ecx,$num # find maximum |
| test %ecx,%ecx |
| mov %ecx,`4*$i`(%rbx) # initialize counters |
| cmovle $Tbl,@ptr[$i] # cancel input |
| ___ |
| } |
| $code.=<<___; |
| test $num,$num |
| jz .Ldone_avx |
| |
| vmovdqu 0x00($ctx),$A # load context |
| lea 128(%rsp),%rax |
| vmovdqu 0x20($ctx),$B |
| vmovdqu 0x40($ctx),$C |
| vmovdqu 0x60($ctx),$D |
| vmovdqu 0x80($ctx),$E |
| vmovdqu 0x60($Tbl),$tx # pbswap_mask |
| jmp .Loop_avx |
| |
| .align 32 |
| .Loop_avx: |
| ___ |
| $code.=" vmovdqa -0x20($Tbl),$K\n"; # K_00_19 |
| for($i=0;$i<20;$i++) { &BODY_00_19_avx($i,@V); unshift(@V,pop(@V)); } |
| $code.=" vmovdqa 0x00($Tbl),$K\n"; # K_20_39 |
| for(;$i<40;$i++) { &BODY_20_39_avx($i,@V); unshift(@V,pop(@V)); } |
| $code.=" vmovdqa 0x20($Tbl),$K\n"; # K_40_59 |
| for(;$i<60;$i++) { &BODY_40_59_avx($i,@V); unshift(@V,pop(@V)); } |
| $code.=" vmovdqa 0x40($Tbl),$K\n"; # K_60_79 |
| for(;$i<80;$i++) { &BODY_20_39_avx($i,@V); unshift(@V,pop(@V)); } |
| $code.=<<___; |
| mov \$1,%ecx |
| ___ |
| for($i=0;$i<4;$i++) { |
| $code.=<<___; |
| cmp `4*$i`(%rbx),%ecx # examine counters |
| cmovge $Tbl,@ptr[$i] # cancel input |
| ___ |
| } |
| $code.=<<___; |
| vmovdqu (%rbx),$t0 # pull counters |
| vpxor $t2,$t2,$t2 |
| vmovdqa $t0,$t1 |
| vpcmpgtd $t2,$t1,$t1 # mask value |
| vpaddd $t1,$t0,$t0 # counters-- |
| |
| vpand $t1,$A,$A |
| vpand $t1,$B,$B |
| vpaddd 0x00($ctx),$A,$A |
| vpand $t1,$C,$C |
| vpaddd 0x20($ctx),$B,$B |
| vpand $t1,$D,$D |
| vpaddd 0x40($ctx),$C,$C |
| vpand $t1,$E,$E |
| vpaddd 0x60($ctx),$D,$D |
| vpaddd 0x80($ctx),$E,$E |
| vmovdqu $A,0x00($ctx) |
| vmovdqu $B,0x20($ctx) |
| vmovdqu $C,0x40($ctx) |
| vmovdqu $D,0x60($ctx) |
| vmovdqu $E,0x80($ctx) |
| |
| vmovdqu $t0,(%rbx) # save counters |
| vmovdqu 0x60($Tbl),$tx # pbswap_mask |
| dec $num |
| jnz .Loop_avx |
| |
| mov `$REG_SZ*17+8`(%rsp),$num |
| lea $REG_SZ($ctx),$ctx |
| lea `$inp_elm_size*$REG_SZ/4`($inp),$inp |
| dec $num |
| jnz .Loop_grande_avx |
| |
| .Ldone_avx: |
| mov `$REG_SZ*17`(%rsp),%rax # original %rsp |
| .cfi_def_cfa %rax,8 |
| vzeroupper |
| ___ |
| $code.=<<___ if ($win64); |
| movaps -0xb8(%rax),%xmm6 |
| movaps -0xa8(%rax),%xmm7 |
| movaps -0x98(%rax),%xmm8 |
| movaps -0x88(%rax),%xmm9 |
| movaps -0x78(%rax),%xmm10 |
| movaps -0x68(%rax),%xmm11 |
| movaps -0x58(%rax),%xmm12 |
| movaps -0x48(%rax),%xmm13 |
| movaps -0x38(%rax),%xmm14 |
| movaps -0x28(%rax),%xmm15 |
| ___ |
| $code.=<<___; |
| mov -16(%rax),%rbp |
| .cfi_restore %rbp |
| mov -8(%rax),%rbx |
| .cfi_restore %rbx |
| lea (%rax),%rsp |
| .cfi_def_cfa_register %rsp |
| .Lepilogue_avx: |
| ret |
| .cfi_endproc |
| .size sha1_multi_block_avx,.-sha1_multi_block_avx |
| ___ |
| |
| if ($avx>1) { |
| $code =~ s/\`([^\`]*)\`/eval $1/gem; |
| |
| $REG_SZ=32; |
| |
| @ptr=map("%r$_",(12..15,8..11)); |
| |
| @V=($A,$B,$C,$D,$E)=map("%ymm$_",(0..4)); |
| ($t0,$t1,$t2,$t3,$tx)=map("%ymm$_",(5..9)); |
| @Xi=map("%ymm$_",(10..14)); |
| $K="%ymm15"; |
| |
| $code.=<<___; |
| .type sha1_multi_block_avx2,\@function,3 |
| .align 32 |
| sha1_multi_block_avx2: |
| .cfi_startproc |
| _avx2_shortcut: |
| mov %rsp,%rax |
| .cfi_def_cfa_register %rax |
| push %rbx |
| .cfi_push %rbx |
| push %rbp |
| .cfi_push %rbp |
| push %r12 |
| .cfi_push %r12 |
| push %r13 |
| .cfi_push %r13 |
| push %r14 |
| .cfi_push %r14 |
| push %r15 |
| .cfi_push %r15 |
| ___ |
| $code.=<<___ if ($win64); |
| lea -0xa8(%rsp),%rsp |
| movaps %xmm6,(%rsp) |
| movaps %xmm7,0x10(%rsp) |
| movaps %xmm8,0x20(%rsp) |
| movaps %xmm9,0x30(%rsp) |
| movaps %xmm10,0x40(%rsp) |
| movaps %xmm11,0x50(%rsp) |
| movaps %xmm12,-0x78(%rax) |
| movaps %xmm13,-0x68(%rax) |
| movaps %xmm14,-0x58(%rax) |
| movaps %xmm15,-0x48(%rax) |
| ___ |
| $code.=<<___; |
| sub \$`$REG_SZ*18`, %rsp |
| and \$-256,%rsp |
| mov %rax,`$REG_SZ*17`(%rsp) # original %rsp |
| .cfi_cfa_expression %rsp+`$REG_SZ*17`,deref,+8 |
| .Lbody_avx2: |
| lea K_XX_XX(%rip),$Tbl |
| shr \$1,$num |
| |
| vzeroupper |
| .Loop_grande_avx2: |
| mov $num,`$REG_SZ*17+8`(%rsp) # original $num |
| xor $num,$num |
| lea `$REG_SZ*16`(%rsp),%rbx |
| ___ |
| for($i=0;$i<8;$i++) { |
| $ptr_reg=&pointer_register($flavour,@ptr[$i]); |
| $code.=<<___; |
| # input pointer |
| mov `$inp_elm_size*$i+0`($inp),$ptr_reg |
| # number of blocks |
| mov `$inp_elm_size*$i+$ptr_size`($inp),%ecx |
| cmp $num,%ecx |
| cmovg %ecx,$num # find maximum |
| test %ecx,%ecx |
| mov %ecx,`4*$i`(%rbx) # initialize counters |
| cmovle $Tbl,@ptr[$i] # cancel input |
| ___ |
| } |
| $code.=<<___; |
| vmovdqu 0x00($ctx),$A # load context |
| lea 128(%rsp),%rax |
| vmovdqu 0x20($ctx),$B |
| lea 256+128(%rsp),%rbx |
| vmovdqu 0x40($ctx),$C |
| vmovdqu 0x60($ctx),$D |
| vmovdqu 0x80($ctx),$E |
| vmovdqu 0x60($Tbl),$tx # pbswap_mask |
| jmp .Loop_avx2 |
| |
| .align 32 |
| .Loop_avx2: |
| ___ |
| $code.=" vmovdqa -0x20($Tbl),$K\n"; # K_00_19 |
| for($i=0;$i<20;$i++) { &BODY_00_19_avx($i,@V); unshift(@V,pop(@V)); } |
| $code.=" vmovdqa 0x00($Tbl),$K\n"; # K_20_39 |
| for(;$i<40;$i++) { &BODY_20_39_avx($i,@V); unshift(@V,pop(@V)); } |
| $code.=" vmovdqa 0x20($Tbl),$K\n"; # K_40_59 |
| for(;$i<60;$i++) { &BODY_40_59_avx($i,@V); unshift(@V,pop(@V)); } |
| $code.=" vmovdqa 0x40($Tbl),$K\n"; # K_60_79 |
| for(;$i<80;$i++) { &BODY_20_39_avx($i,@V); unshift(@V,pop(@V)); } |
| $code.=<<___; |
| mov \$1,%ecx |
| lea `$REG_SZ*16`(%rsp),%rbx |
| ___ |
| for($i=0;$i<8;$i++) { |
| $code.=<<___; |
| cmp `4*$i`(%rbx),%ecx # examine counters |
| cmovge $Tbl,@ptr[$i] # cancel input |
| ___ |
| } |
| $code.=<<___; |
| vmovdqu (%rbx),$t0 # pull counters |
| vpxor $t2,$t2,$t2 |
| vmovdqa $t0,$t1 |
| vpcmpgtd $t2,$t1,$t1 # mask value |
| vpaddd $t1,$t0,$t0 # counters-- |
| |
| vpand $t1,$A,$A |
| vpand $t1,$B,$B |
| vpaddd 0x00($ctx),$A,$A |
| vpand $t1,$C,$C |
| vpaddd 0x20($ctx),$B,$B |
| vpand $t1,$D,$D |
| vpaddd 0x40($ctx),$C,$C |
| vpand $t1,$E,$E |
| vpaddd 0x60($ctx),$D,$D |
| vpaddd 0x80($ctx),$E,$E |
| vmovdqu $A,0x00($ctx) |
| vmovdqu $B,0x20($ctx) |
| vmovdqu $C,0x40($ctx) |
| vmovdqu $D,0x60($ctx) |
| vmovdqu $E,0x80($ctx) |
| |
| vmovdqu $t0,(%rbx) # save counters |
| lea 256+128(%rsp),%rbx |
| vmovdqu 0x60($Tbl),$tx # pbswap_mask |
| dec $num |
| jnz .Loop_avx2 |
| |
| #mov `$REG_SZ*17+8`(%rsp),$num |
| #lea $REG_SZ($ctx),$ctx |
| #lea `$inp_elm_size*$REG_SZ/4`($inp),$inp |
| #dec $num |
| #jnz .Loop_grande_avx2 |
| |
| .Ldone_avx2: |
| mov `$REG_SZ*17`(%rsp),%rax # original %rsp |
| .cfi_def_cfa %rax,8 |
| vzeroupper |
| ___ |
| $code.=<<___ if ($win64); |
| movaps -0xd8(%rax),%xmm6 |
| movaps -0xc8(%rax),%xmm7 |
| movaps -0xb8(%rax),%xmm8 |
| movaps -0xa8(%rax),%xmm9 |
| movaps -0x98(%rax),%xmm10 |
| movaps -0x88(%rax),%xmm11 |
| movaps -0x78(%rax),%xmm12 |
| movaps -0x68(%rax),%xmm13 |
| movaps -0x58(%rax),%xmm14 |
| movaps -0x48(%rax),%xmm15 |
| ___ |
| $code.=<<___; |
| mov -48(%rax),%r15 |
| .cfi_restore %r15 |
| mov -40(%rax),%r14 |
| .cfi_restore %r14 |
| mov -32(%rax),%r13 |
| .cfi_restore %r13 |
| mov -24(%rax),%r12 |
| .cfi_restore %r12 |
| mov -16(%rax),%rbp |
| .cfi_restore %rbp |
| mov -8(%rax),%rbx |
| .cfi_restore %rbx |
| lea (%rax),%rsp |
| .cfi_def_cfa_register %rsp |
| .Lepilogue_avx2: |
| ret |
| .cfi_endproc |
| .size sha1_multi_block_avx2,.-sha1_multi_block_avx2 |
| ___ |
| } }}} |
| $code.=<<___; |
| |
| .align 256 |
| .long 0x5a827999,0x5a827999,0x5a827999,0x5a827999 # K_00_19 |
| .long 0x5a827999,0x5a827999,0x5a827999,0x5a827999 # K_00_19 |
| K_XX_XX: |
| .long 0x6ed9eba1,0x6ed9eba1,0x6ed9eba1,0x6ed9eba1 # K_20_39 |
| .long 0x6ed9eba1,0x6ed9eba1,0x6ed9eba1,0x6ed9eba1 # K_20_39 |
| .long 0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc # K_40_59 |
| .long 0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc # K_40_59 |
| .long 0xca62c1d6,0xca62c1d6,0xca62c1d6,0xca62c1d6 # K_60_79 |
| .long 0xca62c1d6,0xca62c1d6,0xca62c1d6,0xca62c1d6 # K_60_79 |
| .long 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f # pbswap |
| .long 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f # pbswap |
| .byte 0xf,0xe,0xd,0xc,0xb,0xa,0x9,0x8,0x7,0x6,0x5,0x4,0x3,0x2,0x1,0x0 |
| .asciz "SHA1 multi-block transform for x86_64, CRYPTOGAMS by <appro\@openssl.org>" |
| ___ |
| |
| if ($win64) { |
| # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, |
| # CONTEXT *context,DISPATCHER_CONTEXT *disp) |
| $rec="%rcx"; |
| $frame="%rdx"; |
| $context="%r8"; |
| $disp="%r9"; |
| |
| $code.=<<___; |
| .extern __imp_RtlVirtualUnwind |
| .type se_handler,\@abi-omnipotent |
| .align 16 |
| se_handler: |
| push %rsi |
| push %rdi |
| push %rbx |
| push %rbp |
| push %r12 |
| push %r13 |
| push %r14 |
| push %r15 |
| pushfq |
| sub \$64,%rsp |
| |
| mov 120($context),%rax # pull context->Rax |
| mov 248($context),%rbx # pull context->Rip |
| |
| mov 8($disp),%rsi # disp->ImageBase |
| mov 56($disp),%r11 # disp->HandlerData |
| |
| mov 0(%r11),%r10d # HandlerData[0] |
| lea (%rsi,%r10),%r10 # end of prologue label |
| cmp %r10,%rbx # context->Rip<.Lbody |
| jb .Lin_prologue |
| |
| mov 152($context),%rax # pull context->Rsp |
| |
| mov 4(%r11),%r10d # HandlerData[1] |
| lea (%rsi,%r10),%r10 # epilogue label |
| cmp %r10,%rbx # context->Rip>=.Lepilogue |
| jae .Lin_prologue |
| |
| mov `16*17`(%rax),%rax # pull saved stack pointer |
| |
| mov -8(%rax),%rbx |
| mov -16(%rax),%rbp |
| mov %rbx,144($context) # restore context->Rbx |
| mov %rbp,160($context) # restore context->Rbp |
| |
| lea -24-10*16(%rax),%rsi |
| lea 512($context),%rdi # &context.Xmm6 |
| mov \$20,%ecx |
| .long 0xa548f3fc # cld; rep movsq |
| |
| .Lin_prologue: |
| mov 8(%rax),%rdi |
| mov 16(%rax),%rsi |
| mov %rax,152($context) # restore context->Rsp |
| mov %rsi,168($context) # restore context->Rsi |
| mov %rdi,176($context) # restore context->Rdi |
| |
| mov 40($disp),%rdi # disp->ContextRecord |
| mov $context,%rsi # context |
| mov \$154,%ecx # sizeof(CONTEXT) |
| .long 0xa548f3fc # cld; rep movsq |
| |
| mov $disp,%rsi |
| xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER |
| mov 8(%rsi),%rdx # arg2, disp->ImageBase |
| mov 0(%rsi),%r8 # arg3, disp->ControlPc |
| mov 16(%rsi),%r9 # arg4, disp->FunctionEntry |
| mov 40(%rsi),%r10 # disp->ContextRecord |
| lea 56(%rsi),%r11 # &disp->HandlerData |
| lea 24(%rsi),%r12 # &disp->EstablisherFrame |
| mov %r10,32(%rsp) # arg5 |
| mov %r11,40(%rsp) # arg6 |
| mov %r12,48(%rsp) # arg7 |
| mov %rcx,56(%rsp) # arg8, (NULL) |
| call *__imp_RtlVirtualUnwind(%rip) |
| |
| mov \$1,%eax # ExceptionContinueSearch |
| add \$64,%rsp |
| popfq |
| pop %r15 |
| pop %r14 |
| pop %r13 |
| pop %r12 |
| pop %rbp |
| pop %rbx |
| pop %rdi |
| pop %rsi |
| ret |
| .size se_handler,.-se_handler |
| ___ |
| $code.=<<___ if ($avx>1); |
| .type avx2_handler,\@abi-omnipotent |
| .align 16 |
| avx2_handler: |
| push %rsi |
| push %rdi |
| push %rbx |
| push %rbp |
| push %r12 |
| push %r13 |
| push %r14 |
| push %r15 |
| pushfq |
| sub \$64,%rsp |
| |
| mov 120($context),%rax # pull context->Rax |
| mov 248($context),%rbx # pull context->Rip |
| |
| mov 8($disp),%rsi # disp->ImageBase |
| mov 56($disp),%r11 # disp->HandlerData |
| |
| mov 0(%r11),%r10d # HandlerData[0] |
| lea (%rsi,%r10),%r10 # end of prologue label |
| cmp %r10,%rbx # context->Rip<body label |
| jb .Lin_prologue |
| |
| mov 152($context),%rax # pull context->Rsp |
| |
| mov 4(%r11),%r10d # HandlerData[1] |
| lea (%rsi,%r10),%r10 # epilogue label |
| cmp %r10,%rbx # context->Rip>=epilogue label |
| jae .Lin_prologue |
| |
| mov `32*17`($context),%rax # pull saved stack pointer |
| |
| mov -8(%rax),%rbx |
| mov -16(%rax),%rbp |
| mov -24(%rax),%r12 |
| mov -32(%rax),%r13 |
| mov -40(%rax),%r14 |
| mov -48(%rax),%r15 |
| mov %rbx,144($context) # restore context->Rbx |
| mov %rbp,160($context) # restore context->Rbp |
| mov %r12,216($context) # restore context->R12 |
| mov %r13,224($context) # restore context->R13 |
| mov %r14,232($context) # restore context->R14 |
| mov %r15,240($context) # restore context->R15 |
| |
| lea -56-10*16(%rax),%rsi |
| lea 512($context),%rdi # &context.Xmm6 |
| mov \$20,%ecx |
| .long 0xa548f3fc # cld; rep movsq |
| |
| jmp .Lin_prologue |
| .size avx2_handler,.-avx2_handler |
| ___ |
| $code.=<<___; |
| .section .pdata |
| .align 4 |
| .rva .LSEH_begin_sha1_multi_block |
| .rva .LSEH_end_sha1_multi_block |
| .rva .LSEH_info_sha1_multi_block |
| .rva .LSEH_begin_sha1_multi_block_shaext |
| .rva .LSEH_end_sha1_multi_block_shaext |
| .rva .LSEH_info_sha1_multi_block_shaext |
| ___ |
| $code.=<<___ if ($avx); |
| .rva .LSEH_begin_sha1_multi_block_avx |
| .rva .LSEH_end_sha1_multi_block_avx |
| .rva .LSEH_info_sha1_multi_block_avx |
| ___ |
| $code.=<<___ if ($avx>1); |
| .rva .LSEH_begin_sha1_multi_block_avx2 |
| .rva .LSEH_end_sha1_multi_block_avx2 |
| .rva .LSEH_info_sha1_multi_block_avx2 |
| ___ |
| $code.=<<___; |
| .section .xdata |
| .align 8 |
| .LSEH_info_sha1_multi_block: |
| .byte 9,0,0,0 |
| .rva se_handler |
| .rva .Lbody,.Lepilogue # HandlerData[] |
| .LSEH_info_sha1_multi_block_shaext: |
| .byte 9,0,0,0 |
| .rva se_handler |
| .rva .Lbody_shaext,.Lepilogue_shaext # HandlerData[] |
| ___ |
| $code.=<<___ if ($avx); |
| .LSEH_info_sha1_multi_block_avx: |
| .byte 9,0,0,0 |
| .rva se_handler |
| .rva .Lbody_avx,.Lepilogue_avx # HandlerData[] |
| ___ |
| $code.=<<___ if ($avx>1); |
| .LSEH_info_sha1_multi_block_avx2: |
| .byte 9,0,0,0 |
| .rva avx2_handler |
| .rva .Lbody_avx2,.Lepilogue_avx2 # HandlerData[] |
| ___ |
| } |
| #################################################################### |
| |
| sub rex { |
| local *opcode=shift; |
| my ($dst,$src)=@_; |
| my $rex=0; |
| |
| $rex|=0x04 if ($dst>=8); |
| $rex|=0x01 if ($src>=8); |
| unshift @opcode,$rex|0x40 if ($rex); |
| } |
| |
| sub sha1rnds4 { |
| if (@_[0] =~ /\$([x0-9a-f]+),\s*%xmm([0-9]+),\s*%xmm([0-9]+)/) { |
| my @opcode=(0x0f,0x3a,0xcc); |
| rex(\@opcode,$3,$2); |
| push @opcode,0xc0|($2&7)|(($3&7)<<3); # ModR/M |
| my $c=$1; |
| push @opcode,$c=~/^0/?oct($c):$c; |
| return ".byte\t".join(',',@opcode); |
| } else { |
| return "sha1rnds4\t".@_[0]; |
| } |
| } |
| |
| sub sha1op38 { |
| my $instr = shift; |
| my %opcodelet = ( |
| "sha1nexte" => 0xc8, |
| "sha1msg1" => 0xc9, |
| "sha1msg2" => 0xca ); |
| |
| if (defined($opcodelet{$instr}) && @_[0] =~ /%xmm([0-9]+),\s*%xmm([0-9]+)/) { |
| my @opcode=(0x0f,0x38); |
| rex(\@opcode,$2,$1); |
| push @opcode,$opcodelet{$instr}; |
| push @opcode,0xc0|($1&7)|(($2&7)<<3); # ModR/M |
| return ".byte\t".join(',',@opcode); |
| } else { |
| return $instr."\t".@_[0]; |
| } |
| } |
| |
| foreach (split("\n",$code)) { |
| s/\`([^\`]*)\`/eval($1)/ge; |
| |
| s/\b(sha1rnds4)\s+(.*)/sha1rnds4($2)/geo or |
| s/\b(sha1[^\s]*)\s+(.*)/sha1op38($1,$2)/geo or |
| |
| s/\b(vmov[dq])\b(.+)%ymm([0-9]+)/$1$2%xmm$3/go or |
| s/\b(vmovdqu)\b(.+)%x%ymm([0-9]+)/$1$2%xmm$3/go or |
| s/\b(vpinsr[qd])\b(.+)%ymm([0-9]+),%ymm([0-9]+)/$1$2%xmm$3,%xmm$4/go or |
| s/\b(vpextr[qd])\b(.+)%ymm([0-9]+)/$1$2%xmm$3/go or |
| s/\b(vinserti128)\b(\s+)%ymm/$1$2\$1,%xmm/go or |
| s/\b(vpbroadcast[qd]\s+)%ymm([0-9]+)/$1%xmm$2/go; |
| |
| print $_,"\n"; |
| } |
| |
| close STDOUT or die "error closing STDOUT: $!"; |