[aesni|sha*]-mb-x86_64.pl: add data prefetching.
diff --git a/crypto/aes/asm/aesni-mb-x86_64.pl b/crypto/aes/asm/aesni-mb-x86_64.pl index 17ed3b2..e6ac9f9 100644 --- a/crypto/aes/asm/aesni-mb-x86_64.pl +++ b/crypto/aes/asm/aesni-mb-x86_64.pl
@@ -15,8 +15,8 @@ # asymptotic measured # --------------------------- # Westmere 5.00/4=1.25 5.13/4=1.28 -# Atom 15.0/4=3.75 15.7/4=3.93 -# Sandy Bridge 5.06/4=1.27 5.15/4=1.29 +# Atom 15.0/4=3.75 ?15.7/4=3.93 +# Sandy Bridge 5.06/4=1.27 5.18/4=1.29 # Ivy Bridge 5.06/4=1.27 5.14/4=1.29 # Haswell 4.44/4=1.11 4.44/4=1.11 # Bulldozer 5.75/4=1.44 5.76/4=1.44 @@ -27,8 +27,8 @@ # # asymptotic measured # --------------------------- -# Sandy Bridge 5.06/8=0.64 7.05/8=0.88(*) -# Ivy Bridge 5.06/8=0.64 7.02/8=0.88(*) +# Sandy Bridge 5.06/8=0.64 7.10/8=0.89(*) +# Ivy Bridge 5.06/8=0.64 7.14/8=0.89(*) # Haswell 5.00/8=0.63 5.00/8=0.63 # Bulldozer 5.75/8=0.72 5.77/8=0.72 # @@ -188,7 +188,11 @@ sub $offset,$sink aesenc $rndkey1,@out[0] + prefetcht0 31(@inptr[0],$offset) # prefetch input + prefetcht0 31(@inptr[1],$offset) aesenc $rndkey1,@out[1] + prefetcht0 31(@inptr[2],$offset) + prefetcht0 31(@inptr[2],$offset) aesenc $rndkey1,@out[2] aesenc $rndkey1,@out[3] movups 0x30-0x78($key),$rndkey1 @@ -199,8 +203,8 @@ cmp `32+4*$i`(%rsp),$one aesenc $rndkey,@out[0] aesenc $rndkey,@out[1] - cmovge $sink,@inptr[$i] # cancel input aesenc $rndkey,@out[2] + cmovge $sink,@inptr[$i] # cancel input cmovg $sink,@outptr[$i] # sink output aesenc $rndkey,@out[3] movups `0x40+16*$i-0x78`($key),$rndkey @@ -209,7 +213,11 @@ $code.=<<___; movdqa $counters,$mask aesenc $rndkey0,@out[0] + prefetcht0 15(@outptr[0],$offset) # prefetch output + prefetcht0 15(@outptr[1],$offset) aesenc $rndkey0,@out[1] + prefetcht0 15(@outptr[2],$offset) + prefetcht0 15(@outptr[3],$offset) aesenc $rndkey0,@out[2] aesenc $rndkey0,@out[3] movups 0x80-0x78($key),$rndkey0 @@ -260,13 +268,15 @@ aesenc $rndkey0,@out[2] aesenc $rndkey0,@out[3] movups 0xe0-0x78($key),$rndkey0 + jmp .Lenc4x_tail +.align 32 .Lenc4x_tail: aesenc $rndkey1,@out[0] aesenc $rndkey1,@out[1] aesenc $rndkey1,@out[2] - movdqu (@inptr[0],$offset),@inp[0] aesenc $rndkey1,@out[3] + movdqu (@inptr[0],$offset),@inp[0] movdqu 0x10-0x78($key),$rndkey1 aesenclast $rndkey0,@out[0] @@ -426,7 +436,11 @@ sub $offset,$sink aesdec $rndkey1,@out[0] + prefetcht0 31(@inptr[0],$offset) # prefetch input + prefetcht0 31(@inptr[1],$offset) aesdec $rndkey1,@out[1] + prefetcht0 31(@inptr[2],$offset) + prefetcht0 31(@inptr[3],$offset) aesdec $rndkey1,@out[2] aesdec $rndkey1,@out[3] movups 0x30-0x78($key),$rndkey1 @@ -447,7 +461,11 @@ $code.=<<___; movdqa $counters,$mask aesdec $rndkey0,@out[0] + prefetcht0 15(@outptr[0],$offset) # prefetch output + prefetcht0 15(@outptr[1],$offset) aesdec $rndkey0,@out[1] + prefetcht0 15(@outptr[2],$offset) + prefetcht0 15(@outptr[3],$offset) aesdec $rndkey0,@out[2] aesdec $rndkey0,@out[3] movups 0x80-0x78($key),$rndkey0 @@ -498,7 +516,9 @@ aesdec $rndkey0,@out[2] aesdec $rndkey0,@out[3] movups 0xe0-0x78($key),$rndkey0 + jmp .Ldec4x_tail +.align 32 .Ldec4x_tail: aesdec $rndkey1,@out[0] aesdec $rndkey1,@out[1] @@ -512,12 +532,12 @@ movdqu 0x20-0x78($key),$rndkey0 aesdeclast @inp[0],@out[0] - movdqu -16(@inptr[0],$offset),@inp[0] # load next IV aesdeclast @inp[1],@out[1] + movdqu -16(@inptr[0],$offset),@inp[0] # load next IV movdqu -16(@inptr[1],$offset),@inp[1] aesdeclast @inp[2],@out[2] - movdqu -16(@inptr[2],$offset),@inp[2] aesdeclast @inp[3],@out[3] + movdqu -16(@inptr[2],$offset),@inp[2] movdqu -16(@inptr[3],$offset),@inp[3] movups @out[0],-16(@outptr[0],$offset) @@ -682,7 +702,13 @@ ___ $code.=<<___; vaesenc $rndkey,@out[1],@out[1] + prefetcht0 31(@ptr[$i]) # prefetch input vaesenc $rndkey,@out[2],@out[2] +___ +$code.=<<___ if ($i>1); + prefetcht0 15(@ptr[$i-2]) # prefetch output +___ +$code.=<<___; vaesenc $rndkey,@out[3],@out[3] lea (@ptr[$i],$offset),$offset cmovge %rsp,@ptr[$i] # cancel input @@ -703,6 +729,8 @@ } $code.=<<___; vmovdqu 32(%rsp),$counters + prefetcht0 15(@ptr[$i-2]) # prefetch output + prefetcht0 15(@ptr[$i-1]) cmp \$11,$rounds jb .Lenc8x_tail @@ -958,7 +986,13 @@ ___ $code.=<<___; vaesdec $rndkey,@out[1],@out[1] + prefetcht0 31(@ptr[$i]) # prefetch input vaesdec $rndkey,@out[2],@out[2] +___ +$code.=<<___ if ($i>1); + prefetcht0 15(@ptr[$i-2]) # prefetch output +___ +$code.=<<___; vaesdec $rndkey,@out[3],@out[3] lea (@ptr[$i],$offset),$offset cmovge %rsp,@ptr[$i] # cancel input @@ -979,6 +1013,8 @@ } $code.=<<___; vmovdqu 32(%rsp),$counters + prefetcht0 15(@ptr[$i-2]) # prefetch output + prefetcht0 15(@ptr[$i-1]) cmp \$11,$rounds jb .Ldec8x_tail
diff --git a/crypto/sha/asm/sha1-mb-x86_64.pl b/crypto/sha/asm/sha1-mb-x86_64.pl index 93996e1..33e6620 100644 --- a/crypto/sha/asm/sha1-mb-x86_64.pl +++ b/crypto/sha/asm/sha1-mb-x86_64.pl
@@ -14,20 +14,21 @@ # # this +aesni(i) sha1 aesni-sha1 gain(iv) # ------------------------------------------------------------------- -# Westmere(ii) 10.4/n +1.28=3.88(n=4) 5.44 6.58 +70% -# Atom(ii) 18.9/n +3.93=8.66(n=4) 10.0 14.0 +62% +# Westmere(ii) 10.7/n +1.28=3.96(n=4) 5.30 6.66 +68% +# Atom(ii) 18.9?/n +3.93=8.66(n=4) 10.0 14.0 +62% # Sandy Bridge (8.16 +5.15=13.3)/n 4.99 5.98 +80% -# Ivy Bridge (8.03 +5.14=13.2)/n 4.60 5.54 +68% +# Ivy Bridge (8.08 +5.14=13.2)/n 4.60 5.54 +68% # Haswell(iii) (8.96 +5.00=14.0)/n 3.57 4.55 +160% -# Bulldozer (9.75 +5.76=15.5)/n 5.95 6.37 +64% +# Bulldozer (9.76 +5.76=15.5)/n 5.95 6.37 +64% # # (i) multi-block CBC encrypt with 128-bit key; # (ii) (HASH+AES)/n does not apply to Westmere for n>3 and Atom, # because of lower AES-NI instruction throughput; # (iii) "this" is for n=8, when we gather twice as much data, result -# for n=4 is 7.98+4.44=12.4; -# (iv) improvement coefficients in real-life application are somewhat -# lower and range from 30% to 100% (on Haswell); +# for n=4 is 8.00+4.44=12.4; +# (iv) presented improvement coefficients are asymptotic limits and +# in real-life application are somewhat lower, e.g. for 2KB +# fragments they range from 30% to 100% (on Haswell); $flavour = shift; $output = shift; @@ -80,6 +81,14 @@ @Xi=map("%xmm$_",(10..14)); $K="%xmm15"; +if (1) { + # Atom-specific optimization aiming to eliminate pshufb with high + # registers [and thus get rid of 48 cycles accumulated penalty] + @Xi=map("%xmm$_",(0..4)); + ($tx,$t0,$t1,$t2,$t3)=map("%xmm$_",(5..9)); + @V=($A,$B,$C,$D,$E)=map("%xmm$_",(10..14)); +} + $REG_SZ=16; sub Xi_off { @@ -139,8 +148,8 @@ psrld \$2,$b paddd $t2,$e # e+=rol(a,5) - movd `4*$j-16*4`(@ptr[2]),$t2 pshufb $tx,@Xi[1] + movd `4*$j-16*4`(@ptr[2]),$t2 por $t1,$b # b=rol(b,30) ___ $code.=<<___ if ($i==14); # just load input @@ -152,6 +161,7 @@ movdqa $b,$t1 movdqa $b,$t0 pslld \$5,$t2 + prefetcht0 63(@ptr[0]) pandn $d,$t1 pand $c,$t0 punpckldq $t3,@Xi[1] @@ -162,14 +172,17 @@ psrld \$27,$t3 pxor $t1,$t0 # Ch(b,c,d) movdqa $b,$t1 + prefetcht0 63(@ptr[1]) por $t3,$t2 # rol(a,5) pslld \$30,$t1 paddd $t0,$e # e+=Ch(b,c,d) + prefetcht0 63(@ptr[2]) psrld \$2,$b paddd $t2,$e # e+=rol(a,5) pshufb $tx,@Xi[1] + prefetcht0 63(@ptr[3]) por $t1,$b # b=rol(b,30) ___ $code.=<<___ if ($i>=13 && $i<15); @@ -382,12 +395,12 @@ movdqu 0x60($ctx),$D movdqu 0x80($ctx),$E movdqa 0x60($Tbl),$tx # pbswap_mask + movdqa -0x20($Tbl),$K # K_00_19 jmp .Loop .align 32 .Loop: ___ -$code.=" movdqa -0x20($Tbl),$K\n"; # K_00_19 for($i=0;$i<20;$i++) { &BODY_00_19($i,@V); unshift(@V,pop(@V)); } $code.=" movdqa 0x00($Tbl),$K\n"; # K_20_39 for(;$i<40;$i++) { &BODY_20_39($i,@V); unshift(@V,pop(@V)); } @@ -434,6 +447,7 @@ movdqa @Xi[0],(%rbx) # save counters movdqa 0x60($Tbl),$tx # pbswap_mask + movdqa -0x20($Tbl),$K # K_00_19 dec $num jnz .Loop @@ -551,6 +565,7 @@ ___ $code.=<<___ if ($i==14); vpaddd $K,$e,$e # e+=K_00_19 + prefetcht0 63(@ptr[0]) vpslld \$5,$a,$t2 vpandn $d,$b,$t1 vpand $c,$b,$t0 @@ -559,14 +574,17 @@ vpaddd @Xi[0],$e,$e # e+=X[i] $vpack $t3,@Xi[1],@Xi[1] vpsrld \$27,$a,$t3 + prefetcht0 63(@ptr[1]) vpxor $t1,$t0,$t0 # Ch(b,c,d) vpslld \$30,$b,$t1 vpor $t3,$t2,$t2 # rol(a,5) + prefetcht0 63(@ptr[2]) vpaddd $t0,$e,$e # e+=Ch(b,c,d) vpsrld \$2,$b,$b vpaddd $t2,$e,$e # e+=rol(a,5) + prefetcht0 63(@ptr[3]) vpshufb $tx,@Xi[1],@Xi[1] vpor $t1,$b,$b # b=rol(b,30) ___ @@ -580,6 +598,7 @@ vpaddd $K,$e,$e # e+=K_00_19 vpslld \$5,$a,$t2 vpandn $d,$b,$t1 + `"prefetcht0 63(@ptr[4])" if ($i==15 && $REG_SZ==32)` vpand $c,$b,$t0 vmovdqa @Xi[0],`&Xi_off($i)` @@ -588,14 +607,17 @@ vpsrld \$27,$a,$t3 vpxor $t1,$t0,$t0 # Ch(b,c,d) vpxor @Xi[3],@Xi[1],@Xi[1] + `"prefetcht0 63(@ptr[5])" if ($i==15 && $REG_SZ==32)` vpslld \$30,$b,$t1 vpor $t3,$t2,$t2 # rol(a,5) vpaddd $t0,$e,$e # e+=Ch(b,c,d) + `"prefetcht0 63(@ptr[6])" if ($i==15 && $REG_SZ==32)` vpsrld \$31,@Xi[1],$tx vpaddd @Xi[1],@Xi[1],@Xi[1] vpsrld \$2,$b,$b + `"prefetcht0 63(@ptr[7])" if ($i==15 && $REG_SZ==32)` vpaddd $t2,$e,$e # e+=rol(a,5) vpor $tx,@Xi[1],@Xi[1] # rol \$1,@Xi[1] vpor $t1,$b,$b # b=rol(b,30)
diff --git a/crypto/sha/asm/sha256-mb-x86_64.pl b/crypto/sha/asm/sha256-mb-x86_64.pl index 2e4b102..e86f0bc 100644 --- a/crypto/sha/asm/sha256-mb-x86_64.pl +++ b/crypto/sha/asm/sha256-mb-x86_64.pl
@@ -15,7 +15,7 @@ # this +aesni(i) sha256 aesni-sha256 gain(iv) # ------------------------------------------------------------------- # Westmere(ii) 23.3/n +1.28=7.11(n=4) 12.3 +3.75=16.1 +126% -# Atom(ii) 39.1/n +3.93=13.7(n=4) 20.8 +5.69=26.5 +93% +# Atom(ii) ?39.1/n +3.93=13.7(n=4) 20.8 +5.69=26.5 +93% # Sandy Bridge (20.5 +5.15=25.7)/n 11.6 13.0 +103% # Ivy Bridge (20.4 +5.14=25.5)/n 10.3 11.6 +82% # Haswell(iii) (21.0 +5.00=26.0)/n 7.80 8.79 +170% @@ -27,8 +27,9 @@ # AES-NI-SHA256 stitch for these processors; # (iii) "this" is for n=8, when we gather twice as much data, result # for n=4 is 20.3+4.44=24.7; -# (iv) improvement coefficients in real-life application are somewhat -# lower and range from 75% to 130% (on Haswell); +# (iv) presented improvement coefficients are asymptotic limits and +# in real-life application are somewhat lower, e.g. for 2KB +# fragments they range from 75% to 13% (on Haswell); $flavour = shift; $output = shift; @@ -135,6 +136,7 @@ psrld \$25-11,$t2 movdqa $e,$t1 + `"prefetch 63(@ptr[0])" if ($i==15)` pxor $t3,$sigma movdqa $e,$axb # borrow $axb pslld \$26-21,$t3 @@ -142,6 +144,7 @@ pand $f,$axb pxor $t2,$sigma + `"prefetch 63(@ptr[1])" if ($i==15)` movdqa $a,$t2 pxor $t3,$sigma # Sigma1(e) movdqa $a,$t3 @@ -153,6 +156,7 @@ pslld \$10,$t3 pxor $a,$axb # a^b, b^c in next round + `"prefetch 63(@ptr[2])" if ($i==15)` psrld \$13,$sigma pxor $t3,$t2 paddd $t1,$Xi # Xi+=Ch(e,f,g) @@ -160,6 +164,7 @@ pand $axb,$bxc pxor $sigma,$t2 + `"prefetch 63(@ptr[3])" if ($i==15)` psrld \$22-13,$sigma pxor $t3,$t2 movdqa $b,$h @@ -465,30 +470,38 @@ vpsrld \$25,$e,$t2 vpxor $t3,$sigma,$sigma + `"prefetch 63(@ptr[0])" if ($i==15)` vpslld \$7,$e,$t3 vpandn $g,$e,$t1 vpand $f,$e,$axb # borrow $axb + `"prefetch 63(@ptr[1])" if ($i==15)` vpxor $t2,$sigma,$sigma vpsrld \$2,$a,$h # borrow $h vpxor $t3,$sigma,$sigma # Sigma1(e) + `"prefetch 63(@ptr[2])" if ($i==15)` vpslld \$30,$a,$t2 vpxor $axb,$t1,$t1 # Ch(e,f,g) vpxor $a,$b,$axb # a^b, b^c in next round + `"prefetch 63(@ptr[3])" if ($i==15)` vpxor $t2,$h,$h vpaddd $sigma,$Xi,$Xi # Xi+=Sigma1(e) vpsrld \$13,$a,$t2 + `"prefetch 63(@ptr[4])" if ($i==15 && $REG_SZ==32)` vpslld \$19,$a,$t3 vpaddd $t1,$Xi,$Xi # Xi+=Ch(e,f,g) vpand $axb,$bxc,$bxc + `"prefetch 63(@ptr[5])" if ($i==15 && $REG_SZ==32)` vpxor $t2,$h,$sigma vpsrld \$22,$a,$t2 vpxor $t3,$sigma,$sigma + `"prefetch 63(@ptr[6])" if ($i==15 && $REG_SZ==32)` vpslld \$10,$a,$t3 vpxor $bxc,$b,$h # h=Maj(a,b,c)=Ch(a^b,c,b) vpaddd $Xi,$d,$d # d+=Xi + `"prefetch 63(@ptr[7])" if ($i==15 && $REG_SZ==32)` vpxor $t2,$sigma,$sigma vpxor $t3,$sigma,$sigma # Sigma0(a)