#! /usr/bin/env perl
# Copyright 2020-2022 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the Apache License 2.0 (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

#
# This module implements SM4 with ASIMD on aarch64
#
# Feb 2022
#

# $output is the last argument if it looks like a file (it has an extension)
# $flavour is the first argument if it doesn't look like a file
$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
$flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;

$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
die "can't locate arm-xlate.pl";

open OUT,"| \"$^X\" $xlate $flavour \"$output\""
    or die "can't call $xlate: $!";
*STDOUT=*OUT;

$prefix="vpsm4";
my @vtmp=map("v$_",(0..3));
my @data=map("v$_",(4..7));
my @datax=map("v$_",(8..11));
my ($rk0,$rk1)=("v12","v13");
my ($rka,$rkb)=("v14","v15");
my @vtmpx=map("v$_",(12..15));
my @sbox=map("v$_",(16..31));
my ($inp,$outp,$blocks,$rks)=("x0","x1","w2","x3");
my ($tmpw,$tmp,$wtmp0,$wtmp1,$wtmp2)=("w6","x6","w7","w8","w9");
my ($ptr,$counter)=("x10","w11");
my ($word0,$word1,$word2,$word3)=("w12","w13","w14","w15");

sub rev32() {
	my $dst = shift;
	my $src = shift;

	if ($src and ("$src" ne "$dst")) {
$code.=<<___;
#ifndef __ARMEB__
	rev32	$dst.16b,$src.16b
#else
	mov	$dst.16b,$src.16b
#endif
___
	} else {
$code.=<<___;
#ifndef __ARMEB__
	rev32	$dst.16b,$dst.16b
#endif
___
	}
}

sub transpose() {
	my ($dat0,$dat1,$dat2,$dat3,$vt0,$vt1,$vt2,$vt3) = @_;

$code.=<<___;
	zip1	$vt0.4s,$dat0.4s,$dat1.4s
	zip2	$vt1.4s,$dat0.4s,$dat1.4s
	zip1	$vt2.4s,$dat2.4s,$dat3.4s
	zip2	$vt3.4s,$dat2.4s,$dat3.4s
	zip1	$dat0.2d,$vt0.2d,$vt2.2d
	zip2	$dat1.2d,$vt0.2d,$vt2.2d
	zip1	$dat2.2d,$vt1.2d,$vt3.2d
	zip2	$dat3.2d,$vt1.2d,$vt3.2d
___
}

# sbox operations for 4-lane of words
sub sbox() {
	my $dat = shift;

$code.=<<___;
	movi	@vtmp[0].16b,#64
	movi	@vtmp[1].16b,#128
	movi	@vtmp[2].16b,#192
	sub	@vtmp[0].16b,$dat.16b,@vtmp[0].16b
	sub	@vtmp[1].16b,$dat.16b,@vtmp[1].16b
	sub	@vtmp[2].16b,$dat.16b,@vtmp[2].16b
	tbl	$dat.16b,{@sbox[0].16b,@sbox[1].16b,@sbox[2].16b,@sbox[3].16b},$dat.16b
	tbl	@vtmp[0].16b,{@sbox[4].16b,@sbox[5].16b,@sbox[6].16b,@sbox[7].16b},@vtmp[0].16b
	tbl	@vtmp[1].16b,{@sbox[8].16b,@sbox[9].16b,@sbox[10].16b,@sbox[11].16b},@vtmp[1].16b
	tbl	@vtmp[2].16b,{@sbox[12].16b,@sbox[13].16b,@sbox[14].16b,@sbox[15].16b},@vtmp[2].16b
	add	@vtmp[0].2d,@vtmp[0].2d,@vtmp[1].2d
	add	@vtmp[2].2d,@vtmp[2].2d,$dat.2d
	add	$dat.2d,@vtmp[0].2d,@vtmp[2].2d

	ushr	@vtmp[0].4s,$dat.4s,32-2
	sli	@vtmp[0].4s,$dat.4s,2
	ushr	@vtmp[2].4s,$dat.4s,32-10
	eor	@vtmp[1].16b,@vtmp[0].16b,$dat.16b
	sli	@vtmp[2].4s,$dat.4s,10
	eor	@vtmp[1].16b,@vtmp[2].16b,$vtmp[1].16b
	ushr	@vtmp[0].4s,$dat.4s,32-18
	sli	@vtmp[0].4s,$dat.4s,18
	ushr	@vtmp[2].4s,$dat.4s,32-24
	eor	@vtmp[1].16b,@vtmp[0].16b,$vtmp[1].16b
	sli	@vtmp[2].4s,$dat.4s,24
	eor	$dat.16b,@vtmp[2].16b,@vtmp[1].16b
___
}

# sbox operation for 8-lane of words
sub sbox_double() {
	my $dat = shift;
	my $datx = shift;

$code.=<<___;
	movi	@vtmp[3].16b,#64
	sub	@vtmp[0].16b,$dat.16b,@vtmp[3].16b
	sub	@vtmp[1].16b,@vtmp[0].16b,@vtmp[3].16b
	sub	@vtmp[2].16b,@vtmp[1].16b,@vtmp[3].16b
	tbl	$dat.16b,{@sbox[0].16b,@sbox[1].16b,@sbox[2].16b,@sbox[3].16b},$dat.16b
	tbl	@vtmp[0].16b,{@sbox[4].16b,@sbox[5].16b,@sbox[6].16b,@sbox[7].16b},@vtmp[0].16b
	tbl	@vtmp[1].16b,{@sbox[8].16b,@sbox[9].16b,@sbox[10].16b,@sbox[11].16b},@vtmp[1].16b
	tbl	@vtmp[2].16b,{@sbox[12].16b,@sbox[13].16b,@sbox[14].16b,@sbox[15].16b},@vtmp[2].16b
	add	@vtmp[1].2d,@vtmp[0].2d,@vtmp[1].2d
	add	$dat.2d,@vtmp[2].2d,$dat.2d
	add	$dat.2d,@vtmp[1].2d,$dat.2d

	sub	@vtmp[0].16b,$datx.16b,@vtmp[3].16b
	sub	@vtmp[1].16b,@vtmp[0].16b,@vtmp[3].16b
	sub	@vtmp[2].16b,@vtmp[1].16b,@vtmp[3].16b
	tbl	$datx.16b,{@sbox[0].16b,@sbox[1].16b,@sbox[2].16b,@sbox[3].16b},$datx.16b
	tbl	@vtmp[0].16b,{@sbox[4].16b,@sbox[5].16b,@sbox[6].16b,@sbox[7].16b},@vtmp[0].16b
	tbl	@vtmp[1].16b,{@sbox[8].16b,@sbox[9].16b,@sbox[10].16b,@sbox[11].16b},@vtmp[1].16b
	tbl	@vtmp[2].16b,{@sbox[12].16b,@sbox[13].16b,@sbox[14].16b,@sbox[15].16b},@vtmp[2].16b
	add	@vtmp[1].2d,@vtmp[0].2d,@vtmp[1].2d
	add	$datx.2d,@vtmp[2].2d,$datx.2d
	add	$datx.2d,@vtmp[1].2d,$datx.2d

	ushr	@vtmp[0].4s,$dat.4s,32-2
	sli	@vtmp[0].4s,$dat.4s,2
	ushr	@vtmp[2].4s,$datx.4s,32-2
	eor	@vtmp[1].16b,@vtmp[0].16b,$dat.16b
	sli	@vtmp[2].4s,$datx.4s,2

	ushr	@vtmp[0].4s,$dat.4s,32-10
	eor	@vtmp[3].16b,@vtmp[2].16b,$datx.16b
	sli	@vtmp[0].4s,$dat.4s,10
	ushr	@vtmp[2].4s,$datx.4s,32-10
	eor	@vtmp[1].16b,@vtmp[0].16b,$vtmp[1].16b
	sli	@vtmp[2].4s,$datx.4s,10

	ushr	@vtmp[0].4s,$dat.4s,32-18
	eor	@vtmp[3].16b,@vtmp[2].16b,$vtmp[3].16b
	sli	@vtmp[0].4s,$dat.4s,18
	ushr	@vtmp[2].4s,$datx.4s,32-18
	eor	@vtmp[1].16b,@vtmp[0].16b,$vtmp[1].16b
	sli	@vtmp[2].4s,$datx.4s,18

	ushr	@vtmp[0].4s,$dat.4s,32-24
	eor	@vtmp[3].16b,@vtmp[2].16b,$vtmp[3].16b
	sli	@vtmp[0].4s,$dat.4s,24
	ushr	@vtmp[2].4s,$datx.4s,32-24
	eor	$dat.16b,@vtmp[0].16b,@vtmp[1].16b
	sli	@vtmp[2].4s,$datx.4s,24
	eor	$datx.16b,@vtmp[2].16b,@vtmp[3].16b
___
}

# sbox operation for one single word
sub sbox_1word () {
	my $word = shift;

$code.=<<___;
	movi	@vtmp[1].16b,#64
	movi	@vtmp[2].16b,#128
	movi	@vtmp[3].16b,#192
	mov	@vtmp[0].s[0],$word

	sub	@vtmp[1].16b,@vtmp[0].16b,@vtmp[1].16b
	sub	@vtmp[2].16b,@vtmp[0].16b,@vtmp[2].16b
	sub	@vtmp[3].16b,@vtmp[0].16b,@vtmp[3].16b

	tbl	@vtmp[0].16b,{@sbox[0].16b,@sbox[1].16b,@sbox[2].16b,@sbox[3].16b},@vtmp[0].16b
	tbl	@vtmp[1].16b,{@sbox[4].16b,@sbox[5].16b,@sbox[6].16b,@sbox[7].16b},@vtmp[1].16b
	tbl	@vtmp[2].16b,{@sbox[8].16b,@sbox[9].16b,@sbox[10].16b,@sbox[11].16b},@vtmp[2].16b
	tbl	@vtmp[3].16b,{@sbox[12].16b,@sbox[13].16b,@sbox[14].16b,@sbox[15].16b},@vtmp[3].16b

	mov	$word,@vtmp[0].s[0]
	mov	$wtmp0,@vtmp[1].s[0]
	mov	$wtmp2,@vtmp[2].s[0]
	add	$wtmp0,$word,$wtmp0
	mov	$word,@vtmp[3].s[0]
	add	$wtmp0,$wtmp0,$wtmp2
	add	$wtmp0,$wtmp0,$word

	eor	$word,$wtmp0,$wtmp0,ror #32-2
	eor	$word,$word,$wtmp0,ror #32-10
	eor	$word,$word,$wtmp0,ror #32-18
	eor	$word,$word,$wtmp0,ror #32-24
___
}

# sm4 for one block of data, in scalar registers word0/word1/word2/word3
sub sm4_1blk () {
	my $kptr = shift;

$code.=<<___;
	ldp	$wtmp0,$wtmp1,[$kptr],8
	// B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0)
	eor	$tmpw,$word2,$word3
	eor	$wtmp2,$wtmp0,$word1
	eor	$tmpw,$tmpw,$wtmp2
___
	&sbox_1word($tmpw);
$code.=<<___;
	eor	$word0,$word0,$tmpw
	// B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1)
	eor	$tmpw,$word2,$word3
	eor	$wtmp2,$word0,$wtmp1
	eor	$tmpw,$tmpw,$wtmp2
___
	&sbox_1word($tmpw);
$code.=<<___;
	ldp	$wtmp0,$wtmp1,[$kptr],8
	eor	$word1,$word1,$tmpw
	// B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2)
	eor	$tmpw,$word0,$word1
	eor	$wtmp2,$wtmp0,$word3
	eor	$tmpw,$tmpw,$wtmp2
___
	&sbox_1word($tmpw);
$code.=<<___;
	eor	$word2,$word2,$tmpw
	// B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3)
	eor	$tmpw,$word0,$word1
	eor	$wtmp2,$word2,$wtmp1
	eor	$tmpw,$tmpw,$wtmp2
___
	&sbox_1word($tmpw);
$code.=<<___;
	eor	$word3,$word3,$tmpw
___
}

# sm4 for 4-lanes of data, in neon registers data0/data1/data2/data3
sub sm4_4blks () {
	my $kptr = shift;

$code.=<<___;
	ldp	$wtmp0,$wtmp1,[$kptr],8
	dup	$rk0.4s,$wtmp0
	dup	$rk1.4s,$wtmp1

	// B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0)
	eor	$rka.16b,@data[2].16b,@data[3].16b
	eor	$rk0.16b,@data[1].16b,$rk0.16b
	eor	$rk0.16b,$rka.16b,$rk0.16b
___
	&sbox($rk0);
$code.=<<___;
	eor	@data[0].16b,@data[0].16b,$rk0.16b

	// B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1)
	eor	$rka.16b,$rka.16b,@data[0].16b
	eor	$rk1.16b,$rka.16b,$rk1.16b
___
	&sbox($rk1);
$code.=<<___;
	ldp	$wtmp0,$wtmp1,[$kptr],8
	eor	@data[1].16b,@data[1].16b,$rk1.16b

	dup	$rk0.4s,$wtmp0
	dup	$rk1.4s,$wtmp1

	// B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2)
	eor	$rka.16b,@data[0].16b,@data[1].16b
	eor	$rk0.16b,@data[3].16b,$rk0.16b
	eor	$rk0.16b,$rka.16b,$rk0.16b
___
	&sbox($rk0);
$code.=<<___;
	eor	@data[2].16b,@data[2].16b,$rk0.16b

	// B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3)
	eor	$rka.16b,$rka.16b,@data[2].16b
	eor	$rk1.16b,$rka.16b,$rk1.16b
___
	&sbox($rk1);
$code.=<<___;
	eor	@data[3].16b,@data[3].16b,$rk1.16b
___
}

# sm4 for 8 lanes of data, in neon registers
# data0/data1/data2/data3 datax0/datax1/datax2/datax3
sub sm4_8blks () {
	my $kptr = shift;

$code.=<<___;
	ldp	$wtmp0,$wtmp1,[$kptr],8
	// B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0)
	dup	$rk0.4s,$wtmp0
	eor	$rka.16b,@data[2].16b,@data[3].16b
	eor	$rkb.16b,@datax[2].16b,@datax[3].16b
	eor	@vtmp[0].16b,@data[1].16b,$rk0.16b
	eor	@vtmp[1].16b,@datax[1].16b,$rk0.16b
	eor	$rk0.16b,$rka.16b,@vtmp[0].16b
	eor	$rk1.16b,$rkb.16b,@vtmp[1].16b
___
	&sbox_double($rk0,$rk1);
$code.=<<___;
	eor	@data[0].16b,@data[0].16b,$rk0.16b
	eor	@datax[0].16b,@datax[0].16b,$rk1.16b

	// B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1)
	dup	$rk1.4s,$wtmp1
	eor	$rka.16b,$rka.16b,@data[0].16b
	eor	$rkb.16b,$rkb.16b,@datax[0].16b
	eor	$rk0.16b,$rka.16b,$rk1.16b
	eor	$rk1.16b,$rkb.16b,$rk1.16b
___
	&sbox_double($rk0,$rk1);
$code.=<<___;
	ldp	$wtmp0,$wtmp1,[$kptr],8
	eor	@data[1].16b,@data[1].16b,$rk0.16b
	eor	@datax[1].16b,@datax[1].16b,$rk1.16b

	// B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2)
	dup	$rk0.4s,$wtmp0
	eor	$rka.16b,@data[0].16b,@data[1].16b
	eor	$rkb.16b,@datax[0].16b,@datax[1].16b
	eor	@vtmp[0].16b,@data[3].16b,$rk0.16b
	eor	@vtmp[1].16b,@datax[3].16b,$rk0.16b
	eor	$rk0.16b,$rka.16b,@vtmp[0].16b
	eor	$rk1.16b,$rkb.16b,@vtmp[1].16b
___
	&sbox_double($rk0,$rk1);
$code.=<<___;
	eor	@data[2].16b,@data[2].16b,$rk0.16b
	eor	@datax[2].16b,@datax[2].16b,$rk1.16b

	// B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3)
	dup	$rk1.4s,$wtmp1
	eor	$rka.16b,$rka.16b,@data[2].16b
	eor	$rkb.16b,$rkb.16b,@datax[2].16b
	eor	$rk0.16b,$rka.16b,$rk1.16b
	eor	$rk1.16b,$rkb.16b,$rk1.16b
___
	&sbox_double($rk0,$rk1);
$code.=<<___;
	eor	@data[3].16b,@data[3].16b,$rk0.16b
	eor	@datax[3].16b,@datax[3].16b,$rk1.16b
___
}

sub encrypt_1blk_norev() {
	my $dat = shift;

$code.=<<___;
	mov	$ptr,$rks
	mov	$counter,#8
	mov	$word0,$dat.s[0]
	mov	$word1,$dat.s[1]
	mov	$word2,$dat.s[2]
	mov	$word3,$dat.s[3]
10:
___
	&sm4_1blk($ptr);
$code.=<<___;
	subs	$counter,$counter,#1
	b.ne	10b
	mov	$dat.s[0],$word3
	mov	$dat.s[1],$word2
	mov	$dat.s[2],$word1
	mov	$dat.s[3],$word0
___
}

sub encrypt_1blk() {
	my $dat = shift;

	&encrypt_1blk_norev($dat);
	&rev32($dat,$dat);
}

sub encrypt_4blks() {
$code.=<<___;
	mov	$ptr,$rks
	mov	$counter,#8
10:
___
	&sm4_4blks($ptr);
$code.=<<___;
	subs	$counter,$counter,#1
	b.ne	10b
___
	&rev32(@vtmp[3],@data[0]);
	&rev32(@vtmp[2],@data[1]);
	&rev32(@vtmp[1],@data[2]);
	&rev32(@vtmp[0],@data[3]);
}

sub encrypt_8blks() {
$code.=<<___;
	mov	$ptr,$rks
	mov	$counter,#8
10:
___
	&sm4_8blks($ptr);
$code.=<<___;
	subs	$counter,$counter,#1
	b.ne	10b
___
	&rev32(@vtmp[3],@data[0]);
	&rev32(@vtmp[2],@data[1]);
	&rev32(@vtmp[1],@data[2]);
	&rev32(@vtmp[0],@data[3]);
	&rev32(@data[3],@datax[0]);
	&rev32(@data[2],@datax[1]);
	&rev32(@data[1],@datax[2]);
	&rev32(@data[0],@datax[3]);
}

sub load_sbox () {
	my $data = shift;

$code.=<<___;
	adr	$ptr,.Lsbox
	ld1	{@sbox[0].4s,@sbox[1].4s,@sbox[2].4s,@sbox[3].4s},[$ptr],#64
	ld1	{@sbox[4].4s,@sbox[5].4s,@sbox[6].4s,@sbox[7].4s},[$ptr],#64
	ld1	{@sbox[8].4s,@sbox[9].4s,@sbox[10].4s,@sbox[11].4s},[$ptr],#64
	ld1	{@sbox[12].4s,@sbox[13].4s,@sbox[14].4s,@sbox[15].4s},[$ptr]
___
}

$code=<<___;
#include "arm_arch.h"
.arch	armv8-a
.text

.type	_vpsm4_consts,%object
.align	7
_vpsm4_consts:
.Lsbox:
	.byte 0xD6,0x90,0xE9,0xFE,0xCC,0xE1,0x3D,0xB7,0x16,0xB6,0x14,0xC2,0x28,0xFB,0x2C,0x05
	.byte 0x2B,0x67,0x9A,0x76,0x2A,0xBE,0x04,0xC3,0xAA,0x44,0x13,0x26,0x49,0x86,0x06,0x99
	.byte 0x9C,0x42,0x50,0xF4,0x91,0xEF,0x98,0x7A,0x33,0x54,0x0B,0x43,0xED,0xCF,0xAC,0x62
	.byte 0xE4,0xB3,0x1C,0xA9,0xC9,0x08,0xE8,0x95,0x80,0xDF,0x94,0xFA,0x75,0x8F,0x3F,0xA6
	.byte 0x47,0x07,0xA7,0xFC,0xF3,0x73,0x17,0xBA,0x83,0x59,0x3C,0x19,0xE6,0x85,0x4F,0xA8
	.byte 0x68,0x6B,0x81,0xB2,0x71,0x64,0xDA,0x8B,0xF8,0xEB,0x0F,0x4B,0x70,0x56,0x9D,0x35
	.byte 0x1E,0x24,0x0E,0x5E,0x63,0x58,0xD1,0xA2,0x25,0x22,0x7C,0x3B,0x01,0x21,0x78,0x87
	.byte 0xD4,0x00,0x46,0x57,0x9F,0xD3,0x27,0x52,0x4C,0x36,0x02,0xE7,0xA0,0xC4,0xC8,0x9E
	.byte 0xEA,0xBF,0x8A,0xD2,0x40,0xC7,0x38,0xB5,0xA3,0xF7,0xF2,0xCE,0xF9,0x61,0x15,0xA1
	.byte 0xE0,0xAE,0x5D,0xA4,0x9B,0x34,0x1A,0x55,0xAD,0x93,0x32,0x30,0xF5,0x8C,0xB1,0xE3
	.byte 0x1D,0xF6,0xE2,0x2E,0x82,0x66,0xCA,0x60,0xC0,0x29,0x23,0xAB,0x0D,0x53,0x4E,0x6F
	.byte 0xD5,0xDB,0x37,0x45,0xDE,0xFD,0x8E,0x2F,0x03,0xFF,0x6A,0x72,0x6D,0x6C,0x5B,0x51
	.byte 0x8D,0x1B,0xAF,0x92,0xBB,0xDD,0xBC,0x7F,0x11,0xD9,0x5C,0x41,0x1F,0x10,0x5A,0xD8
	.byte 0x0A,0xC1,0x31,0x88,0xA5,0xCD,0x7B,0xBD,0x2D,0x74,0xD0,0x12,0xB8,0xE5,0xB4,0xB0
	.byte 0x89,0x69,0x97,0x4A,0x0C,0x96,0x77,0x7E,0x65,0xB9,0xF1,0x09,0xC5,0x6E,0xC6,0x84
	.byte 0x18,0xF0,0x7D,0xEC,0x3A,0xDC,0x4D,0x20,0x79,0xEE,0x5F,0x3E,0xD7,0xCB,0x39,0x48
.Lck:
	.long 0x00070E15, 0x1C232A31, 0x383F464D, 0x545B6269
	.long 0x70777E85, 0x8C939AA1, 0xA8AFB6BD, 0xC4CBD2D9
	.long 0xE0E7EEF5, 0xFC030A11, 0x181F262D, 0x343B4249
	.long 0x50575E65, 0x6C737A81, 0x888F969D, 0xA4ABB2B9
	.long 0xC0C7CED5, 0xDCE3EAF1, 0xF8FF060D, 0x141B2229
	.long 0x30373E45, 0x4C535A61, 0x686F767D, 0x848B9299
	.long 0xA0A7AEB5, 0xBCC3CAD1, 0xD8DFE6ED, 0xF4FB0209
	.long 0x10171E25, 0x2C333A41, 0x484F565D, 0x646B7279
.Lfk:
	.dword 0x56aa3350a3b1bac6,0xb27022dc677d9197
.Lshuffles:
	.dword 0x0B0A090807060504,0x030201000F0E0D0C

.size	_vpsm4_consts,.-_vpsm4_consts
___

{{{
my ($key,$keys,$enc)=("x0","x1","w2");
my ($pointer,$schedules,$wtmp,$roundkey)=("x5","x6","w7","w8");
my ($vkey,$vfk,$vmap)=("v5","v6","v7");
$code.=<<___;
.type	_vpsm4_set_key,%function
.align	4
_vpsm4_set_key:
	AARCH64_VALID_CALL_TARGET
	ld1	{$vkey.4s},[$key]
___
	&load_sbox();
	&rev32($vkey,$vkey);
$code.=<<___;
	adr	$pointer,.Lshuffles
	ld1	{$vmap.4s},[$pointer]
	adr	$pointer,.Lfk
	ld1	{$vfk.4s},[$pointer]
	eor	$vkey.16b,$vkey.16b,$vfk.16b
	mov	$schedules,#32
	adr	$pointer,.Lck
	movi	@vtmp[0].16b,#64
	cbnz	$enc,1f
	add	$keys,$keys,124
1:
	mov	$wtmp,$vkey.s[1]
	ldr	$roundkey,[$pointer],#4
	eor	$roundkey,$roundkey,$wtmp
	mov	$wtmp,$vkey.s[2]
	eor	$roundkey,$roundkey,$wtmp
	mov	$wtmp,$vkey.s[3]
	eor	$roundkey,$roundkey,$wtmp
	// sbox lookup
	mov	@data[0].s[0],$roundkey
	tbl	@vtmp[1].16b,{@sbox[0].16b,@sbox[1].16b,@sbox[2].16b,@sbox[3].16b},@data[0].16b
	sub	@data[0].16b,@data[0].16b,@vtmp[0].16b
	tbx	@vtmp[1].16b,{@sbox[4].16b,@sbox[5].16b,@sbox[6].16b,@sbox[7].16b},@data[0].16b
	sub	@data[0].16b,@data[0].16b,@vtmp[0].16b
	tbx	@vtmp[1].16b,{@sbox[8].16b,@sbox[9].16b,@sbox[10].16b,@sbox[11].16b},@data[0].16b
	sub	@data[0].16b,@data[0].16b,@vtmp[0].16b
	tbx	@vtmp[1].16b,{@sbox[12].16b,@sbox[13].16b,@sbox[14].16b,@sbox[15].16b},@data[0].16b
	mov	$wtmp,@vtmp[1].s[0]
	eor	$roundkey,$wtmp,$wtmp,ror #19
	eor	$roundkey,$roundkey,$wtmp,ror #9
	mov	$wtmp,$vkey.s[0]
	eor	$roundkey,$roundkey,$wtmp
	mov	$vkey.s[0],$roundkey
	cbz	$enc,2f
	str	$roundkey,[$keys],#4
	b	3f
2:
	str	$roundkey,[$keys],#-4
3:
	tbl	$vkey.16b,{$vkey.16b},$vmap.16b
	subs	$schedules,$schedules,#1
	b.ne	1b
	ret
.size	_vpsm4_set_key,.-_vpsm4_set_key
___
}}}


{{{
$code.=<<___;
.type	_vpsm4_enc_4blks,%function
.align	4
_vpsm4_enc_4blks:
	AARCH64_VALID_CALL_TARGET
___
	&encrypt_4blks();
$code.=<<___;
	ret
.size	_vpsm4_enc_4blks,.-_vpsm4_enc_4blks
___
}}}

{{{
$code.=<<___;
.type	_vpsm4_enc_8blks,%function
.align	4
_vpsm4_enc_8blks:
	AARCH64_VALID_CALL_TARGET
___
	&encrypt_8blks();
$code.=<<___;
	ret
.size	_vpsm4_enc_8blks,.-_vpsm4_enc_8blks
___
}}}


{{{
my ($key,$keys)=("x0","x1");
$code.=<<___;
.globl	${prefix}_set_encrypt_key
.type	${prefix}_set_encrypt_key,%function
.align	5
${prefix}_set_encrypt_key:
	AARCH64_SIGN_LINK_REGISTER
	stp	x29,x30,[sp,#-16]!
	mov	w2,1
	bl	_vpsm4_set_key
	ldp	x29,x30,[sp],#16
	AARCH64_VALIDATE_LINK_REGISTER
	ret
.size	${prefix}_set_encrypt_key,.-${prefix}_set_encrypt_key
___
}}}

{{{
my ($key,$keys)=("x0","x1");
$code.=<<___;
.globl	${prefix}_set_decrypt_key
.type	${prefix}_set_decrypt_key,%function
.align	5
${prefix}_set_decrypt_key:
	AARCH64_SIGN_LINK_REGISTER
	stp	x29,x30,[sp,#-16]!
	mov	w2,0
	bl	_vpsm4_set_key
	ldp	x29,x30,[sp],#16
	AARCH64_VALIDATE_LINK_REGISTER
	ret
.size	${prefix}_set_decrypt_key,.-${prefix}_set_decrypt_key
___
}}}

{{{
sub gen_block () {
	my $dir = shift;
	my ($inp,$outp,$rk)=map("x$_",(0..2));

$code.=<<___;
.globl	${prefix}_${dir}crypt
.type	${prefix}_${dir}crypt,%function
.align	5
${prefix}_${dir}crypt:
	AARCH64_VALID_CALL_TARGET
	ld1	{@data[0].16b},[$inp]
___
	&load_sbox();
	&rev32(@data[0],@data[0]);
$code.=<<___;
	mov	$rks,x2
___
	&encrypt_1blk(@data[0]);
$code.=<<___;
	st1	{@data[0].16b},[$outp]
	ret
.size	${prefix}_${dir}crypt,.-${prefix}_${dir}crypt
___
}
&gen_block("en");
&gen_block("de");
}}}

{{{
my ($enc) = ("w4");
my @dat=map("v$_",(16..23));

$code.=<<___;
.globl	${prefix}_ecb_encrypt
.type	${prefix}_ecb_encrypt,%function
.align	5
${prefix}_ecb_encrypt:
	AARCH64_SIGN_LINK_REGISTER
	// convert length into blocks
	lsr	x2,x2,4
	stp	d8,d9,[sp,#-80]!
	stp	d10,d11,[sp,#16]
	stp	d12,d13,[sp,#32]
	stp	d14,d15,[sp,#48]
	stp	x29,x30,[sp,#64]
___
	&load_sbox();
$code.=<<___;
.Lecb_8_blocks_process:
	cmp	$blocks,#8
	b.lt	.Lecb_4_blocks_process
	ld4	{@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$inp],#64
	ld4	{@datax[0].4s,$datax[1].4s,@datax[2].4s,@datax[3].4s},[$inp],#64
___
	&rev32(@data[0],@data[0]);
	&rev32(@data[1],@data[1]);
	&rev32(@data[2],@data[2]);
	&rev32(@data[3],@data[3]);
	&rev32(@datax[0],@datax[0]);
	&rev32(@datax[1],@datax[1]);
	&rev32(@datax[2],@datax[2]);
	&rev32(@datax[3],@datax[3]);
$code.=<<___;
	bl	_vpsm4_enc_8blks
	st4	{@vtmp[0].4s,@vtmp[1].4s,@vtmp[2].4s,@vtmp[3].4s},[$outp],#64
	st4	{@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$outp],#64
	subs	$blocks,$blocks,#8
	b.gt	.Lecb_8_blocks_process
	b	100f
.Lecb_4_blocks_process:
	cmp	$blocks,#4
	b.lt	1f
	ld4	{@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$inp],#64
___
	&rev32(@data[0],@data[0]);
	&rev32(@data[1],@data[1]);
	&rev32(@data[2],@data[2]);
	&rev32(@data[3],@data[3]);
$code.=<<___;
	bl	_vpsm4_enc_4blks
	st4	{@vtmp[0].4s,@vtmp[1].4s,@vtmp[2].4s,@vtmp[3].4s},[$outp],#64
	sub	$blocks,$blocks,#4
1:
	// process last block
	cmp	$blocks,#1
	b.lt	100f
	b.gt	1f
	ld1	{@data[0].16b},[$inp]
___
	&rev32(@data[0],@data[0]);
	&encrypt_1blk(@data[0]);
$code.=<<___;
	st1	{@data[0].16b},[$outp]
	b	100f
1:	// process last 2 blocks
	ld4	{@data[0].s,@data[1].s,@data[2].s,@data[3].s}[0],[$inp],#16
	ld4	{@data[0].s,@data[1].s,@data[2].s,@data[3].s}[1],[$inp],#16
	cmp	$blocks,#2
	b.gt	1f
___
	&rev32(@data[0],@data[0]);
	&rev32(@data[1],@data[1]);
	&rev32(@data[2],@data[2]);
	&rev32(@data[3],@data[3]);
$code.=<<___;
	bl	_vpsm4_enc_4blks
	st4	{@vtmp[0].s-@vtmp[3].s}[0],[$outp],#16
	st4	{@vtmp[0].s-@vtmp[3].s}[1],[$outp]
	b	100f
1:	// process last 3 blocks
	ld4	{@data[0].s,@data[1].s,@data[2].s,@data[3].s}[2],[$inp],#16
___
	&rev32(@data[0],@data[0]);
	&rev32(@data[1],@data[1]);
	&rev32(@data[2],@data[2]);
	&rev32(@data[3],@data[3]);
$code.=<<___;
	bl	_vpsm4_enc_4blks
	st4	{@vtmp[0].s-@vtmp[3].s}[0],[$outp],#16
	st4	{@vtmp[0].s-@vtmp[3].s}[1],[$outp],#16
	st4	{@vtmp[0].s-@vtmp[3].s}[2],[$outp]
100:
	ldp	d10,d11,[sp,#16]
	ldp	d12,d13,[sp,#32]
	ldp	d14,d15,[sp,#48]
	ldp	x29,x30,[sp,#64]
	ldp	d8,d9,[sp],#80
	AARCH64_VALIDATE_LINK_REGISTER
	ret
.size	${prefix}_ecb_encrypt,.-${prefix}_ecb_encrypt
___
}}}

{{{
my ($len,$ivp,$enc)=("x2","x4","w5");
my $ivec0=("v3");
my $ivec1=("v15");

$code.=<<___;
.globl	${prefix}_cbc_encrypt
.type	${prefix}_cbc_encrypt,%function
.align	5
${prefix}_cbc_encrypt:
	AARCH64_VALID_CALL_TARGET
	lsr	$len,$len,4
___
	&load_sbox();
$code.=<<___;
	cbz	$enc,.Ldec
	ld1	{$ivec0.4s},[$ivp]
.Lcbc_4_blocks_enc:
	cmp	$blocks,#4
	b.lt	1f
	ld1	{@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$inp],#64
	eor	@data[0].16b,@data[0].16b,$ivec0.16b
___
	&rev32(@data[1],@data[1]);
	&rev32(@data[0],@data[0]);
	&rev32(@data[2],@data[2]);
	&rev32(@data[3],@data[3]);
	&encrypt_1blk_norev(@data[0]);
$code.=<<___;
	eor	@data[1].16b,@data[1].16b,@data[0].16b
___
	&encrypt_1blk_norev(@data[1]);
	&rev32(@data[0],@data[0]);

$code.=<<___;
	eor	@data[2].16b,@data[2].16b,@data[1].16b
___
	&encrypt_1blk_norev(@data[2]);
	&rev32(@data[1],@data[1]);
$code.=<<___;
	eor	@data[3].16b,@data[3].16b,@data[2].16b
___
	&encrypt_1blk_norev(@data[3]);
	&rev32(@data[2],@data[2]);
	&rev32(@data[3],@data[3]);
$code.=<<___;
	orr	$ivec0.16b,@data[3].16b,@data[3].16b
	st1	{@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$outp],#64
	subs	$blocks,$blocks,#4
	b.ne	.Lcbc_4_blocks_enc
	b	2f
1:
	subs	$blocks,$blocks,#1
	b.lt	2f
	ld1	{@data[0].4s},[$inp],#16
	eor	$ivec0.16b,$ivec0.16b,@data[0].16b
___
	&rev32($ivec0,$ivec0);
	&encrypt_1blk($ivec0);
$code.=<<___;
	st1	{$ivec0.16b},[$outp],#16
	b	1b
2:
	// save back IV
	st1	{$ivec0.16b},[$ivp]
	ret

.Ldec:
	// decryption mode starts
	AARCH64_SIGN_LINK_REGISTER
	stp	d8,d9,[sp,#-80]!
	stp	d10,d11,[sp,#16]
	stp	d12,d13,[sp,#32]
	stp	d14,d15,[sp,#48]
	stp	x29,x30,[sp,#64]
.Lcbc_8_blocks_dec:
	cmp	$blocks,#8
	b.lt	1f
	ld4	{@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$inp]
	add	$ptr,$inp,#64
	ld4	{@datax[0].4s,@datax[1].4s,@datax[2].4s,@datax[3].4s},[$ptr]
___
	&rev32(@data[0],@data[0]);
	&rev32(@data[1],@data[1]);
	&rev32(@data[2],@data[2]);
	&rev32(@data[3],$data[3]);
	&rev32(@datax[0],@datax[0]);
	&rev32(@datax[1],@datax[1]);
	&rev32(@datax[2],@datax[2]);
	&rev32(@datax[3],$datax[3]);
$code.=<<___;
	bl	_vpsm4_enc_8blks
___
	&transpose(@vtmp,@datax);
	&transpose(@data,@datax);
$code.=<<___;
	ld1	{$ivec1.16b},[$ivp]
	ld1	{@datax[0].4s,@datax[1].4s,@datax[2].4s,@datax[3].4s},[$inp],#64
	// note ivec1 and vtmpx[3] are resuing the same register
	// care needs to be taken to avoid conflict
	eor	@vtmp[0].16b,@vtmp[0].16b,$ivec1.16b
	ld1	{@vtmpx[0].4s,@vtmpx[1].4s,@vtmpx[2].4s,@vtmpx[3].4s},[$inp],#64
	eor	@vtmp[1].16b,@vtmp[1].16b,@datax[0].16b
	eor	@vtmp[2].16b,@vtmp[2].16b,@datax[1].16b
	eor	@vtmp[3].16b,$vtmp[3].16b,@datax[2].16b
	// save back IV
	st1	{$vtmpx[3].16b}, [$ivp]
	eor	@data[0].16b,@data[0].16b,$datax[3].16b
	eor	@data[1].16b,@data[1].16b,@vtmpx[0].16b
	eor	@data[2].16b,@data[2].16b,@vtmpx[1].16b
	eor	@data[3].16b,$data[3].16b,@vtmpx[2].16b
	st1	{@vtmp[0].4s,@vtmp[1].4s,@vtmp[2].4s,@vtmp[3].4s},[$outp],#64
	st1	{@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$outp],#64
	subs	$blocks,$blocks,#8
	b.gt	.Lcbc_8_blocks_dec
	b.eq	100f
1:
	ld1	{$ivec1.16b},[$ivp]
.Lcbc_4_blocks_dec:
	cmp	$blocks,#4
	b.lt	1f
	ld4	{@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$inp]
___
	&rev32(@data[0],@data[0]);
	&rev32(@data[1],@data[1]);
	&rev32(@data[2],@data[2]);
	&rev32(@data[3],$data[3]);
$code.=<<___;
	bl	_vpsm4_enc_4blks
	ld1	{@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$inp],#64
___
	&transpose(@vtmp,@datax);
$code.=<<___;
	eor	@vtmp[0].16b,@vtmp[0].16b,$ivec1.16b
	eor	@vtmp[1].16b,@vtmp[1].16b,@data[0].16b
	orr	$ivec1.16b,@data[3].16b,@data[3].16b
	eor	@vtmp[2].16b,@vtmp[2].16b,@data[1].16b
	eor	@vtmp[3].16b,$vtmp[3].16b,@data[2].16b
	st1	{@vtmp[0].4s,@vtmp[1].4s,@vtmp[2].4s,@vtmp[3].4s},[$outp],#64
	subs	$blocks,$blocks,#4
	b.gt	.Lcbc_4_blocks_dec
	// save back IV
	st1	{@vtmp[3].16b}, [$ivp]
	b	100f
1:	// last block
	subs	$blocks,$blocks,#1
	b.lt	100f
	b.gt	1f
	ld1	{@data[0].4s},[$inp],#16
	// save back IV
	st1	{$data[0].16b}, [$ivp]
___
	&rev32(@datax[0],@data[0]);
	&encrypt_1blk(@datax[0]);
$code.=<<___;
	eor	@datax[0].16b,@datax[0].16b,$ivec1.16b
	st1	{@datax[0].16b},[$outp],#16
	b	100f
1:	// last two blocks
	ld4	{@data[0].s,@data[1].s,@data[2].s,@data[3].s}[0],[$inp]
	add	$ptr,$inp,#16
	ld4	{@data[0].s,@data[1].s,@data[2].s,@data[3].s}[1],[$ptr],#16
	subs	$blocks,$blocks,1
	b.gt	1f
___
	&rev32(@data[0],@data[0]);
	&rev32(@data[1],@data[1]);
	&rev32(@data[2],@data[2]);
	&rev32(@data[3],@data[3]);
$code.=<<___;
	bl	_vpsm4_enc_4blks
	ld1	{@data[0].4s,@data[1].4s},[$inp],#32
___
	&transpose(@vtmp,@datax);
$code.=<<___;
	eor	@vtmp[0].16b,@vtmp[0].16b,$ivec1.16b
	eor	@vtmp[1].16b,@vtmp[1].16b,@data[0].16b
	st1	{@vtmp[0].4s,@vtmp[1].4s},[$outp],#32
	// save back IV
	st1	{@data[1].16b}, [$ivp]
	b	100f
1:	// last 3 blocks
	ld4	{@data[0].s,@data[1].s,@data[2].s,@data[3].s}[2],[$ptr]
___
	&rev32(@data[0],@data[0]);
	&rev32(@data[1],@data[1]);
	&rev32(@data[2],@data[2]);
	&rev32(@data[3],@data[3]);
$code.=<<___;
	bl	_vpsm4_enc_4blks
	ld1	{@data[0].4s,@data[1].4s,@data[2].4s},[$inp],#48
___
	&transpose(@vtmp,@datax);
$code.=<<___;
	eor	@vtmp[0].16b,@vtmp[0].16b,$ivec1.16b
	eor	@vtmp[1].16b,@vtmp[1].16b,@data[0].16b
	eor	@vtmp[2].16b,@vtmp[2].16b,@data[1].16b
	st1	{@vtmp[0].4s,@vtmp[1].4s,@vtmp[2].4s},[$outp],#48
	// save back IV
	st1	{@data[2].16b}, [$ivp]
100:
	ldp	d10,d11,[sp,#16]
	ldp	d12,d13,[sp,#32]
	ldp	d14,d15,[sp,#48]
	ldp	x29,x30,[sp,#64]
	ldp	d8,d9,[sp],#80
	AARCH64_VALIDATE_LINK_REGISTER
	ret
.size	${prefix}_cbc_encrypt,.-${prefix}_cbc_encrypt
___
}}}

{{{
my ($ivp)=("x4");
my ($ctr)=("w5");
my $ivec=("v3");

$code.=<<___;
.globl	${prefix}_ctr32_encrypt_blocks
.type	${prefix}_ctr32_encrypt_blocks,%function
.align	5
${prefix}_ctr32_encrypt_blocks:
	AARCH64_VALID_CALL_TARGET
	ld1	{$ivec.4s},[$ivp]
___
	&rev32($ivec,$ivec);
	&load_sbox();
$code.=<<___;
	cmp	$blocks,#1
	b.ne	1f
	// fast processing for one single block without
	// context saving overhead
___
	&encrypt_1blk($ivec);
$code.=<<___;
	ld1	{@data[0].16b},[$inp]
	eor	@data[0].16b,@data[0].16b,$ivec.16b
	st1	{@data[0].16b},[$outp]
	ret
1:
	AARCH64_SIGN_LINK_REGISTER
	stp	d8,d9,[sp,#-80]!
	stp	d10,d11,[sp,#16]
	stp	d12,d13,[sp,#32]
	stp	d14,d15,[sp,#48]
	stp	x29,x30,[sp,#64]
	mov	$word0,$ivec.s[0]
	mov	$word1,$ivec.s[1]
	mov	$word2,$ivec.s[2]
	mov	$ctr,$ivec.s[3]
.Lctr32_4_blocks_process:
	cmp	$blocks,#4
	b.lt	1f
	dup	@data[0].4s,$word0
	dup	@data[1].4s,$word1
	dup	@data[2].4s,$word2
	mov	@data[3].s[0],$ctr
	add	$ctr,$ctr,#1
	mov	$data[3].s[1],$ctr
	add	$ctr,$ctr,#1
	mov	@data[3].s[2],$ctr
	add	$ctr,$ctr,#1
	mov	@data[3].s[3],$ctr
	add	$ctr,$ctr,#1
	cmp	$blocks,#8
	b.ge	.Lctr32_8_blocks_process
	bl	_vpsm4_enc_4blks
	ld4	{@vtmpx[0].4s,@vtmpx[1].4s,@vtmpx[2].4s,@vtmpx[3].4s},[$inp],#64
	eor	@vtmp[0].16b,@vtmp[0].16b,@vtmpx[0].16b
	eor	@vtmp[1].16b,@vtmp[1].16b,@vtmpx[1].16b
	eor	@vtmp[2].16b,@vtmp[2].16b,@vtmpx[2].16b
	eor	@vtmp[3].16b,@vtmp[3].16b,@vtmpx[3].16b
	st4	{@vtmp[0].4s,@vtmp[1].4s,@vtmp[2].4s,@vtmp[3].4s},[$outp],#64
	subs	$blocks,$blocks,#4
	b.ne	.Lctr32_4_blocks_process
	b	100f
.Lctr32_8_blocks_process:
	dup	@datax[0].4s,$word0
	dup	@datax[1].4s,$word1
	dup	@datax[2].4s,$word2
	mov	@datax[3].s[0],$ctr
	add	$ctr,$ctr,#1
	mov	$datax[3].s[1],$ctr
	add	$ctr,$ctr,#1
	mov	@datax[3].s[2],$ctr
	add	$ctr,$ctr,#1
	mov	@datax[3].s[3],$ctr
	add	$ctr,$ctr,#1
	bl	_vpsm4_enc_8blks
	ld4	{@vtmpx[0].4s,@vtmpx[1].4s,@vtmpx[2].4s,@vtmpx[3].4s},[$inp],#64
	ld4	{@datax[0].4s,@datax[1].4s,@datax[2].4s,@datax[3].4s},[$inp],#64
	eor	@vtmp[0].16b,@vtmp[0].16b,@vtmpx[0].16b
	eor	@vtmp[1].16b,@vtmp[1].16b,@vtmpx[1].16b
	eor	@vtmp[2].16b,@vtmp[2].16b,@vtmpx[2].16b
	eor	@vtmp[3].16b,@vtmp[3].16b,@vtmpx[3].16b
	eor	@data[0].16b,@data[0].16b,@datax[0].16b
	eor	@data[1].16b,@data[1].16b,@datax[1].16b
	eor	@data[2].16b,@data[2].16b,@datax[2].16b
	eor	@data[3].16b,@data[3].16b,@datax[3].16b
	st4	{@vtmp[0].4s,@vtmp[1].4s,@vtmp[2].4s,@vtmp[3].4s},[$outp],#64
	st4	{@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$outp],#64
	subs	$blocks,$blocks,#8
	b.ne	.Lctr32_4_blocks_process
	b	100f
1:	// last block processing
	subs	$blocks,$blocks,#1
	b.lt	100f
	b.gt	1f
	mov	$ivec.s[0],$word0
	mov	$ivec.s[1],$word1
	mov	$ivec.s[2],$word2
	mov	$ivec.s[3],$ctr
___
	&encrypt_1blk($ivec);
$code.=<<___;
	ld1	{@data[0].16b},[$inp]
	eor	@data[0].16b,@data[0].16b,$ivec.16b
	st1	{@data[0].16b},[$outp]
	b	100f
1:	// last 2 blocks processing
	dup	@data[0].4s,$word0
	dup	@data[1].4s,$word1
	dup	@data[2].4s,$word2
	mov	@data[3].s[0],$ctr
	add	$ctr,$ctr,#1
	mov	@data[3].s[1],$ctr
	subs	$blocks,$blocks,#1
	b.ne	1f
	bl	_vpsm4_enc_4blks
	ld4	{@vtmpx[0].s,@vtmpx[1].s,@vtmpx[2].s,@vtmpx[3].s}[0],[$inp],#16
	ld4	{@vtmpx[0].s,@vtmpx[1].s,@vtmpx[2].s,@vtmpx[3].s}[1],[$inp],#16
	eor	@vtmp[0].16b,@vtmp[0].16b,@vtmpx[0].16b
	eor	@vtmp[1].16b,@vtmp[1].16b,@vtmpx[1].16b
	eor	@vtmp[2].16b,@vtmp[2].16b,@vtmpx[2].16b
	eor	@vtmp[3].16b,@vtmp[3].16b,@vtmpx[3].16b
	st4	{@vtmp[0].s,@vtmp[1].s,@vtmp[2].s,@vtmp[3].s}[0],[$outp],#16
	st4	{@vtmp[0].s,@vtmp[1].s,@vtmp[2].s,@vtmp[3].s}[1],[$outp],#16
	b	100f
1:	// last 3 blocks processing
	add	$ctr,$ctr,#1
	mov	@data[3].s[2],$ctr
	bl	_vpsm4_enc_4blks
	ld4	{@vtmpx[0].s,@vtmpx[1].s,@vtmpx[2].s,@vtmpx[3].s}[0],[$inp],#16
	ld4	{@vtmpx[0].s,@vtmpx[1].s,@vtmpx[2].s,@vtmpx[3].s}[1],[$inp],#16
	ld4	{@vtmpx[0].s,@vtmpx[1].s,@vtmpx[2].s,@vtmpx[3].s}[2],[$inp],#16
	eor	@vtmp[0].16b,@vtmp[0].16b,@vtmpx[0].16b
	eor	@vtmp[1].16b,@vtmp[1].16b,@vtmpx[1].16b
	eor	@vtmp[2].16b,@vtmp[2].16b,@vtmpx[2].16b
	eor	@vtmp[3].16b,@vtmp[3].16b,@vtmpx[3].16b
	st4	{@vtmp[0].s,@vtmp[1].s,@vtmp[2].s,@vtmp[3].s}[0],[$outp],#16
	st4	{@vtmp[0].s,@vtmp[1].s,@vtmp[2].s,@vtmp[3].s}[1],[$outp],#16
	st4	{@vtmp[0].s,@vtmp[1].s,@vtmp[2].s,@vtmp[3].s}[2],[$outp],#16
100:
	ldp	d10,d11,[sp,#16]
	ldp	d12,d13,[sp,#32]
	ldp	d14,d15,[sp,#48]
	ldp	x29,x30,[sp,#64]
	ldp	d8,d9,[sp],#80
	AARCH64_VALIDATE_LINK_REGISTER
	ret
.size	${prefix}_ctr32_encrypt_blocks,.-${prefix}_ctr32_encrypt_blocks
___
}}}
########################################
open SELF,$0;
while(<SELF>) {
        next if (/^#!/);
        last if (!s/^#/\/\// and !/^$/);
        print;
}
close SELF;

foreach(split("\n",$code)) {
	s/\`([^\`]*)\`/eval($1)/ge;
	print $_,"\n";
}

close STDOUT or die "error closing STDOUT: $!";