crypto/modes/asm/ghash-riscv64.pl - third_party/openssl - Git at Google

 #! /usr/bin/env perl
 # This file is dual-licensed, meaning that you can use it under your
 # choice of either of the following two licenses:
 #
 # Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved.
 #
 # Licensed under the Apache License 2.0 (the "License"). You can obtain
 # a copy in the file LICENSE in the source distribution or at
 # https://www.openssl.org/source/license.html
 #
 # or
 #
 # Copyright (c) 2023, Christoph Müllner <christoph.muellner@vrull.eu>
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
 # are met:
 # 1. Redistributions of source code must retain the above copyright
 #    notice, this list of conditions and the following disclaimer.
 # 2. Redistributions in binary form must reproduce the above copyright
 #    notice, this list of conditions and the following disclaimer in the
 #    documentation and/or other materials provided with the distribution.
 #
 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

 use strict;
 use warnings;

 use FindBin qw($Bin);
 use lib "$Bin";
 use lib "$Bin/../../perlasm";
 use riscv;

 # $output is the last argument if it looks like a file (it has an extension)
 # $flavour is the first argument if it doesn't look like a file
 my $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
 my $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;

 $output and open STDOUT,">$output";

 my $code=<<___;
 .text
 ___

 ################################################################################
 # void gcm_init_rv64i_zbc(u128 Htable[16], const u64 H[2]);
 # void gcm_init_rv64i_zbc__zbb(u128 Htable[16], const u64 H[2]);
 # void gcm_init_rv64i_zbc__zbkb(u128 Htable[16], const u64 H[2]);
 #
 # input:  H: 128-bit H - secret parameter E(K, 0^128)
 # output: Htable: Preprocessed key data for gcm_gmult_rv64i_zbc* and
 #                 gcm_ghash_rv64i_zbc*
 #
 # All callers of this function revert the byte-order unconditionally
 # on little-endian machines. So we need to revert the byte-order back.
 # Additionally we reverse the bits of each byte.

 {
 my ($Htable,$H,$VAL0,$VAL1,$TMP0,$TMP1,$TMP2) = ("a0","a1","a2","a3","t0","t1","t2");

 $code .= <<___;
 .p2align 3
 .globl gcm_init_rv64i_zbc
 .type gcm_init_rv64i_zbc,\@function
 gcm_init_rv64i_zbc:
     ld      $VAL0,0($H)
     ld      $VAL1,8($H)
     @{[brev8_rv64i   $VAL0, $TMP0, $TMP1, $TMP2]}
     @{[brev8_rv64i   $VAL1, $TMP0, $TMP1, $TMP2]}
     @{[sd_rev8_rv64i $VAL0, $Htable, 0, $TMP0]}
     @{[sd_rev8_rv64i $VAL1, $Htable, 8, $TMP0]}
     ret
 .size gcm_init_rv64i_zbc,.-gcm_init_rv64i_zbc
 ___
 }

 {
 my ($Htable,$H,$VAL0,$VAL1,$TMP0,$TMP1,$TMP2) = ("a0","a1","a2","a3","t0","t1","t2");

 $code .= <<___;
 .p2align 3
 .globl gcm_init_rv64i_zbc__zbb
 .type gcm_init_rv64i_zbc__zbb,\@function
 gcm_init_rv64i_zbc__zbb:
     ld      $VAL0,0($H)
     ld      $VAL1,8($H)
     @{[brev8_rv64i $VAL0, $TMP0, $TMP1, $TMP2]}
     @{[brev8_rv64i $VAL1, $TMP0, $TMP1, $TMP2]}
     @{[rev8 $VAL0, $VAL0]}
     @{[rev8 $VAL1, $VAL1]}
     sd      $VAL0,0($Htable)
     sd      $VAL1,8($Htable)
     ret
 .size gcm_init_rv64i_zbc__zbb,.-gcm_init_rv64i_zbc__zbb
 ___
 }

 {
 my ($Htable,$H,$TMP0,$TMP1) = ("a0","a1","t0","t1");

 $code .= <<___;
 .p2align 3
 .globl gcm_init_rv64i_zbc__zbkb
 .type gcm_init_rv64i_zbc__zbkb,\@function
 gcm_init_rv64i_zbc__zbkb:
     ld      $TMP0,0($H)
     ld      $TMP1,8($H)
     @{[brev8 $TMP0, $TMP0]}
     @{[brev8 $TMP1, $TMP1]}
     @{[rev8 $TMP0, $TMP0]}
     @{[rev8 $TMP1, $TMP1]}
     sd      $TMP0,0($Htable)
     sd      $TMP1,8($Htable)
     ret
 .size gcm_init_rv64i_zbc__zbkb,.-gcm_init_rv64i_zbc__zbkb
 ___
 }

 ################################################################################
 # void gcm_gmult_rv64i_zbc(u64 Xi[2], const u128 Htable[16]);
 # void gcm_gmult_rv64i_zbc__zbkb(u64 Xi[2], const u128 Htable[16]);
 #
 # input:  Xi: current hash value
 #         Htable: copy of H
 # output: Xi: next hash value Xi
 #
 # Compute GMULT (Xi*H mod f) using the Zbc (clmul) and Zbb (basic bit manip)
 # extensions. Using the no-Karatsuba approach and clmul for the final reduction.
 # This results in an implementation with minimized number of instructions.
 # HW with clmul latencies higher than 2 cycles might observe a performance
 # improvement with Karatsuba. HW with clmul latencies higher than 6 cycles
 # might observe a performance improvement with additionally converting the
 # reduction to shift&xor. For a full discussion of this estimates see
 # https://github.com/riscv/riscv-crypto/blob/master/doc/supp/gcm-mode-cmul.adoc
 {
 my ($Xi,$Htable,$x0,$x1,$y0,$y1) = ("a0","a1","a4","a5","a6","a7");
 my ($z0,$z1,$z2,$z3,$t0,$t1,$polymod) = ("t0","t1","t2","t3","t4","t5","t6");

 $code .= <<___;
 .p2align 3
 .globl gcm_gmult_rv64i_zbc
 .type gcm_gmult_rv64i_zbc,\@function
 gcm_gmult_rv64i_zbc:
     # Load Xi and bit-reverse it
     ld        $x0, 0($Xi)
     ld        $x1, 8($Xi)
     @{[brev8_rv64i $x0, $z0, $z1, $z2]}
     @{[brev8_rv64i $x1, $z0, $z1, $z2]}

     # Load the key (already bit-reversed)
     ld        $y0, 0($Htable)
     ld        $y1, 8($Htable)

     # Load the reduction constant
     la        $polymod, Lpolymod
     lbu       $polymod, 0($polymod)

     # Multiplication (without Karatsuba)
     @{[clmulh $z3, $x1, $y1]}
     @{[clmul  $z2, $x1, $y1]}
     @{[clmulh $t1, $x0, $y1]}
     @{[clmul  $z1, $x0, $y1]}
     xor       $z2, $z2, $t1
     @{[clmulh $t1, $x1, $y0]}
     @{[clmul  $t0, $x1, $y0]}
     xor       $z2, $z2, $t1
     xor       $z1, $z1, $t0
     @{[clmulh $t1, $x0, $y0]}
     @{[clmul  $z0, $x0, $y0]}
     xor       $z1, $z1, $t1

     # Reduction with clmul
     @{[clmulh $t1, $z3, $polymod]}
     @{[clmul  $t0, $z3, $polymod]}
     xor       $z2, $z2, $t1
     xor       $z1, $z1, $t0
     @{[clmulh $t1, $z2, $polymod]}
     @{[clmul  $t0, $z2, $polymod]}
     xor       $x1, $z1, $t1
     xor       $x0, $z0, $t0

     # Bit-reverse Xi back and store it
     @{[brev8_rv64i $x0, $z0, $z1, $z2]}
     @{[brev8_rv64i $x1, $z0, $z1, $z2]}
     sd        $x0, 0($Xi)
     sd        $x1, 8($Xi)
     ret
 .size gcm_gmult_rv64i_zbc,.-gcm_gmult_rv64i_zbc
 ___
 }

 {
 my ($Xi,$Htable,$x0,$x1,$y0,$y1) = ("a0","a1","a4","a5","a6","a7");
 my ($z0,$z1,$z2,$z3,$t0,$t1,$polymod) = ("t0","t1","t2","t3","t4","t5","t6");

 $code .= <<___;
 .p2align 3
 .globl gcm_gmult_rv64i_zbc__zbkb
 .type gcm_gmult_rv64i_zbc__zbkb,\@function
 gcm_gmult_rv64i_zbc__zbkb:
     # Load Xi and bit-reverse it
     ld        $x0, 0($Xi)
     ld        $x1, 8($Xi)
     @{[brev8  $x0, $x0]}
     @{[brev8  $x1, $x1]}

     # Load the key (already bit-reversed)
     ld        $y0, 0($Htable)
     ld        $y1, 8($Htable)

     # Load the reduction constant
     la        $polymod, Lpolymod
     lbu       $polymod, 0($polymod)

     # Multiplication (without Karatsuba)
     @{[clmulh $z3, $x1, $y1]}
     @{[clmul  $z2, $x1, $y1]}
     @{[clmulh $t1, $x0, $y1]}
     @{[clmul  $z1, $x0, $y1]}
     xor       $z2, $z2, $t1
     @{[clmulh $t1, $x1, $y0]}
     @{[clmul  $t0, $x1, $y0]}
     xor       $z2, $z2, $t1
     xor       $z1, $z1, $t0
     @{[clmulh $t1, $x0, $y0]}
     @{[clmul  $z0, $x0, $y0]}
     xor       $z1, $z1, $t1

     # Reduction with clmul
     @{[clmulh $t1, $z3, $polymod]}
     @{[clmul  $t0, $z3, $polymod]}
     xor       $z2, $z2, $t1
     xor       $z1, $z1, $t0
     @{[clmulh $t1, $z2, $polymod]}
     @{[clmul  $t0, $z2, $polymod]}
     xor       $x1, $z1, $t1
     xor       $x0, $z0, $t0

     # Bit-reverse Xi back and store it
     @{[brev8  $x0, $x0]}
     @{[brev8  $x1, $x1]}
     sd        $x0, 0($Xi)
     sd        $x1, 8($Xi)
     ret
 .size gcm_gmult_rv64i_zbc__zbkb,.-gcm_gmult_rv64i_zbc__zbkb
 ___
 }

 ################################################################################
 # void gcm_ghash_rv64i_zbc(u64 Xi[2], const u128 Htable[16],
 #                          const u8 *inp, size_t len);
 # void gcm_ghash_rv64i_zbc__zbkb(u64 Xi[2], const u128 Htable[16],
 #                                const u8 *inp, size_t len);
 #
 # input:  Xi: current hash value
 #         Htable: copy of H
 #         inp: pointer to input data
 #         len: length of input data in bytes (multiple of block size)
 # output: Xi: Xi+1 (next hash value Xi)
 {
 my ($Xi,$Htable,$inp,$len,$x0,$x1,$y0,$y1) = ("a0","a1","a2","a3","a4","a5","a6","a7");
 my ($z0,$z1,$z2,$z3,$t0,$t1,$polymod) = ("t0","t1","t2","t3","t4","t5","t6");

 $code .= <<___;
 .p2align 3
 .globl gcm_ghash_rv64i_zbc
 .type gcm_ghash_rv64i_zbc,\@function
 gcm_ghash_rv64i_zbc:
     # Load Xi and bit-reverse it
     ld        $x0, 0($Xi)
     ld        $x1, 8($Xi)
     @{[brev8_rv64i $x0, $z0, $z1, $z2]}
     @{[brev8_rv64i $x1, $z0, $z1, $z2]}

     # Load the key (already bit-reversed)
     ld        $y0, 0($Htable)
     ld        $y1, 8($Htable)

     # Load the reduction constant
     la        $polymod, Lpolymod
     lbu       $polymod, 0($polymod)

 Lstep:
     # Load the input data, bit-reverse them, and XOR them with Xi
     ld        $t0, 0($inp)
     ld        $t1, 8($inp)
     add       $inp, $inp, 16
     add       $len, $len, -16
     @{[brev8_rv64i $t0, $z0, $z1, $z2]}
     @{[brev8_rv64i $t1, $z0, $z1, $z2]}
     xor       $x0, $x0, $t0
     xor       $x1, $x1, $t1

     # Multiplication (without Karatsuba)
     @{[clmulh $z3, $x1, $y1]}
     @{[clmul  $z2, $x1, $y1]}
     @{[clmulh $t1, $x0, $y1]}
     @{[clmul  $z1, $x0, $y1]}
     xor       $z2, $z2, $t1
     @{[clmulh $t1, $x1, $y0]}
     @{[clmul  $t0, $x1, $y0]}
     xor       $z2, $z2, $t1
     xor       $z1, $z1, $t0
     @{[clmulh $t1, $x0, $y0]}
     @{[clmul  $z0, $x0, $y0]}
     xor       $z1, $z1, $t1

     # Reduction with clmul
     @{[clmulh $t1, $z3, $polymod]}
     @{[clmul  $t0, $z3, $polymod]}
     xor       $z2, $z2, $t1
     xor       $z1, $z1, $t0
     @{[clmulh $t1, $z2, $polymod]}
     @{[clmul  $t0, $z2, $polymod]}
     xor       $x1, $z1, $t1
     xor       $x0, $z0, $t0

     # Iterate over all blocks
     bnez      $len, Lstep

     # Bit-reverse final Xi back and store it
     @{[brev8_rv64i $x0, $z0, $z1, $z2]}
     @{[brev8_rv64i $x1, $z0, $z1, $z2]}
     sd        $x0, 0($Xi)
     sd        $x1, 8($Xi)
     ret
 .size gcm_ghash_rv64i_zbc,.-gcm_ghash_rv64i_zbc
 ___
 }

 {
 my ($Xi,$Htable,$inp,$len,$x0,$x1,$y0,$y1) = ("a0","a1","a2","a3","a4","a5","a6","a7");
 my ($z0,$z1,$z2,$z3,$t0,$t1,$polymod) = ("t0","t1","t2","t3","t4","t5","t6");

 $code .= <<___;
 .p2align 3
 .globl gcm_ghash_rv64i_zbc__zbkb
 .type gcm_ghash_rv64i_zbc__zbkb,\@function
 gcm_ghash_rv64i_zbc__zbkb:
     # Load Xi and bit-reverse it
     ld        $x0, 0($Xi)
     ld        $x1, 8($Xi)
     @{[brev8  $x0, $x0]}
     @{[brev8  $x1, $x1]}

     # Load the key (already bit-reversed)
     ld        $y0, 0($Htable)
     ld        $y1, 8($Htable)

     # Load the reduction constant
     la        $polymod, Lpolymod
     lbu       $polymod, 0($polymod)

 Lstep_zkbk:
     # Load the input data, bit-reverse them, and XOR them with Xi
     ld        $t0, 0($inp)
     ld        $t1, 8($inp)
     add       $inp, $inp, 16
     add       $len, $len, -16
     @{[brev8  $t0, $t0]}
     @{[brev8  $t1, $t1]}
     xor       $x0, $x0, $t0
     xor       $x1, $x1, $t1

     # Multiplication (without Karatsuba)
     @{[clmulh $z3, $x1, $y1]}
     @{[clmul  $z2, $x1, $y1]}
     @{[clmulh $t1, $x0, $y1]}
     @{[clmul  $z1, $x0, $y1]}
     xor       $z2, $z2, $t1
     @{[clmulh $t1, $x1, $y0]}
     @{[clmul  $t0, $x1, $y0]}
     xor       $z2, $z2, $t1
     xor       $z1, $z1, $t0
     @{[clmulh $t1, $x0, $y0]}
     @{[clmul  $z0, $x0, $y0]}
     xor       $z1, $z1, $t1

     # Reduction with clmul
     @{[clmulh $t1, $z3, $polymod]}
     @{[clmul  $t0, $z3, $polymod]}
     xor       $z2, $z2, $t1
     xor       $z1, $z1, $t0
     @{[clmulh $t1, $z2, $polymod]}
     @{[clmul  $t0, $z2, $polymod]}
     xor       $x1, $z1, $t1
     xor       $x0, $z0, $t0

     # Iterate over all blocks
     bnez      $len, Lstep_zkbk

     # Bit-reverse final Xi back and store it
     @{[brev8  $x0, $x0]}
     @{[brev8  $x1, $x1]}
     sd $x0,  0($Xi)
     sd $x1,  8($Xi)
     ret
 .size gcm_ghash_rv64i_zbc__zbkb,.-gcm_ghash_rv64i_zbc__zbkb
 ___
 }

 $code .= <<___;
 .p2align 3
 Lbrev8_const:
     .dword  0xAAAAAAAAAAAAAAAA
     .dword  0xCCCCCCCCCCCCCCCC
     .dword  0xF0F0F0F0F0F0F0F0
 .size Lbrev8_const,.-Lbrev8_const

 Lpolymod:
     .byte 0x87
 .size Lpolymod,.-Lpolymod
 ___

 print $code;

 close STDOUT or die "error closing STDOUT: $!";
	#! /usr/bin/env perl
	# This file is dual-licensed, meaning that you can use it under your
	# choice of either of the following two licenses:
	#
	# Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved.
	#
	# Licensed under the Apache License 2.0 (the "License"). You can obtain
	# a copy in the file LICENSE in the source distribution or at
	# https://www.openssl.org/source/license.html
	#
	# or
	#
	# Copyright (c) 2023, Christoph Müllner <christoph.muellner@vrull.eu>
	# All rights reserved.
	#
	# Redistribution and use in source and binary forms, with or without
	# modification, are permitted provided that the following conditions
	# are met:
	# 1. Redistributions of source code must retain the above copyright
	# notice, this list of conditions and the following disclaimer.
	# 2. Redistributions in binary form must reproduce the above copyright
	# notice, this list of conditions and the following disclaimer in the
	# documentation and/or other materials provided with the distribution.
	#
	# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

	use strict;
	use warnings;

	use FindBin qw($Bin);
	use lib "$Bin";
	use lib "$Bin/../../perlasm";
	use riscv;

	# $output is the last argument if it looks like a file (it has an extension)
	# $flavour is the first argument if it doesn't look like a file
	my $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m\|\.\w+$\| ? pop : undef;
	my $flavour = $#ARGV >= 0 && $ARGV[0] !~ m\|\.\| ? shift : undef;

	$output and open STDOUT,">$output";

	my $code=<<___;
	.text
	___

	################################################################################
	# void gcm_init_rv64i_zbc(u128 Htable[16], const u64 H[2]);
	# void gcm_init_rv64i_zbc__zbb(u128 Htable[16], const u64 H[2]);
	# void gcm_init_rv64i_zbc__zbkb(u128 Htable[16], const u64 H[2]);
	#
	# input: H: 128-bit H - secret parameter E(K, 0^128)
	# output: Htable: Preprocessed key data for gcm_gmult_rv64i_zbc* and
	# gcm_ghash_rv64i_zbc*
	#
	# All callers of this function revert the byte-order unconditionally
	# on little-endian machines. So we need to revert the byte-order back.
	# Additionally we reverse the bits of each byte.

	{
	my ($Htable,$H,$VAL0,$VAL1,$TMP0,$TMP1,$TMP2) = ("a0","a1","a2","a3","t0","t1","t2");

	$code .= <<___;
	.p2align 3
	.globl gcm_init_rv64i_zbc
	.type gcm_init_rv64i_zbc,\@function
	gcm_init_rv64i_zbc:
	ld $VAL0,0($H)
	ld $VAL1,8($H)
	@{[brev8_rv64i $VAL0, $TMP0, $TMP1, $TMP2]}
	@{[brev8_rv64i $VAL1, $TMP0, $TMP1, $TMP2]}
	@{[sd_rev8_rv64i $VAL0, $Htable, 0, $TMP0]}
	@{[sd_rev8_rv64i $VAL1, $Htable, 8, $TMP0]}
	ret
	.size gcm_init_rv64i_zbc,.-gcm_init_rv64i_zbc
	___
	}

	{
	my ($Htable,$H,$VAL0,$VAL1,$TMP0,$TMP1,$TMP2) = ("a0","a1","a2","a3","t0","t1","t2");

	$code .= <<___;
	.p2align 3
	.globl gcm_init_rv64i_zbc__zbb
	.type gcm_init_rv64i_zbc__zbb,\@function
	gcm_init_rv64i_zbc__zbb:
	ld $VAL0,0($H)
	ld $VAL1,8($H)
	@{[brev8_rv64i $VAL0, $TMP0, $TMP1, $TMP2]}
	@{[brev8_rv64i $VAL1, $TMP0, $TMP1, $TMP2]}
	@{[rev8 $VAL0, $VAL0]}
	@{[rev8 $VAL1, $VAL1]}
	sd $VAL0,0($Htable)
	sd $VAL1,8($Htable)
	ret
	.size gcm_init_rv64i_zbc__zbb,.-gcm_init_rv64i_zbc__zbb
	___
	}

	{
	my ($Htable,$H,$TMP0,$TMP1) = ("a0","a1","t0","t1");

	$code .= <<___;
	.p2align 3
	.globl gcm_init_rv64i_zbc__zbkb
	.type gcm_init_rv64i_zbc__zbkb,\@function
	gcm_init_rv64i_zbc__zbkb:
	ld $TMP0,0($H)
	ld $TMP1,8($H)
	@{[brev8 $TMP0, $TMP0]}
	@{[brev8 $TMP1, $TMP1]}
	@{[rev8 $TMP0, $TMP0]}
	@{[rev8 $TMP1, $TMP1]}
	sd $TMP0,0($Htable)
	sd $TMP1,8($Htable)
	ret
	.size gcm_init_rv64i_zbc__zbkb,.-gcm_init_rv64i_zbc__zbkb
	___
	}

	################################################################################
	# void gcm_gmult_rv64i_zbc(u64 Xi[2], const u128 Htable[16]);
	# void gcm_gmult_rv64i_zbc__zbkb(u64 Xi[2], const u128 Htable[16]);
	#
	# input: Xi: current hash value
	# Htable: copy of H
	# output: Xi: next hash value Xi
	#
	# Compute GMULT (Xi*H mod f) using the Zbc (clmul) and Zbb (basic bit manip)
	# extensions. Using the no-Karatsuba approach and clmul for the final reduction.
	# This results in an implementation with minimized number of instructions.
	# HW with clmul latencies higher than 2 cycles might observe a performance
	# improvement with Karatsuba. HW with clmul latencies higher than 6 cycles
	# might observe a performance improvement with additionally converting the
	# reduction to shift&xor. For a full discussion of this estimates see
	# https://github.com/riscv/riscv-crypto/blob/master/doc/supp/gcm-mode-cmul.adoc
	{
	my ($Xi,$Htable,$x0,$x1,$y0,$y1) = ("a0","a1","a4","a5","a6","a7");
	my ($z0,$z1,$z2,$z3,$t0,$t1,$polymod) = ("t0","t1","t2","t3","t4","t5","t6");

	$code .= <<___;
	.p2align 3
	.globl gcm_gmult_rv64i_zbc
	.type gcm_gmult_rv64i_zbc,\@function
	gcm_gmult_rv64i_zbc:
	# Load Xi and bit-reverse it
	ld $x0, 0($Xi)
	ld $x1, 8($Xi)
	@{[brev8_rv64i $x0, $z0, $z1, $z2]}
	@{[brev8_rv64i $x1, $z0, $z1, $z2]}

	# Load the key (already bit-reversed)
	ld $y0, 0($Htable)
	ld $y1, 8($Htable)

	# Load the reduction constant
	la $polymod, Lpolymod
	lbu $polymod, 0($polymod)

	# Multiplication (without Karatsuba)
	@{[clmulh $z3, $x1, $y1]}
	@{[clmul $z2, $x1, $y1]}
	@{[clmulh $t1, $x0, $y1]}
	@{[clmul $z1, $x0, $y1]}
	xor $z2, $z2, $t1
	@{[clmulh $t1, $x1, $y0]}
	@{[clmul $t0, $x1, $y0]}
	xor $z2, $z2, $t1
	xor $z1, $z1, $t0
	@{[clmulh $t1, $x0, $y0]}
	@{[clmul $z0, $x0, $y0]}
	xor $z1, $z1, $t1

	# Reduction with clmul
	@{[clmulh $t1, $z3, $polymod]}
	@{[clmul $t0, $z3, $polymod]}
	xor $z2, $z2, $t1
	xor $z1, $z1, $t0
	@{[clmulh $t1, $z2, $polymod]}
	@{[clmul $t0, $z2, $polymod]}
	xor $x1, $z1, $t1
	xor $x0, $z0, $t0

	# Bit-reverse Xi back and store it
	@{[brev8_rv64i $x0, $z0, $z1, $z2]}
	@{[brev8_rv64i $x1, $z0, $z1, $z2]}
	sd $x0, 0($Xi)
	sd $x1, 8($Xi)
	ret
	.size gcm_gmult_rv64i_zbc,.-gcm_gmult_rv64i_zbc
	___
	}

	{
	my ($Xi,$Htable,$x0,$x1,$y0,$y1) = ("a0","a1","a4","a5","a6","a7");
	my ($z0,$z1,$z2,$z3,$t0,$t1,$polymod) = ("t0","t1","t2","t3","t4","t5","t6");

	$code .= <<___;
	.p2align 3
	.globl gcm_gmult_rv64i_zbc__zbkb
	.type gcm_gmult_rv64i_zbc__zbkb,\@function
	gcm_gmult_rv64i_zbc__zbkb:
	# Load Xi and bit-reverse it
	ld $x0, 0($Xi)
	ld $x1, 8($Xi)
	@{[brev8 $x0, $x0]}
	@{[brev8 $x1, $x1]}

	# Load the key (already bit-reversed)
	ld $y0, 0($Htable)
	ld $y1, 8($Htable)

	# Load the reduction constant
	la $polymod, Lpolymod
	lbu $polymod, 0($polymod)

	# Multiplication (without Karatsuba)
	@{[clmulh $z3, $x1, $y1]}
	@{[clmul $z2, $x1, $y1]}
	@{[clmulh $t1, $x0, $y1]}
	@{[clmul $z1, $x0, $y1]}
	xor $z2, $z2, $t1
	@{[clmulh $t1, $x1, $y0]}
	@{[clmul $t0, $x1, $y0]}
	xor $z2, $z2, $t1
	xor $z1, $z1, $t0
	@{[clmulh $t1, $x0, $y0]}
	@{[clmul $z0, $x0, $y0]}
	xor $z1, $z1, $t1

	# Reduction with clmul
	@{[clmulh $t1, $z3, $polymod]}
	@{[clmul $t0, $z3, $polymod]}
	xor $z2, $z2, $t1
	xor $z1, $z1, $t0
	@{[clmulh $t1, $z2, $polymod]}
	@{[clmul $t0, $z2, $polymod]}
	xor $x1, $z1, $t1
	xor $x0, $z0, $t0

	# Bit-reverse Xi back and store it
	@{[brev8 $x0, $x0]}
	@{[brev8 $x1, $x1]}
	sd $x0, 0($Xi)
	sd $x1, 8($Xi)
	ret
	.size gcm_gmult_rv64i_zbc__zbkb,.-gcm_gmult_rv64i_zbc__zbkb
	___
	}

	################################################################################
	# void gcm_ghash_rv64i_zbc(u64 Xi[2], const u128 Htable[16],
	# const u8 *inp, size_t len);
	# void gcm_ghash_rv64i_zbc__zbkb(u64 Xi[2], const u128 Htable[16],
	# const u8 *inp, size_t len);
	#
	# input: Xi: current hash value
	# Htable: copy of H
	# inp: pointer to input data
	# len: length of input data in bytes (multiple of block size)
	# output: Xi: Xi+1 (next hash value Xi)
	{
	my ($Xi,$Htable,$inp,$len,$x0,$x1,$y0,$y1) = ("a0","a1","a2","a3","a4","a5","a6","a7");
	my ($z0,$z1,$z2,$z3,$t0,$t1,$polymod) = ("t0","t1","t2","t3","t4","t5","t6");

	$code .= <<___;
	.p2align 3
	.globl gcm_ghash_rv64i_zbc
	.type gcm_ghash_rv64i_zbc,\@function
	gcm_ghash_rv64i_zbc:
	# Load Xi and bit-reverse it
	ld $x0, 0($Xi)
	ld $x1, 8($Xi)
	@{[brev8_rv64i $x0, $z0, $z1, $z2]}
	@{[brev8_rv64i $x1, $z0, $z1, $z2]}

	# Load the key (already bit-reversed)
	ld $y0, 0($Htable)
	ld $y1, 8($Htable)

	# Load the reduction constant
	la $polymod, Lpolymod
	lbu $polymod, 0($polymod)

	Lstep:
	# Load the input data, bit-reverse them, and XOR them with Xi
	ld $t0, 0($inp)
	ld $t1, 8($inp)
	add $inp, $inp, 16
	add $len, $len, -16
	@{[brev8_rv64i $t0, $z0, $z1, $z2]}
	@{[brev8_rv64i $t1, $z0, $z1, $z2]}
	xor $x0, $x0, $t0
	xor $x1, $x1, $t1

	# Multiplication (without Karatsuba)
	@{[clmulh $z3, $x1, $y1]}
	@{[clmul $z2, $x1, $y1]}
	@{[clmulh $t1, $x0, $y1]}
	@{[clmul $z1, $x0, $y1]}
	xor $z2, $z2, $t1
	@{[clmulh $t1, $x1, $y0]}
	@{[clmul $t0, $x1, $y0]}
	xor $z2, $z2, $t1
	xor $z1, $z1, $t0
	@{[clmulh $t1, $x0, $y0]}
	@{[clmul $z0, $x0, $y0]}
	xor $z1, $z1, $t1

	# Reduction with clmul
	@{[clmulh $t1, $z3, $polymod]}
	@{[clmul $t0, $z3, $polymod]}
	xor $z2, $z2, $t1
	xor $z1, $z1, $t0
	@{[clmulh $t1, $z2, $polymod]}
	@{[clmul $t0, $z2, $polymod]}
	xor $x1, $z1, $t1
	xor $x0, $z0, $t0

	# Iterate over all blocks
	bnez $len, Lstep

	# Bit-reverse final Xi back and store it
	@{[brev8_rv64i $x0, $z0, $z1, $z2]}
	@{[brev8_rv64i $x1, $z0, $z1, $z2]}
	sd $x0, 0($Xi)
	sd $x1, 8($Xi)
	ret
	.size gcm_ghash_rv64i_zbc,.-gcm_ghash_rv64i_zbc
	___
	}

	{
	my ($Xi,$Htable,$inp,$len,$x0,$x1,$y0,$y1) = ("a0","a1","a2","a3","a4","a5","a6","a7");
	my ($z0,$z1,$z2,$z3,$t0,$t1,$polymod) = ("t0","t1","t2","t3","t4","t5","t6");

	$code .= <<___;
	.p2align 3
	.globl gcm_ghash_rv64i_zbc__zbkb
	.type gcm_ghash_rv64i_zbc__zbkb,\@function
	gcm_ghash_rv64i_zbc__zbkb:
	# Load Xi and bit-reverse it
	ld $x0, 0($Xi)
	ld $x1, 8($Xi)
	@{[brev8 $x0, $x0]}
	@{[brev8 $x1, $x1]}

	# Load the key (already bit-reversed)
	ld $y0, 0($Htable)
	ld $y1, 8($Htable)

	# Load the reduction constant
	la $polymod, Lpolymod
	lbu $polymod, 0($polymod)

	Lstep_zkbk:
	# Load the input data, bit-reverse them, and XOR them with Xi
	ld $t0, 0($inp)
	ld $t1, 8($inp)
	add $inp, $inp, 16
	add $len, $len, -16
	@{[brev8 $t0, $t0]}
	@{[brev8 $t1, $t1]}
	xor $x0, $x0, $t0
	xor $x1, $x1, $t1

	# Multiplication (without Karatsuba)
	@{[clmulh $z3, $x1, $y1]}
	@{[clmul $z2, $x1, $y1]}
	@{[clmulh $t1, $x0, $y1]}
	@{[clmul $z1, $x0, $y1]}
	xor $z2, $z2, $t1
	@{[clmulh $t1, $x1, $y0]}
	@{[clmul $t0, $x1, $y0]}
	xor $z2, $z2, $t1
	xor $z1, $z1, $t0
	@{[clmulh $t1, $x0, $y0]}
	@{[clmul $z0, $x0, $y0]}
	xor $z1, $z1, $t1

	# Reduction with clmul
	@{[clmulh $t1, $z3, $polymod]}
	@{[clmul $t0, $z3, $polymod]}
	xor $z2, $z2, $t1
	xor $z1, $z1, $t0
	@{[clmulh $t1, $z2, $polymod]}
	@{[clmul $t0, $z2, $polymod]}
	xor $x1, $z1, $t1
	xor $x0, $z0, $t0

	# Iterate over all blocks
	bnez $len, Lstep_zkbk

	# Bit-reverse final Xi back and store it
	@{[brev8 $x0, $x0]}
	@{[brev8 $x1, $x1]}
	sd $x0, 0($Xi)
	sd $x1, 8($Xi)
	ret
	.size gcm_ghash_rv64i_zbc__zbkb,.-gcm_ghash_rv64i_zbc__zbkb
	___
	}

	$code .= <<___;
	.p2align 3
	Lbrev8_const:
	.dword 0xAAAAAAAAAAAAAAAA
	.dword 0xCCCCCCCCCCCCCCCC
	.dword 0xF0F0F0F0F0F0F0F0
	.size Lbrev8_const,.-Lbrev8_const

	Lpolymod:
	.byte 0x87
	.size Lpolymod,.-Lpolymod
	___

	print $code;

	close STDOUT or die "error closing STDOUT: $!";