Add support for the TLS 1.3 signature_algorithms_cert extension The new extension is like signature_algorithms, but only for the signature *on* the certificate we will present to the peer (the old signature_algorithms extension is still used for signatures that we *generate*, i.e., those over TLS data structures). We do not need to generate this extension, since we are the same implementation as our X.509 stack and can handle the same types of signatures, but we need to be prepared to receive it, and use the received information when selecting what certificate to present. There is a lot of interplay between signature_algorithms_cert and signature_algorithms, since both affect what certificate we can use, and thus the resulting signature algorithm used for TLS messages. So, apply signature_algorithms_cert (if present) as a filter on what certificates we can consider when choosing a certificate+sigalg pair. As part of this addition, we also remove the fallback code that let keys of type EVP_PKEY_RSA be used to generate RSA-PSS signatures -- the new rsa_pss_pss_* and rsa_pss_rsae_* signature schemes have pulled the key type into what is covered by the signature algorithm, so we should not apply this sort of compatibility workaround. Reviewed-by: Matt Caswell <matt@openssl.org> (Merged from https://github.com/openssl/openssl/pull/5068)

diff --git a/ssl/statem/extensions.c b/ssl/statem/extensions.c
index 5a0fa25..5ad86f2 100644
--- a/ssl/statem/extensions.c
+++ b/ssl/statem/extensions.c

@@ -29,6 +29,7 @@
 #endif
 static int init_alpn(SSL *s, unsigned int context);
 static int final_alpn(SSL *s, unsigned int context, int sent);
+static int init_sig_algs_cert(SSL *s, unsigned int context);
 static int init_sig_algs(SSL *s, unsigned int context);
 static int init_certificate_authorities(SSL *s, unsigned int context);
 static EXT_RETURN tls_construct_certificate_authorities(SSL *s, WPACKET *pkt,
@@ -281,6 +282,14 @@
         tls_construct_stoc_ems, tls_construct_ctos_ems, final_ems
     },
     {
+        TLSEXT_TYPE_signature_algorithms_cert,
+        SSL_EXT_CLIENT_HELLO | SSL_EXT_TLS1_3_CERTIFICATE_REQUEST,
+        init_sig_algs_cert, tls_parse_ctos_sig_algs_cert,
+        tls_parse_ctos_sig_algs_cert,
+        /* We do not generate signature_algorithms_cert at present. */
+        NULL, NULL, NULL
+    },
+    {
         TLSEXT_TYPE_signature_algorithms,
         SSL_EXT_CLIENT_HELLO | SSL_EXT_TLS1_3_CERTIFICATE_REQUEST,
         init_sig_algs, tls_parse_ctos_sig_algs,
@@ -1090,6 +1099,15 @@
     return 1;
 }
 
+static int init_sig_algs_cert(SSL *s, unsigned int context)
+{
+    /* Clear any signature algorithms extension received */
+    OPENSSL_free(s->s3->tmp.peer_cert_sigalgs);
+    s->s3->tmp.peer_cert_sigalgs = NULL;
+
+    return 1;
+}
+
 #ifndef OPENSSL_NO_SRP
 static int init_srp(SSL *s, unsigned int context)
 {

diff --git a/ssl/statem/extensions_srvr.c b/ssl/statem/extensions_srvr.c
index fadc6a7..0a7bac4 100644
--- a/ssl/statem/extensions_srvr.c
+++ b/ssl/statem/extensions_srvr.c

@@ -276,6 +276,27 @@
     return 1;
 }
 
+int tls_parse_ctos_sig_algs_cert(SSL *s, PACKET *pkt, unsigned int context,
+                                 X509 *x, size_t chainidx)
+{
+    PACKET supported_sig_algs;
+
+    if (!PACKET_as_length_prefixed_2(pkt, &supported_sig_algs)
+            || PACKET_remaining(&supported_sig_algs) == 0) {
+        SSLfatal(s, SSL_AD_DECODE_ERROR,
+                 SSL_F_TLS_PARSE_CTOS_SIG_ALGS_CERT, SSL_R_BAD_EXTENSION);
+        return 0;
+    }
+
+    if (!s->hit && !tls1_save_sigalgs(s, &supported_sig_algs, 1)) {
+        SSLfatal(s, SSL_AD_DECODE_ERROR,
+                 SSL_F_TLS_PARSE_CTOS_SIG_ALGS_CERT, SSL_R_BAD_EXTENSION);
+        return 0;
+    }
+
+    return 1;
+}
+
 int tls_parse_ctos_sig_algs(SSL *s, PACKET *pkt, unsigned int context, X509 *x,
                             size_t chainidx)
 {
@@ -288,7 +309,7 @@
         return 0;
     }
 
-    if (!s->hit && !tls1_save_sigalgs(s, &supported_sig_algs)) {
+    if (!s->hit && !tls1_save_sigalgs(s, &supported_sig_algs, 0)) {
         SSLfatal(s, SSL_AD_DECODE_ERROR,
                  SSL_F_TLS_PARSE_CTOS_SIG_ALGS, SSL_R_BAD_EXTENSION);
         return 0;

diff --git a/ssl/statem/statem_clnt.c b/ssl/statem/statem_clnt.c
index 3129138..8a33386 100644
--- a/ssl/statem/statem_clnt.c
+++ b/ssl/statem/statem_clnt.c

@@ -2458,7 +2458,11 @@
                 return MSG_PROCESS_ERROR;
             }
 
-            if (!tls1_save_sigalgs(s, &sigalgs)) {
+            /*
+             * Despite this being for certificates, preserve compatibility
+             * with pre-TLS 1.3 and use the regular sigalgs field.
+             */
+            if (!tls1_save_sigalgs(s, &sigalgs, 0)) {
                 SSLfatal(s, SSL_AD_INTERNAL_ERROR,
                          SSL_F_TLS_PROCESS_CERTIFICATE_REQUEST,
                          SSL_R_SIGNATURE_ALGORITHMS_ERROR);

diff --git a/ssl/statem/statem_locl.h b/ssl/statem/statem_locl.h
index 38b0ce8..f16d3cb 100644
--- a/ssl/statem/statem_locl.h
+++ b/ssl/statem/statem_locl.h

@@ -205,6 +205,8 @@
 #endif
 int tls_parse_ctos_session_ticket(SSL *s, PACKET *pkt, unsigned int context,
                                   X509 *x, size_t chainidx);
+int tls_parse_ctos_sig_algs_cert(SSL *s, PACKET *pkt, unsigned int context,
+                                 X509 *x, size_t chainidx);
 int tls_parse_ctos_sig_algs(SSL *s, PACKET *pkt, unsigned int context, X509 *x,
                             size_t chainidx);
 #ifndef OPENSSL_NO_OCSP