Reorganise supported signature algorithm extension processing.

Only store encoded versions of peer and configured signature algorithms.
Determine shared signature algorithms and cache the result along with NID
equivalents of each algorithm.
diff --git a/ssl/t1_lib.c b/ssl/t1_lib.c
index dcfecf4..5fae9fb 100644
--- a/ssl/t1_lib.c
+++ b/ssl/t1_lib.c
@@ -631,35 +631,26 @@
 
 size_t tls12_get_sig_algs(SSL *s, unsigned char *p)
 	{
-	TLS_SIGALGS *sptr = s->cert->conf_sigalgs;
-	size_t slen;
+	const unsigned char *sigs;
+	size_t sigslen;
+	sigs = s->cert->conf_sigalgs;
 
-	/* Use custom signature algorithms if any are set */
-
-	if (sptr)
+	if (sigs)
+		sigslen = s->cert->conf_sigalgslen;
+	else
 		{
-		slen = s->cert->conf_sigalgslen;
-		if (p)
-			{
-			size_t i;
-			for (i = 0; i < slen; i++, sptr++)
-				{
-				*p++ = sptr->rhash;
-				*p++ = sptr->rsign;
-				}
-			}
-		return slen * 2;
-		}
-		
-	slen = sizeof(tls12_sigalgs);
+		sigs = tls12_sigalgs;
+		sigslen = sizeof(tls12_sigalgs);
 #ifdef OPENSSL_FIPS
-	/* If FIPS mode don't include MD5 which is last */
-	if (FIPS_mode())
-		slen -= 2;
+		/* If FIPS mode don't include MD5 which is last */
+		if (FIPS_mode())
+			sigslen -= 2;
 #endif
+		}
+
 	if (p)
-		memcpy(p, tls12_sigalgs, slen);
-	return slen;
+		memcpy(p, sigs, sigslen);
+	return sigslen;
 	}
 
 /* byte_compare is a compare function for qsort(3) that compares bytes. */
@@ -1312,7 +1303,6 @@
 	unsigned short len;
 	unsigned char *data = *p;
 	int renegotiate_seen = 0;
-	int sigalg_seen = 0;
 
 	s->servername_done = 0;
 	s->tlsext_status_type = -1;
@@ -1324,6 +1314,18 @@
 	s->tlsext_heartbeat &= ~(SSL_TLSEXT_HB_ENABLED |
 	                       SSL_TLSEXT_HB_DONT_SEND_REQUESTS);
 #endif
+	/* Clear any signature algorithms extension received */
+	if (s->cert->peer_sigalgs)
+		{
+		OPENSSL_free(s->cert->peer_sigalgs);
+		s->cert->peer_sigalgs = NULL;
+		}
+	/* Clear any shared sigtnature algorithms */
+	if (s->cert->shared_sigalgs)
+		{
+		OPENSSL_free(s->cert->shared_sigalgs);
+		s->cert->shared_sigalgs = NULL;
+		}
 
 	if (data >= (d+n-2))
 		goto ri_check;
@@ -1599,15 +1601,14 @@
 		else if (type == TLSEXT_TYPE_signature_algorithms)
 			{
 			int dsize;
-			if (sigalg_seen || size < 2) 
+			if (s->cert->peer_sigalgs || size < 2) 
 				{
 				*al = SSL_AD_DECODE_ERROR;
 				return 0;
 				}
-			sigalg_seen = 1;
 			n2s(data,dsize);
 			size -= 2;
-			if (dsize != size || dsize & 1) 
+			if (dsize != size || dsize & 1 || !dsize) 
 				{
 				*al = SSL_AD_DECODE_ERROR;
 				return 0;
@@ -1617,6 +1618,16 @@
 				*al = SSL_AD_DECODE_ERROR;
 				return 0;
 				}
+			/* If sigalgs received and no shared algorithms fatal
+			 * error.
+			 */
+			if (s->cert->peer_sigalgs && !s->cert->shared_sigalgs)
+				{
+				SSLerr(SSL_F_SSL_SCAN_CLIENTHELLO_TLSEXT,
+					SSL_R_NO_SHARED_SIGATURE_ALGORITHMS);
+				*al = SSL_AD_ILLEGAL_PARAMETER;
+				return 0;
+				}
 			}
 		else if (type == TLSEXT_TYPE_status_request &&
 		         s->version != DTLS1_VERSION && s->ctx->tlsext_status_cb)
@@ -1855,6 +1866,9 @@
 				SSL_R_UNSAFE_LEGACY_RENEGOTIATION_DISABLED);
 		return 0;
 		}
+	/* If no signature algorithms extension set default values */
+	if (!s->cert->peer_sigalgs)
+		ssl_cert_set_default_md(s->cert);
 
 	return 1;
 	}
@@ -2861,11 +2875,145 @@
 		}
 	}
 
+static int tls12_get_pkey_idx(unsigned char sig_alg)
+	{
+	switch(sig_alg)
+		{
+#ifndef OPENSSL_NO_RSA
+	case TLSEXT_signature_rsa:
+		return SSL_PKEY_RSA_SIGN;
+#endif
+#ifndef OPENSSL_NO_DSA
+	case TLSEXT_signature_dsa:
+		return SSL_PKEY_DSA_SIGN;
+#endif
+#ifndef OPENSSL_NO_ECDSA
+	case TLSEXT_signature_ecdsa:
+		return SSL_PKEY_ECC;
+#endif
+		}
+	return -1;
+	}
+
+/* Convert TLS 1.2 signature algorithm extension values into NIDs */
+static void tls1_lookup_sigalg(int *phash_nid, int *psign_nid,
+			int *psignhash_nid, const unsigned char *data)
+	{
+	int sign_nid, hash_nid;
+	if (!phash_nid && !psign_nid && !psignhash_nid)
+		return;
+	if (phash_nid || psignhash_nid)
+		{
+		hash_nid = tls12_find_nid(data[0], tls12_md,
+					sizeof(tls12_md)/sizeof(tls12_lookup));
+		if (phash_nid)
+			*phash_nid = hash_nid;
+		}
+	if (psign_nid || psignhash_nid)
+		{
+		sign_nid = tls12_find_nid(data[1], tls12_sig,
+					sizeof(tls12_sig)/sizeof(tls12_lookup));
+		if (psign_nid)
+			*psign_nid = sign_nid;
+		}
+	if (psignhash_nid)
+		{
+		if (sign_nid && hash_nid)
+			OBJ_find_sigid_by_algs(psignhash_nid,
+							hash_nid, sign_nid);
+		else
+			*psignhash_nid = NID_undef;
+		}
+	}
+/* Given preference and allowed sigalgs set shared sigalgs */
+static int tls12_do_shared_sigalgs(TLS_SIGALGS *shsig,
+				const unsigned char *pref, size_t preflen,
+				const unsigned char *allow, size_t allowlen)
+	{
+	const unsigned char *ptmp, *atmp;
+	size_t i, j, nmatch = 0;
+	for (i = 0, ptmp = pref; i < preflen; i+=2, ptmp+=2)
+		{
+		/* Skip disabled hashes or signature algorithms */
+		if (tls12_get_hash(ptmp[0]) == NULL)
+			continue;
+		if (tls12_get_pkey_idx(ptmp[1]) == -1)
+			continue;
+		for (j = 0, atmp = allow; j < allowlen; j+=2, atmp+=2)
+			{
+			if (ptmp[0] == atmp[0] && ptmp[1] == atmp[1])
+				{
+				nmatch++;
+				if (shsig)
+					{
+					shsig->rhash = ptmp[0];
+					shsig->rsign = ptmp[1];
+					tls1_lookup_sigalg(&shsig->hash_nid,
+						&shsig->sign_nid,
+						&shsig->signandhash_nid,
+						ptmp);
+					shsig++;
+					}
+				break;
+				}
+			}
+		}
+	return nmatch;
+	}
+
+/* Set shared signature algorithms for SSL structures */
+static int tls1_set_shared_sigalgs(SSL *s)
+	{
+	const unsigned char *pref, *allow, *conf;
+	size_t preflen, allowlen, conflen;
+	size_t nmatch;
+	TLS_SIGALGS *salgs = NULL;
+	CERT *c = s->cert;
+	conf = c->conf_sigalgs;
+	if (conf)
+		conflen = c->conf_sigalgslen;
+	else
+		{
+		conf = tls12_sigalgs;
+		conflen = sizeof(tls12_sigalgs);
+#ifdef OPENSSL_FIPS
+		if (FIPS_mode())
+			conflen -= 2;
+#endif
+		}
+	if(s->options & SSL_OP_CIPHER_SERVER_PREFERENCE)
+		{
+		pref = conf;
+		preflen = conflen;
+		allow = c->peer_sigalgs;
+		allowlen = c->peer_sigalgslen;
+		}
+	else
+		{
+		allow = conf;
+		allowlen = conflen;
+		pref = c->peer_sigalgs;
+		preflen = c->peer_sigalgslen;
+		}
+	nmatch = tls12_do_shared_sigalgs(NULL, pref, preflen, allow, allowlen);
+	if (!nmatch)
+		return 1;
+	salgs = OPENSSL_malloc(nmatch * sizeof(TLS_SIGALGS));
+	if (!salgs)
+		return 0;
+	nmatch = tls12_do_shared_sigalgs(salgs, pref, preflen, allow, allowlen);
+	c->shared_sigalgs = salgs;
+	c->shared_sigalgslen = nmatch;
+	return 1;
+	}
+		
+
 /* Set preferred digest for each key type */
 
 int tls1_process_sigalgs(SSL *s, const unsigned char *data, int dsize)
 	{
-	int i, idx;
+	int idx;
+	size_t i;
 	const EVP_MD *md;
 	CERT *c = s->cert;
 	TLS_SIGALGS *sigptr;
@@ -2881,60 +3029,27 @@
 	c->pkeys[SSL_PKEY_RSA_ENC].digest = NULL;
 	c->pkeys[SSL_PKEY_ECC].digest = NULL;
 
-	if (c->peer_sigalgs)
-		OPENSSL_free(c->peer_sigalgs);
-	c->peer_sigalgs = OPENSSL_malloc((dsize/2) * sizeof(TLS_SIGALGS));
+	c->peer_sigalgs = OPENSSL_malloc(dsize);
 	if (!c->peer_sigalgs)
 		return 0;
-	c->peer_sigalgslen = dsize/2;
+	c->peer_sigalgslen = dsize;
+	memcpy(c->peer_sigalgs, data, dsize);
 
-	for (i = 0, sigptr = c->peer_sigalgs; i < dsize; i += 2, sigptr++)
+	tls1_set_shared_sigalgs(s);
+
+	for (i = 0, sigptr = c->shared_sigalgs;
+			i < c->shared_sigalgslen; i++, sigptr++)
 		{
-		sigptr->rhash = data[i];
-		sigptr->rsign = data[i + 1];
-		sigptr->hash_nid = tls12_find_nid(sigptr->rhash, tls12_md,
-					sizeof(tls12_md)/sizeof(tls12_lookup));
-		sigptr->sign_nid = tls12_find_nid(sigptr->rsign, tls12_sig,
-					sizeof(tls12_sig)/sizeof(tls12_lookup));
-		if (!OBJ_find_sigid_by_algs(&sigptr->signandhash_nid,
-						sigptr->hash_nid,
-						sigptr->sign_nid))
-			sigptr->signandhash_nid = NID_undef;
-		switch(sigptr->rsign)
-			{
-#ifndef OPENSSL_NO_RSA
-			case TLSEXT_signature_rsa:
-			idx = SSL_PKEY_RSA_SIGN;
-			break;
-#endif
-#ifndef OPENSSL_NO_DSA
-			case TLSEXT_signature_dsa:
-			idx = SSL_PKEY_DSA_SIGN;
-			break;
-#endif
-#ifndef OPENSSL_NO_ECDSA
-			case TLSEXT_signature_ecdsa:
-			idx = SSL_PKEY_ECC;
-			break;
-#endif
-			default:
-			continue;
-			}
-
-		if (c->pkeys[idx].digest == NULL)
+		idx = tls12_get_pkey_idx(sigptr->rsign);
+		if (idx > 0 && c->pkeys[idx].digest == NULL)
 			{
 			md = tls12_get_hash(sigptr->rhash);
-			if (md)
-				{
-				c->pkeys[idx].digest = md;
-				if (idx == SSL_PKEY_RSA_SIGN)
-					c->pkeys[SSL_PKEY_RSA_ENC].digest = md;
-				}
+			c->pkeys[idx].digest = md;
+			if (idx == SSL_PKEY_RSA_SIGN)
+				c->pkeys[SSL_PKEY_RSA_ENC].digest = md;
 			}
 
 		}
-
-
 	/* Set any remaining keys to default values. NOTE: if alg is not
 	 * supported it stays as NULL.
 	 */
@@ -2956,32 +3071,48 @@
 	return 1;
 	}
 
-#endif
 
 int SSL_get_sigalgs(SSL *s, int idx,
-			int *psign, int *phash, int *psignandhash,
+			int *psign, int *phash, int *psignhash,
 			unsigned char *rsig, unsigned char *rhash)
 	{
-	if (s->cert->peer_sigalgs == NULL)
+	const unsigned char *psig = s->cert->peer_sigalgs;
+	if (psig == NULL)
 		return 0;
 	if (idx >= 0)
 		{
-		TLS_SIGALGS *psig;
+		idx <<= 1;
 		if (idx >= (int)s->cert->peer_sigalgslen)
 			return 0;
-		psig = s->cert->peer_sigalgs + idx;
-		if (psign)
-			*psign = psig->sign_nid;
-		if (phash)
-			*phash = psig->hash_nid;
-		if (psignandhash)
-			*psignandhash = psig->signandhash_nid;
-		if (rsig)
-			*rsig = psig->rsign;
+		psig += idx;
 		if (rhash)
-			*rhash = psig->rhash;
+			*rhash = psig[0];
+		if (rsig)
+			*rsig = psig[1];
+		tls1_lookup_sigalg(phash, psign, psignhash, psig);
 		}
-	return s->cert->peer_sigalgslen;
+	return s->cert->peer_sigalgslen / 2;
+	}
+
+int SSL_get_shared_sigalgs(SSL *s, int idx,
+			int *psign, int *phash, int *psignhash,
+			unsigned char *rsig, unsigned char *rhash)
+	{
+	TLS_SIGALGS *shsigalgs = s->cert->shared_sigalgs;
+	if (!shsigalgs || idx >= (int)s->cert->shared_sigalgslen)
+		return 0;
+	shsigalgs += idx;
+	if (phash)
+		*phash = shsigalgs->hash_nid;
+	if (psign)
+		*psign = shsigalgs->sign_nid;
+	if (psignhash)
+		*psignhash = shsigalgs->signandhash_nid;
+	if (rsig)
+		*rsig = shsigalgs->rsign;
+	if (rhash)
+		*rhash = shsigalgs->rhash;
+	return s->cert->shared_sigalgslen;
 	}
 	
 
@@ -3130,7 +3261,7 @@
 	}
 #endif
 
-#define MAX_SIGALGLEN	(TLSEXT_hash_num * TLSEXT_signature_num *2)
+#define MAX_SIGALGLEN	(TLSEXT_hash_num * TLSEXT_signature_num * 2)
 
 typedef struct
 	{
@@ -3194,35 +3325,27 @@
 	return tls1_set_sigalgs(c, sig.sigalgs, sig.sigalgcnt);
 	}
 
-int tls1_set_sigalgs(CERT *c, const int *salg, size_t salglen)
+int tls1_set_sigalgs(CERT *c, const int *psig_nids, size_t salglen)
 	{
-	TLS_SIGALGS *sigalgs, *sptr;
+	unsigned char *sigalgs, *sptr;
 	int rhash, rsign;
 	size_t i;
 	if (salglen & 1)
 		return 0;
-	salglen /= 2;
-	sigalgs = OPENSSL_malloc(sizeof(TLS_SIGALGS) * salglen);
+	sigalgs = OPENSSL_malloc(salglen);
 	if (sigalgs == NULL)
 		return 0;
-	for (i = 0, sptr = sigalgs; i < salglen; i++, sptr++)
+	for (i = 0, sptr = sigalgs; i < salglen; i+=2)
 		{
-		sptr->hash_nid = *salg++;
-		sptr->sign_nid = *salg++;
-		rhash = tls12_find_id(sptr->hash_nid, tls12_md,
+		rhash = tls12_find_id(*psig_nids++, tls12_md,
 					sizeof(tls12_md)/sizeof(tls12_lookup));
-		rsign = tls12_find_id(sptr->sign_nid, tls12_sig,
+		rsign = tls12_find_id(*psig_nids++, tls12_sig,
 				sizeof(tls12_sig)/sizeof(tls12_lookup));
 
 		if (rhash == -1 || rsign == -1)
 			goto err;
-
-		if (!OBJ_find_sigid_by_algs(&sptr->signandhash_nid,
-						sptr->hash_nid,
-						sptr->sign_nid))
-			sptr->signandhash_nid = NID_undef;
-		sptr->rhash = rhash;
-		sptr->rsign = rsign;
+		*sptr++ = rhash;
+		*sptr++ = rsign;
 		}
 
 	if (c->conf_sigalgs)
@@ -3236,3 +3359,5 @@
 	OPENSSL_free(sigalgs);
 	return 0;
 	}
+
+#endif