Reject compressed point format with TLS 1.3

Reviewed-by: Rich Salz <rsalz@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/2739)

ssl/ssl_err.c

diff --git a/ssl/ssl_err.c b/ssl/ssl_err.c
index addc3de..444de8e 100644
--- a/ssl/ssl_err.c
+++ b/ssl/ssl_err.c
@@ -570,6 +570,8 @@
     {ERR_REASON(SSL_R_GOT_A_FIN_BEFORE_A_CCS), "got a fin before a ccs"},
     {ERR_REASON(SSL_R_HTTPS_PROXY_REQUEST), "https proxy request"},
     {ERR_REASON(SSL_R_HTTP_REQUEST), "http request"},
+    {ERR_REASON(SSL_R_ILLEGAL_POINT_COMPRESSION),
+     "illegal point compression"},
     {ERR_REASON(SSL_R_ILLEGAL_SUITEB_DIGEST), "illegal Suite B digest"},
     {ERR_REASON(SSL_R_INAPPROPRIATE_FALLBACK), "inappropriate fallback"},
     {ERR_REASON(SSL_R_INCONSISTENT_COMPRESSION), "inconsistent compression"},

ssl/t1_lib.c

diff --git a/ssl/t1_lib.c b/ssl/t1_lib.c
index 100c97c..a7b4679 100644
--- a/ssl/t1_lib.c
+++ b/ssl/t1_lib.c
@@ -916,8 +916,12 @@
         int curve = EC_GROUP_get_curve_name(EC_KEY_get0_group(ec));
 
         if (SSL_IS_TLS13(s)) {
+            if (EC_KEY_get_conv_form(ec) != POINT_CONVERSION_UNCOMPRESSED) {
+                SSLerr(SSL_F_TLS12_CHECK_PEER_SIGALG,
+                       SSL_R_ILLEGAL_POINT_COMPRESSION);
+                return 0;
+            }
             /* For TLS 1.3 check curve matches signature algorithm */
-
             if (lu->curve != NID_undef && curve != lu->curve) {
                 SSLerr(SSL_F_TLS12_CHECK_PEER_SIGALG, SSL_R_WRONG_CURVE);
                 return 0;
@@ -2237,7 +2241,7 @@
     if (SSL_IS_TLS13(s)) {
         size_t i;
 #ifndef OPENSSL_NO_EC
-        int curve = -1;
+        int curve = -1, skip_ec = 0;
 #endif
 
         /* Look for a certificate matching shared sigaglgs */
@@ -2258,8 +2262,11 @@
                     EC_KEY *ec = EVP_PKEY_get0_EC_KEY(s->cert->pkeys[idx].privatekey);
 
                     curve = EC_GROUP_get_curve_name(EC_KEY_get0_group(ec));
+                    if (EC_KEY_get_conv_form(ec)
+                        != POINT_CONVERSION_UNCOMPRESSED)
+                        skip_ec = 1;
                 }
-                if (lu->curve != NID_undef && curve != lu->curve)
+                if (skip_ec || (lu->curve != NID_undef && curve != lu->curve))
                     continue;
 #else
                 continue;