EVP_MD_size() can return an error Fix some instances where we weren't checking the error return. Reviewed-by: Rich Salz <rsalz@openssl.org> (Merged from https://github.com/openssl/openssl/pull/6373)

diff --git a/ssl/statem/extensions.c b/ssl/statem/extensions.c
index 7f9fd5f..209b4df 100644
--- a/ssl/statem/extensions.c
+++ b/ssl/statem/extensions.c

@@ -1427,10 +1427,19 @@
     const char external_label[] = "ext binder";
     const char nonce_label[] = "resumption";
     const char *label;
-    size_t bindersize, labelsize, psklen, hashsize = EVP_MD_size(md);
+    size_t bindersize, labelsize, psklen, hashsize;
+    int hashsizei = EVP_MD_size(md);
     int ret = -1;
     int usepskfored = 0;
 
+    /* Ensure cast to size_t is safe */
+    if (!ossl_assert(hashsizei >= 0)) {
+        SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_F_TLS_PSK_DO_BINDER,
+                 ERR_R_INTERNAL_ERROR);
+        goto err;
+    }
+    hashsize = (size_t)hashsizei;
+
     if (external
             && s->early_data_state == SSL_EARLY_DATA_CONNECTING
             && s->session->ext.max_early_data == 0

diff --git a/ssl/tls13_enc.c b/ssl/tls13_enc.c
index 1e6db92..3fc8e96 100644
--- a/ssl/tls13_enc.c
+++ b/ssl/tls13_enc.c

@@ -129,6 +129,7 @@
                           unsigned char *outsecret)
 {
     size_t mdlen, prevsecretlen;
+    int mdleni;
     int ret;
     EVP_PKEY_CTX *pctx = EVP_PKEY_CTX_new_id(EVP_PKEY_HKDF, NULL);
     static const char derived_secret_label[] = "derived";
@@ -140,7 +141,14 @@
         return 0;
     }
 
-    mdlen = EVP_MD_size(md);
+    mdleni = EVP_MD_size(md);
+    /* Ensure cast to size_t is safe */
+    if (!ossl_assert(mdleni >= 0)) {
+        SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_F_TLS13_GENERATE_SECRET,
+                 ERR_R_INTERNAL_ERROR);
+        return 0;
+    }
+    mdlen = (size_t)mdleni;
 
     if (insecret == NULL) {
         insecret = default_zeros;
@@ -316,7 +324,16 @@
 {
     unsigned char key[EVP_MAX_KEY_LENGTH];
     size_t ivlen, keylen, taglen;
-    size_t hashlen = EVP_MD_size(md);
+    int hashleni = EVP_MD_size(md);
+    size_t hashlen;
+
+    /* Ensure cast to size_t is safe */
+    if (!ossl_assert(hashleni >= 0)) {
+        SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_F_DERIVE_SECRET_KEY_AND_IV,
+                 ERR_R_EVP_LIB);
+        goto err;
+    }
+    hashlen = (size_t)hashleni;
 
     if (!tls13_hkdf_expand(s, md, insecret, label, labellen, hash, hashlen,
                            secret, hashlen)) {