ssl/statem/*.c: address "enum mixed with another type" warnings.

This is actually not all warnings, only return values.

Reviewed-by: Rich Salz <rsalz@openssl.org>
Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/4297)

ssl/statem/extensions_clnt.c

diff --git a/ssl/statem/extensions_clnt.c b/ssl/statem/extensions_clnt.c
index d8a9f7c..cb3d218 100644
--- a/ssl/statem/extensions_clnt.c
+++ b/ssl/statem/extensions_clnt.c
@@ -851,7 +851,7 @@
         if (!WPACKET_put_bytes_u16(pkt, TLSEXT_TYPE_padding)
                 || !WPACKET_sub_allocate_bytes_u16(pkt, hlen, &padbytes)) {
             SSLerr(SSL_F_TLS_CONSTRUCT_CTOS_PADDING, ERR_R_INTERNAL_ERROR);
-            return 0;
+            return EXT_RETURN_FAIL;
         }
         memset(padbytes, 0, hlen);
     }
@@ -1052,7 +1052,7 @@
  err:
     return ret;
 #else
-    return 1;
+    return EXT_RETURN_SENT;
 #endif
 }
 

ssl/statem/statem_clnt.c

diff --git a/ssl/statem/statem_clnt.c b/ssl/statem/statem_clnt.c
index 5f2855b..ff907aa 100644
--- a/ssl/statem/statem_clnt.c
+++ b/ssl/statem/statem_clnt.c
@@ -1658,7 +1658,8 @@
 
 MSG_PROCESS_RETURN tls_process_server_certificate(SSL *s, PACKET *pkt)
 {
-    int al, i, ret = MSG_PROCESS_ERROR;
+    int al, i;
+    MSG_PROCESS_RETURN ret = MSG_PROCESS_ERROR;
     unsigned long cert_list_len, cert_len;
     X509 *x = NULL;
     const unsigned char *certstart, *certbytes;
@@ -2314,7 +2315,7 @@
 
 MSG_PROCESS_RETURN tls_process_certificate_request(SSL *s, PACKET *pkt)
 {
-    int ret = MSG_PROCESS_ERROR;
+    MSG_PROCESS_RETURN ret = MSG_PROCESS_ERROR;
     int al = SSL_AD_DECODE_ERROR;
     size_t i;
 
@@ -3246,7 +3247,7 @@
             if (i == 0) {
                 ssl3_send_alert(s, SSL3_AL_FATAL, SSL_AD_INTERNAL_ERROR);
                 ossl_statem_set_error(s);
-                return 0;
+                return WORK_ERROR;
             }
             s->rwstate = SSL_NOTHING;
         }
@@ -3292,7 +3293,7 @@
                 if (!ssl3_digest_cached_records(s, 0)) {
                     ssl3_send_alert(s, SSL3_AL_FATAL, SSL_AD_INTERNAL_ERROR);
                     ossl_statem_set_error(s);
-                    return 0;
+                    return WORK_ERROR;
                 }
             }
         }

ssl/statem/statem_lib.c

diff --git a/ssl/statem/statem_lib.c b/ssl/statem/statem_lib.c
index c92c8b5..d296243 100644
--- a/ssl/statem/statem_lib.c
+++ b/ssl/statem/statem_lib.c
@@ -316,7 +316,8 @@
 #ifndef OPENSSL_NO_GOST
     unsigned char *gost_data = NULL;
 #endif
-    int al = SSL_AD_INTERNAL_ERROR, ret = MSG_PROCESS_ERROR;
+    int al = SSL_AD_INTERNAL_ERROR;
+    MSG_PROCESS_RETURN ret = MSG_PROCESS_ERROR;
     int j;
     unsigned int len;
     X509 *peer;

ssl/statem/statem_srvr.c

diff --git a/ssl/statem/statem_srvr.c b/ssl/statem/statem_srvr.c
index a3a6bdf..d2f8f90 100644
--- a/ssl/statem/statem_srvr.c
+++ b/ssl/statem/statem_srvr.c
@@ -1353,7 +1353,7 @@
              */
             if (SSL_get_options(s) & SSL_OP_COOKIE_EXCHANGE) {
                 if (clienthello->dtls_cookie_len == 0)
-                    return 1;
+                    return MSG_PROCESS_FINISHED_READING;
             }
         }
 
@@ -3251,7 +3251,8 @@
 
 MSG_PROCESS_RETURN tls_process_client_certificate(SSL *s, PACKET *pkt)
 {
-    int i, al = SSL_AD_INTERNAL_ERROR, ret = MSG_PROCESS_ERROR;
+    int i, al = SSL_AD_INTERNAL_ERROR;
+    MSG_PROCESS_RETURN ret = MSG_PROCESS_ERROR;
     X509 *x = NULL;
     unsigned long l, llen;
     const unsigned char *certstart, *certbytes;