Give more information in the SSL_stateless return code

Allow users to distinguish between an error occurring and an HRR being
issued.

Fixes #5549

Reviewed-by: Rich Salz <rsalz@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/5562)
diff --git a/ssl/ssl_err.c b/ssl/ssl_err.c
index f0bde60..34e8ec4 100644
--- a/ssl/ssl_err.c
+++ b/ssl/ssl_err.c
@@ -952,6 +952,8 @@
     "no client cert method"},
     {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_NO_COMPRESSION_SPECIFIED),
     "no compression specified"},
+    {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_NO_COOKIE_CALLBACK_SET),
+    "no cookie callback set"},
     {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_NO_GOST_CERTIFICATE_SENT_BY_PEER),
     "Peer haven't sent GOST certificate, required for selected ciphersuite"},
     {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_NO_METHOD_SPECIFIED),
diff --git a/ssl/ssl_lib.c b/ssl/ssl_lib.c
index accef0c..f5219c2 100644
--- a/ssl/ssl_lib.c
+++ b/ssl/ssl_lib.c
@@ -5352,7 +5352,10 @@
     if (ret > 0 && s->ext.cookieok)
         return 1;
 
-    return 0;
+    if (s->hello_retry_request == SSL_HRR_PENDING && !ossl_statem_in_error(s))
+        return 0;
+
+    return -1;
 }
 
 void SSL_force_post_handshake_auth(SSL *ssl)
diff --git a/ssl/statem/extensions_srvr.c b/ssl/statem/extensions_srvr.c
index bcabb85..b9692f4 100644
--- a/ssl/statem/extensions_srvr.c
+++ b/ssl/statem/extensions_srvr.c
@@ -1682,10 +1682,15 @@
     EVP_PKEY *pkey;
     int ret = EXT_RETURN_FAIL;
 
-    if (s->ctx->app_gen_cookie_cb == NULL
-            || (s->s3->flags & TLS1_FLAGS_STATELESS) == 0)
+    if ((s->s3->flags & TLS1_FLAGS_STATELESS) == 0)
         return EXT_RETURN_NOT_SENT;
 
+    if (s->ctx->app_gen_cookie_cb == NULL) {
+        SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_F_TLS_CONSTRUCT_STOC_COOKIE,
+                 SSL_R_NO_COOKIE_CALLBACK_SET);
+        return EXT_RETURN_FAIL;
+    }
+
     if (!WPACKET_put_bytes_u16(pkt, TLSEXT_TYPE_cookie)
             || !WPACKET_start_sub_packet_u16(pkt)
             || !WPACKET_start_sub_packet_u16(pkt)