evp_md_init_internal: Avoid reallocating algctx if digest unchanged

Fixes #16947

Also refactor out algctx freeing into a separate function.

Reviewed-by: Dmitry Belyavskiy <beldmit@gmail.com>
Reviewed-by: Ben Kaduk <kaduk@mit.edu>
Reviewed-by: Paul Dale <pauli@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/18105)

crypto/evp/digest.c

diff --git a/crypto/evp/digest.c b/crypto/evp/digest.c
index cd930eb..6567921 100644
--- a/crypto/evp/digest.c
+++ b/crypto/evp/digest.c
@@ -141,6 +141,20 @@
     OPENSSL_free(ctx);
 }
 
+int evp_md_ctx_free_algctx(EVP_MD_CTX *ctx)
+{
+    if (ctx->algctx != NULL) {
+        if (!ossl_assert(ctx->digest != NULL)) {
+            ERR_raise(ERR_LIB_EVP, EVP_R_INITIALIZATION_ERROR);
+            return 0;
+        }
+        if (ctx->digest->freectx != NULL)
+            ctx->digest->freectx(ctx->algctx);
+        ctx->algctx = NULL;
+    }
+    return 1;
+}
+
 static int evp_md_init_internal(EVP_MD_CTX *ctx, const EVP_MD *type,
                                 const OSSL_PARAM params[], ENGINE *impl)
 {
@@ -169,16 +183,6 @@
 
     EVP_MD_CTX_clear_flags(ctx, EVP_MD_CTX_FLAG_CLEANED);
 
-    if (ctx->algctx != NULL) {
-        if (!ossl_assert(ctx->digest != NULL)) {
-            ERR_raise(ERR_LIB_EVP, EVP_R_INITIALIZATION_ERROR);
-            return 0;
-        }
-        if (ctx->digest->freectx != NULL)
-            ctx->digest->freectx(ctx->algctx);
-        ctx->algctx = NULL;
-    }
-
     if (type != NULL) {
         ctx->reqdigest = type;
     } else {
@@ -227,6 +231,10 @@
 #endif
             || (ctx->flags & EVP_MD_CTX_FLAG_NO_INIT) != 0
             || type->origin == EVP_ORIG_METH) {
+        /* If we were using provided hash before, cleanup algctx */
+        if (!evp_md_ctx_free_algctx(ctx))
+            return 0;
+
         if (ctx->digest == ctx->fetched_digest)
             ctx->digest = NULL;
         EVP_MD_free(ctx->fetched_digest);
@@ -237,6 +245,8 @@
     cleanup_old_md_data(ctx, 1);
 
     /* Start of non-legacy code below */
+    if (ctx->digest != type && !evp_md_ctx_free_algctx(ctx))
+        return 0;
 
     if (type->prov == NULL) {
 #ifdef FIPS_MODULE
@@ -259,11 +269,6 @@
 #endif
     }
 
-    if (ctx->algctx != NULL && ctx->digest != NULL && ctx->digest != type) {
-        if (ctx->digest->freectx != NULL)
-            ctx->digest->freectx(ctx->algctx);
-        ctx->algctx = NULL;
-    }
     if (type->prov != NULL && ctx->fetched_digest != type) {
         if (!EVP_MD_up_ref((EVP_MD *)type)) {
             ERR_raise(ERR_LIB_EVP, EVP_R_INITIALIZATION_ERROR);

crypto/evp/m_sigver.c

diff --git a/crypto/evp/m_sigver.c b/crypto/evp/m_sigver.c
index 371bca5..a949398 100644
--- a/crypto/evp/m_sigver.c
+++ b/crypto/evp/m_sigver.c
@@ -51,15 +51,8 @@
     void *provkey = NULL;
     int ret, iter, reinit = 1;
 
-    if (ctx->algctx != NULL) {
-        if (!ossl_assert(ctx->digest != NULL)) {
-            ERR_raise(ERR_LIB_EVP, EVP_R_INITIALIZATION_ERROR);
-            return 0;
-        }
-        if (ctx->digest->freectx != NULL)
-            ctx->digest->freectx(ctx->algctx);
-        ctx->algctx = NULL;
-    }
+    if (!evp_md_ctx_free_algctx(ctx))
+        return 0;
 
     if (ctx->pctx == NULL) {
         reinit = 0;

include/crypto/evp.h

diff --git a/include/crypto/evp.h b/include/crypto/evp.h
index 6fb0530..9c20ead 100644
--- a/include/crypto/evp.h
+++ b/include/crypto/evp.h
@@ -901,6 +901,8 @@
 char *evp_get_global_properties_str(OSSL_LIB_CTX *libctx, int loadconfig);
 
 void evp_md_ctx_clear_digest(EVP_MD_CTX *ctx, int force, int keep_digest);
+/* just free the algctx if set, returns 0 on inconsistent state of ctx */
+int evp_md_ctx_free_algctx(EVP_MD_CTX *ctx);
 
 /* Three possible states: */
 # define EVP_PKEY_STATE_UNKNOWN         0