commit 680827a15f12c3b37a6335fcb992555cf300730e
author PW Hu <jlu.hpw@foxmail.com> Fri Nov 05 17:33:32 2021 +0800
committer Tomas Mraz <tomas@openssl.org> Fri Nov 12 11:20:48 2021 +0100
tree eb992df2faf2d55a5cfd75228605938f4341637e
parent ed5b26ce0b34ec00bdd53d15854a22bccbb4d415

Fix return value checking of BN_check_prime invocations

Negative return value indicates an error so we bail out.

Reviewed-by: Paul Dale <pauli@openssl.org>
Reviewed-by: Kurt Roeckx <kurt@roeckx.be>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/16975)

crypto/bn/bn_rsa_fips186_4.c

diff --git a/crypto/bn/bn_rsa_fips186_4.c b/crypto/bn/bn_rsa_fips186_4.c
index bde9ffa..8faaaef 100644
--- a/crypto/bn/bn_rsa_fips186_4.c
+++ b/crypto/bn/bn_rsa_fips186_4.c
@@ -106,6 +106,7 @@
 {
     int ret = 0;
     int i = 0;
+    int tmp = 0;
 
     if (BN_copy(p1, Xp1) == NULL)
         return 0;
@@ -116,8 +117,11 @@
         i++;
         BN_GENCB_call(cb, 0, i);
         /* MR test with trial division */
-        if (BN_check_prime(p1, ctx, cb))
+        tmp = BN_check_prime(p1, ctx, cb);
+        if (tmp > 0)
             break;
+        if (tmp < 0)
+            goto err;
         /* Get next odd number */
         if (!BN_add_word(p1, 2))
             goto err;
@@ -329,8 +333,14 @@
                     || !BN_sub_word(y1, 1)
                     || !BN_gcd(tmp, y1, e, ctx))
                 goto err;
-            if (BN_is_one(tmp) && BN_check_prime(Y, ctx, cb))
-                goto end;
+            if (BN_is_one(tmp)) {
+                int rv = BN_check_prime(Y, ctx, cb);
+
+                if (rv > 0)
+                    goto end;
+                if (rv < 0)
+                    goto err;
+            }
             /* (Step 8-10) */
             if (++i >= imax || !BN_add(Y, Y, r1r2x2))
                 goto err;