Check range of RSA plaintext and ciphertext when using no padding.

Fixes #24051

RSA with 'no padding' corresponds to RSAEP/RSADP.
The code was not checking the lower bounds.
The bounds are specified in SP800-56Br2, section 7.1.1.1 and 7.1.2.1
Note that RFC8017 expresses the range in a sentence using the word
between, and there is some ambiguity in this.
The upper bounds have change to match the definition in SP800.

Reviewed-by: Paul Dale <ppzgs1@gmail.com>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/24061)

crypto/rsa/rsa_ossl.c

@@ -155,10 +155,35 @@ static int rsa_ossl_public_encrypt(int flen, const unsigned char *from,
     if (BN_bin2bn(buf, num, f) == NULL)
         goto err;
 
-    if (BN_ucmp(f, rsa->n) >= 0) {
-        /* usually the padding functions would catch this */
-        ERR_raise(ERR_LIB_RSA, RSA_R_DATA_TOO_LARGE_FOR_MODULUS);
-        goto err;
+#ifdef FIPS_MODULE
+    /*
+     * See SP800-56Br2, section 7.1.1.1
+     * RSAEP: 1 < f < (n – 1).
+     * (where f is the plaintext).
+     */
+    if (padding == RSA_NO_PADDING) {
+        BIGNUM *nminus1 = BN_CTX_get(ctx);
+
+        if (BN_ucmp(f, BN_value_one()) <= 0) {
+            ERR_raise(ERR_LIB_RSA, RSA_R_DATA_TOO_SMALL);
+            goto err;
+        }
+        if (nminus1 == NULL
+                || BN_copy(nminus1, rsa->n) == NULL
+                || !BN_sub_word(nminus1, 1))
+            goto err;
+        if (BN_ucmp(f, nminus1) >= 0) {
+            ERR_raise(ERR_LIB_RSA, RSA_R_DATA_TOO_LARGE_FOR_MODULUS);
+            goto err;
+        }
+    } else
+#endif
+    {
+        if (BN_ucmp(f, rsa->n) >= 0) {
+            /* usually the padding functions would catch this */
+            ERR_raise(ERR_LIB_RSA, RSA_R_DATA_TOO_LARGE_FOR_MODULUS);
+            goto err;
+        }
     }
 
     if (rsa->flags & RSA_FLAG_CACHE_PUBLIC)
@@ -546,11 +571,35 @@ static int rsa_ossl_private_decrypt(int flen, const unsigned char *from,
     if (BN_bin2bn(from, (int)flen, f) == NULL)
         goto err;
 
-    if (BN_ucmp(f, rsa->n) >= 0) {
-        ERR_raise(ERR_LIB_RSA, RSA_R_DATA_TOO_LARGE_FOR_MODULUS);
-        goto err;
-    }
+#ifdef FIPS_MODULE
+    /*
+     * See SP800-56Br2, section 7.1.2.1
+     * RSADP: 1 < f < (n – 1)
+     * (where f is the ciphertext).
+     */
+    if (padding == RSA_NO_PADDING) {
+        BIGNUM *nminus1 = BN_CTX_get(ctx);
 
+        if (BN_ucmp(f, BN_value_one()) <= 0) {
+            ERR_raise(ERR_LIB_RSA, RSA_R_DATA_TOO_SMALL);
+            goto err;
+        }
+        if (nminus1 == NULL
+                || BN_copy(nminus1, rsa->n) == NULL
+                || !BN_sub_word(nminus1, 1))
+            goto err;
+        if (BN_ucmp(f, nminus1) >= 0) {
+            ERR_raise(ERR_LIB_RSA, RSA_R_DATA_TOO_LARGE_FOR_MODULUS);
+            goto err;
+        }
+    } else
+#endif
+    {
+        if (BN_ucmp(f, rsa->n) >= 0) {
+            ERR_raise(ERR_LIB_RSA, RSA_R_DATA_TOO_LARGE_FOR_MODULUS);
+            goto err;
+        }
+    }
     if (rsa->flags & RSA_FLAG_CACHE_PUBLIC)
         if (!BN_MONT_CTX_set_locked(&rsa->_method_mod_n, rsa->lock,
                                     rsa->n, ctx))

test/recipes/30-test_evp_data/evppkey_rsa_common.txt

@@ -279,6 +279,49 @@ Input = 550AF55A2904E7B9762352F8FB7FA235A9CB053AACB2D5FCB8CA48453CB2EE3619746C70
 Output = "Hello World"
 Result = KEYOP_ERROR
 
+# RSADP Ciphertext = 0 should fail
+Availablein = fips
+FIPSversion = >=3.4.0
+Decrypt = RSA-2048
+Ctrl = rsa_padding_mode:none
+Input = 0000000000000000000000000000000000000000
+Result = KEYOP_ERROR
+
+# RSADP Ciphertext = 1 should fail
+Availablein = fips
+FIPSversion = >=3.4.0
+Decrypt = RSA-2048
+Ctrl = rsa_padding_mode:none
+Input = 0000000000000000000000000000000000000001
+Result = KEYOP_ERROR
+
+# RSADP Ciphertext = 2 should pass
+Decrypt = RSA-2048
+Ctrl = rsa_padding_mode:none
+Input = 0000000000000000000000000000000000000002
+Output = 93d0bae8ad0d94de400eb078dd10edd7418ef1bf11b8e8b5d2b86b142e77d603e108fbcca2b976aa7b5326e5369db3bb73bf74f8d47c36a6318e913888c873502a561fc69329e7c24a0a016d81310449a52b29e49a6a41bdfe6c10a8d90072d64b4486756fd007c0071da2a8c7107a904621c11f0d81aa80b655a713c28170594ece28133dfbfddd61d4e4dad0d6781f6145a351a994054993fd57cd1330966ce97d7ac259b15616fd7235e2cac29fdc1c05f1612c61785614b80e7b650c03ef77d64163d75fa637cc2a9a7e570b3176fdcfb6ad6d25e8515f6ced02cfb3a441c87220044110fd27dcb53888f0377e1797bf297b7da27d3f033cd8b5d60ececc
+
+# RSADP Ciphertext = n-2 should pass
+Availablein = fips
+Decrypt = RSA-2048
+Ctrl = rsa_padding_mode:none
+Input = cd0081ea7b2ae1ea06d59f7c73d9ffb94a09615c2e4ba7c636cef08dd3533ec3185525b015c769b99a77d6725bf9c3532a9b6e5f6627d5fb85160768d3dda9cbd35974511717dc3d309d2fc47ee41f97e32adb7f9dd864a1c4767a666ecd71bc1aacf5e7517f4b38594fea9b05e42d5ada9912008013e45316a4d9bb8ed086b88d28758bacaf922d46a868b485d239c9baeb0e2b64592710f42b2d1ea0a4b4802c0becab328f8a68b0073bdb546feea9809d2849912b390c1532bc7e29c7658f8175fae46f34332ff87bcab3e40649b98577869da0ea718353f0722754886913648760d122be676e0fc483dd20ffc31bda96a31966c9aa2e75ad03de47e1c44d
+Output = 392fc701ce1d4d0bc6c6ef0396c911e2087a6f9d1c92bf1064168579a4db68bf374c29e3730df30f1f24af8d255c0f97b6dbf96691ab9f55538776304b15367ba903548a83edf47ae6932e56fdb31b4e3dffb19b036e22e3c60a69bd95ccfee5cf686f71e1af4378523247f23ed3b2ca947750e1729239d2604f32a7cc4f165f3e5a4d786eb3944fe4d383d9b4fbc1aa59a56ad9bac521c7602dd5518d741e13428e71e8d8de3451b29505f889ad4ecd649736e864c9c0b6007aae02c4bb61a0099fb98097d48cf82c5130358cfb184287a7cff033c48931f483852484d4c4d19c1540cce1ad6a46330f4b5430c8450442d7799de9272cef72702b2871d2f583
+
+# RSADP Ciphertext = n-1 should fail
+Availablein = fips
+FIPSversion = >=3.4.0
+Decrypt = RSA-2048
+Ctrl = rsa_padding_mode:none
+Input = cd0081ea7b2ae1ea06d59f7c73d9ffb94a09615c2e4ba7c636cef08dd3533ec3185525b015c769b99a77d6725bf9c3532a9b6e5f6627d5fb85160768d3dda9cbd35974511717dc3d309d2fc47ee41f97e32adb7f9dd864a1c4767a666ecd71bc1aacf5e7517f4b38594fea9b05e42d5ada9912008013e45316a4d9bb8ed086b88d28758bacaf922d46a868b485d239c9baeb0e2b64592710f42b2d1ea0a4b4802c0becab328f8a68b0073bdb546feea9809d2849912b390c1532bc7e29c7658f8175fae46f34332ff87bcab3e40649b98577869da0ea718353f0722754886913648760d122be676e0fc483dd20ffc31bda96a31966c9aa2e75ad03de47e1c44e
+Result = KEYOP_ERROR
+
+# RSADP Ciphertext = n should fail
+Decrypt = RSA-2048
+Ctrl = rsa_padding_mode:none
+Input = cd0081ea7b2ae1ea06d59f7c73d9ffb94a09615c2e4ba7c636cef08dd3533ec3185525b015c769b99a77d6725bf9c3532a9b6e5f6627d5fb85160768d3dda9cbd35974511717dc3d309d2fc47ee41f97e32adb7f9dd864a1c4767a666ecd71bc1aacf5e7517f4b38594fea9b05e42d5ada9912008013e45316a4d9bb8ed086b88d28758bacaf922d46a868b485d239c9baeb0e2b64592710f42b2d1ea0a4b4802c0becab328f8a68b0073bdb546feea9809d2849912b390c1532bc7e29c7658f8175fae46f34332ff87bcab3e40649b98577869da0ea718353f0722754886913648760d122be676e0fc483dd20ffc31bda96a31966c9aa2e75ad03de47e1c44f
+Result = KEYOP_ERROR
+
 # OAEP padding
 Decrypt = RSA-2048
 Ctrl = rsa_padding_mode:oaep