rsa/rsa_ameth.c: Add the check for the EVP_MD_get_size()

Add the check for the EVP_MD_get_size() to avoid invalid negative numbers.

Fixes: 17c63d1cca ("RSA PSS ASN1 signing method")
Signed-off-by: Jiasheng Jiang <jiasheng@purdue.edu>

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Tom Cosgrove <tom.cosgrove@arm.com>
Reviewed-by: Neil Horman <nhorman@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/23973)
This commit is contained in:
Jiasheng Jiang 2024-03-25 21:07:46 +00:00 committed by Neil Horman
parent 1967539e21
commit e582b2b22b
1 changed files with 13 additions and 6 deletions

View File

@ -454,15 +454,19 @@ static RSA_PSS_PARAMS *rsa_ctx_to_pss(EVP_PKEY_CTX *pkctx)
EVP_PKEY *pk = EVP_PKEY_CTX_get0_pkey(pkctx);
int saltlen;
int saltlenMax = -1;
int md_size;
if (EVP_PKEY_CTX_get_signature_md(pkctx, &sigmd) <= 0)
return NULL;
md_size = EVP_MD_get_size(sigmd);
if (md_size <= 0)
return NULL;
if (EVP_PKEY_CTX_get_rsa_mgf1_md(pkctx, &mgf1md) <= 0)
return NULL;
if (EVP_PKEY_CTX_get_rsa_pss_saltlen(pkctx, &saltlen) <= 0)
return NULL;
if (saltlen == RSA_PSS_SALTLEN_DIGEST) {
saltlen = EVP_MD_get_size(sigmd);
saltlen = md_size;
} else if (saltlen == RSA_PSS_SALTLEN_AUTO_DIGEST_MAX) {
/* FIPS 186-4 section 5 "The RSA Digital Signature Algorithm",
* subsection 5.5 "PKCS #1" says: "For RSASSA-PSS […] the length (in
@ -472,10 +476,10 @@ static RSA_PSS_PARAMS *rsa_ctx_to_pss(EVP_PKEY_CTX *pkctx)
* Provide a way to use at most the digest length, so that the default
* does not violate FIPS 186-4. */
saltlen = RSA_PSS_SALTLEN_MAX;
saltlenMax = EVP_MD_get_size(sigmd);
saltlenMax = md_size;
}
if (saltlen == RSA_PSS_SALTLEN_MAX || saltlen == RSA_PSS_SALTLEN_AUTO) {
saltlen = EVP_PKEY_get_size(pk) - EVP_MD_get_size(sigmd) - 2;
saltlen = EVP_PKEY_get_size(pk) - md_size - 2;
if ((EVP_PKEY_get_bits(pk) & 0x7) == 1)
saltlen--;
if (saltlen < 0)
@ -719,7 +723,7 @@ static int rsa_sig_info_set(X509_SIG_INFO *siginf, const X509_ALGOR *sigalg,
const ASN1_STRING *sig)
{
int rv = 0;
int mdnid, saltlen;
int mdnid, saltlen, md_size;
uint32_t flags;
const EVP_MD *mgf1md = NULL, *md = NULL;
RSA_PSS_PARAMS *pss;
@ -732,6 +736,9 @@ static int rsa_sig_info_set(X509_SIG_INFO *siginf, const X509_ALGOR *sigalg,
pss = ossl_rsa_pss_decode(sigalg);
if (!ossl_rsa_pss_get_param(pss, &md, &mgf1md, &saltlen))
goto err;
md_size = EVP_MD_get_size(md);
if (md_size <= 0)
goto err;
mdnid = EVP_MD_get_type(md);
/*
* For TLS need SHA256, SHA384 or SHA512, digest and MGF1 digest must
@ -739,12 +746,12 @@ static int rsa_sig_info_set(X509_SIG_INFO *siginf, const X509_ALGOR *sigalg,
*/
if ((mdnid == NID_sha256 || mdnid == NID_sha384 || mdnid == NID_sha512)
&& mdnid == EVP_MD_get_type(mgf1md)
&& saltlen == EVP_MD_get_size(md))
&& saltlen == md_size)
flags = X509_SIG_INFO_TLS;
else
flags = 0;
/* Note: security bits half number of digest bits */
secbits = EVP_MD_get_size(md) * 4;
secbits = md_size * 4;
/*
* SHA1 and MD5 are known to be broken. Reduce security bits so that
* they're no longer accepted at security level 1. The real values don't