X509: clean up doc and implementation of X509{,_REQ}_check_private_key()

Also constify X509_REQ_get0_pubkey() and X509_REQ_check_private_key().

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Dmitry Belyavskiy <beldmit@gmail.com>
Reviewed-by: David von Oheimb <david.von.oheimb@siemens.com>
(Merged from https://github.com/openssl/openssl/pull/18930)

crypto/x509/x509_cmp.c

@@ -389,30 +389,38 @@ EVP_PKEY *X509_get_pubkey(X509 *x)
     return X509_PUBKEY_get(x->cert_info.key);
 }
 
-int X509_check_private_key(const X509 *x, const EVP_PKEY *k)
+int X509_check_private_key(const X509 *cert, const EVP_PKEY *pkey)
 {
-    const EVP_PKEY *xk;
-    int ret;
+    const EVP_PKEY *xk = X509_get0_pubkey(cert);
 
-    xk = X509_get0_pubkey(x);
     if (xk == NULL) {
         ERR_raise(ERR_LIB_X509, X509_R_UNABLE_TO_GET_CERTS_PUBLIC_KEY);
         return 0;
     }
+    return ossl_x509_check_private_key(xk, pkey);
+}
 
-    switch (ret = EVP_PKEY_eq(xk, k)) {
+int ossl_x509_check_private_key(const EVP_PKEY *x, const EVP_PKEY *pkey)
+{
+    if (x == NULL) {
+        ERR_raise(ERR_LIB_X509, ERR_R_PASSED_NULL_PARAMETER);
+        return 0;
+    }
+    switch (EVP_PKEY_eq(x, pkey)) {
+    case 1:
+        return 1;
     case 0:
         ERR_raise(ERR_LIB_X509, X509_R_KEY_VALUES_MISMATCH);
-        break;
+        return 0;
     case -1:
         ERR_raise(ERR_LIB_X509, X509_R_KEY_TYPE_MISMATCH);
-        break;
+        return 0;
     case -2:
         ERR_raise(ERR_LIB_X509, X509_R_UNKNOWN_KEY_TYPE);
-        break;
+        /* fall thru */
+    default:
+        return 0;
     }
-
-    return ret > 0;
 }
 
 /*

crypto/x509/x509_req.c

@@ -67,7 +67,7 @@ EVP_PKEY *X509_REQ_get_pubkey(X509_REQ *req)
     return X509_PUBKEY_get(req->req_info.pubkey);
 }
 
-EVP_PKEY *X509_REQ_get0_pubkey(X509_REQ *req)
+EVP_PKEY *X509_REQ_get0_pubkey(const X509_REQ *req)
 {
     if (req == NULL)
         return NULL;
@@ -79,28 +79,9 @@ X509_PUBKEY *X509_REQ_get_X509_PUBKEY(X509_REQ *req)
     return req->req_info.pubkey;
 }
 
-int X509_REQ_check_private_key(X509_REQ *x, EVP_PKEY *k)
+int X509_REQ_check_private_key(const X509_REQ *req, EVP_PKEY *pkey)
 {
-    EVP_PKEY *xk = NULL;
-    int ok = 0;
-
-    xk = X509_REQ_get_pubkey(x);
-    switch (EVP_PKEY_eq(xk, k)) {
-    case 1:
-        ok = 1;
-        break;
-    case 0:
-        ERR_raise(ERR_LIB_X509, X509_R_KEY_VALUES_MISMATCH);
-        break;
-    case -1:
-        ERR_raise(ERR_LIB_X509, X509_R_KEY_TYPE_MISMATCH);
-        break;
-    case -2:
-        ERR_raise(ERR_LIB_X509, X509_R_UNKNOWN_KEY_TYPE);
-    }
-
-    EVP_PKEY_free(xk);
-    return ok;
+    return ossl_x509_check_private_key(X509_REQ_get0_pubkey(req), pkey);
 }
 
 /*

doc/man3/X509_check_private_key.pod

@@ -10,17 +10,17 @@ request
 
  #include <openssl/x509.h>
 
- int X509_check_private_key(X509 *x, EVP_PKEY *k);
+ int X509_check_private_key(const X509 *cert, EVP_PKEY *pkey);
 
- int X509_REQ_check_private_key(X509_REQ *x, EVP_PKEY *k);
+ int X509_REQ_check_private_key(X509_REQ *req, EVP_PKEY *pkey);
 
 =head1 DESCRIPTION
 
 X509_check_private_key() function checks the consistency of private
-key B<k> with the public key in B<x>.
+key I<pkey> with the public key in I<cert>.
 
 X509_REQ_check_private_key() is equivalent to X509_check_private_key()
-except that B<x> represents a certificate request of structure B<X509_REQ>.
+except that I<req> represents a certificate request of structure B<X509_REQ>.
 
 =head1 RETURN VALUES
 
@@ -32,11 +32,11 @@ obtained using L<ERR_get_error(3)>.
 
 =head1 BUGS
 
-The B<check_private_key> functions don't check if B<k> itself is indeed
-a private key or not. It merely compares the public materials (e.g. exponent
-and modulus of an RSA key) and/or key parameters (e.g. EC params of an EC key)
-of a key pair. So if you pass a public key to these functions in B<k>, it will
-return success.
+The X509_check_private_key() and X509_REQ_check_private_key() functions
+do not check if I<pkey> itself is indeed a private key or not.
+They merely compare the public materials (e.g., exponent and modulus of an RSA
+key) and/or key parameters (e.g. EC params of an EC key) of a key pair.
+So they also return success if I<pkey> is a matching public key.
 
 =head1 SEE ALSO
 

include/crypto/x509.h

@@ -361,6 +361,7 @@ int ossl_i2d_X448_PUBKEY(const ECX_KEY *a, unsigned char **pp);
 # endif /* OPENSSL_NO_EC */
 EVP_PKEY *ossl_d2i_PUBKEY_legacy(EVP_PKEY **a, const unsigned char **pp,
                                  long length);
+int ossl_x509_check_private_key(const EVP_PKEY *k, const EVP_PKEY *pkey);
 
 int x509v3_add_len_value_uchar(const char *name, const unsigned char *value,
                                size_t vallen, STACK_OF(CONF_VALUE) **extlist);

include/openssl/x509.h.in

@@ -693,7 +693,7 @@ int X509_REQ_get_signature_nid(const X509_REQ *req);
 int i2d_re_X509_REQ_tbs(X509_REQ *req, unsigned char **pp);
 int X509_REQ_set_pubkey(X509_REQ *x, EVP_PKEY *pkey);
 EVP_PKEY *X509_REQ_get_pubkey(X509_REQ *req);
-EVP_PKEY *X509_REQ_get0_pubkey(X509_REQ *req);
+EVP_PKEY *X509_REQ_get0_pubkey(const X509_REQ *req);
 X509_PUBKEY *X509_REQ_get_X509_PUBKEY(X509_REQ *req);
 int X509_REQ_extension_nid(int nid);
 int *X509_REQ_get_extension_nids(void);
@@ -759,9 +759,9 @@ X509_REVOKED_get0_extensions(const X509_REVOKED *r);
 X509_CRL *X509_CRL_diff(X509_CRL *base, X509_CRL *newer,
                         EVP_PKEY *skey, const EVP_MD *md, unsigned int flags);
 
-int X509_REQ_check_private_key(X509_REQ *x509, EVP_PKEY *pkey);
+int X509_REQ_check_private_key(const X509_REQ *req, EVP_PKEY *pkey);
 
-int X509_check_private_key(const X509 *x509, const EVP_PKEY *pkey);
+int X509_check_private_key(const X509 *cert, const EVP_PKEY *pkey);
 int X509_chain_check_suiteb(int *perror_depth,
                             X509 *x, STACK_OF(X509) *chain,
                             unsigned long flags);