Add some testing for the case where the FIPS provider fails to load

Ensure we get correct behaviour in the event that an attempt is made
to load the fips provider but it fails to load.

Reviewed-by: Tim Hudson <tjh@openssl.org>
Reviewed-by: Paul Dale <pauli@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/16168)

test/defltfips_test.c

@@ -4,6 +4,7 @@
 #include "testutil.h"
 
 static int is_fips;
+static int bad_fips;
 
 static int test_is_fips_enabled(void)
 {
@@ -24,8 +25,8 @@ static int test_is_fips_enabled(void)
      * on the default properties. However we only set those properties if also
      * loading the FIPS provider.
      */
-    if (!TEST_int_eq(is_fips, is_fips_enabled)
-            || !TEST_int_eq(is_fips, is_fips_loaded))
+    if (!TEST_int_eq(is_fips || bad_fips, is_fips_enabled)
+            || !TEST_int_eq(is_fips && !bad_fips, is_fips_loaded))
         return 0;
 
     /*
@@ -33,19 +34,26 @@ static int test_is_fips_enabled(void)
      * expected provider.
      */
     sha256 = EVP_MD_fetch(NULL, "SHA2-256", NULL);
-    if (!TEST_ptr(sha256))
-        return 0;
-    if (is_fips
-        && !TEST_str_eq(OSSL_PROVIDER_get0_name(EVP_MD_get0_provider(sha256)),
-                        "fips")) {
+    if (bad_fips) {
+        if (!TEST_ptr_null(sha256)) {
+            EVP_MD_free(sha256);
+            return 0;
+        }
+    } else {
+        if (!TEST_ptr(sha256))
+            return 0;
+        if (is_fips
+            && !TEST_str_eq(OSSL_PROVIDER_get0_name(EVP_MD_get0_provider(sha256)),
+                            "fips")) {
+            EVP_MD_free(sha256);
+            return 0;
+        }
         EVP_MD_free(sha256);
-        return 0;
     }
-    EVP_MD_free(sha256);
 
     /* State should still be consistent */
     is_fips_enabled = EVP_default_properties_is_fips_enabled(NULL);
-    if (!TEST_int_eq(is_fips, is_fips_enabled))
+    if (!TEST_int_eq(is_fips || bad_fips, is_fips_enabled))
         return 0;
 
     return 1;
@@ -54,6 +62,7 @@ static int test_is_fips_enabled(void)
 int setup_tests(void)
 {
     size_t argc;
+    char *arg1;
 
     if (!test_skip_common_options()) {
         TEST_error("Error parsing test options\n");
@@ -64,10 +73,18 @@ int setup_tests(void)
     switch(argc) {
     case 0:
         is_fips = 0;
+        bad_fips = 0;
         break;
     case 1:
-        if (strcmp(test_get_argument(0), "fips") == 0) {
+        arg1 = test_get_argument(0);
+        if (strcmp(arg1, "fips") == 0) {
             is_fips = 1;
+            bad_fips = 0;
+            break;
+        } else if (strcmp(arg1, "badfips") == 0) {
+            /* Configured for FIPS, but the module fails to load */
+            is_fips = 0;
+            bad_fips = 1;
             break;
         }
         /* fall through */

test/fips-alt.cnf

@@ -0,0 +1,16 @@
+openssl_conf = openssl_init
+
+.include fipsmodule.cnf
+
+[openssl_init]
+providers = provider_sect
+alg_section = evp_properties
+
+[evp_properties]
+# Ensure FIPS non-approved algorithms in the FIPS module are suppressed (e.g.
+# TEST-RAND). This also means that EVP_default_properties_is_fips_enabled()
+# returns the expected value
+fips_mode = true
+
+[provider_sect]
+fips = fips_sect

test/recipes/30-test_defltfips.t

@@ -10,12 +10,12 @@
 use strict;
 use warnings;
 
-use OpenSSL::Test qw/:DEFAULT srctop_file srctop_dir bldtop_file bldtop_dir/;
+use OpenSSL::Test qw/:DEFAULT srctop_file srctop_dir bldtop_file bldtop_dir data_dir/;
 use OpenSSL::Test::Utils;
 use Cwd qw(abs_path);
 
 BEGIN {
-    setup("test_evp");
+    setup("test_defltfips");
 }
 
 use lib srctop_dir('Configurations');
@@ -24,11 +24,24 @@ use lib bldtop_dir('.');
 my $no_fips = disabled('fips') || ($ENV{NO_FIPS} // 0);
 
 plan tests =>
-    ($no_fips ? 1 : 2);
+    ($no_fips ? 1 : 5);
 
 unless ($no_fips) {
     $ENV{OPENSSL_CONF} = abs_path(srctop_file("test", "fips.cnf"));
     ok(run(test(["defltfips_test", "fips"])), "running defltfips_test fips");
+
+    #Test an alternative way of configuring fips
+    $ENV{OPENSSL_CONF} = abs_path(srctop_file("test", "fips-alt.cnf"));
+    ok(run(test(["defltfips_test", "fips"])), "running defltfips_test fips");
+
+    #Configured to run FIPS but the module-mac is bad
+    $ENV{OPENSSL_CONF} = abs_path(srctop_file("test", "fips.cnf"));
+    $ENV{OPENSSL_CONF_INCLUDE} = srctop_file("test", "recipes", "30-test_defltfips");
+    ok(run(test(["defltfips_test", "badfips"])), "running defltfips_test badfips");
+
+    #Test an alternative way of configuring fips (but still with bad module-mac)
+    $ENV{OPENSSL_CONF} = abs_path(srctop_file("test", "fips-alt.cnf"));
+    ok(run(test(["defltfips_test", "badfips"])), "running defltfips_test badfips");
 }
 
 $ENV{OPENSSL_CONF} = abs_path(srctop_file("test", "default.cnf"));

test/recipes/30-test_defltfips/fipsmodule.cnf

@@ -0,0 +1,5 @@
+[fips_sect]
+activate = 1
+conditional-errors = 1
+security-checks = 1
+module-mac = B9:C9:E1:F5:B7:49:18:1B:BF:63:68:DF:1A:66:40:2E:04:2A:8F:E2:B1:D9:F7:7C:08:6F:80:A0:1D:47:F2:00