fix: don't enable custom random for openssl fips

crypto/s2n_libcrypto.c

@@ -125,6 +125,19 @@ bool s2n_libcrypto_is_openssl(void)
     return !is_other_libcrypto_variant;
 }
 
+/* Check if the linked libcrypto is a fips-capable openssl variant.
+ *
+ * This method does not check whether the linked libcrypto has fips mode enabled.
+ */
+bool s2n_libcrypto_is_openssl_fips(void)
+{
+#if defined(OPENSSL_FIPS)
+    return true;
+#else
+    return false;
+#endif
+}
+
 bool s2n_libcrypto_is_awslc()
 {
 #if defined(OPENSSL_IS_AWSLC)

crypto/s2n_libcrypto.h

@@ -21,3 +21,4 @@ uint64_t s2n_libcrypto_awslc_api_version(void);
 S2N_RESULT s2n_libcrypto_validate_runtime(void);
 const char *s2n_libcrypto_get_version_name(void);
 bool s2n_libcrypto_supports_flag_no_check_time();
+bool s2n_libcrypto_is_openssl_fips(void);

tests/unit/s2n_random_test.c

@@ -795,8 +795,7 @@ static int s2n_random_rand_bytes_after_cleanup_cb(struct random_test_case *test_
 
 static int s2n_random_rand_bytes_before_init(struct random_test_case *test_case)
 {
-    /* s2n_libcrypto_is_fips() is used since we are testing `s2n_init()` */
-    if (s2n_supports_custom_rand() && !s2n_libcrypto_is_fips()) {
+    if (s2n_supports_custom_rand()) {
         /* Calling RAND_bytes will set a global random method */
         unsigned char rndbytes[16] = { 0 };
         EXPECT_EQUAL(RAND_bytes(rndbytes, sizeof(rndbytes)), 1);

utils/s2n_random.c

@@ -71,6 +71,7 @@
 #include "api/s2n.h"
 #include "crypto/s2n_drbg.h"
 #include "crypto/s2n_fips.h"
+#include "crypto/s2n_libcrypto.h"
 #include "error/s2n_errno.h"
 #include "s2n_io.h"
 #include "stuffer/s2n_stuffer.h"
@@ -557,7 +558,13 @@ bool s2n_supports_custom_rand(void)
 #if !defined(S2N_LIBCRYPTO_SUPPORTS_ENGINE)
     return false;
 #else
-    return s2n_libcrypto_is_openssl() && !s2n_is_in_fips_mode();
+    /* AWS-LC-FIPS supports custom rand unless s2n-tls is in FIPS mode */
+    /* OpenSSL-FIPS never supports custom rand, regardless of mode */
+    /* OpenSSL non-fips always supports custom rand */
+    /* other libcryptos never support custom rand */
+    bool awslc_fips_with_fips_enabled = s2n_libcrypto_is_awslc() && s2n_is_in_fips_mode();
+    bool disable_for_fips = s2n_libcrypto_is_openssl_fips() || awslc_fips_with_fips_enabled;
+    return s2n_libcrypto_is_openssl() && !disable_for_fips;
 #endif
 }