Author: vadim <tutumbul@gmail.com>
Author: Robie Basak <robie.basak@canonical.com>
Description: add support for OpenSSL 3.0
 This patch originates from https://github.com/mysql/mysql-server/pull/320 with
 thanks to macvk and vadim <tutumbul@gmail.com>.
 .
 The original commits followed the "stream-of-consciousness" pattern, with
 later commits correcting mis-steps in previous commits, rather than the usual
 separation of logical steps that we're used to. Since the original individual
 commits therefore weren't useful indepedently, I squashed them together here.
 .
 Further, I found that some additional changes were required to build the
 package the way we do it in Debian and Ubuntu, including in some plugins that
 were perhaps not being built by the original author, and one or two other
 tweaks.
 .
 So this patch is derived from that work, but the original authors should not
 be attributed for any mistakes in it.
 .
Bug: https://bugs.mysql.com/bug.php?id=102405
Bug-Ubuntu: https://launchpad.net/bugs/1945956
Origin: other, https://github.com/mysql/mysql-server/pull/320
Last-Update: 2022-02-04

--- a/cmake/ssl.cmake
+++ b/cmake/ssl.cmake
@@ -206,25 +206,52 @@ MACRO (MYSQL_CHECK_SSL)
                  HINTS ${OPENSSL_ROOT_DIR}/lib)
 
     IF(OPENSSL_INCLUDE_DIR)
-      # Verify version number. Version information looks like:
-      #   #define OPENSSL_VERSION_NUMBER 0x1000103fL
-      # Encoded as MNNFFPPS: major minor fix patch status
       FILE(STRINGS "${OPENSSL_INCLUDE_DIR}/openssl/opensslv.h"
-        OPENSSL_VERSION_NUMBER
-        REGEX "^#[ ]*define[\t ]+OPENSSL_VERSION_NUMBER[\t ]+0x[0-9].*"
-        )
-      STRING(REGEX REPLACE
-        "^.*OPENSSL_VERSION_NUMBER[\t ]+0x([0-9]).*$" "\\1"
-        OPENSSL_MAJOR_VERSION "${OPENSSL_VERSION_NUMBER}"
-        )
-      STRING(REGEX REPLACE
-        "^.*OPENSSL_VERSION_NUMBER[\t ]+0x[0-9]([0-9][0-9]).*$" "\\1"
-        OPENSSL_MINOR_VERSION "${OPENSSL_VERSION_NUMBER}"
-        )
-      STRING(REGEX REPLACE
-        "^.*OPENSSL_VERSION_NUMBER[\t ]+0x[0-9][0-9][0-9]([0-9][0-9]).*$" "\\1"
-        OPENSSL_FIX_VERSION "${OPENSSL_VERSION_NUMBER}"
+        OPENSSL_MAJOR_VERSION
+        REGEX "^#[ ]*define[\t ]+OPENSSL_VERSION_MAJOR[\t ]+[0-9].*"
         )
+      IF(OPENSSL_MAJOR_VERSION STREQUAL "")
+        # Verify version number. Version information looks like:
+        #   #define OPENSSL_VERSION_NUMBER 0x1000103fL
+        # Encoded as MNNFFPPS: major minor fix patch status
+        FILE(STRINGS "${OPENSSL_INCLUDE_DIR}/openssl/opensslv.h"
+          OPENSSL_VERSION_NUMBER
+          REGEX "^#[ ]*define[\t ]+OPENSSL_VERSION_NUMBER[\t ]+0x[0-9].*"
+          )
+        STRING(REGEX REPLACE
+          "^.*OPENSSL_VERSION_NUMBER[\t ]+0x([0-9]).*$" "\\1"
+          OPENSSL_MAJOR_VERSION "${OPENSSL_VERSION_NUMBER}"
+          )
+        STRING(REGEX REPLACE
+          "^.*OPENSSL_VERSION_NUMBER[\t ]+0x[0-9]([0-9][0-9]).*$" "\\1"
+          OPENSSL_MINOR_VERSION "${OPENSSL_VERSION_NUMBER}"
+          )
+        STRING(REGEX REPLACE
+          "^.*OPENSSL_VERSION_NUMBER[\t ]+0x[0-9][0-9][0-9]([0-9][0-9]).*$" "\\1"
+          OPENSSL_FIX_VERSION "${OPENSSL_VERSION_NUMBER}"
+          )
+      ELSE()
+        STRING(REGEX REPLACE
+          "^.*OPENSSL_VERSION_MAJOR[\t ]+([0-9]).*$" "\\1"
+          OPENSSL_MAJOR_VERSION "${OPENSSL_MAJOR_VERSION}"
+          )
+        FILE(STRINGS "${OPENSSL_INCLUDE_DIR}/openssl/opensslv.h"
+          OPENSSL_MINOR_VERSION
+          REGEX "^#[ ]*define[\t ]+OPENSSL_VERSION_MINOR[\t ]+[0-9].*"
+          )
+        STRING(REGEX REPLACE
+          "^.*OPENSSL_VERSION_MINOR[\t ]+([0-9]).*$" "\\1"
+          OPENSSL_MINOR_VERSION "${OPENSSL_MINOR_VERSION}"
+          )
+        FILE(STRINGS "${OPENSSL_INCLUDE_DIR}/openssl/opensslv.h"
+          OPENSSL_FIX_VERSION
+          REGEX "^#[ ]*define[\t ]+OPENSSL_VERSION_PATCH[\t ]+[0-9].*"
+          )
+        STRING(REGEX REPLACE
+          "^.*OPENSSL_VERSION_PATCH[\t ]+([0-9]).*$" "\\1"
+          OPENSSL_FIX_VERSION "${OPENSSL_FIX_VERSION}"
+          )
+      ENDIF()
     ENDIF()
     IF("${OPENSSL_MAJOR_VERSION}.${OPENSSL_MINOR_VERSION}.${OPENSSL_FIX_VERSION}" VERSION_GREATER "1.1.0")
        ADD_DEFINITIONS(-DHAVE_TLSv13)
@@ -232,7 +259,7 @@ MACRO (MYSQL_CHECK_SSL)
     IF(OPENSSL_INCLUDE_DIR AND
        OPENSSL_LIBRARY   AND
        CRYPTO_LIBRARY      AND
-       OPENSSL_MAJOR_VERSION STREQUAL "1"
+       (OPENSSL_MAJOR_VERSION STREQUAL "1" OR OPENSSL_MAJOR_VERSION STREQUAL "3")
       )
       SET(OPENSSL_FOUND TRUE)
       FIND_PROGRAM(OPENSSL_EXECUTABLE openssl
--- a/mysys/my_md5.cc
+++ b/mysys/my_md5.cc
@@ -34,6 +34,10 @@
 
 #include <openssl/crypto.h>
 #include <openssl/md5.h>
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+#include <openssl/evp.h>
+#include <openssl/provider.h>
+#endif
 
 static void my_md5_hash(unsigned char *digest, unsigned const char *buf,
                         int len) {
@@ -56,7 +60,11 @@ static void my_md5_hash(unsigned char *d
 int compute_md5_hash(char *digest, const char *buf, int len) {
   int retval = 0;
   int fips_mode = 0;
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+  fips_mode = EVP_default_properties_is_fips_enabled(NULL) && OSSL_PROVIDER_available(NULL, "fips");
+#else
   fips_mode = FIPS_mode();
+#endif
   /* If fips mode is ON/STRICT restricted method calls will result into abort,
    * skipping call. */
   if (fips_mode == 0) {
--- a/plugin/group_replication/libmysqlgcs/src/bindings/xcom/xcom/network/xcom_network_provider_ssl_native_lib.cc
+++ b/plugin/group_replication/libmysqlgcs/src/bindings/xcom/xcom/network/xcom_network_provider_ssl_native_lib.cc
@@ -40,6 +40,11 @@
 
 #include "openssl/engine.h"
 
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+#include <openssl/evp.h>
+#include <openssl/provider.h>
+#endif
+
 #include "xcom/task_debug.h"
 #include "xcom/x_platform.h"
 
@@ -325,12 +330,20 @@ static int configure_ssl_fips_mode(const
   if (fips_mode > 2) {
     goto EXIT;
   }
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+  fips_mode_old = EVP_default_properties_is_fips_enabled(NULL) && OSSL_PROVIDER_available(NULL, "fips");
+#else
   fips_mode_old = FIPS_mode();
+#endif
   if (fips_mode_old == fips_mode) {
     rc = 1;
     goto EXIT;
   }
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+  if (!(rc = EVP_default_properties_enable_fips(NULL, fips_mode))) {
+#else
   if (!(rc = FIPS_mode_set(fips_mode))) {
+#endif
     err_library = ERR_get_error();
     ERR_error_string_n(err_library, err_string, sizeof(err_string) - 1);
     err_string[sizeof(err_string) - 1] = '\0';
--- a/plugin/x/client/xconnection_impl.cc
+++ b/plugin/x/client/xconnection_impl.cc
@@ -29,6 +29,10 @@
 #include <netinet/in.h>
 #endif  // HAVE_NETINET_IN_H
 #include <openssl/x509v3.h>
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+#include <openssl/evp.h>
+#include <openssl/provider.h>
+#endif
 #include <cassert>
 #include <chrono>  // NOLINT(build/c++11)
 #include <future>  // NOLINT(build/c++11)
@@ -644,12 +648,20 @@ int set_fips_mode(const uint32_t fips_mo
   if (fips_mode > 2) {
     goto EXIT;
   }
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+  fips_mode_old = EVP_default_properties_is_fips_enabled(NULL) && OSSL_PROVIDER_available(NULL, "fips");
+#else
   fips_mode_old = FIPS_mode();
+#endif
   if (fips_mode_old == fips_mode) {
     rc = 1;
     goto EXIT;
   }
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+  if (!(rc = EVP_default_properties_enable_fips(NULL, fips_mode))) {
+#else
   if (!(rc = FIPS_mode_set(fips_mode))) {
+#endif
     err_library = ERR_get_error();
     ERR_error_string_n(err_library, err_string, OPENSSL_ERROR_LENGTH - 1);
     err_string[OPENSSL_ERROR_LENGTH - 1] = '\0';
--- a/vio/viosslfactories.cc
+++ b/vio/viosslfactories.cc
@@ -45,6 +45,11 @@
 #include <openssl/ec.h>
 #endif /* OPENSSL_VERSION_NUMBER < 0x10002000L */
 
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+#include <openssl/evp.h>
+#include <openssl/provider.h>
+#endif
+
 #define TLS_VERSION_OPTION_SIZE 256
 
 /*
@@ -497,12 +502,20 @@ int set_fips_mode(const uint fips_mode,
   if (fips_mode > 2) {
     goto EXIT;
   }
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+  fips_mode_old = EVP_default_properties_is_fips_enabled(NULL);
+#else
   fips_mode_old = FIPS_mode();
+#endif
   if (fips_mode_old == fips_mode) {
     rc = 1;
     goto EXIT;
   }
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+  if (!(rc = EVP_default_properties_enable_fips(NULL, fips_mode))) {
+#else
   if (!(rc = FIPS_mode_set(fips_mode))) {
+#endif
     /*
       If OS doesn't have FIPS enabled openssl library and user sets FIPS mode
       ON, It fails with proper error. But in the same time it doesn't allow to
@@ -510,7 +523,11 @@ int set_fips_mode(const uint fips_mode,
       error, setting old working FIPS mode value in the OpenSSL library. It will
       allow successful cryptographic operation and will not abort the server.
     */
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+    EVP_default_properties_enable_fips(NULL, fips_mode_old);
+#else
     FIPS_mode_set(fips_mode_old);
+#endif
     err_library = ERR_get_error();
     ERR_error_string_n(err_library, err_string, OPENSSL_ERROR_LENGTH - 1);
     err_string[OPENSSL_ERROR_LENGTH - 1] = '\0';
@@ -524,7 +541,14 @@ EXIT:
 
   @returns openssl current fips mode
 */
-uint get_fips_mode() { return FIPS_mode(); }
+uint get_fips_mode()
+{
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+  return EVP_default_properties_is_fips_enabled(NULL) && OSSL_PROVIDER_available(NULL, "fips");
+#else
+  return FIPS_mode();
+#endif
+}
 
 /**
   Toggle FIPS mode, to see whether it is available with the current SSL library.
@@ -532,7 +556,12 @@ uint get_fips_mode() { return FIPS_mode(
   @retval non-zero: FIPS is supported.
 */
 int test_ssl_fips_mode(char *err_string) {
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+  int ret = EVP_default_properties_enable_fips(NULL,
+    !(EVP_default_properties_is_fips_enabled(NULL) && OSSL_PROVIDER_available(NULL, "fips")));
+#else
   int ret = FIPS_mode_set(FIPS_mode() == 0 ? 1 : 0);
+#endif
   unsigned long err = (ret == 0) ? ERR_get_error() : 0;
 
   if (err != 0) {
