Support for OpenSSL 3.0.x

CHANGELOG.md

@@ -4,6 +4,37 @@ librdkafka v1.9.3 is a maintenance release:
 
  * Self-contained static libraries can now be built on Linux arm64 (#4005).
  * Fix for using PKCS#12 keystores on Windows.
+ * OpenSSL 3.0.x support - the maximum bundled OpenSSL version is now 3.0.5 (previously 1.1.1q).
+ * Updated to zlib 1.2.13 in self-contained librdkafka bundles.
+
+
+## Upgrade considerations
+
+### OpenSSL 3.0.x
+
+#### OpenSSL default ciphers
+
+The introduction of OpenSSL 3.0.x in the self-contained librdkafka bundles
+changes the default set of available ciphers, in particular all obsolete
+or insecure ciphers and algorithms as listed in the OpenSSL [legacy](https://www.openssl.org/docs/man3.0/man7/OSSL_PROVIDER-legacy.html)
+are now disabled by default.
+
+**WARNING**: These ciphers are disabled for security reasons and it is
+highly recommended NOT to use them.
+
+Should you need to use any of these old ciphers you'll need to explicitly
+enable the `legacy` provider by configuring `ssl.providers=default,legacy`
+on the librdkafka client.
+
+#### OpenSSL engines and providers
+
+OpenSSL 3.0.x deprecates the use of engines, which is being replaced by
+providers. As such librdkafka will emit a deprecation warning if
+`ssl.engine.location` is configured.
+
+OpenSSL providers may be configured with the new `ssl.providers`
+configuration property.
+
 
 
 ## Fixes

CONFIGURATION.md

@@ -75,7 +75,8 @@ ssl.ca.certificate.stores | * | | Root
 ssl.crl.location | * | | | low | Path to CRL for verifying broker's certificate validity. <br>*Type: string*
 ssl.keystore.location | * | | | low | Path to client's keystore (PKCS#12) used for authentication. <br>*Type: string*
 ssl.keystore.password | * | | | low | Client's keystore (PKCS#12) password. <br>*Type: string*
-ssl.engine.location | * | | | low | Path to OpenSSL engine library. OpenSSL >= 1.1.0 required. <br>*Type: string*
+ssl.providers | * | | | low | Comma-separated list of OpenSSL 3.0.x implementation providers. E.g., "default,legacy". <br>*Type: string*
+ssl.engine.location | * | | | low | **DEPRECATED** Path to OpenSSL engine library. OpenSSL >= 1.1.x required. DEPRECATED: OpenSSL engine support is deprecated and should be replaced by OpenSSL 3 providers. <br>*Type: string*
 ssl.engine.id | * | | dynamic | low | OpenSSL engine id is the name used for loading engine. <br>*Type: string*
 ssl_engine_callback_data | * | | | low | OpenSSL engine callback data (set with rd_kafka_conf_set_engine_callback_data()). <br>*Type: see dedicated API*
 enable.ssl.certificate.verification | * | true, false | true | low | Enable OpenSSL's builtin broker (server) certificate verification. This verification can be extended by the application by implementing a certificate_verify_cb. <br>*Type: boolean*

mklove/modules/configure.libssl

@@ -81,15 +81,29 @@ function manual_checks {
 function libcrypto_install_source {
  local name=$1
  local destdir=$2
- local ver=1.1.1q
- local checksum="d7939ce614029cdff0b6c20f0e2e5703158a489a72b2507b8bd51bf8c8fd10ca"
+ local ver=3.0.5
+ local checksum="aa7d8d9bef71ad6525c55ba11e5f4397889ce49c2c9349dcea6d3e4f0b024a7a"
  local url=https://www.openssl.org/source/openssl-${ver}.tar.gz
 
- local conf_args="--prefix=/usr --openssldir=/usr/lib/ssl no-shared no-zlib no-deprecated"
+ local conf_args="--prefix=/usr --openssldir=/usr/lib/ssl no-shared no-zlib"
+
  if [[ $ver == 1.0.* ]]; then
  conf_args="${conf_args} no-krb5"
  fi
 
+ if [[ $ver == 3.* ]]; then
+ # Silence OpenSSL 3.0.0 deprecation warnings since they'll make
+ # -Werror fail.
+ mkl_define_set "libcrypto" OPENSSL_SUPPRESS_DEPRECATED
+ # Make sure legacy provider (et.al) are built-in, since we're building
+ # a static library we don't want to rely on dynamically loaded modules.
+ conf_args="${conf_args} no-module"
+ else
+ # OpenSSL 3 deprecates ENGINE support, but we still need it, so only
+ # add no-deprecated to non-3.x builds.
+ conf_args="${conf_args} no-deprecated"
+ fi
+
  # 1.1.1q tests fail to build on OSX/M1, so disable them.
  if [[ $MKL_DISTRO == osx && $ver == 1.1.1q ]]; then
  conf_args="${conf_args} no-tests"

src-cpp/rdkafkacpp.h

@@ -1326,6 +1326,14 @@ class RD_EXPORT Conf {
  *
  * @remark CA certificate in PEM format may also be set with the
  * `ssl.ca.pem` configuration property.
+ *
+ * @remark When librdkafka is linked to OpenSSL 3.0 and the certificate is
+ * encoded using an obsolete cipher, it might be necessary to set up
+ * an OpenSSL configuration file to load the "legacy" provider and
+ * set the OPENSSL_CONF environment variable.
+ * See
+ * https://github.com/openssl/openssl/blob/master/README-PROVIDERS.md for more
+ * information.
  */
  virtual Conf::ConfResult set_ssl_cert(RdKafka::CertificateType cert_type,
  RdKafka::CertificateEncoding cert_enc,

src/rdkafka.c

@@ -957,6 +957,7 @@ void rd_kafka_destroy_final(rd_kafka_t *rk) {
  rd_kafka_dbg(rk, GENERIC, "TERMINATE", "Destroying SSL CTX");
  rd_kafka_ssl_ctx_term(rk);
  }
+ rd_list_destroy(&rk->rk_conf.ssl.loaded_providers);
 #endif
 
  /* It is not safe to log after this point. */

src/rdkafka.h

@@ -2363,6 +2363,14 @@ typedef enum rd_kafka_cert_enc_t {
  *
  * @remark CA certificate in PEM format may also be set with the
  * `ssl.ca.pem` configuration property.
+ *
+ * @remark When librdkafka is linked to OpenSSL 3.0 and the certificate is
+ * encoded using an obsolete cipher, it might be necessary to set up
+ * an OpenSSL configuration file to load the "legacy" provider and
+ * set the OPENSSL_CONF environment variable.
+ * See
+ * https://github.com/openssl/openssl/blob/master/README-PROVIDERS.md for more
+ * information.
  */
 RD_EXPORT rd_kafka_conf_res_t
 rd_kafka_conf_set_ssl_cert(rd_kafka_conf_t *conf,

src/rdkafka_cert.c

@@ -95,8 +95,10 @@ static rd_kafka_cert_t *rd_kafka_cert_dup(rd_kafka_cert_t *src) {
  return src;
 }
 
+
+#if OPENSSL_VERSION_NUMBER < 0x30000000
 /**
- * @brief Print the OpenSSL error stack do stdout, for development use.
+ * @brief Print the OpenSSL error stack to stdout, for development use.
  */
 static RD_UNUSED void rd_kafka_print_ssl_errors(void) {
  unsigned long l;
@@ -121,6 +123,8 @@ static RD_UNUSED void rd_kafka_print_ssl_errors(void) {
  flags & ERR_TXT_STRING);
  }
 }
+#endif
+
 
 /**
  * @returns a cert structure with a copy of the memory in \p buffer on success,
@@ -150,7 +154,7 @@ static rd_kafka_cert_t *rd_kafka_cert_new(const rd_kafka_conf_t *conf,
  [RD_KAFKA_CERT_ENC_DER] = rd_true,
  [RD_KAFKA_CERT_ENC_PEM] = rd_true},
  };
- const char *action = "";
+ const char *action = "", *ssl_errstr = NULL, *extra = "";
  BIO *bio;
  rd_kafka_cert_t *cert = NULL;
  PKCS12 *p12 = NULL;
@@ -398,10 +402,22 @@ static rd_kafka_cert_t *rd_kafka_cert_new(const rd_kafka_conf_t *conf,
  return cert;
 
 fail:
- rd_snprintf(errstr, errstr_size, "Failed to %s %s (encoding %s): %s",
+ ssl_errstr = rd_kafka_ssl_last_error_str();
+
+ /* OpenSSL 3.x does not provide obsolete ciphers out of the box, so
+ * let's try to identify such an error message and guide the user
+ * to what to do (set up a provider config file and point to it
+ * through the OPENSSL_CONF environment variable).
+ * We could call OSSL_PROVIDER_load("legacy") here, but that would be
+ * a non-obvious side-effect of calling this set function. */
+ if (strstr(action, "parse") && strstr(ssl_errstr, "Algorithm"))
+ extra =
+ ": legacy ciphers may require loading OpenSSL's \"legacy\" "
+ "provider through an OPENSSL_CONF configuration file";
+
+ rd_snprintf(errstr, errstr_size, "Failed to %s %s (encoding %s): %s%s",
  action, rd_kafka_cert_type_names[type],
- rd_kafka_cert_enc_names[encoding],
- rd_kafka_ssl_last_error_str());
+ rd_kafka_cert_enc_names[encoding], ssl_errstr, extra);
 
  if (cert)
  rd_kafka_cert_destroy(cert);

src/rdkafka_conf.c

@@ -149,6 +149,20 @@ struct rd_kafka_property {
  .unsupported = "OpenSSL >= 1.1.0 not available at build time"
 #endif
 
+#if WITH_SSL_ENGINE
+#define _UNSUPPORTED_SSL_ENGINE .unsupported = NULL
+#else
+#define _UNSUPPORTED_SSL_ENGINE \
+ .unsupported = "OpenSSL >= 1.1.x not available at build time"
+#endif
+
+#if OPENSSL_VERSION_NUMBER >= 0x30000000 && defined(WITH_SSL)
+#define _UNSUPPORTED_SSL_3 .unsupported = NULL
+#else
+#define _UNSUPPORTED_SSL_3 \
+ .unsupported = "OpenSSL >= 3.0.0 not available at build time"
+#endif
+
 
 #if WITH_ZLIB
 #define _UNSUPPORTED_ZLIB .unsupported = NULL
@@ -821,17 +835,24 @@ static const struct rd_kafka_property rd_kafka_properties[] = {
  {_RK_GLOBAL | _RK_SENSITIVE, "ssl.keystore.password", _RK_C_STR,
  _RK(ssl.keystore_password), "Client's keystore (PKCS#12) password.",
  _UNSUPPORTED_SSL},
- {_RK_GLOBAL, "ssl.engine.location", _RK_C_STR, _RK(ssl.engine_location),
- "Path to OpenSSL engine library. OpenSSL >= 1.1.0 required.",
- _UNSUPPORTED_OPENSSL_1_1_0},
+ {_RK_GLOBAL, "ssl.providers", _RK_C_STR, _RK(ssl.providers),
+ "Comma-separated list of OpenSSL 3.0.x implementation providers. "
+ "E.g., \"default,legacy\".",
+ _UNSUPPORTED_SSL_3},
+ {_RK_GLOBAL | _RK_DEPRECATED, "ssl.engine.location", _RK_C_STR,
+ _RK(ssl.engine_location),
+ "Path to OpenSSL engine library. OpenSSL >= 1.1.x required. "
+ "DEPRECATED: OpenSSL engine support is deprecated and should be "
+ "replaced by OpenSSL 3 providers.",
+ _UNSUPPORTED_SSL_ENGINE},
  {_RK_GLOBAL, "ssl.engine.id", _RK_C_STR, _RK(ssl.engine_id),
  "OpenSSL engine id is the name used for loading engine.",
- .sdef = "dynamic", _UNSUPPORTED_OPENSSL_1_1_0},
+ .sdef = "dynamic", _UNSUPPORTED_SSL_ENGINE},
  {_RK_GLOBAL, "ssl_engine_callback_data", _RK_C_PTR,
  _RK(ssl.engine_callback_data),
  "OpenSSL engine callback data (set "
  "with rd_kafka_conf_set_engine_callback_data()).",
- _UNSUPPORTED_OPENSSL_1_1_0},
+ _UNSUPPORTED_SSL_ENGINE},
  {_RK_GLOBAL, "enable.ssl.certificate.verification", _RK_C_BOOL,
  _RK(ssl.enable_verify),
  "Enable OpenSSL's builtin broker (server) certificate verification. "

src/rdkafka_conf.h

@@ -33,6 +33,8 @@
 #include "rdkafka_cert.h"
 
 #if WITH_SSL && OPENSSL_VERSION_NUMBER >= 0x10100000
+#define WITH_SSL_ENGINE 1
+/* Deprecated in OpenSSL 3 */
 #include <openssl/engine.h>
 #endif /* WITH_SSL && OPENSSL_VERSION_NUMBER >= 0x10100000 */
 
@@ -248,6 +250,8 @@ struct rd_kafka_conf_s {
  char *engine_location;
  char *engine_id;
  void *engine_callback_data;
+ char *providers;
+ rd_list_t loaded_providers; /**< (SSL_PROVIDER*) */
  char *keystore_location;
  char *keystore_password;
  int endpoint_identification;