Skip to content

Commit

Permalink
Dynamic loading of OpenSSL (facebookincubator#302)
Browse files Browse the repository at this point in the history
Summary: Pull Request resolved: facebookincubator#302

Test Plan: Imported from OSS

Reviewed By: malfet

Differential Revision: D27781296

Pulled By: pbelevich

fbshipit-source-id: 4544d62518ea22017df11d728cc530888360d398
  • Loading branch information
pbelevich authored and facebook-github-bot committed Apr 15, 2021
1 parent 1b62018 commit 2e0fe46
Show file tree
Hide file tree
Showing 15 changed files with 342 additions and 69 deletions.
30 changes: 24 additions & 6 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,13 @@ workflows:
docker_image: ubuntu:18.04
apt_get: "gcc-5 g++-5"
cmake_compiler: -DCMAKE_C_COMPILER=gcc-5 -DCMAKE_CXX_COMPILER=g++-5
cmake_args: -DUSE_IBVERBS=ON -DUSE_LIBUV=ON -DUSE_TCP_OPENSSL=ON
cmake_args: -DUSE_IBVERBS=ON -DUSE_LIBUV=ON -DUSE_TCP_OPENSSL_LINK=ON
- build:
name: gcc5-load-openssl
docker_image: ubuntu:18.04
apt_get: "gcc-5 g++-5"
cmake_compiler: -DCMAKE_C_COMPILER=gcc-5 -DCMAKE_CXX_COMPILER=g++-5
cmake_args: -DUSE_TCP_OPENSSL_LOAD=ON
- build:
name: gcc7
docker_image: ubuntu:18.04
Expand All @@ -190,7 +196,13 @@ workflows:
docker_image: ubuntu:18.04
apt_get: "gcc-7 g++-7"
cmake_compiler: -DCMAKE_C_COMPILER=gcc-7 -DCMAKE_CXX_COMPILER=g++-7
cmake_args: -DUSE_IBVERBS=ON -DUSE_LIBUV=ON -DUSE_TCP_OPENSSL=ON
cmake_args: -DUSE_IBVERBS=ON -DUSE_LIBUV=ON -DUSE_TCP_OPENSSL_LINK=ON
- build:
name: gcc7-load-openssl
docker_image: ubuntu:18.04
apt_get: "gcc-7 g++-7"
cmake_compiler: -DCMAKE_C_COMPILER=gcc-7 -DCMAKE_CXX_COMPILER=g++-7
cmake_args: -DUSE_TCP_OPENSSL_LOAD=ON
- build:
name: clang6
docker_image: ubuntu:18.04
Expand All @@ -201,26 +213,32 @@ workflows:
docker_image: ubuntu:18.04
apt_get: "clang-6.0"
cmake_compiler: -DCMAKE_C_COMPILER=clang-6.0 -DCMAKE_CXX_COMPILER=clang++-6.0
cmake_args: -DUSE_IBVERBS=ON -DUSE_LIBUV=ON -DUSE_TCP_OPENSSL=ON
cmake_args: -DUSE_IBVERBS=ON -DUSE_LIBUV=ON -DUSE_TCP_OPENSSL_LINK=ON
- build:
name: clang6-load-openssl
docker_image: ubuntu:18.04
apt_get: "clang-6.0"
cmake_compiler: -DCMAKE_C_COMPILER=clang-6.0 -DCMAKE_CXX_COMPILER=clang++-6.0
cmake_args: -DUSE_TCP_OPENSSL_LOAD=ON
- build:
name: cuda9.2
docker_image: nvidia/cuda:9.2-devel-ubuntu18.04
cmake_args: -DUSE_CUDA=ON
- build:
name: cuda9.2-all-transports
docker_image: nvidia/cuda:9.2-devel-ubuntu18.04
cmake_args: -DUSE_IBVERBS=ON -DUSE_LIBUV=ON -DUSE_TCP_OPENSSL=ON -DUSE_CUDA=ON
cmake_args: -DUSE_IBVERBS=ON -DUSE_LIBUV=ON -DUSE_TCP_OPENSSL_LINK=ON -DUSE_CUDA=ON
- build:
name: cuda10.1
docker_image: nvidia/cuda:10.1-devel-ubuntu18.04
cmake_args: -DUSE_CUDA=ON
- build:
name: cuda10.1-all-transports
docker_image: nvidia/cuda:10.1-devel-ubuntu18.04
cmake_args: -DUSE_IBVERBS=ON -DUSE_LIBUV=ON -DUSE_TCP_OPENSSL=ON -DUSE_CUDA=ON
cmake_args: -DUSE_IBVERBS=ON -DUSE_LIBUV=ON -DUSE_TCP_OPENSSL_LINK=ON -DUSE_CUDA=ON
- build:
name: tsan-all-transports
docker_image: ubuntu:18.04
apt_get: "gcc-7 g++-7"
cmake_compiler: -DCMAKE_C_COMPILER=gcc-7 -DCMAKE_CXX_COMPILER=g++-7
cmake_args: -DUSE_IBVERBS=ON -DUSE_LIBUV=ON -DUSE_TCP_OPENSSL=ON -DCMAKE_BUILD_TYPE=Debug -DSANITIZE=thread
cmake_args: -DUSE_IBVERBS=ON -DUSE_LIBUV=ON -DUSE_TCP_OPENSSL_LINK=ON -DCMAKE_BUILD_TYPE=Debug -DSANITIZE=thread
9 changes: 7 additions & 2 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,20 @@ set(USE_IBVERBS_DEFAULT OFF)
set(USE_NCCL_DEFAULT OFF)
set(USE_RCCL_DEFAULT OFF)
set(USE_LIBUV_DEFAULT OFF)
set(USE_TCP_OPENSSL_DEFAULT OFF)
set(USE_TCP_OPENSSL_LINK_DEFAULT OFF)
set(USE_TCP_OPENSSL_LOAD_DEFAULT OFF)

# Options
option(USE_REDIS "Support using Redis for rendezvous" ${USE_REDIS_DEFAULT})
option(USE_IBVERBS "Support ibverbs transport" ${USE_IBVERBS_DEFAULT})
option(USE_NCCL "Support using NCCL for local collectives" ${USE_NCCL_DEFAULT})
option(USE_RCCL "Support using RCCL for local collectives" ${USE_RCCL_DEFAULT})
option(USE_LIBUV "Build libuv transport" ${USE_LIBUV_DEFAULT})
option(USE_TCP_OPENSSL "Build tcp+openssl transport" ${USE_TCP_OPENSSL_DEFAULT})
option(USE_TCP_OPENSSL_LINK "Build TCP-TLS transport with dynamically linked OpenSSL (Linux only)" ${USE_TCP_OPENSSL_LINK_DEFAULT})
option(USE_TCP_OPENSSL_LOAD "Build TCP-TLS transport with OpenSSL dynamically loaded during runtime (Linux only)" ${USE_TCP_OPENSSL_LOAD_DEFAULT})
if(${USE_TCP_OPENSSL_LINK} AND ${USE_TCP_OPENSSL_LOAD})
message(FATAL_ERROR "USE_TCP_OPENSSL_LINK and USE_TCP_OPENSSL_LOAD are mutually exclusive")
endif()

if(MSVC)
message(STATUS "MSVC detected")
Expand Down
8 changes: 6 additions & 2 deletions gloo/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ endif()
# Compile TCP transport only on Linux (it uses epoll(2))
if(${CMAKE_SYSTEM_NAME} STREQUAL "Linux")
set(GLOO_HAVE_TRANSPORT_TCP 1)
if(${USE_TCP_OPENSSL})
if(${USE_TCP_OPENSSL_LINK} OR ${USE_TCP_OPENSSL_LOAD})
set(GLOO_HAVE_TRANSPORT_TCP_TLS 1)
else()
set(GLOO_HAVE_TRANSPORT_TCP_TLS 0)
Expand Down Expand Up @@ -151,8 +151,12 @@ endif()
if(USE_LIBUV)
target_link_libraries(gloo PRIVATE uv_a)
endif()
if(USE_TCP_OPENSSL)
if(USE_TCP_OPENSSL_LINK)
target_compile_definitions(gloo PRIVATE USE_TCP_OPENSSL_LINK)
target_link_libraries(gloo PRIVATE ssl crypto)
elseif(USE_TCP_OPENSSL_LOAD)
target_compile_definitions(gloo PRIVATE USE_TCP_OPENSSL_LOAD)
target_link_libraries(gloo PRIVATE dl)
endif()

target_link_libraries(gloo PRIVATE ${gloo_DEPENDENCY_LIBS})
Expand Down
4 changes: 2 additions & 2 deletions gloo/transport/tcp/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
message(WARNING "Gloo doesn't support OSX (uses epoll)")
if(NOT ${CMAKE_SYSTEM_NAME} STREQUAL "Linux")
message(WARNING "Gloo TCP transport is supported only on Linux(uses epoll API)")
else()
list(APPEND GLOO_TRANSPORT_SRCS
"${CMAKE_CURRENT_SOURCE_DIR}/address.cc"
Expand Down
30 changes: 20 additions & 10 deletions gloo/transport/tcp/tls/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,17 +1,27 @@
if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
message(WARNING "Gloo doesn't support OSX (uses epoll)")
else()
if (NOT ${CMAKE_SYSTEM_NAME} STREQUAL "Linux")
message(WARNING "Gloo TCP-TLS transport is supported only on Linux(uses epoll API)")
else ()
list(APPEND GLOO_TRANSPORT_SRCS
"${CMAKE_CURRENT_SOURCE_DIR}/context.cc"
"${CMAKE_CURRENT_SOURCE_DIR}/device.cc"
"${CMAKE_CURRENT_SOURCE_DIR}/openssl.cc"
"${CMAKE_CURRENT_SOURCE_DIR}/pair.cc"
)
list(APPEND GLOO_TRANSPORT_HDRS
"${CMAKE_CURRENT_SOURCE_DIR}/context.h"
"${CMAKE_CURRENT_SOURCE_DIR}/device.h"
"${CMAKE_CURRENT_SOURCE_DIR}/openssl.h"
"${CMAKE_CURRENT_SOURCE_DIR}/pair.h"
)
if (USE_TCP_OPENSSL_LOAD)
list(APPEND GLOO_TRANSPORT_SRCS
"${CMAKE_CURRENT_SOURCE_DIR}/context.cc"
"${CMAKE_CURRENT_SOURCE_DIR}/device.cc"
"${CMAKE_CURRENT_SOURCE_DIR}/pair.cc"
"${CMAKE_CURRENT_SOURCE_DIR}/dynamic_library.cc"
)
list(APPEND GLOO_TRANSPORT_HDRS
"${CMAKE_CURRENT_SOURCE_DIR}/context.h"
"${CMAKE_CURRENT_SOURCE_DIR}/device.h"
"${CMAKE_CURRENT_SOURCE_DIR}/pair.h"
"${CMAKE_CURRENT_SOURCE_DIR}/dynamic_library.h"
)
endif()
endif ()
endif ()

set(GLOO_TRANSPORT_SRCS ${GLOO_TRANSPORT_SRCS} PARENT_SCOPE)
set(GLOO_TRANSPORT_HDRS ${GLOO_TRANSPORT_HDRS} PARENT_SCOPE)
47 changes: 28 additions & 19 deletions gloo/transport/tcp/tls/context.cc
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,11 @@

#include "gloo/common/logging.h"
#include "gloo/transport/tcp/tls/device.h"
#include "gloo/transport/tcp/tls/openssl.h"
#include "gloo/transport/tcp/tls/pair.h"

namespace {
const char* c_str_or_null(const std::string &s) {
const char *c_str_or_null(const std::string &s) {
return s.empty() ? nullptr : s.c_str();
}
} // namespace
Expand All @@ -25,7 +26,7 @@ namespace tls {

std::string getSSLErrorMessage() {
std::stringstream ss;
ERR_print_errors_cb(
_glootls::ERR_print_errors_cb(
[](const char *str, size_t sz, void *ss) -> int {
reinterpret_cast<std::stringstream *>(ss)->write(str, sz);
return 1; // if (callback(...) <= 0) break
Expand All @@ -42,7 +43,7 @@ Context::Context(std::shared_ptr<Device> device, int rank, int size)
c_str_or_null(device->getCertFile()),
c_str_or_null(device->getCAFile()),
c_str_or_null(device->getCAPath())),
[](::SSL_CTX *x) { ::SSL_CTX_free(x); }) {}
[](::SSL_CTX *x) { ::_glootls::SSL_CTX_free(x); }) {}

SSL_CTX *Context::create_ssl_ctx(const char *pkey, const char *cert,
const char *ca_file, const char *ca_path) {
Expand All @@ -52,19 +53,23 @@ SSL_CTX *Context::create_ssl_ctx(const char *pkey, const char *cert,
"CAfile or CApath must be specified");
static std::once_flag ssl_ctx_init_;
std::call_once(ssl_ctx_init_, [] {
SSL_load_error_strings();
SSL_library_init();
// SSL_load_error_strings();
// SSL_library_init();
_glootls::OPENSSL_init_ssl(
OPENSSL_INIT_LOAD_SSL_STRINGS | OPENSSL_INIT_LOAD_CRYPTO_STRINGS, NULL);
_glootls::OPENSSL_init_ssl(0, NULL);
});
SSL_CTX *ssl_ctx = SSL_CTX_new(TLS_method());
SSL_CTX *ssl_ctx = _glootls::SSL_CTX_new(_glootls::TLS_method());
GLOO_ENFORCE(ssl_ctx != nullptr, getSSLErrorMessage());
GLOO_ENFORCE(SSL_CTX_set_min_proto_version(ssl_ctx, TLS_MAX_VERSION) == 1,
getSSLErrorMessage());
GLOO_ENFORCE(
_glootls::SSL_CTX_set_min_proto_version(ssl_ctx, TLS_MAX_VERSION) == 1,
getSSLErrorMessage());

// As we don't need to handle legacy clients,
// let's remove support for legacy renegotiation:
SSL_CTX_clear_options(ssl_ctx, SSL_OP_LEGACY_SERVER_CONNECT);
_glootls::SSL_CTX_clear_options(ssl_ctx, SSL_OP_LEGACY_SERVER_CONNECT);

SSL_CTX_set_verify_depth(ssl_ctx, 1);
_glootls::SSL_CTX_set_verify_depth(ssl_ctx, 1);

// To enforcing a higher security level, set it to 3.
//
Expand All @@ -83,15 +88,18 @@ SSL_CTX *Context::create_ssl_ctx(const char *pkey, const char *cert,
// TLS versions below 1.1 are not permitted. Session tickets are disabled.
//
// TODO: should be 3, but it doesn't work yet :(
SSL_CTX_set_security_level(ssl_ctx, 2);
_glootls::SSL_CTX_set_security_level(ssl_ctx, 2);

GLOO_ENFORCE(SSL_CTX_load_verify_locations(ssl_ctx, ca_file, ca_path) == 1,
GLOO_ENFORCE(
_glootls::SSL_CTX_load_verify_locations(ssl_ctx, ca_file, ca_path) == 1,
getSSLErrorMessage());
GLOO_ENFORCE(_glootls::SSL_CTX_use_certificate_chain_file(ssl_ctx, cert) == 1,
getSSLErrorMessage());
GLOO_ENFORCE(SSL_CTX_use_certificate_chain_file(ssl_ctx, cert) == 1,
GLOO_ENFORCE(_glootls::SSL_CTX_use_PrivateKey_file(ssl_ctx, pkey,
SSL_FILETYPE_PEM) == 1,
getSSLErrorMessage());
GLOO_ENFORCE(SSL_CTX_use_PrivateKey_file(ssl_ctx, pkey, SSL_FILETYPE_PEM) == 1,
GLOO_ENFORCE(_glootls::SSL_CTX_check_private_key(ssl_ctx) == 1,
getSSLErrorMessage());
GLOO_ENFORCE(SSL_CTX_check_private_key(ssl_ctx) == 1, getSSLErrorMessage());
// SSL_VERIFY_PEER
//
// Server mode: the server sends a client certificate request to the client.
Expand All @@ -106,10 +114,11 @@ SSL_CTX *Context::create_ssl_ctx(const char *pkey, const char *cert,
// an alert message containing the reason for the verification failure.
// If no server certificate is sent, because an anonymous cipher is used,
// SSL_VERIFY_PEER is ignored.
SSL_CTX_set_verify(ssl_ctx,
SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT |
SSL_VERIFY_CLIENT_ONCE,
nullptr);
_glootls::SSL_CTX_set_verify(ssl_ctx,
SSL_VERIFY_PEER |
SSL_VERIFY_FAIL_IF_NO_PEER_CERT |
SSL_VERIFY_CLIENT_ONCE,
nullptr);
return ssl_ctx;
}

Expand Down
3 changes: 1 addition & 2 deletions gloo/transport/tcp/tls/context.h
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,7 @@
#pragma once

#include "gloo/transport/tcp/context.h"

#include <openssl/ssl.h>
#include "gloo/transport/tcp/tls/openssl.h"

namespace gloo {
namespace transport {
Expand Down
8 changes: 4 additions & 4 deletions gloo/transport/tcp/tls/device.cc
Original file line number Diff line number Diff line change
Expand Up @@ -37,13 +37,13 @@ std::shared_ptr<transport::Context> Device::createContext(int rank, int size) {
std::dynamic_pointer_cast<Device>(shared_from_this()), rank, size));
}

const std::string& Device::getPKeyFile() const { return pkey_file_; }
const std::string &Device::getPKeyFile() const { return pkey_file_; }

const std::string& Device::getCertFile() const { return cert_file_; }
const std::string &Device::getCertFile() const { return cert_file_; }

const std::string& Device::getCAFile() const { return ca_file_; }
const std::string &Device::getCAFile() const { return ca_file_; }

const std::string& Device::getCAPath() const { return ca_path_; }
const std::string &Device::getCAPath() const { return ca_path_; }

} // namespace tls
} // namespace tcp
Expand Down
8 changes: 4 additions & 4 deletions gloo/transport/tcp/tls/device.h
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,13 @@ class Device : public ::gloo::transport::tcp::Device {
std::shared_ptr<::gloo::transport::Context> createContext(int rank,
int size) override;

const std::string& getPKeyFile() const;
const std::string &getPKeyFile() const;

const std::string& getCertFile() const;
const std::string &getCertFile() const;

const std::string& getCAFile() const;
const std::string &getCAFile() const;

const std::string& getCAPath() const;
const std::string &getCAPath() const;

protected:
const std::string pkey_file_;
Expand Down
34 changes: 34 additions & 0 deletions gloo/transport/tcp/tls/dynamic_library.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
#include "dynamic_library.h"

#include <dlfcn.h>
#include <stdexcept>

DynamicLibrary::DynamicLibrary(const char *name, const char *alt_name)
: lib_name(name) {
handle = dlopen(name, RTLD_LOCAL | RTLD_NOW);
if (!handle) {
if (alt_name == nullptr) {
throw std::runtime_error(dlerror());
} else {
handle = dlopen(alt_name, RTLD_LOCAL | RTLD_NOW);
if (!handle) {
throw std::runtime_error(dlerror());
}
}
}
}

void *DynamicLibrary::sym(const char *name) {
void *res = dlsym(handle, name);
if (res == nullptr) {
throw std::runtime_error("Can't find " + std::string(name) + " in " +
lib_name + ":" + dlerror());
}
return res;
}

DynamicLibrary::~DynamicLibrary() {
if (!handle)
return;
dlclose(handle);
}
27 changes: 27 additions & 0 deletions gloo/transport/tcp/tls/dynamic_library.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
/**
* Copyright (c) 2021-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree.
*/

#pragma once

#include <string>

struct DynamicLibrary {
DynamicLibrary(const DynamicLibrary &) = delete;

void operator=(const DynamicLibrary &) = delete;

DynamicLibrary(const char *name, const char *alt_name);

void *sym(const char *name);

~DynamicLibrary();

private:
const std::string lib_name;
void *handle = nullptr;
};
Loading

0 comments on commit 2e0fe46

Please sign in to comment.