C++20 compatibility (facebook#6697)

Summary:
Based on facebook#6648 (CLA Signed), but heavily modified / extended:

* Implicit capture of this via [=] deprecated in C++20, and [=,this] not standard before C++20 -> now using explicit capture lists
* Implicit copy operator deprecated in gcc 9 -> add explicit '= default' definition
* std::random_shuffle deprecated in C++17 and removed in C++20 -> migrated to a replacement in RocksDB random.h API
* Add the ability to build with different std version though -DCMAKE_CXX_STANDARD=11/14/17/20 on the cmake command line
* Minimal rebuild flag of MSVC is deprecated and is forbidden with /std:c++latest (C++20)
* Added MSVC 2019 C++11 & MSVC 2019 C++20 in AppVeyor
* Added GCC 9 C++11 & GCC9 C++20 in Travis
Pull Request resolved: facebook#6697

Test Plan: make check and CI

Reviewed By: cheng-chang

Differential Revision: D21020318

Pulled By: pdillinger

fbshipit-source-id: 12311be5dbd8675a0e2c817f7ec50fa11c18ab91

.travis.yml

@@ -50,12 +50,18 @@ env:
  - JOB_NAME=examples # 5-7 minutes
  - JOB_NAME=cmake # 3-5 minutes
  - JOB_NAME=cmake-gcc8 # 3-5 minutes
+ - JOB_NAME=cmake-gcc9 # 3-5 minutes
+ - JOB_NAME=cmake-gcc9-c++20 # 3-5 minutes
  - JOB_NAME=cmake-mingw # 3 minutes
 
 matrix:
  exclude:
  - os: osx
  env: JOB_NAME=cmake-gcc8
+ - os: osx
+ env: JOB_NAME=cmake-gcc9
+ - os: osx
+ env: JOB_NAME=cmake-gcc9-c++20
  - os: osx
  env: JOB_NAME=cmake-mingw
  - os: osx
@@ -70,6 +76,19 @@ matrix:
  env: JOB_NAME=cmake-mingw
  - os: linux
  compiler: clang
+ # Exclude all but most unique cmake variants for pull requests, but build all in branches
+ - if: type = pull_request
+ os : linux
+ arch: amd64
+ env: JOB_NAME=cmake
+ - if: type = pull_request
+ os : linux
+ arch: amd64
+ env: JOB_NAME=cmake-gcc8
+ - if: type = pull_request
+ os : linux
+ arch: amd64
+ env: JOB_NAME=cmake-gcc9
  # Exclude most osx, arm64 and ppc64le tests for pull requests, but build in branches
  # Temporarily disable ppc64le unit tests in PRs until Travis gets its act together (#6653)
  - if: type = pull_request
@@ -162,6 +181,22 @@ matrix:
  os: linux
  arch: ppc64le
  env: JOB_NAME=cmake-gcc8
+ - if: type = pull_request
+ os : linux
+ arch: arm64
+ env: JOB_NAME=cmake-gcc9
+ - if: type = pull_request
+ os: linux
+ arch: ppc64le
+ env: JOB_NAME=cmake-gcc9
+ - if: type = pull_request
+ os : linux
+ arch: arm64
+ env: JOB_NAME=cmake-gcc9-c++20
+ - if: type = pull_request
+ os: linux
+ arch: ppc64le
+ env: JOB_NAME=cmake-gcc9-c++20
 
 install:
  - if [ "${TRAVIS_OS_NAME}" == osx ]; then
@@ -171,6 +206,10 @@ install:
  sudo apt-get install -y g++-8;
  CC=gcc-8 && CXX=g++-8;
  fi
+ - if [ "${JOB_NAME}" == cmake-gcc9 ] || [ "${JOB_NAME}" == cmake-gcc9-c++20 ]; then
+ sudo apt-get install -y g++-9;
+ CC=gcc-9 && CXX=g++-9;
+ fi
  - if [ "${JOB_NAME}" == cmake-mingw ]; then
  sudo apt-get install -y mingw-w64 ;
  fi
@@ -240,7 +279,13 @@ script:
  mkdir build && cd build && cmake -DJNI=1 -DWITH_GFLAGS=OFF .. -DCMAKE_C_COMPILER=x86_64-w64-mingw32-gcc -DCMAKE_CXX_COMPILER=x86_64-w64-mingw32-g++ -DCMAKE_SYSTEM_NAME=Windows && make -j4 rocksdb rocksdbjni
  ;;
  cmake*)
- mkdir build && cd build && cmake -DJNI=1 .. -DCMAKE_BUILD_TYPE=Release && make -j4 rocksdb rocksdbjni
+ case $JOB_NAME in
+ *-c++20)
+ OPT=-DCMAKE_CXX_STANDARD=20
+ ;;
+ esac
+
+ mkdir build && cd build && cmake -DJNI=1 .. -DCMAKE_BUILD_TYPE=Release $OPT && make -j4 rocksdb rocksdbjni
  ;;
  esac
 notifications:

CMakeLists.txt

@@ -83,6 +83,10 @@ else()
  option(WITH_FOLLY_DISTRIBUTED_MUTEX "build with folly::DistributedMutex" OFF)
 endif()
 
+if( NOT DEFINED CMAKE_CXX_STANDARD )
+ set(CMAKE_CXX_STANDARD 11)
+endif()
+
 include(CMakeDependentOption)
 CMAKE_DEPENDENT_OPTION(WITH_GFLAGS "build with GFlags" ON
  "NOT MSVC;NOT MINGW" OFF)
@@ -191,7 +195,6 @@ else()
  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-format -fno-asynchronous-unwind-tables")
  add_definitions(-D_POSIX_C_SOURCE=1)
  endif()
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
  if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer")
  include(CheckCXXCompilerFlag)
@@ -254,6 +257,7 @@ include(CheckCXXSourceCompiles)
 if(NOT MSVC)
  set(CMAKE_REQUIRED_FLAGS "-msse4.2 -mpclmul")
 endif()
+
 CHECK_CXX_SOURCE_COMPILES("
 #include <cstdint>
 #include <nmmintrin.h>
@@ -386,7 +390,15 @@ if(MSVC)
  message(STATUS "Debug optimization is enabled")
  set(CMAKE_CXX_FLAGS_DEBUG "/Oxt")
  else()
- set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /Od /RTC1 /Gm")
+ set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /Od /RTC1")
+
+ # Minimal Build is deprecated after MSVC 2015
+ if( MSVC_VERSION GREATER 1900 )
+ set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /Gm-")
+ else()
+ set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /Gm")
+ endif()
+
  endif()
  if(WITH_RUNTIME_DEBUG)
  set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /${RUNTIME_LIBRARY}d")
@@ -849,7 +861,6 @@ if(ROCKSDB_BUILD_SHARED)
  LINKER_LANGUAGE CXX
  VERSION ${rocksdb_VERSION}
  SOVERSION ${rocksdb_VERSION_MAJOR}
- CXX_STANDARD 11
  OUTPUT_NAME "rocksdb")
  endif()
 endif()
@@ -1130,7 +1141,7 @@ if(WITH_TESTS)
  PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD_RELEASE 1
  EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1
  EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1
-  )
+ )
 
  foreach(sourcefile ${TESTS})
  get_filename_component(exename ${sourcefile} NAME_WE)
@@ -1140,7 +1151,7 @@ if(WITH_TESTS)
  EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1
  EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1
  OUTPUT_NAME ${exename}${ARTIFACT_SUFFIX}
-  )
+ )
  target_link_libraries(${CMAKE_PROJECT_NAME}_${exename}${ARTIFACT_SUFFIX} testutillib${ARTIFACT_SUFFIX} testharness gtest ${ROCKSDB_LIB})
  if(NOT "${exename}" MATCHES "db_sanity_test")
  add_test(NAME ${exename} COMMAND ${exename}${ARTIFACT_SUFFIX})

appveyor.yml

@@ -1,6 +1,6 @@
 version: 1.0.{build}
 
-image: Visual Studio 2017
+image: Visual Studio 2019
 
 environment:
  JAVA_HOME: C:\Program Files\Java\jdk1.8.0
@@ -24,6 +24,15 @@ environment:
  - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
  CMAKE_GENERATOR: Visual Studio 15 Win64
  DEV_ENV: C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\Common7\IDE\devenv.com
+ - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019
+ CMAKE_GENERATOR: Visual Studio 16
+ CMAKE_PLATEFORM_NAME: x64
+ DEV_ENV: C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\Common7\IDE\devenv.com
+ - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019
+ CMAKE_GENERATOR: Visual Studio 16
+ CMAKE_PLATEFORM_NAME: x64
+ CMAKE_OPT: -DCMAKE_CXX_STANDARD=20
+ DEV_ENV: C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\Common7\IDE\devenv.com
 
 install:
  - md %THIRDPARTY_HOME%
@@ -34,7 +43,8 @@ install:
  - cd snappy-1.1.7
  - mkdir build
  - cd build
- - cmake -G "%CMAKE_GENERATOR%" ..
+ - if DEFINED CMAKE_PLATEFORM_NAME (set "PLATEFORM_OPT=-A %CMAKE_PLATEFORM_NAME%")
+ - cmake .. -G "%CMAKE_GENERATOR%" %PLATEFORM_OPT%
  - msbuild Snappy.sln /p:Configuration=Debug /p:Platform=x64
  - msbuild Snappy.sln /p:Configuration=Release /p:Platform=x64
  - echo "Building LZ4 dependency..."
@@ -57,7 +67,8 @@ install:
 before_build:
  - md %APPVEYOR_BUILD_FOLDER%\build
  - cd %APPVEYOR_BUILD_FOLDER%\build
- - cmake -G "%CMAKE_GENERATOR%" -DCMAKE_BUILD_TYPE=Debug -DOPTDBG=1 -DPORTABLE=1 -DSNAPPY=1 -DLZ4=1 -DZSTD=1 -DXPRESS=1 -DJNI=1 ..
+ - if DEFINED CMAKE_PLATEFORM_NAME (set "PLATEFORM_OPT=-A %CMAKE_PLATEFORM_NAME%")
+ - cmake .. -G "%CMAKE_GENERATOR%" %PLATEFORM_OPT% %CMAKE_OPT% -DCMAKE_BUILD_TYPE=Debug -DOPTDBG=1 -DPORTABLE=1 -DSNAPPY=1 -DLZ4=1 -DZSTD=1 -DXPRESS=1 -DJNI=1
  - cd ..
 
 build:

build_tools/build_detect_platform

@@ -705,6 +705,7 @@ EOF
  fi
 fi
 
+
 if [ "$FBCODE_BUILD" != "true" -a "$PLATFORM" = OS_LINUX ]; then
  $CXX $COMMON_FLAGS $PLATFORM_SHARED_CFLAGS -x c++ -c - -o test_dl.o 2>/dev/null <<EOF
  void dummy_func() {}

cmake/modules/CxxFlags.cmake

@@ -0,0 +1,7 @@
+macro(get_cxx_std_flags FLAGS_VARIABLE)
+ if( CMAKE_CXX_STANDARD_REQUIRED )
+ set(${FLAGS_VARIABLE} ${CMAKE_CXX${CMAKE_CXX_STANDARD}_STANDARD_COMPILE_OPTION})
+ else()
+ set(${FLAGS_VARIABLE} ${CMAKE_CXX${CMAKE_CXX_STANDARD}_EXTENSION_COMPILE_OPTION})
+ endif()
+endmacro()

db/db_bloom_filter_test.cc

@@ -1343,8 +1343,7 @@ TEST_F(DBBloomFilterTest, OptimizeFiltersForHits) {
  for (int i = 0; i < numkeys; i += 2) {
  keys.push_back(i);
  }
- std::random_shuffle(std::begin(keys), std::end(keys));
-
+ RandomShuffle(std::begin(keys), std::end(keys));
  int num_inserted = 0;
  for (int key : keys) {
  ASSERT_OK(Put(1, Key(key), "val"));

db/db_compaction_test.cc

@@ -4762,7 +4762,7 @@ TEST_P(CompactionPriTest, Test) {
  for (int i = 0; i < kNKeys; i++) {
  keys[i] = i;
  }
- std::random_shuffle(std::begin(keys), std::end(keys));
+ RandomShuffle(std::begin(keys), std::end(keys), rnd.Next());
 
  for (int i = 0; i < kNKeys; i++) {
  ASSERT_OK(Put(Key(keys[i]), RandomString(&rnd, 102)));

db/db_dynamic_level_test.cc

@@ -50,7 +50,7 @@ TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesBase) {
  keys[i] = i;
  }
  if (ordered_insert == 0) {
- std::random_shuffle(std::begin(keys), std::end(keys));
+ RandomShuffle(std::begin(keys), std::end(keys), rnd.Next());
  }
  for (int max_background_compactions = 1; max_background_compactions < 4;
  max_background_compactions += 2) {

db/db_test.cc

@@ -1421,7 +1421,7 @@ TEST_F(DBTest, ApproximateSizesMemTable) {
  keys[i * 3 + 1] = i * 5 + 1;
  keys[i * 3 + 2] = i * 5 + 2;
  }
- std::random_shuffle(std::begin(keys), std::end(keys));
+ RandomShuffle(std::begin(keys), std::end(keys));
 
  for (int i = 0; i < N * 3; i++) {
  ASSERT_OK(Put(Key(keys[i] + 1000), RandomString(&rnd, 1024)));
@@ -4530,7 +4530,7 @@ TEST_F(DBTest, DynamicLevelCompressionPerLevel) {
  for (int i = 0; i < kNKeys; i++) {
  keys[i] = i;
  }
- std::random_shuffle(std::begin(keys), std::end(keys));
+ RandomShuffle(std::begin(keys), std::end(keys));
 
  Random rnd(301);
  Options options;
@@ -4613,7 +4613,7 @@ TEST_F(DBTest, DynamicLevelCompressionPerLevel2) {
  for (int i = 0; i < kNKeys; i++) {
  keys[i] = i;
  }
- std::random_shuffle(std::begin(keys), std::end(keys));
+ RandomShuffle(std::begin(keys), std::end(keys));
 
  Random rnd(301);
  Options options;

db/log_reader.cc

@@ -11,11 +11,11 @@
 
 #include <stdio.h>
 #include "file/sequence_file_reader.h"
+#include "port/lang.h"
 #include "rocksdb/env.h"
 #include "test_util/sync_point.h"
 #include "util/coding.h"
 #include "util/crc32c.h"
-#include "util/util.h"
 
 namespace ROCKSDB_NAMESPACE {
 namespace log {

db/memtable.cc

@@ -23,6 +23,7 @@
 #include "memory/memory_usage.h"
 #include "monitoring/perf_context_imp.h"
 #include "monitoring/statistics.h"
+#include "port/lang.h"
 #include "port/port.h"
 #include "rocksdb/comparator.h"
 #include "rocksdb/env.h"
@@ -36,7 +37,6 @@
 #include "util/autovector.h"
 #include "util/coding.h"
 #include "util/mutexlock.h"
-#include "util/util.h"
 
 namespace ROCKSDB_NAMESPACE {
 

db/perf_context_test.cc

@@ -251,7 +251,7 @@ void ProfileQueries(bool enabled_time = false) {
  }
 
  if (FLAGS_random_key) {
- std::random_shuffle(keys.begin(), keys.end());
+ RandomShuffle(std::begin(keys), std::end(keys));
  }
 #ifndef NDEBUG
  ThreadStatusUtil::TEST_SetStateDelay(ThreadStatus::STATE_MUTEX_WAIT, 1U);
@@ -524,7 +524,7 @@ TEST_F(PerfContextTest, SeekKeyComparison) {
  }
 
  if (FLAGS_random_key) {
- std::random_shuffle(keys.begin(), keys.end());
+ RandomShuffle(std::begin(keys), std::end(keys));
  }
 
  HistogramImpl hist_put_time;

db/write_batch.cc

@@ -53,13 +53,13 @@
 #include "db/write_batch_internal.h"
 #include "monitoring/perf_context_imp.h"
 #include "monitoring/statistics.h"
+#include "port/lang.h"
 #include "rocksdb/merge_operator.h"
 #include "util/autovector.h"
 #include "util/cast_util.h"
 #include "util/coding.h"
 #include "util/duplicate_detector.h"
 #include "util/string_util.h"
-#include "util/util.h"
 
 namespace ROCKSDB_NAMESPACE {
 

include/rocksdb/file_system.h

@@ -105,6 +105,8 @@ struct FileOptions : EnvOptions {
 
  FileOptions(const FileOptions& opts)
  : EnvOptions(opts), io_options(opts.io_options) {}
+
+ FileOptions& operator=(const FileOptions& opts) = default;
 };
 
 // A structure to pass back some debugging information from the FileSystem

memory/concurrent_arena.h

@@ -13,6 +13,7 @@
 #include <utility>
 #include "memory/allocator.h"
 #include "memory/arena.h"
+#include "port/lang.h"
 #include "port/likely.h"
 #include "util/core_local.h"
 #include "util/mutexlock.h"
@@ -49,7 +50,7 @@ class ConcurrentArena : public Allocator {
 
  char* Allocate(size_t bytes) override {
  return AllocateImpl(bytes, false /*force_arena*/,
- [=]() { return arena_.Allocate(bytes); });
+ [this, bytes]() { return arena_.Allocate(bytes); });
  }
 
  char* AllocateAligned(size_t bytes, size_t huge_page_size = 0,
@@ -58,9 +59,11 @@ class ConcurrentArena : public Allocator {
  assert(rounded_up >= bytes && rounded_up < bytes + sizeof(void*) &&
  (rounded_up % sizeof(void*)) == 0);
 
- return AllocateImpl(rounded_up, huge_page_size != 0 /*force_arena*/, [=]() {
- return arena_.AllocateAligned(rounded_up, huge_page_size, logger);
- });
+ return AllocateImpl(rounded_up, huge_page_size != 0 /*force_arena*/,
+ [this, rounded_up, huge_page_size, logger]() {
+ return arena_.AllocateAligned(rounded_up,
+ huge_page_size, logger);
+ });
  }
 
  size_t ApproximateMemoryUsage() const {

memtable/memtablerep_bench.cc

@@ -170,9 +170,8 @@ class KeyGenerator {
  for (uint64_t i = 0; i < num_; ++i) {
  values_[i] = i;
  }
- std::shuffle(
- values_.begin(), values_.end(),
- std::default_random_engine(static_cast<unsigned int>(FLAGS_seed)));
+ RandomShuffle(values_.begin(), values_.end(),
+ static_cast<uint32_t>(FLAGS_seed));
  }
  }