Removed non-standard CMake build types and fixed other bugs. (open-qu…

…antum-safe#726) * Removed non-standard CMake build types and introduced new variables to customize the build. * Added handling for new build variables to all KEMS and signatures and updated CI. * Added clang-9 test to CI.
mraavi432 · Apr 16, 2020 · 8950275 · 8950275
1 parent 866f409
commit 8950275
Show file tree

Hide file tree

Showing 32 changed files with 256 additions and 102 deletions.
diff --git a/.CMake/alg_support.cmake b/.CMake/alg_support.cmake
@@ -128,15 +128,15 @@ cmake_dependent_option(OQS_ENABLE_KEM_threebears_papabear_ephem "" ON "OQS_ENABL
 
 option(OQS_ENABLE_SIG_DILITHIUM "" ON)
 cmake_dependent_option(OQS_ENABLE_SIG_dilithium_2 "" ON "OQS_ENABLE_SIG_DILITHIUM" OFF)
-if(ARCH STREQUAL "x86_64" AND CMAKE_SYSTEM_NAME MATCHES "Darwin|Linux" AND OQS_USE_AVX2_INSTRUCTIONS AND OQS_USE_BMI2_INSTRUCTIONS)
+if(ARCH STREQUAL "x86_64" AND CMAKE_SYSTEM_NAME MATCHES "Linux" AND OQS_USE_AVX2_INSTRUCTIONS AND OQS_USE_BMI_INSTRUCTIONS AND OQS_USE_POPCNT_INSTRUCTIONS)
     cmake_dependent_option(OQS_ENABLE_SIG_dilithium_2_avx2 "" ON "OQS_ENABLE_SIG_dilithium_2" OFF)
 endif()
 cmake_dependent_option(OQS_ENABLE_SIG_dilithium_3 "" ON "OQS_ENABLE_SIG_DILITHIUM" OFF)
-if(ARCH STREQUAL "x86_64" AND CMAKE_SYSTEM_NAME MATCHES "Darwin|Linux" AND OQS_USE_AVX2_INSTRUCTIONS AND OQS_USE_BMI2_INSTRUCTIONS)
+if(ARCH STREQUAL "x86_64" AND CMAKE_SYSTEM_NAME MATCHES "Linux" AND OQS_USE_AVX2_INSTRUCTIONS AND OQS_USE_BMI_INSTRUCTIONS AND OQS_USE_POPCNT_INSTRUCTIONS)
     cmake_dependent_option(OQS_ENABLE_SIG_dilithium_3_avx2 "" ON "OQS_ENABLE_SIG_dilithium_3" OFF)
 endif()
 cmake_dependent_option(OQS_ENABLE_SIG_dilithium_4 "" ON "OQS_ENABLE_SIG_DILITHIUM" OFF)
-if(ARCH STREQUAL "x86_64" AND CMAKE_SYSTEM_NAME MATCHES "Darwin|Linux" AND OQS_USE_AVX2_INSTRUCTIONS AND OQS_USE_BMI2_INSTRUCTIONS)
+if(ARCH STREQUAL "x86_64" AND CMAKE_SYSTEM_NAME MATCHES "Linux" AND OQS_USE_AVX2_INSTRUCTIONS AND OQS_USE_BMI_INSTRUCTIONS AND OQS_USE_POPCNT_INSTRUCTIONS)
     cmake_dependent_option(OQS_ENABLE_SIG_dilithium_4_avx2 "" ON "OQS_ENABLE_SIG_dilithium_4" OFF)
 endif()
 

diff --git a/.CMake/compiler_opts.cmake b/.CMake/compiler_opts.cmake
@@ -4,7 +4,16 @@ if(CMAKE_C_COMPILER_ID MATCHES "Clang")
     add_compile_options(-Wextra)
     add_compile_options(-Wpedantic)
     add_compile_options(-Wno-unused-command-line-argument)
-    if(CMAKE_BUILD_TYPE STREQUAL "Debug")
+
+    set(THREADS_PREFER_PTHREAD_FLAG ON)
+    find_package(Threads REQUIRED)
+    set(OQS_USE_PTHREADS_IN_TESTS 1)
+
+    if(OQS_USE_CPU_EXTENSIONS)
+        include(${CMAKE_CURRENT_LIST_DIR}/gcc_clang_intrinsics.cmake)
+    endif()
+
+    if(CMAKE_BUILD_TYPE STREQUAL "Debug" OR CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo")
         add_compile_options(-g3)
         add_compile_options(-fno-omit-frame-pointer)
         if(USE_SANITIZER STREQUAL "Address")
@@ -32,19 +41,11 @@ if(CMAKE_C_COMPILER_ID MATCHES "Clang")
             add_compile_options(-fsanitize=leak)
             set(SANITIZER_LD_FLAGS "-fsanitize=leak")
         endif()
-    elseif(CMAKE_BUILD_TYPE STREQUAL "Optimized")
-        add_compile_options(-O3)
-        add_compile_options(-fomit-frame-pointer)
-        include(${CMAKE_CURRENT_LIST_DIR}/gcc_clang_intrinsics.cmake)
-    else() #Build type = Generic/Dependency
+    else()
         add_compile_options(-O3)
         add_compile_options(-fomit-frame-pointer)
     endif()
 
-    set(THREADS_PREFER_PTHREAD_FLAG ON)
-    find_package(Threads REQUIRED)
-    set(OQS_USE_PTHREADS_IN_TESTS 1)
-
 elseif(CMAKE_C_COMPILER_ID STREQUAL "GNU")
     add_compile_options(-Werror)
     add_compile_options(-Wall)
@@ -55,21 +56,19 @@ elseif(CMAKE_C_COMPILER_ID STREQUAL "GNU")
     add_compile_options(-Wformat=2)
     add_compile_options(-Wfloat-equal)
     add_compile_options(-Wwrite-strings)
-    if(CMAKE_BUILD_TYPE STREQUAL "Debug")
+
+    set(THREADS_PREFER_PTHREAD_FLAG ON)
+    find_package(Threads REQUIRED)
+    set(OQS_USE_PTHREADS_IN_TESTS 1)
+
+    if(OQS_USE_CPU_EXTENSIONS)
+        include(${CMAKE_CURRENT_LIST_DIR}/gcc_clang_intrinsics.cmake)
+    endif()
+
+    if(CMAKE_BUILD_TYPE STREQUAL "Debug" OR CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo")
         add_compile_options (-Wstrict-overflow)
         add_compile_options(-ggdb3)
-    elseif(CMAKE_BUILD_TYPE STREQUAL "Optimized")
-        add_compile_options(-O3)
-        add_compile_options(-fomit-frame-pointer)
-        add_compile_options(-fdata-sections)
-        add_compile_options(-ffunction-sections)
-        if (CMAKE_SYSTEM_NAME STREQUAL "Darwin")
-            add_compile_options(-Wl,-dead_strip)
-        else ()
-            add_compile_options(-Wl,--gc-sections)
-        endif ()
-        include(${CMAKE_CURRENT_LIST_DIR}/gcc_clang_intrinsics.cmake)
-    else() #Build type = Generic/Dependency
+    else()
         add_compile_options(-O3)
         add_compile_options(-fomit-frame-pointer)
         add_compile_options(-fdata-sections)
@@ -81,10 +80,6 @@ elseif(CMAKE_C_COMPILER_ID STREQUAL "GNU")
         endif ()
     endif()
 
-    set(THREADS_PREFER_PTHREAD_FLAG ON)
-    find_package(Threads REQUIRED)
-    set(OQS_USE_PTHREADS_IN_TESTS 1)
-
 elseif(CMAKE_C_COMPILER_ID STREQUAL "MSVC")
     # Warning C4146 is raised when a unary minus operator is applied to an
     # unsigned type; this has nonetheless been standard and portable for as

diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -10,7 +10,7 @@ localCheckout: &localCheckout
     PROJECT_PATH=$(cd ${CIRCLE_WORKING_DIRECTORY}; pwd)
     mkdir -p ${PROJECT_PATH}
     cd /tmp/_circleci_local_build_repo
-    git ls-files -z | xargs -0 -s 2090860 tar -c | tar -x -C ${PROJECT_PATH} 
+    git ls-files -z | xargs -0 -s 2090860 tar -c | tar -x -C ${PROJECT_PATH}
     cp -a /tmp/_circleci_local_build_repo/.git ${PROJECT_PATH}
 
 .oqsjob: &oqsjob
@@ -20,10 +20,10 @@ localCheckout: &localCheckout
     - checkout # change this from "checkout" to "*localCheckout" when running CircleCI locally
     - run:
         name: Configure
-        command: scripts/git_no_checkin_in_last_day.sh || (mkdir build && cd build  && cmake -GNinja ${CONFIGURE_ARGS} ..)
+        command: scripts/git_no_checkin_in_last_day.sh || (mkdir build && cd build && source ~/.bashrc && cmake -GNinja ${CONFIGURE_ARGS} ..)
     - run:
         name: Build
-        command: ../scripts/git_no_checkin_in_last_day.sh || (source ~/.bashrc && ninja)
+        command: ../scripts/git_no_checkin_in_last_day.sh || ninja
         working_directory: build
     - run:
         name: Run tests
@@ -101,13 +101,12 @@ jobs:
     <<: *oqsjob
     environment:
       IMAGE: openquantumsafe/ci-centos-8-amd64:latest
-      CONFIGURE_ARGS: -DCMAKE_BUILD_TYPE=Optimized
+      CONFIGURE_ARGS: -DOQS_PORTABLE_BUILD=OFF
       SKIP_TESTS: style
   debian-buster-amd64:
     <<: *oqsjob
     environment:
       IMAGE: openquantumsafe/ci-debian-buster-amd64:latest
-      CONFIGURE_ARGS: -DCMAKE_BUILD_TYPE=Optimized
       SKIP_TESTS: style
   debian-buster-arm64:
     <<: *emulatedjob
@@ -124,31 +123,31 @@ jobs:
     environment:
       CONFIGURE_ARGS: -DOQS_ENABLE_SIG_SPHINCS=OFF # sig-sphincs exhausts memory on CircleCI servers
       ARCH: armel
-  ubuntu-bionic-x86_64-gcc7:
+  ubuntu-bionic-x86_64-gcc8:
     <<: *oqsjob
     environment:
       IMAGE: openquantumsafe/ci-ubuntu-bionic-x86_64:latest
-      CC: gcc-7
+      CC: gcc-8
   ubuntu-bionic-x86_64-gcc7-noopenssl:
     <<: *oqsjob
     environment:
       IMAGE: openquantumsafe/ci-ubuntu-bionic-x86_64:latest
       CC: gcc-7
-      CONFIGURE_ARGS: -DCMAKE_BUILD_TYPE=Optimized -DOQS_USE_OPENSSL=OFF
+      CONFIGURE_ARGS: -DOQS_USE_OPENSSL=OFF
   ubuntu-bionic-x86_64-gcc7-shared:
     <<: *oqsjob
     environment:
       IMAGE: openquantumsafe/ci-ubuntu-bionic-x86_64:latest
       CC: gcc-7
-      CONFIGURE_ARGS: -DCMAKE_BUILD_TYPE=Optimized -DBUILD_SHARED_LIBS=ON
+      CONFIGURE_ARGS: -DBUILD_SHARED_LIBS=ON
       #TODO: _init and _fini trip test_namespace.py
       SKIP_TESTS: namespace
-  ubuntu-bionic-x86_64-gcc8:
+  ubuntu-bionic-x86_64-clang9:
     <<: *oqsjob
     environment:
       IMAGE: openquantumsafe/ci-ubuntu-bionic-x86_64:latest
-      CC: gcc-8
-      CONFIGURE_ARGS: -DCMAKE_BUILD_TYPE=Optimized
+      CC: clang-9
+      #TODO: Adding -DOQS_PORTABLE_BUILD=OFF causes kat_sig picnic to fail
   ubuntu-bionic-x86_64-asan:
     <<: *oqsjob
     environment:
@@ -165,7 +164,7 @@ jobs:
     macos:
         xcode: "11.3.0"
     environment:
-        CONFIGURE_ARGS: -DOQS_USE_OPENSSL=OFF -DCMAKE_BUILD_TYPE=Optimized
+        CONFIGURE_ARGS: -DOQS_USE_OPENSSL=OFF
         SKIP_TESTS: style
     steps:
         - checkout
@@ -225,8 +224,8 @@ workflows:
       - centos-8-amd64
       - debian-buster-amd64
       - macOS
-      - ubuntu-bionic-x86_64-gcc7
-      - ubuntu-bionic-x86_64-gcc7-noopenssl
+      - ubuntu-bionic-x86_64-gcc8
+      - ubuntu-bionic-x86_64-clang9
       - ubuntu-bionic-x86_64-gcc7-shared
   nightly:
     triggers:
@@ -244,8 +243,8 @@ workflows:
       - debian-buster-armhf
       - debian-buster-armel
       - macOS
-      - ubuntu-bionic-x86_64-gcc7
       - ubuntu-bionic-x86_64-gcc7-noopenssl
       - ubuntu-bionic-x86_64-gcc7-shared
       - ubuntu-bionic-x86_64-gcc8
+      - ubuntu-bionic-x86_64-clang9
       - ubuntu-bionic-x86_64-asan
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -18,6 +18,7 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 set(CMAKE_C_VISIBILITY_PRESET hidden)
 set(OQS_VERSION_TEXT "0.2.1-dev")
 set(OQS_COMPILE_BUILD_TARGET "${CMAKE_SYSTEM_PROCESSOR}-${CMAKE_HOST_SYSTEM}")
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
 if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|amd64|AMD64")
     set(ARCH "x86_64")
@@ -29,18 +30,13 @@ else()
     message(FATAL_ERROR "Unknown or unsupported processor: " ${CMAKE_SYSTEM_PROCESSOR})
 endif()
 
-if(DEFINED OQS_BUILD_TYPE)
-    set(CMAKE_BUILD_TYPE ${OQS_BUILD_TYPE})
-endif()
 if(WIN32)
     set(CMAKE_GENERATOR_CC cl)
 endif()
-if(CMAKE_BUILD_TYPE STREQUAL "Debug")
-    set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
-endif()
-if(CMAKE_BUILD_TYPE STREQUAL "Optimized")
-    set(OQS_OPTIMIZED_BUILD ON)
-endif()
+
+option(OQS_USE_CPU_EXTENSIONS "Enable compile and run-time support for CPU extensions such as AVX2, SSE, etc." ON)
+option(OQS_PORTABLE_BUILD "Ensure the resulting library is portable. This implies having run-time checks for CPU extensions." ON)
+option(OQS_BUILD_TESTS "Build liboqs test programs." ON)
 include(.CMake/compiler_opts.cmake)
 
 include(.CMake/alg_support.cmake)
@@ -126,7 +122,7 @@ set(PUBLIC_HEADERS ${PUBLIC_HEADERS} ${PROJECT_BINARY_DIR}/include/oqs/oqsconfig
 include_directories(${PROJECT_BINARY_DIR}/include)
 add_subdirectory(src)
 
-if(NOT CMAKE_BUILD_TYPE STREQUAL "Dependency")
+if(OQS_BUILD_TESTS)
     add_subdirectory(tests)
 
     find_package(Doxygen)

diff --git a/README.md b/README.md
@@ -103,7 +103,9 @@ Various options can be passed to `cmake` to customize the build. Some of them in
 - `-DBUILD_SHARED_LIBS=<val>`: `<val>` can be `ON` or `OFF`; when `ON`, CMake generates instructions for building a shared library, otherwise it generates instructions for building a static library.
 - `OPENSSL_ROOT_DIR=<dir>`: `<dir>` specifies the directory in which CMake will look for OpenSSL.
 
-All supported options are listed in the `.CMake/alg-support.cmake` file, and can be viewed by running `cmake -LAH ..` in the `build` directory. All subsequent instructions assume we are in `build`.
+All supported options are listed in the `.CMake/alg-support.cmake` file, and can be viewed by running `cmake -LAH ..` in the `build` directory. They are also listed and explained in [the wiki](https://github.com/open-quantum-safe/liboqs/wiki/Customizing-liboqs).
+
+The following instructions assume we are in `build`.
 
 3. The main build result is `lib/liboqs.a`, a static library. The public headers are located in the `include` directory. There are also a variety of programs built under the `tests` directory:
 

diff --git a/appveyor_build.bat b/appveyor_build.bat
@@ -2,13 +2,14 @@
 IF %COMPILER%==msys2 (
     @echo on
     SET "PATH=C:\msys64\mingw64\bin;%PATH%"
-    bash -lc "cd ${APPVEYOR_BUILD_FOLDER} && mkdir build && cd build && cmake .. -GNinja -DCMAKE_BUILD_TYPE=Optimized -DBUILD_SHARED_LIBS=%BUILD_SHARED% && ninja"
+    bash -lc "cd ${APPVEYOR_BUILD_FOLDER} && mkdir build && cd build && cmake .. -GNinja -DOQS_ENABLE_SIG_SPHINCS=OFF -DOQS_ENABLE_SIG_RAINBOW=OFF -DBUILD_SHARED_LIBS=%BUILD_SHARED% && ninja"
 )
 IF %COMPILER%==msvc2019 (
     @echo on
     CALL "C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvars64.bat"
     mkdir build
     cd build
-    cmake -DCMAKE_BUILD_TYPE=Optimized .. -GNinja -DBUILD_SHARED_LIBS=%BUILD_SHARED%
+    REM SPHINCS and Rainbow cause a big slowdown in the tests
+    cmake .. -GNinja -DOQS_ENABLE_SIG_SPHINCS=OFF -DOQS_ENABLE_SIG_RAINBOW=OFF -DBUILD_SHARED_LIBS=%BUILD_SHARED%
     ninja
 )
diff --git a/scripts/copy_from_pqclean/copy_from_pqclean.py b/scripts/copy_from_pqclean/copy_from_pqclean.py
@@ -67,6 +67,13 @@ def load_instructions():
         family['family'] = family['name']
         for scheme in family['schemes']:
             scheme['metadata'] = yaml.safe_load(file_get_contents(os.path.join(os.environ['PQCLEAN_DIR'], 'crypto_sign', scheme['pqclean_scheme'], 'META.yml')))
+            # This is a temporary hack to work around the fact that
+            # the PQClean's META.ymls for the Dilithium AVX2 variants
+            # are not properly specified.
+            if scheme['pretty_name_full'].startswith('DILITHIUM_'):
+                scheme['metadata']['implementations'][1]['supported_platforms'][0]['operating_systems'] = ['Linux']
+                scheme['metadata']['implementations'][1]['supported_platforms'][0]['required_flags'] = ['avx2', 'bmi', 'popcnt']
+
             scheme['metadata']['euf_cma'] = 'true'
             scheme['pqclean_scheme_c'] = scheme['pqclean_scheme'].replace('-', '')
             scheme['scheme_c'] = scheme['scheme'].replace('-', '')

diff --git a/scripts/copy_from_pqclean/copy_from_pqclean.yml b/scripts/copy_from_pqclean/copy_from_pqclean.yml
@@ -224,58 +224,58 @@ sigs:
     name: dilithium
     default_implementation: clean
     schemes:
-      - 
+      -
         scheme: "2"
         pqclean_scheme: dilithium2
         pretty_name_full: DILITHIUM_2
         signed_msg_order: sig_then_msg
-      - 
+      -
         scheme: "3"
         pqclean_scheme: dilithium3
         pretty_name_full: DILITHIUM_3
         signed_msg_order: sig_then_msg
-      - 
+      -
         scheme: "4"
         pqclean_scheme: dilithium4
         pretty_name_full: DILITHIUM_4
         signed_msg_order: sig_then_msg
-  - 
+  -
     name: falcon
     default_implementation: clean
     schemes:
-      - 
+      -
         scheme: "512"
         pqclean_scheme: falcon-512
         pretty_name_full: Falcon-512
         implementation: clean
         sources: ['codec.c', 'common.c', 'fft.c', 'fpr.c', 'keygen.c', 'pqclean.c', 'rng.c', 'sign.c', 'vrfy.c']
         signed_msg_order: falcon
-      - 
+      -
         scheme: "1024"
         pqclean_scheme: falcon-1024
         pretty_name_full: Falcon-1024
         implementation: clean
         sources: ['codec.c', 'common.c', 'fft.c', 'fpr.c', 'keygen.c', 'pqclean.c', 'rng.c', 'sign.c', 'vrfy.c']
         signed_msg_order: falcon
-  - 
+  -
     name: mqdss
     default_implementation: clean
     schemes:
-      - 
+      -
         scheme: "31_48"
         pqclean_scheme: mqdss-48
         pretty_name_full: MQDSS-31-48
         implementation: clean
         sources: ['gf31.c', 'mq.c', 'sign.c']
         signed_msg_order: sig_then_msg
-      - 
+      -
         scheme: "31_64"
         pqclean_scheme: mqdss-64
         pretty_name_full: MQDSS-31-64
         implementation: clean
         sources: ['gf31.c', 'mq.c', 'sign.c']
         signed_msg_order: sig_then_msg
-  - 
+  -
     name: rainbow
     default_implementation: clean
     schemes: