node: constexpr snapshot configs (erigontech#1667)

cmake/copyright.cmake

@@ -56,7 +56,7 @@ list(FILTER SRC EXCLUDE REGEX [[silkworm/core/crypto/kzg\.cpp$]])
 list(FILTER SRC EXCLUDE REGEX [[silkworm/infra/concurrency/thread_pool\.hpp$]])
 list(FILTER SRC EXCLUDE REGEX [[silkworm/interfaces/]])
 list(FILTER SRC EXCLUDE REGEX [[silkworm/node/common/preverified_hashes_[a-z]+\.cpp$]])
-list(FILTER SRC EXCLUDE REGEX [[silkworm/node/snapshot/config/[a-z_]+\.cpp$]])
+list(FILTER SRC EXCLUDE REGEX [[silkworm/node/snapshot/config/[a-z_]+\.hpp$]])
 
 foreach(F IN LISTS SRC)
   check("${F}")

cmake/format.cmake

@@ -42,7 +42,5 @@ list(FILTER SRC EXCLUDE REGEX "silkworm/interfaces/")
 list(FILTER SRC EXCLUDE REGEX "silkworm/core/chain/genesis_[a-z]+.cpp\$")
 list(FILTER SRC EXCLUDE REGEX "silkworm/core/chain/dao.hpp$")
 list(FILTER SRC EXCLUDE REGEX "silkworm/node/common/preverified_hashes_[a-z]+.cpp\$")
-list(FILTER SRC EXCLUDE REGEX "silkworm/node/snapshot/config/[a-z_]+.cpp\$")
-list(FILTER SRC EXCLUDE REGEX "silkworm/node/snapshot/toml.hpp$$")
 
 execute_process(COMMAND ${CLANG_FORMAT} -style=file -i ${SRC})

cmd/dev/CMakeLists.txt

@@ -20,6 +20,7 @@ find_package(CLI11 REQUIRED)
 find_package(gRPC REQUIRED)
 find_package(magic_enum REQUIRED)
 find_package(Protobuf REQUIRED)
+find_package(tomlplusplus REQUIRED)
 
 # cmake-format: off
 set(BACKEND_KV_SERVER_SRC
@@ -53,6 +54,9 @@ target_link_libraries(check_senders PRIVATE silkworm_node cmd_common)
 add_executable(check_tx_lookup check_tx_lookup.cpp)
 target_link_libraries(check_tx_lookup PRIVATE silkworm_node CLI11::CLI11)
 
+add_executable(embed_toml embed_toml.cpp)
+target_link_libraries(embed_toml PRIVATE CLI11::CLI11 tomlplusplus::tomlplusplus)
+
 add_executable(genesistool genesistool.cpp)
 target_link_libraries(genesistool PRIVATE CLI11::CLI11 Boost::headers)
 

cmd/dev/embed.cpp → cmd/dev/embed_toml.cpp

@@ -14,25 +14,38 @@
    limitations under the License.
 */
 
-#include <cassert>
+#include <cctype>
 #include <filesystem>
 #include <iostream>
 #include <regex>
 #include <string>
+#include <string_view>
 
 #include <CLI/CLI.hpp>
+#include <toml.hpp>
 
 namespace fs = std::filesystem;
 
+std::string snake_to_camel(std::string_view snake) {
+    std::string camel;
+    camel += static_cast<char>(std::toupper(static_cast<unsigned char>(snake[0])));
+    for (std::size_t i = 1; i < snake.length(); ++i) {
+        if (snake[i] == '_' && (i + 1) < snake.length()) {
+            camel += static_cast<char>(std::toupper(static_cast<unsigned char>(snake[++i])));
+        } else {
+            camel += snake[i];
+        }
+    }
+    return camel;
+}
+
 int main(int argc, char* argv[]) {
     // Define the command line options
     CLI::App app{"Embed files as binary data"};
 
-    std::string input_folder, output_folder, extension, suffix;
+    std::string input_folder, output_folder;
     app.add_option("-i,--input", input_folder, "input folder where to look for files")->required();
-    app.add_option("-o,--output", output_folder, "output folder where to place generated .cpp files")->required();
-    app.add_option("-e,--extension", extension, "extension of input files to look for")->required();
-    app.add_option("-s,--suffix", suffix, "suffix for data structures in generated .cpp files")->required();
+    app.add_option("-o,--output", output_folder, "output folder where to place generated .hpp files")->required();
 
     // Parse the command line arguments
     try {
@@ -51,41 +64,32 @@ int main(int argc, char* argv[]) {
             continue;
         }
         // Match only required extension
-        if (entry_path.extension() == extension) {
-            // Open the input .toml file
-            std::ifstream input{entry_path.string(), std::ios::binary};
-            if (!input.good()) continue;
-            size_t input_size{fs::file_size(entry_path)};
-            if (!input_size) continue;
-
-            // Read the entire file in memory (this *must* be OK because we're embedding the file)
-            std::vector<char> bytes(input_size);
-            input.read(bytes.data(), static_cast<std::streamsize>(input_size));
-            assert(static_cast<size_t>(input.gcount()) == input_size);
-
+        if (entry_path.extension() == ".toml") {
             std::cout << "Processing TOML file: " << entry_path.string() << "\n";
+            const auto table = toml::parse_file(entry_path.string());
 
-            // Open the output .cpp file
+            // Open the output .hpp file
             fs::path entry_filename = entry.path().stem();
             entry_filename.replace_filename(std::regex_replace(entry_filename.string(), hyphen_re, "_"));
-            fs::path output_path = fs::path{output_folder} / entry_filename.replace_extension(".cpp");
+            fs::path output_path = fs::path{output_folder} / entry_filename.replace_extension(".hpp");
             std::ofstream output{output_path};
 
-            // Write bytes from the input file to the output file as built-in array of characters
-            std::string output_file_name = entry_filename.stem().string();
-
-            output << "/* Generated from " << entry.path().filename().string() << " using Silkworm embed */\n\n";
-            output << "#include <cstddef>\n\n";
-            output << "static const char " << output_file_name << "_data[] = {\n";
-            auto count{1u};
-            for (auto& b : bytes) {
-                output << "0x" << std::setfill('0') << std::hex << std::setw(2) << static_cast<int>(b)
-                       << ((count == bytes.size()) ? "" : ",") << ((count % 16 == 0) ? "\n" : " ");
-                ++count;
+            // Write the snapshots as a constexpr std::array
+            std::string snapshot_name = snake_to_camel(entry_filename.stem().string());
+
+            output << "/* Generated from " << entry.path().filename().string() << " using Silkworm embed_toml */\n\n";
+            output << "#pragma once\n\n";
+            output << "#include <array>\n\n";
+            output << "#include <silkworm/node/snapshot/entry.hpp>\n\n";
+            output << "namespace silkworm::snapshot {\n\n";
+            output << "inline constexpr std::array<Entry, " << table.size() << "> k" << snapshot_name << "Snapshots{\n";
+            for (auto&& [key, value] : table) {
+                std::string key_str{key.begin(), key.end()};
+                std::string val_str{value.as_string()->get()};
+                output << "    Entry{\"" << key_str << "\"sv, \"" << val_str << "\"sv},\n";
             }
             output << "};\n\n";
-            output << "const char* " << output_file_name << "_" << suffix << "_data() { return &" << output_file_name << "_data[0]; }\n";
-            output << "size_t " << output_file_name << "_" << suffix << "_size() { return sizeof(" << output_file_name << "_data); }\n";
+            output << "}\n";
         }
     }
 

docs/CONTRIBUTING.md

@@ -10,6 +10,8 @@ We follow the [C++ Core Guidelines][cpp-core-guidelines] as much as possible.
 
 ## Style Guide
 
+Every file should contain the Apache license boilerplate. Use `make lint` to check the proper license formatting.
+
 We adhere to [Google's C++ Style Guide][cpp-google-style-guide] with the following differences:
 
 * `snake_case()` for function names.
@@ -18,9 +20,9 @@ We adhere to [Google's C++ Style Guide][cpp-google-style-guide] with the followi
 * Exceptions are allowed outside the `core` library.
 * User-defined literals are allowed.
 * Maximum line length is 120, indentation is 4 spaces. Use `make fmt` to reformat according to the code style.
-* Add Apache copyright banners. Use `make lint` to check the proper banner style.
 * Use `#pragma once` in the headers instead of the classic `#ifndef` guards.
 * `template <Concept T>` syntax is allowed.
+* `<filesystem>` is allowed.
 
 In addition to the [Boost libraries permitted in the style guide](https://google.github.io/styleguide/cppguide.html#Boost), we allow:
 * Algorithm

silkworm/core/chain/config.hpp

@@ -149,15 +149,6 @@ SILKWORM_CONSTINIT extern const ChainConfig kPolygonConfig;
 inline constexpr evmc::bytes32 kMumbaiGenesisHash{0x7b66506a9ebdbf30d32b43c5f15a3b1216269a1ec3a75aa3182b86176a2b1ca7_bytes32};
 SILKWORM_CONSTINIT extern const ChainConfig kMumbaiConfig;
 
-//! \brief Known chain IDs mapped to their respective chain configs
-inline constexpr SmallMap<ChainId, const ChainConfig*> kKnownChainConfigs{
-    {1, &kMainnetConfig},
-    {5, &kGoerliConfig},
-    {11155111, &kSepoliaConfig},
-    {137, &kPolygonConfig},
-    {80001, &kMumbaiConfig},
-};
-
 //! \brief Known chain names mapped to their respective chain IDs
 inline constexpr SmallMap<std::string_view, ChainId> kKnownChainNameToId{
     {"mainnet"sv, 1},
@@ -167,4 +158,13 @@ inline constexpr SmallMap<std::string_view, ChainId> kKnownChainNameToId{
     {"mumbai"sv, 80001},
 };
 
+//! \brief Known chain IDs mapped to their respective chain configs
+inline constexpr SmallMap<ChainId, const ChainConfig*> kKnownChainConfigs{
+    {*kKnownChainNameToId.find("mainnet"sv), &kMainnetConfig},
+    {*kKnownChainNameToId.find("goerli"sv), &kGoerliConfig},
+    {*kKnownChainNameToId.find("sepolia"sv), &kSepoliaConfig},
+    {*kKnownChainNameToId.find("polygon"sv), &kPolygonConfig},
+    {*kKnownChainNameToId.find("mumbai"sv), &kMumbaiConfig},
+};
+
 }  // namespace silkworm

silkworm/core/common/small_map.hpp

@@ -67,8 +67,6 @@ class SmallMap {
         return *this;
     }
 
-    constexpr bool operator==(const SmallMap&) const = default;
-
     [[nodiscard]] constexpr bool empty() const noexcept {
         return size_ == 0;
     }
@@ -114,4 +112,9 @@ class SmallMap {
     std::size_t size_{0};
 };
 
+template <std::totally_ordered Key, std::equality_comparable T>
+constexpr bool operator==(const SmallMap<Key, T>& a, const SmallMap<Key, T>& b) {
+    return std::equal(a.begin(), a.end(), b.begin(), b.end());
+}
+
 }  // namespace silkworm

silkworm/node/CMakeLists.txt

@@ -22,9 +22,6 @@ find_package(gRPC REQUIRED)
 find_package(magic_enum REQUIRED)
 find_package(Protobuf REQUIRED)
 find_package(roaring REQUIRED)
-find_package(tomlplusplus REQUIRED)
-
-add_subdirectory(snapshot/config)
 
 file(
   GLOB_RECURSE
@@ -67,8 +64,6 @@ set(SILKWORM_NODE_PRIVATE_LIBS
     evmone
     magic_enum::magic_enum
     silkworm_interfaces
-    silkworm_node_snapshot_config
-    tomlplusplus::tomlplusplus
 )
 # cmake-format: on
 if(MSVC)

silkworm/node/bittorrent/client.cpp

@@ -82,7 +82,7 @@ BitTorrentClient::~BitTorrentClient() {
     stop();
 }
 
-void BitTorrentClient::add_info_hash(const std::string& name, const std::string& info_hash) {
+void BitTorrentClient::add_info_hash(std::string_view name, std::string_view info_hash) {
     lt::sha1_hash sha1_info_hash;
     lt::aux::from_hex(info_hash, sha1_info_hash.data());
     lt::info_hash_t info_hashes{sha1_info_hash};

silkworm/node/bittorrent/client.hpp

@@ -25,7 +25,7 @@
 #include <optional>
 #include <set>
 #include <span>
-#include <string>
+#include <string_view>
 #include <thread>
 #include <vector>
 
@@ -70,7 +70,7 @@ class BitTorrentClient {
     [[nodiscard]] const std::vector<lt::stats_metric>& stats_metrics() const { return stats_metrics_; }
 
     //! Add the specified info hash to the download list
-    void add_info_hash(const std::string& name, const std::string& info_hash);
+    void add_info_hash(std::string_view name, std::string_view info_hash);
 
     //! Run the client execution loop until it is stopped or has finished downloading and seeding is not required
     void execute_loop();

silkworm/node/snapshot/config.cpp

@@ -16,68 +16,30 @@
 
 #include "config.hpp"
 
-#include <algorithm>
 #include <map>
 #include <utility>
 
-#include <toml.hpp>
-
-#include <silkworm/core/chain/config.hpp>
 #include <silkworm/infra/common/log.hpp>
-#include <silkworm/node/snapshot/config_toml.hpp>
 #include <silkworm/node/snapshot/path.hpp>
 
 namespace silkworm::snapshot {
 
-PreverifiedList from_toml(std::string_view preverified_toml_doc) {
-    const auto table = toml::parse(preverified_toml_doc);
-    SILK_DEBUG << "from_toml #size: " << preverified_toml_doc.size() << " #snapshots: " << table.size();
-
-    PreverifiedList preverified;
-    preverified.reserve(table.size());
-    for (auto&& [key, value] : table) {
-        preverified.emplace_back(PreverifiedEntry{{key.begin(), key.end()}, value.as_string()->get()});
-    }
-    std::sort(preverified.begin(), preverified.end(), [](auto& p1, auto& p2) { return p1.file_name < p2.file_name; });
-
-    std::for_each(preverified.begin(), preverified.end(), [](auto& p) {
-        SILK_DEBUG << "name: " << p.file_name << " hash: " << p.torrent_hash;
-    });
-
-    return preverified;
-}
-
-const Config Config::kGoerliSnapshotConfig{from_toml({goerli_toml_data(), goerli_toml_size()})};
-const Config Config::kMainnetSnapshotConfig{from_toml({mainnet_toml_data(), mainnet_toml_size()})};
-const Config Config::kSepoliaSnapshotConfig{from_toml({sepolia_toml_data(), sepolia_toml_size()})};
-
-const std::map<uint64_t, const Config*> Config::kKnownSnapshotConfigs{
-    {kGoerliConfig.chain_id, &kGoerliSnapshotConfig},
-    {kMainnetConfig.chain_id, &kMainnetSnapshotConfig},
-    {kSepoliaConfig.chain_id, &kSepoliaSnapshotConfig},
-    // TODO(yperbasis): add Polygon
-};
-
-struct NullDeleter {
-    void operator()(void const*) const {}
-};
-
-std::shared_ptr<const Config> Config::lookup_known_config(uint64_t chain_id, const std::vector<std::string>& whitelist) {
-    const auto config_it = kKnownSnapshotConfigs.find(chain_id);
-    if (config_it == kKnownSnapshotConfigs.end()) {
-        return std::make_shared<Config>(PreverifiedList{});
+Config Config::lookup_known_config(ChainId chain_id, const std::vector<std::string>& whitelist) {
+    const auto config = kKnownSnapshotConfigs.find(chain_id);
+    if (!config) {
+        return Config{PreverifiedList{}};
     }
     if (whitelist.empty()) {
-        return std::shared_ptr<const Config>{config_it->second, NullDeleter{}};
+        return Config{PreverifiedList(config->begin(), config->end())};
     }
 
     PreverifiedList filtered_preverified;
-    for (const auto& preverified_entry : config_it->second->preverified_snapshots()) {
+    for (const auto& preverified_entry : *config) {
         if (std::find(whitelist.cbegin(), whitelist.cend(), preverified_entry.file_name) != whitelist.cend()) {
             filtered_preverified.push_back(preverified_entry);
         }
     }
-    return std::make_shared<Config>(filtered_preverified);
+    return Config{filtered_preverified};
 }
 
 Config::Config(PreverifiedList preverified_snapshots)

silkworm/node/snapshot/config.hpp

@@ -16,45 +16,45 @@
 
 #pragma once
 
-#include <map>
-#include <memory>
+#include <span>
 #include <string>
-#include <string_view>
 #include <vector>
 
-#include <silkworm/core/common/base.hpp>
+#include <silkworm/core/chain/config.hpp>
+#include <silkworm/core/common/small_map.hpp>
+#include <silkworm/node/snapshot/config/bor_mainnet.hpp>
+#include <silkworm/node/snapshot/config/goerli.hpp>
+#include <silkworm/node/snapshot/config/mainnet.hpp>
+#include <silkworm/node/snapshot/config/mumbai.hpp>
+#include <silkworm/node/snapshot/config/sepolia.hpp>
+#include <silkworm/node/snapshot/entry.hpp>
 
 namespace silkworm::snapshot {
 
-struct PreverifiedEntry {
-    std::string file_name;
-    std::string torrent_hash;
-};
-
-using PreverifiedList = std::vector<PreverifiedEntry>;
-
-PreverifiedList from_toml(std::string_view preverified_toml_doc);
+using PreverifiedList = std::vector<Entry>;
 
 class Config {
   public:
-    static std::shared_ptr<const Config> lookup_known_config(uint64_t chain_id, const std::vector<std::string>& whitelist);
+    static Config lookup_known_config(ChainId chain_id, const std::vector<std::string>& whitelist);
 
     explicit Config(PreverifiedList preverified_snapshots);
 
     [[nodiscard]] const PreverifiedList& preverified_snapshots() const { return preverified_snapshots_; }
     [[nodiscard]] BlockNum max_block_number() const { return max_block_number_; }
 
   private:
-    static const Config kGoerliSnapshotConfig;
-    static const Config kMainnetSnapshotConfig;
-    static const Config kSepoliaSnapshotConfig;
-
-    static const std::map<uint64_t, const Config*> kKnownSnapshotConfigs;
-
     BlockNum compute_max_block();
 
     PreverifiedList preverified_snapshots_;
     BlockNum max_block_number_;
 };
 
+inline constexpr SmallMap<ChainId, std::span<const Entry>> kKnownSnapshotConfigs{
+    {*kKnownChainNameToId.find("mainnet"sv), {kMainnetSnapshots.data(), kMainnetSnapshots.size()}},
+    {*kKnownChainNameToId.find("goerli"sv), {kGoerliSnapshots.data(), kGoerliSnapshots.size()}},
+    {*kKnownChainNameToId.find("sepolia"sv), {kSepoliaSnapshots.data(), kSepoliaSnapshots.size()}},
+    {*kKnownChainNameToId.find("polygon"sv), {kBorMainnetSnapshots.data(), kBorMainnetSnapshots.size()}},
+    {*kKnownChainNameToId.find("mumbai"sv), {kMumbaiSnapshots.data(), kMumbaiSnapshots.size()}},
+};
+
 }  // namespace silkworm::snapshot