diff --git a/mcrouter/CarbonRouterInstanceBase.cpp b/mcrouter/CarbonRouterInstanceBase.cpp index 2290eafd6..7af131f5d 100644 --- a/mcrouter/CarbonRouterInstanceBase.cpp +++ b/mcrouter/CarbonRouterInstanceBase.cpp @@ -79,6 +79,27 @@ CarbonRouterInstanceBase::CarbonRouterInstanceBase(McrouterOptions inputOptions) statsLogger->makeQueueSizeUnlimited(); } } + + if (!opts_.pool_stats_config_file.empty()) { + try { + folly::dynamic poolStatJson = + readStaticJsonFile(opts_.pool_stats_config_file); + if (poolStatJson != nullptr) { + auto jStatsEnabledPools = poolStatJson.get_ptr("stats_enabled_pools"); + if (jStatsEnabledPools && jStatsEnabledPools->isArray()) { + for (const auto& it : *jStatsEnabledPools) { + if (it.isString()) { + statsEnabledPools_.push_back(it.asString()); + } else { + LOG(ERROR) << "Pool Name is not a string"; + } + } + } + } + } catch (const std::exception& e) { + LOG(ERROR) << "Invalid pool-stats-config-file : " << e.what(); + } + } } void CarbonRouterInstanceBase::setUpCompressionDictionaries( @@ -170,6 +191,30 @@ CarbonRouterInstanceBase::functionScheduler() { return globalFunctionScheduler.try_get(); } +int32_t CarbonRouterInstanceBase::getStatsEnabledPoolIndex( + const folly::StringPiece poolName) const { + if (statsEnabledPools_.size() == 0) { + return -1; + } + + int longestPrefixMatchIndex = -1; + // Do sequential search for longest matching name. Since this is done + // only once during the initialization and the size of the array is + // expected to be small, linear search should be OK. + for (size_t i = 0; i < statsEnabledPools_.size(); i++) { + if (poolName.subpiece(0, statsEnabledPools_[i].length()) + .compare(statsEnabledPools_[i]) == 0) { + if ((longestPrefixMatchIndex == -1) || + (statsEnabledPools_[longestPrefixMatchIndex].length() < + statsEnabledPools_[i].length())) { + longestPrefixMatchIndex = i; + } + } + } + + return longestPrefixMatchIndex; +} + } // mcrouter } // memcache } // facebook diff --git a/mcrouter/CarbonRouterInstanceBase.h b/mcrouter/CarbonRouterInstanceBase.h index 887ba03dc..a33503056 100644 --- a/mcrouter/CarbonRouterInstanceBase.h +++ b/mcrouter/CarbonRouterInstanceBase.h @@ -22,6 +22,7 @@ #include "mcrouter/ConfigApi.h" #include "mcrouter/LeaseTokenMap.h" #include "mcrouter/Observable.h" +#include "mcrouter/PoolStats.h" #include "mcrouter/TkoTracker.h" #include "mcrouter/options.h" @@ -145,6 +146,23 @@ class CarbonRouterInstanceBase { return disableRxmitReconnection_; } + /** + * This function finds the index of poolName in the statsEnabledPools_ + * sorted array by doing binary search. If exact match is not found, + * index with maximum prefix match is returned. + * + * @return index of the pool in the statsEnabledPools_ vector + * -1 if not found + */ + int32_t getStatsEnabledPoolIndex(folly::StringPiece poolName) const; + + /** + * @return reference to the statsEnabledPools_ vector + */ + const std::vector& getStatsEnabledPools() const { + return statsEnabledPools_; + } + /** * @return nullptr if index is >= opts.num_proxies, * pointer to the proxy otherwise. @@ -224,6 +242,8 @@ class CarbonRouterInstanceBase { // Name of the stats update function registered with the function scheduler. const std::string statsUpdateFunctionHandle_; + std::vector statsEnabledPools_; + // Aggregates stats for all associated proxies. Should be called periodically. void updateStats(); }; diff --git a/mcrouter/McrouterLogger.cpp b/mcrouter/McrouterLogger.cpp index 81fded021..d662cebd3 100644 --- a/mcrouter/McrouterLogger.cpp +++ b/mcrouter/McrouterLogger.cpp @@ -208,6 +208,7 @@ void McrouterLogger::log() { std::vector stats(num_stats); prepare_stats(router_, stats.data()); + append_pool_stats(router_, stats); folly::dynamic requestStats(folly::dynamic::object()); for (size_t i = 0; i < router_.opts().num_proxies; ++i) { diff --git a/mcrouter/PoolStats.h b/mcrouter/PoolStats.h new file mode 100644 index 000000000..856061abe --- /dev/null +++ b/mcrouter/PoolStats.h @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2017-present, Facebook, Inc. + * + * This source code is licensed under the MIT license found in the LICENSE + * file in the root directory of this source tree. + * + */ +#pragma once + +#include + +#include "mcrouter/stats.h" + +namespace facebook { +namespace memcache { +namespace mcrouter { + +class PoolStats { + public: + PoolStats(folly::StringPiece poolName) + : requestsCountStatName_( + folly::to(poolName, ".requests.sum")), + finalResultErrorStatName_( + folly::to(poolName, ".final_result_error.sum")) { + initStat(requestCountStat_, requestsCountStatName_); + initStat(finalResultErrorStat_, finalResultErrorStatName_); + } + + std::vector getStats() const { + return {requestCountStat_, finalResultErrorStat_}; + } + + void incrementRequestCount(uint64_t amount = 1) { + requestCountStat_.data.uint64 += amount; + } + + void incrementFinalResultErrorCount(uint64_t amount = 1) { + finalResultErrorStat_.data.uint64 += amount; + } + + private: + void initStat(stat_t& stat, folly::StringPiece name) const { + stat.name = name; + stat.group = ods_stats | count_stats; + stat.type = stat_uint64; + stat.aggregate = 0; + stat.data.uint64 = 0; + } + + const std::string requestsCountStatName_; + const std::string finalResultErrorStatName_; + stat_t requestCountStat_; + stat_t finalResultErrorStat_; +}; + +} // namespace mcrouter +} // namespace memcache +} // namespace facebook diff --git a/mcrouter/ProxyBase-inl.h b/mcrouter/ProxyBase-inl.h index e3327a047..a9e9a30a8 100644 --- a/mcrouter/ProxyBase-inl.h +++ b/mcrouter/ProxyBase-inl.h @@ -32,6 +32,7 @@ ProxyBase::ProxyBase( std::make_unique(), getFiberManagerOptions(router_.opts())), asyncLog_(router_.opts()), + stats_(router_.getStatsEnabledPools()), flushCallback_(*this), destinationMap_(std::make_unique(this)) { // Setup a full random seed sequence diff --git a/mcrouter/ProxyRequestContext.h b/mcrouter/ProxyRequestContext.h index 5a5050f0b..3d104ff31 100644 --- a/mcrouter/ProxyRequestContext.h +++ b/mcrouter/ProxyRequestContext.h @@ -87,6 +87,12 @@ class ProxyRequestContext { return failoverDisabled_; } + void setPoolStatsIndex(int32_t index) { + if (poolStatIndex_ == -1) { + poolStatIndex_ = index; + } + } + ProxyRequestPriority priority() const { return priority_; } @@ -136,6 +142,7 @@ class ProxyRequestContext { */ void (*reqComplete_)(ProxyRequestContext& preq){nullptr}; mc_res_t finalResult_{mc_res_unknown}; + int32_t poolStatIndex_{-1}; bool replied_{false}; ProxyRequestContext(ProxyBase& pr, ProxyRequestPriority priority__); diff --git a/mcrouter/ProxyRequestContextTyped.h b/mcrouter/ProxyRequestContextTyped.h index 234b8f18f..caa90b0a0 100644 --- a/mcrouter/ProxyRequestContextTyped.h +++ b/mcrouter/ProxyRequestContextTyped.h @@ -91,6 +91,8 @@ class ProxyRequestContextWithInfo : public ProxyRequestContext { } ~ProxyRequestContextWithInfo() override { + proxy_.stats().incrementPoolStats( + poolStatIndex_, 1, isErrorResult(finalResult_) ? 1 : 0); if (reqComplete_) { fiber_local::runWithoutLocals( [this]() { reqComplete_(*this); }); diff --git a/mcrouter/ProxyStats.cpp b/mcrouter/ProxyStats.cpp index d34af56df..518f973c0 100644 --- a/mcrouter/ProxyStats.cpp +++ b/mcrouter/ProxyStats.cpp @@ -11,8 +11,12 @@ namespace facebook { namespace memcache { namespace mcrouter { -ProxyStats::ProxyStats() { +ProxyStats::ProxyStats(const std::vector& statsEnabledPools) { init_stats(stats_); + poolStats_.reserve(statsEnabledPools.size()); + for (const auto& curPoolName : statsEnabledPools) { + poolStats_.emplace_back(curPoolName); + } } void ProxyStats::aggregate(size_t statId) { diff --git a/mcrouter/ProxyStats.h b/mcrouter/ProxyStats.h index 5d26ff4d8..653f737e1 100644 --- a/mcrouter/ProxyStats.h +++ b/mcrouter/ProxyStats.h @@ -9,7 +9,10 @@ #include +#include + #include "mcrouter/ExponentialSmoothData.h" +#include "mcrouter/PoolStats.h" #include "mcrouter/stats.h" namespace facebook { @@ -18,7 +21,7 @@ namespace mcrouter { class ProxyStats { public: - ProxyStats(); + explicit ProxyStats(const std::vector& statsEnabledPools); /** * Aggregate proxy stat with the given index. @@ -129,9 +132,32 @@ class ProxyStats { return stats_[statId]; } + folly::StringKeyedUnorderedMap getAggregatedPoolStatsMap() const { + folly::StringKeyedUnorderedMap poolStatsMap; + for (const auto& poolStats : poolStats_) { + for (const auto& stat : poolStats.getStats()) { + poolStatsMap.emplace(stat.name, stat); + } + } + return poolStatsMap; + } + + void incrementPoolStats( + int32_t idx, + uint64_t requestCount, + uint64_t finalErrorResultCount) { + if (idx < 0 || static_cast(idx) >= poolStats_.size()) { + return; + } + poolStats_[idx].incrementRequestCount(requestCount); + poolStats_[idx].incrementFinalResultErrorCount(finalErrorResultCount); + } + private: mutable std::mutex mutex_; stat_t stats_[num_stats]{}; + // vector of the PoolStats + std::vector poolStats_; ExponentialSmoothData<64> durationUs_; diff --git a/mcrouter/mcrouter_config.cpp b/mcrouter/mcrouter_config.cpp index 46fb8315b..c5458c74a 100644 --- a/mcrouter/mcrouter_config.cpp +++ b/mcrouter/mcrouter_config.cpp @@ -7,7 +7,9 @@ */ #include +#include #include +#include #include "mcrouter/CarbonRouterInstanceBase.h" #include "mcrouter/McrouterLogger.h" @@ -123,10 +125,20 @@ std::string getBinPath(folly::StringPiece name) { std::string getDefaultPemCertPath() { return ""; } + std::string getDefaultPemCertKey() { return ""; } +folly::dynamic readStaticJsonFile(folly::StringPiece file) { + std::string contents; + if (!folly::readFile(file.str().c_str(), contents)) { + LOG(ERROR) << "Failed to open pool-stats-config-file " << file.str(); + return nullptr; + } + return folly::parseJson(contents); +} + } // mcrouter } // memcache } // facebook diff --git a/mcrouter/mcrouter_config.h b/mcrouter/mcrouter_config.h index 42e32cf77..3c0fcc379 100644 --- a/mcrouter/mcrouter_config.h +++ b/mcrouter/mcrouter_config.h @@ -166,6 +166,19 @@ std::string getBinPath(folly::StringPiece name); std::string getDefaultPemCertPath(); std::string getDefaultPemCertKey(); +/** + * Reads a static json file. Do not monitor for changes. + * May throw if there's an error while parsing file contents. + * + * @params file The path of the json file. + * + * @return folly::dynamic with the contents of the file. + * nullptr if cannot open/read the file + * may throw exception if invalid json + * + */ +folly::dynamic readStaticJsonFile(folly::StringPiece file); + #ifndef MCROUTER_PACKAGE_STRING #define MCROUTER_PACKAGE_STRING "1.0.0 mcrouter" #endif diff --git a/mcrouter/mcrouter_options_list.h b/mcrouter/mcrouter_options_list.h index d66a069ba..d849da3d2 100644 --- a/mcrouter/mcrouter_options_list.h +++ b/mcrouter/mcrouter_options_list.h @@ -424,6 +424,13 @@ MCROUTER_OPTION_STRING( "DEPRECATED. Load configuration from file. This option has no effect if" " --config option is used.") +MCROUTER_OPTION_STRING( + pool_stats_config_file, + "", + "pool-stats-config-file", + no_short, + "File containing stats enabled pool names.") + MCROUTER_OPTION_STRING( config_str, "", diff --git a/mcrouter/routes/DestinationRoute.h b/mcrouter/routes/DestinationRoute.h index 13b993447..b16d1535e 100644 --- a/mcrouter/routes/DestinationRoute.h +++ b/mcrouter/routes/DestinationRoute.h @@ -65,11 +65,13 @@ class DestinationRoute { std::shared_ptr destination, folly::StringPiece poolName, size_t indexInPool, + int32_t poolStatIdx, std::chrono::milliseconds timeout, bool keepRoutingPrefix) : destination_(std::move(destination)), poolName_(poolName), indexInPool_(indexInPool), + poolStatIndex_(poolStatIdx), timeout_(timeout), keepRoutingPrefix_(keepRoutingPrefix) {} @@ -105,6 +107,7 @@ class DestinationRoute { const std::shared_ptr destination_; const folly::StringPiece poolName_; const size_t indexInPool_; + const int poolStatIndex_{-1}; const std::chrono::milliseconds timeout_; size_t pendingShadowReqs_{0}; const bool keepRoutingPrefix_; @@ -127,6 +130,9 @@ class DestinationRoute { return constructAndLog(req, *ctx, BusyReply); } + if (poolStatIndex_ >= 0) { + ctx->setPoolStatsIndex(poolStatIndex_); + } auto requestClass = fiber_local::getRequestClass(); if (ctx->recording()) { bool isShadow = requestClass.is(RequestClass::kShadow); @@ -257,12 +263,14 @@ std::shared_ptr makeDestinationRoute( std::shared_ptr destination, folly::StringPiece poolName, size_t indexInPool, + int32_t poolStatsIndex, std::chrono::milliseconds timeout, bool keepRoutingPrefix) { return makeRouteHandleWithInfo( std::move(destination), poolName, indexInPool, + poolStatsIndex, timeout, keepRoutingPrefix); } diff --git a/mcrouter/routes/McRouteHandleProvider-inl.h b/mcrouter/routes/McRouteHandleProvider-inl.h index 9db08354a..4e37f8356 100644 --- a/mcrouter/routes/McRouteHandleProvider-inl.h +++ b/mcrouter/routes/McRouteHandleProvider-inl.h @@ -206,6 +206,8 @@ McRouteHandleProvider::makePool( jservers->size(), jhostnames ? jhostnames->size() : 0); + int32_t poolStatIndex = proxy_.router().getStatsEnabledPoolIndex(name); + std::vector destinations; destinations.reserve(jservers->size()); for (size_t i = 0; i < jservers->size(); ++i) { @@ -252,7 +254,12 @@ McRouteHandleProvider::makePool( pdstn->updateShortestTimeout(timeout); destinations.push_back(makeDestinationRoute( - std::move(pdstn), nameSp, i, timeout, keepRoutingPrefix)); + std::move(pdstn), + nameSp, + i, + poolStatIndex, + timeout, + keepRoutingPrefix)); } // servers return pools_.emplace(std::move(name), std::move(destinations)) diff --git a/mcrouter/stats.cpp b/mcrouter/stats.cpp index 666f40570..5401efb40 100644 --- a/mcrouter/stats.cpp +++ b/mcrouter/stats.cpp @@ -17,6 +17,7 @@ #include #include +#include #include #include "mcrouter/CarbonRouterInstanceBase.h" @@ -336,6 +337,32 @@ static int get_proc_stat(pid_t pid, proc_stat_data_t* data) { return 0; } +void append_pool_stats( + CarbonRouterInstanceBase& router, + std::vector& stats) { + folly::StringKeyedUnorderedMap mergedPoolStatsMap; + + auto mergeMaps = [&mergedPoolStatsMap]( + folly::StringKeyedUnorderedMap&& poolStatMap) { + for (auto& poolStatMapEntry : poolStatMap) { + auto it = mergedPoolStatsMap.find(poolStatMapEntry.first); + if (it != mergedPoolStatsMap.end()) { + it->second.data.uint64 += poolStatMapEntry.second.data.uint64; + } else { + mergedPoolStatsMap.insert(std::move(poolStatMapEntry)); + } + } + }; + + for (size_t j = 0; j < router.opts().num_proxies; ++j) { + auto pr = router.getProxyBase(j); + mergeMaps(pr->stats().getAggregatedPoolStatsMap()); + } + for (const auto& mergedPoolStatMapEntry : mergedPoolStatsMap) { + stats.emplace_back(mergedPoolStatMapEntry.second); + } +} + void prepare_stats(CarbonRouterInstanceBase& router, stat_t* stats) { init_stats(stats); @@ -552,9 +579,9 @@ McStatsReply stats_reply(ProxyBase* proxy, folly::StringPiece group_str) { return errorReply; } - stat_t stats[num_stats]; + std::vector stats(num_stats); - prepare_stats(proxy->router(), stats); + prepare_stats(proxy->router(), stats.data()); for (unsigned int ii = 0; ii < num_stats; ii++) { stat_t* stat = &stats[ii]; @@ -570,6 +597,7 @@ McStatsReply stats_reply(ProxyBase* proxy, folly::StringPiece group_str) { } } } + append_pool_stats(proxy->router(), stats); if (groups & (mcproxy_stats | all_stats | detailed_stats | ods_stats)) { folly::dynamic requestStats(folly::dynamic::object()); diff --git a/mcrouter/stats.h b/mcrouter/stats.h index 6c0e16acf..7655884a4 100644 --- a/mcrouter/stats.h +++ b/mcrouter/stats.h @@ -131,6 +131,9 @@ uint64_t stat_get_uint64(const stat_t*, stat_name_t); uint64_t stat_get_config_age(const stat_t* stats, uint64_t now); McStatsReply stats_reply(ProxyBase*, folly::StringPiece); void prepare_stats(CarbonRouterInstanceBase& router, stat_t* stats); +void append_pool_stats( + CarbonRouterInstanceBase& router, + std::vector& stats); void set_standalone_args(folly::StringPiece args); diff --git a/mcrouter/test/test_poolstats.json b/mcrouter/test/test_poolstats.json new file mode 100644 index 000000000..f41cb233f --- /dev/null +++ b/mcrouter/test/test_poolstats.json @@ -0,0 +1,35 @@ +{ + + "pools": { + "twmemcache.CI.west.1": { + "servers": [ + "127.0.0.1:15001" + ] + }, + "twmemcache.CI.west": { + "servers": [ + "127.0.0.1:15002" + ] + }, + "twmemcache.CI.east": { + "servers": [ + "127.0.0.1:15003" + ] + }, + "wc": { + "servers": [ + "127.0.0.1:15004" + ] + } + }, + + "route": { + "type": "PrefixSelectorRoute", + "policies": { + "twmemcache.CI.west.1": "PoolRoute|twmemcache.CI.west.1", + "twmemcache.CI.west": "PoolRoute|twmemcache.CI.west", + "twmemcache.CI.east": "PoolRoute|twmemcache.CI.east" + }, + "wildcard": "PoolRoute|wc" + } +} diff --git a/mcrouter/test/test_poolstats.py b/mcrouter/test/test_poolstats.py new file mode 100644 index 000000000..2103cddc5 --- /dev/null +++ b/mcrouter/test/test_poolstats.py @@ -0,0 +1,91 @@ +# Copyright (c) 2015-present, Facebook, Inc. +# +# This source code is licensed under the MIT license found in the LICENSE +# file in the root directory of this source tree. + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from mcrouter.test.MCProcess import Mcrouter +from mcrouter.test.McrouterTestCase import McrouterTestCase +from mcrouter.test.mock_servers import SleepServer + +import time +import os + + +class TestPoolStats(McrouterTestCase): + config = './mcrouter/test/test_poolstats.json' + null_route_config = './mcrouter/test/test_nullroute.json' + mcrouter_server_extra_args = [] + extra_args = [ + '--pool-stats-config-file=./mcrouter/test/test_poolstats_config.json', + '--timeouts-until-tko=50', + '--disable-miss-on-get-errors', + '--num-proxies=4'] + stat_prefix = 'libmcrouter.mcrouter.0.' + + def setUp(self): + self.mc = [] + for _i in range(2): + self.mc.append(Mcrouter(self.null_route_config, + extra_args=self.mcrouter_server_extra_args)) + self.add_server(self.mc[_i]) + + # configure SleepServer for the east and wc pools + for _i in range(2): + self.mc.append(SleepServer()) + self.add_server(self.mc[_i + 2]) + + self.mcrouter = self.add_mcrouter( + self.config, + extra_args=self.extra_args) + + def check_pool_stats(self, stats_dir): + file_stat = os.path.join(stats_dir, self.stat_prefix + 'stats') + sprefix = self.stat_prefix + 'twmemcache.CI.' + verifiedEastReqs = False + verifiedWestReqs = False + verifiedEastErrs = False + verifiedWestErrs = False + with open(file_stat, 'r') as f: + for line in f.readlines(): + # Expect all east requests to fail because it + # is running SleepServer + if sprefix + 'east.requests.sum' in line: + s = line.split(':')[1].split(',')[0] + self.assertEqual(int(s), 50) + verifiedEastReqs = True + if sprefix + 'east.final_result_error.sum' in line: + s = line.split(':')[1].split(',')[0] + self.assertEqual(int(s), 50) + verifiedEastErrs = True + if sprefix + 'west.requests.sum' in line: + s = line.split(':')[1].split(',')[0] + self.assertEqual(int(s), 100) + verifiedWestReqs = True + if sprefix + 'west.final_result_error.sum' in line: + s = line.split(':')[1].split(',')[0] + self.assertEqual(int(s), 0) + verifiedWestErrs = True + self.assertTrue(verifiedEastReqs) + self.assertTrue(verifiedEastErrs) + self.assertTrue(verifiedWestReqs) + self.assertTrue(verifiedWestErrs) + + def test_poolstats(self): + n = 150 + for i in range(0, n): + m = i % 3 + if m == 0: + key = 'twmemcache.CI.west:{}:|#|id=123'.format(i) + elif m == 1: + key = 'twmemcache.CI.west.1:{}:|#|id=123'.format(i) + else: + key = 'twmemcache.CI.east.1:{}:|#|id=123'.format(i) + self.mcrouter.get(key) + self.assertTrue(self.mcrouter.stats()['cmd_get_count'] > 0) + time.sleep(11) + self.check_pool_stats(self.mcrouter.stats_dir) diff --git a/mcrouter/test/test_poolstats_config.json b/mcrouter/test/test_poolstats_config.json new file mode 100644 index 000000000..facec1258 --- /dev/null +++ b/mcrouter/test/test_poolstats_config.json @@ -0,0 +1,3 @@ +{ + "stats_enabled_pools" : ["twmemcache.CI.west", "twmemcache.CI.east"] +}