Add Pool stats ODS counters to McRouter

Summary: Add Pool stats ODS counters to McRouter

Reviewed By: andreazevedo

Differential Revision: D6390195

fbshipit-source-id: cc48b87b4c2366883677aceee3e2545ab5509b57

mcrouter/CarbonRouterInstanceBase.cpp

@@ -79,6 +79,27 @@ CarbonRouterInstanceBase::CarbonRouterInstanceBase(McrouterOptions inputOptions)
       statsLogger->makeQueueSizeUnlimited();
     }
   }
+
+  if (!opts_.pool_stats_config_file.empty()) {
+    try {
+      folly::dynamic poolStatJson =
+          readStaticJsonFile(opts_.pool_stats_config_file);
+      if (poolStatJson != nullptr) {
+        auto jStatsEnabledPools = poolStatJson.get_ptr("stats_enabled_pools");
+        if (jStatsEnabledPools && jStatsEnabledPools->isArray()) {
+          for (const auto& it : *jStatsEnabledPools) {
+            if (it.isString()) {
+              statsEnabledPools_.push_back(it.asString());
+            } else {
+              LOG(ERROR) << "Pool Name is not a string";
+            }
+          }
+        }
+      }
+    } catch (const std::exception& e) {
+      LOG(ERROR) << "Invalid pool-stats-config-file : " << e.what();
+    }
+  }
 }
 
 void CarbonRouterInstanceBase::setUpCompressionDictionaries(
@@ -170,6 +191,30 @@ CarbonRouterInstanceBase::functionScheduler() {
   return globalFunctionScheduler.try_get();
 }
 
+int32_t CarbonRouterInstanceBase::getStatsEnabledPoolIndex(
+    const folly::StringPiece poolName) const {
+  if (statsEnabledPools_.size() == 0) {
+    return -1;
+  }
+
+  int longestPrefixMatchIndex = -1;
+  // Do sequential search for longest matching name. Since this is done
+  // only once during the initialization and the size of the array is
+  // expected to be small, linear search should be OK.
+  for (size_t i = 0; i < statsEnabledPools_.size(); i++) {
+    if (poolName.subpiece(0, statsEnabledPools_[i].length())
+            .compare(statsEnabledPools_[i]) == 0) {
+      if ((longestPrefixMatchIndex == -1) ||
+          (statsEnabledPools_[longestPrefixMatchIndex].length() <
+           statsEnabledPools_[i].length())) {
+        longestPrefixMatchIndex = i;
+      }
+    }
+  }
+
+  return longestPrefixMatchIndex;
+}
+
 } // mcrouter
 } // memcache
 } // facebook

mcrouter/CarbonRouterInstanceBase.h

@@ -22,6 +22,7 @@
 #include "mcrouter/ConfigApi.h"
 #include "mcrouter/LeaseTokenMap.h"
 #include "mcrouter/Observable.h"
+#include "mcrouter/PoolStats.h"
 #include "mcrouter/TkoTracker.h"
 #include "mcrouter/options.h"
 
@@ -145,6 +146,23 @@ class CarbonRouterInstanceBase {
     return disableRxmitReconnection_;
   }
 
+  /**
+   * This function finds the index of poolName in the statsEnabledPools_
+   * sorted array by doing binary search. If exact match is not found,
+   * index with maximum prefix match is returned.
+   *
+   * @return index of the pool in the statsEnabledPools_ vector
+   *         -1 if not found
+   */
+  int32_t getStatsEnabledPoolIndex(folly::StringPiece poolName) const;
+
+  /**
+   * @return  reference to the statsEnabledPools_ vector
+   */
+  const std::vector<std::string>& getStatsEnabledPools() const {
+    return statsEnabledPools_;
+  }
+
   /**
    * @return  nullptr if index is >= opts.num_proxies,
    *          pointer to the proxy otherwise.
@@ -224,6 +242,8 @@ class CarbonRouterInstanceBase {
   // Name of the stats update function registered with the function scheduler.
   const std::string statsUpdateFunctionHandle_;
 
+  std::vector<std::string> statsEnabledPools_;
+
   // Aggregates stats for all associated proxies. Should be called periodically.
   void updateStats();
 };

mcrouter/McrouterLogger.cpp

@@ -208,6 +208,7 @@ void McrouterLogger::log() {
 
   std::vector<stat_t> stats(num_stats);
   prepare_stats(router_, stats.data());
+  append_pool_stats(router_, stats);
 
   folly::dynamic requestStats(folly::dynamic::object());
   for (size_t i = 0; i < router_.opts().num_proxies; ++i) {

mcrouter/PoolStats.h

@@ -0,0 +1,58 @@
+/*
+ *  Copyright (c) 2017-present, Facebook, Inc.
+ *
+ *  This source code is licensed under the MIT license found in the LICENSE
+ *  file in the root directory of this source tree.
+ *
+ */
+#pragma once
+
+#include <folly/experimental/StringKeyedUnorderedMap.h>
+
+#include "mcrouter/stats.h"
+
+namespace facebook {
+namespace memcache {
+namespace mcrouter {
+
+class PoolStats {
+ public:
+  PoolStats(folly::StringPiece poolName)
+      : requestsCountStatName_(
+            folly::to<std::string>(poolName, ".requests.sum")),
+        finalResultErrorStatName_(
+            folly::to<std::string>(poolName, ".final_result_error.sum")) {
+    initStat(requestCountStat_, requestsCountStatName_);
+    initStat(finalResultErrorStat_, finalResultErrorStatName_);
+  }
+
+  std::vector<stat_t> getStats() const {
+    return {requestCountStat_, finalResultErrorStat_};
+  }
+
+  void incrementRequestCount(uint64_t amount = 1) {
+    requestCountStat_.data.uint64 += amount;
+  }
+
+  void incrementFinalResultErrorCount(uint64_t amount = 1) {
+    finalResultErrorStat_.data.uint64 += amount;
+  }
+
+ private:
+  void initStat(stat_t& stat, folly::StringPiece name) const {
+    stat.name = name;
+    stat.group = ods_stats | count_stats;
+    stat.type = stat_uint64;
+    stat.aggregate = 0;
+    stat.data.uint64 = 0;
+  }
+
+  const std::string requestsCountStatName_;
+  const std::string finalResultErrorStatName_;
+  stat_t requestCountStat_;
+  stat_t finalResultErrorStat_;
+};
+
+} // namespace mcrouter
+} // namespace memcache
+} // namespace facebook

mcrouter/ProxyBase-inl.h

@@ -32,6 +32,7 @@ ProxyBase::ProxyBase(
           std::make_unique<folly::fibers::EventBaseLoopController>(),
           getFiberManagerOptions(router_.opts())),
       asyncLog_(router_.opts()),
+      stats_(router_.getStatsEnabledPools()),
       flushCallback_(*this),
       destinationMap_(std::make_unique<ProxyDestinationMap>(this)) {
   // Setup a full random seed sequence

mcrouter/ProxyRequestContext.h

@@ -87,6 +87,12 @@ class ProxyRequestContext {
     return failoverDisabled_;
   }
 
+  void setPoolStatsIndex(int32_t index) {
+    if (poolStatIndex_ == -1) {
+      poolStatIndex_ = index;
+    }
+  }
+
   ProxyRequestPriority priority() const {
     return priority_;
   }
@@ -136,6 +142,7 @@ class ProxyRequestContext {
    */
   void (*reqComplete_)(ProxyRequestContext& preq){nullptr};
   mc_res_t finalResult_{mc_res_unknown};
+  int32_t poolStatIndex_{-1};
   bool replied_{false};
 
   ProxyRequestContext(ProxyBase& pr, ProxyRequestPriority priority__);

mcrouter/ProxyRequestContextTyped.h

@@ -91,6 +91,8 @@ class ProxyRequestContextWithInfo : public ProxyRequestContext {
   }
 
   ~ProxyRequestContextWithInfo() override {
+    proxy_.stats().incrementPoolStats(
+        poolStatIndex_, 1, isErrorResult(finalResult_) ? 1 : 0);
     if (reqComplete_) {
       fiber_local<RouterInfo>::runWithoutLocals(
           [this]() { reqComplete_(*this); });

mcrouter/ProxyStats.cpp

@@ -11,8 +11,12 @@ namespace facebook {
 namespace memcache {
 namespace mcrouter {
 
-ProxyStats::ProxyStats() {
+ProxyStats::ProxyStats(const std::vector<std::string>& statsEnabledPools) {
   init_stats(stats_);
+  poolStats_.reserve(statsEnabledPools.size());
+  for (const auto& curPoolName : statsEnabledPools) {
+    poolStats_.emplace_back(curPoolName);
+  }
 }
 
 void ProxyStats::aggregate(size_t statId) {

mcrouter/ProxyStats.h

@@ -9,7 +9,10 @@
 
 #include <mutex>
 
+#include <folly/experimental/StringKeyedUnorderedMap.h>
+
 #include "mcrouter/ExponentialSmoothData.h"
+#include "mcrouter/PoolStats.h"
 #include "mcrouter/stats.h"
 
 namespace facebook {
@@ -18,7 +21,7 @@ namespace mcrouter {
 
 class ProxyStats {
  public:
-  ProxyStats();
+  explicit ProxyStats(const std::vector<std::string>& statsEnabledPools);
 
   /**
    * Aggregate proxy stat with the given index.
@@ -129,9 +132,32 @@ class ProxyStats {
     return stats_[statId];
   }
 
+  folly::StringKeyedUnorderedMap<stat_t> getAggregatedPoolStatsMap() const {
+    folly::StringKeyedUnorderedMap<stat_t> poolStatsMap;
+    for (const auto& poolStats : poolStats_) {
+      for (const auto& stat : poolStats.getStats()) {
+        poolStatsMap.emplace(stat.name, stat);
+      }
+    }
+    return poolStatsMap;
+  }
+
+  void incrementPoolStats(
+      int32_t idx,
+      uint64_t requestCount,
+      uint64_t finalErrorResultCount) {
+    if (idx < 0 || static_cast<size_t>(idx) >= poolStats_.size()) {
+      return;
+    }
+    poolStats_[idx].incrementRequestCount(requestCount);
+    poolStats_[idx].incrementFinalResultErrorCount(finalErrorResultCount);
+  }
+
  private:
   mutable std::mutex mutex_;
   stat_t stats_[num_stats]{};
+  // vector of the PoolStats
+  std::vector<PoolStats> poolStats_;
 
   ExponentialSmoothData<64> durationUs_;
 

mcrouter/mcrouter_config.cpp

@@ -7,7 +7,9 @@
  */
 #include <memory>
 
+#include <folly/FileUtil.h>
 #include <folly/Range.h>
+#include <folly/json.h>
 
 #include "mcrouter/CarbonRouterInstanceBase.h"
 #include "mcrouter/McrouterLogger.h"
@@ -123,10 +125,20 @@ std::string getBinPath(folly::StringPiece name) {
 std::string getDefaultPemCertPath() {
   return "";
 }
+
 std::string getDefaultPemCertKey() {
   return "";
 }
 
+folly::dynamic readStaticJsonFile(folly::StringPiece file) {
+  std::string contents;
+  if (!folly::readFile(file.str().c_str(), contents)) {
+    LOG(ERROR) << "Failed to open pool-stats-config-file " << file.str();
+    return nullptr;
+  }
+  return folly::parseJson(contents);
+}
+
 } // mcrouter
 } // memcache
 } // facebook

mcrouter/mcrouter_config.h

@@ -166,6 +166,19 @@ std::string getBinPath(folly::StringPiece name);
 std::string getDefaultPemCertPath();
 std::string getDefaultPemCertKey();
 
+/**
+ * Reads a static json file. Do not monitor for changes.
+ * May throw if there's an error while parsing file contents.
+ *
+ * @params file   The path of the json file.
+ *
+ * @return        folly::dynamic with the contents of the file.
+ *                nullptr if cannot open/read the file
+ *                may throw exception if invalid json
+ *
+ */
+folly::dynamic readStaticJsonFile(folly::StringPiece file);
+
 #ifndef MCROUTER_PACKAGE_STRING
 #define MCROUTER_PACKAGE_STRING "1.0.0 mcrouter"
 #endif

mcrouter/mcrouter_options_list.h

@@ -424,6 +424,13 @@ MCROUTER_OPTION_STRING(
     "DEPRECATED. Load configuration from file. This option has no effect if"
     " --config option is used.")
 
+MCROUTER_OPTION_STRING(
+    pool_stats_config_file,
+    "",
+    "pool-stats-config-file",
+    no_short,
+    "File containing stats enabled pool names.")
+
 MCROUTER_OPTION_STRING(
     config_str,
     "",

mcrouter/routes/DestinationRoute.h

@@ -65,11 +65,13 @@ class DestinationRoute {
       std::shared_ptr<ProxyDestination> destination,
       folly::StringPiece poolName,
       size_t indexInPool,
+      int32_t poolStatIdx,
       std::chrono::milliseconds timeout,
       bool keepRoutingPrefix)
       : destination_(std::move(destination)),
         poolName_(poolName),
         indexInPool_(indexInPool),
+        poolStatIndex_(poolStatIdx),
         timeout_(timeout),
         keepRoutingPrefix_(keepRoutingPrefix) {}
 
@@ -105,6 +107,7 @@ class DestinationRoute {
   const std::shared_ptr<ProxyDestination> destination_;
   const folly::StringPiece poolName_;
   const size_t indexInPool_;
+  const int poolStatIndex_{-1};
   const std::chrono::milliseconds timeout_;
   size_t pendingShadowReqs_{0};
   const bool keepRoutingPrefix_;
@@ -127,6 +130,9 @@ class DestinationRoute {
       return constructAndLog(req, *ctx, BusyReply);
     }
 
+    if (poolStatIndex_ >= 0) {
+      ctx->setPoolStatsIndex(poolStatIndex_);
+    }
     auto requestClass = fiber_local<RouterInfo>::getRequestClass();
     if (ctx->recording()) {
       bool isShadow = requestClass.is(RequestClass::kShadow);
@@ -257,12 +263,14 @@ std::shared_ptr<typename RouterInfo::RouteHandleIf> makeDestinationRoute(
     std::shared_ptr<ProxyDestination> destination,
     folly::StringPiece poolName,
     size_t indexInPool,
+    int32_t poolStatsIndex,
     std::chrono::milliseconds timeout,
     bool keepRoutingPrefix) {
   return makeRouteHandleWithInfo<RouterInfo, DestinationRoute>(
       std::move(destination),
       poolName,
       indexInPool,
+      poolStatsIndex,
       timeout,
       keepRoutingPrefix);
 }