capture - More accurate memory reporting and added cpu reporting

baubaubau · Feb 9, 2015 · 61aa343 · 61aa343
1 parent ad2acfd
commit 61aa343
Show file tree

Hide file tree

Showing 7 changed files with 107 additions and 37 deletions.
diff --git a/CHANGELOG b/CHANGELOG
@@ -49,6 +49,8 @@
   - Added cert.cnt back
   - Handle bad ip.protocol strings better (issue #330)
   - Added dontSaveBPFs config
+  - Switched capture memory reporting to more accurate getrusage
+  - Added capture cpu reporting to stats (requires db.pl upgrade)
 
 
 

diff --git a/capture/db.c b/capture/db.c
@@ -22,14 +22,15 @@
 #include <unistd.h>
 #include <inttypes.h>
 #include <errno.h>
+#include <sys/resource.h>
 #include <sys/statvfs.h>
 #include "glib.h"
 #include "moloch.h"
 #include "bsb.h"
 #include "patricia.h"
 #include "GeoIP.h"
 
-#define MOLOCH_MIN_DB_VERSION 21
+#define MOLOCH_MIN_DB_VERSION 22
 
 extern uint64_t         totalPackets;
 extern uint64_t         totalBytes;
@@ -906,12 +907,13 @@ void moloch_db_load_stats()
 /******************************************************************************/
 void moloch_db_update_stats()
 {
-    static uint64_t lastPackets = 0;
-    static uint64_t lastBytes = 0;
-    static uint64_t lastSessions = 0;
-    static uint64_t lastDropped = 0;
-    uint64_t        freeSpaceM = 0;
-    int             i;
+    static uint64_t       lastPackets = 0;
+    static uint64_t       lastBytes = 0;
+    static uint64_t       lastSessions = 0;
+    static uint64_t       lastDropped = 0;
+    uint64_t              freeSpaceM = 0;
+    static struct rusage  lastUsage;
+    int                   i;
 
     char *json = moloch_http_get_buffer(MOLOCH_HTTP_BUFFER_SIZE);
     struct timeval currentTime;
@@ -933,15 +935,22 @@ void moloch_db_update_stats()
     dbTotalSessions += (totalSessions - lastSessions);
     dbTotalDropped += (totalDropped - lastDropped);
     dbTotalK += (totalBytes - lastBytes)/1024;
-    uint64_t mem = (uint64_t)sbrk(0);
+
+    struct rusage usage;
+    getrusage(RUSAGE_SELF, &usage);
+
     int diffms = (currentTime.tv_sec - dbLastTime.tv_sec)*1000 + (currentTime.tv_usec/1000 - dbLastTime.tv_usec/1000);
+    uint64_t diffusage = (usage.ru_utime.tv_sec - lastUsage.ru_utime.tv_sec)*1000 + (usage.ru_utime.tv_usec/1000 - lastUsage.ru_utime.tv_usec/1000) +
+                         (usage.ru_stime.tv_sec - lastUsage.ru_stime.tv_sec)*1000 + (usage.ru_stime.tv_usec/1000 - lastUsage.ru_stime.tv_usec/1000);
+
     int json_len = snprintf(json, MOLOCH_HTTP_BUFFER_SIZE,
         "{"
         "\"hostname\": \"%s\", "
         "\"currentTime\": %u, "
         "\"freeSpaceM\": %" PRIu64 ", "
         "\"monitoring\": %u, "
         "\"memory\": %" PRIu64 ", "
+        "\"cpu\": %" PRIu64 ", "
         "\"diskQueue\": %u, "
         "\"totalPackets\": %" PRIu64 ", "
         "\"totalK\": %" PRIu64 ", "
@@ -957,7 +966,12 @@ void moloch_db_update_stats()
         (uint32_t)currentTime.tv_sec,
         freeSpaceM,
         moloch_nids_monitoring_sessions(),
-        mem,
+#if defined(__APPLE__) && defined(__MACH__)
+        usage.ru_maxrss,
+#else
+        usage.ru_maxrss * 1024UL,
+#endif
+        diffusage*10000/diffms,
         moloch_nids_disk_queue(),
         dbTotalPackets,
         dbTotalK,
@@ -974,23 +988,25 @@ void moloch_db_update_stats()
     lastPackets  = totalPackets;
     lastSessions = totalSessions;
     lastDropped  = totalDropped;
+    lastUsage    = usage;
 
     moloch_http_set(esServer, stats_key, stats_key_len, json, json_len, NULL, NULL);
 }
 
 /******************************************************************************/
 void moloch_db_update_dstats(int n)
 {
-    static uint64_t lastPackets[2] = {0, 0};
-    static uint64_t lastBytes[2] = {0, 0};
-    static uint64_t lastSessions[2] = {0, 0};
-    static uint64_t lastDropped[2] = {0, 0};
+    static uint64_t       lastPackets[2] = {0, 0};
+    static uint64_t       lastBytes[2] = {0, 0};
+    static uint64_t       lastSessions[2] = {0, 0};
+    static uint64_t       lastDropped[2] = {0, 0};
+    static struct rusage  lastUsage[2];
     static struct timeval lastTime[2];
-    static int      intervals[2] = {5, 60};
-    uint64_t        freeSpaceM = 0;
-    int             i;
-    char            key[200];
-    int             key_len = 0;
+    static int            intervals[2] = {5, 60};
+    uint64_t              freeSpaceM = 0;
+    int                   i;
+    char                  key[200];
+    int                   key_len = 0;
 
     char *json = moloch_http_get_buffer(MOLOCH_HTTP_BUFFER_SIZE);
     struct timeval currentTime;
@@ -1012,7 +1028,12 @@ void moloch_db_update_dstats(int n)
 
     const uint64_t cursec = currentTime.tv_sec;
     const uint64_t diffms = (currentTime.tv_sec - lastTime[n].tv_sec)*1000 + (currentTime.tv_usec/1000 - lastTime[n].tv_usec/1000);
-    uint64_t mem = (uint64_t)sbrk(0);
+
+    struct rusage usage;
+    getrusage(RUSAGE_SELF, &usage);
+
+    uint64_t diffusage = (usage.ru_utime.tv_sec - lastUsage[n].ru_utime.tv_sec)*1000 + (usage.ru_utime.tv_usec/1000 - lastUsage[n].ru_utime.tv_usec/1000) +
+                         (usage.ru_stime.tv_sec - lastUsage[n].ru_stime.tv_sec)*1000 + (usage.ru_stime.tv_usec/1000 - lastUsage[n].ru_stime.tv_usec/1000);
 
     int json_len = snprintf(json, MOLOCH_HTTP_BUFFER_SIZE,
         "{"
@@ -1022,6 +1043,7 @@ void moloch_db_update_dstats(int n)
         "\"freeSpaceM\": %" PRIu64 ", "
         "\"monitoring\": %u, "
         "\"memory\": %" PRIu64 ", "
+        "\"cpu\": %" PRIu64 ", "
         "\"diskQueue\": %u, "
         "\"deltaPackets\": %" PRIu64 ", "
         "\"deltaBytes\": %" PRIu64 ", "
@@ -1034,7 +1056,12 @@ void moloch_db_update_dstats(int n)
         cursec,
         freeSpaceM,
         moloch_nids_monitoring_sessions(),
-        mem,
+#if defined(__APPLE__) && defined(__MACH__)
+        usage.ru_maxrss,
+#else
+        usage.ru_maxrss * 1024UL,
+#endif
+        diffusage*10000/diffms,
         moloch_nids_disk_queue(),
         (totalPackets - lastPackets[n]),
         (totalBytes - lastBytes[n]),
@@ -1047,6 +1074,7 @@ void moloch_db_update_dstats(int n)
     lastPackets[n]  = totalPackets;
     lastSessions[n] = totalSessions;
     lastDropped[n]  = totalDropped;
+    lastUsage[n]    = usage;
     moloch_http_set(esServer, key, key_len, json, json_len, NULL, NULL);
 }
 /******************************************************************************/

diff --git a/db/db.pl b/db/db.pl
@@ -28,6 +28,7 @@
 # 19 - users_v3
 # 20 - queries
 # 21 - doc_values, new tls fields, starttime/stoptime/view
+# 22 - cpu to stats/dstats
 
 use HTTP::Request::Common;
 use LWP::UserAgent;
@@ -36,7 +37,7 @@
 use POSIX;
 use strict;
 
-my $VERSION = 21;
+my $VERSION = 22;
 my $verbose = 0;
 my $PREFIX = "";
 
@@ -411,6 +412,10 @@ sub statsUpdate
         type: "long",
         index: "no"
       },
+      cpu: {
+        type: "integer",
+        index: "no"
+      },
       diskQueue: {
         type: "long",
         index: "no"
@@ -492,6 +497,10 @@ sub dstatsUpdate
         type: "long",
         index: "no"
       },
+      cpu: {
+        type: "integer",
+        index: "no"
+      },
       diskQueue: {
         type: "long",
         index: "no"
@@ -2211,19 +2220,25 @@ sub dataNodes
     fieldsUpdate();
     sessionsUpdate();
     queriesCreate();
+    statsUpdate();
+    dstatsUpdate();
 
     print "Finished\n";
 } elsif ($main::versionNumber >= 19 && $main::versionNumber < 20) {
     waitFor("UPGRADE", "do you want to upgrade?");
     sessionsUpdate();
     queriesCreate();
     fieldsUpdate();
+    statsUpdate();
+    dstatsUpdate();
 
     print "Finished\n";
-} elsif ($main::versionNumber >= 20 && $main::versionNumber <= 21) {
+} elsif ($main::versionNumber >= 20 && $main::versionNumber <= 22) {
     waitFor("UPGRADE", "do you want to upgrade?");
     sessionsUpdate();
     fieldsUpdate();
+    statsUpdate();
+    dstatsUpdate();
 
     print "Finished\n";
 } else {

diff --git a/tests/api-stats.t b/tests/api-stats.t
@@ -1,4 +1,4 @@
-use Test::More tests => 23;
+use Test::More tests => 24;
 use Cwd;
 use URI::Escape;
 use MolochTest;
@@ -19,7 +19,7 @@ my $pwd = getcwd() . "/pcap";
         is ($stats->{aaData}->[0]->{$i}, 0, "stats.json $i == 0");
     }
 
-    foreach my $i ("deltaMS", "totalPackets", "deltaSessions", "deltaPackets", "deltaBytes", "memory", "currentTime", "totalK", "totalSessions", "freeSpaceM", "deltaSessionsPerSec", "deltaBytesPerSec", "deltaPacketsPerSec") {
+    foreach my $i ("deltaMS", "totalPackets", "deltaSessions", "deltaPackets", "deltaBytes", "memory", "cpu", "currentTime", "totalK", "totalSessions", "freeSpaceM", "deltaSessionsPerSec", "deltaBytesPerSec", "deltaPacketsPerSec") {
         cmp_ok ($stats->{aaData}->[0]->{$i}, '>', 0, "stats.json $i > 0");
     }
 

diff --git a/viewer/viewer.js b/viewer/viewer.js
@@ -1310,7 +1310,7 @@ function mergeUnarray(to, from) {
 app.get('/stats.json', function(req, res) {
   noCache(req, res);
 
-  var columns = ["_id", "currentTime", "totalPackets", "totalK", "totalSessions", "monitoring", "memory", "diskQueue", "freeSpaceM", "deltaPackets", "deltaBytes", "deltaSessions", "deltaDropped", "deltaMS"];
+  var columns = ["_id", "currentTime", "totalPackets", "totalK", "totalSessions", "monitoring", "memory", "cpu", "diskQueue", "freeSpaceM", "deltaPackets", "deltaBytes", "deltaSessions", "deltaDropped", "deltaMS"];
   var limit = (req.query.iDisplayLength?Math.min(parseInt(req.query.iDisplayLength, 10),1000000):500);
 
   var query = {_source: columns,
@@ -1333,6 +1333,7 @@ app.get('/stats.json', function(req, res) {
             }
             fields.id        = result.hits.hits[i]._id;
             fields.memory    = fields.memory || 0;
+            fields.cpu       = fields.cpu || 0;
             fields.diskQueue = fields.diskQueue || 0;
             fields.deltaBytesPerSec = Math.floor(fields.deltaBytes * 1000.0/fields.deltaMS);
             fields.deltaPacketsPerSec = Math.floor(fields.deltaPackets * 1000.0/fields.deltaMS);

diff --git a/viewer/views/stats.jade b/viewer/views/stats.jade
@@ -16,6 +16,7 @@ block content
       option(value="monitoring") Active Sessions
       option(value="freeSpaceM") Free Space
       option(value="memory") Memory
+      option(value="cpu") CPU
       option(value="diskQueue") Disk Queue
   div#statsGraph
   table#stats(cellpadding="0",cellspacing="0",border="0",class="display",style="table { clear: both }")
@@ -30,6 +31,7 @@ block content
         th Active Sessions
         th Free Space MB
         th Memory MB
+        th CPU
         th Disk Queue
         th Packets / Sec
         th Bytes / Sec
@@ -52,6 +54,7 @@ block content
         th
         th
         th
+        th
       tr#tot
         th
         th Total
@@ -67,6 +70,7 @@ block content
         th
         th
         th
+        th
   br
   h1 Elasticsearch
   table#esstats(cellpadding="0",cellspacing="0",border="0",class="display",style="table { clear: both }")
@@ -128,6 +132,11 @@ block content
                           "&interval=5" +
                           "&name=" + metricName,
             "success": function(data) {
+              if (metricName === "cpu") {
+                for (i = 0; i < data.length; i++) {
+                  data[i] = data[i]/100.0;
+                }
+              }
               callback(null, data);
             }
           });
@@ -240,6 +249,13 @@ block content
             "sClass": "right",
             "sType": "formatted-num"
           },
+          { "mDataProp" : "cpu",
+            "fnRender" : function(oObj) {
+              return "" + oObj.aData[oObj.mDataProp]/100.0;
+            },
+            "sClass": "right",
+            "sType": "formatted-num"
+          },
           { "mDataProp" : "diskQueue",
             "fnRender" : function(oObj) {
               return numberWithCommas(Math.ceil(oObj.aData[oObj.mDataProp]));
@@ -281,7 +297,7 @@ block content
           var avgCells = $('tr#avg > th');
           var totCells = $('tr#tot > th');
 
-          var totals = {totalPackets: 0, totalK: 0, totalSessions:0, deltaPacketsPerSec:0, deltaBytesPerSec:0 , deltaSessionsPerSec: 0, deltaDroppedPerSec: 0, monitoring: 0, memory: 0, diskQueue: 0};
+          var totals = {totalPackets: 0, totalK: 0, totalSessions:0, deltaPacketsPerSec:0, deltaBytesPerSec:0 , deltaSessionsPerSec: 0, deltaDroppedPerSec: 0, monitoring: 0, memory: 0, cpu: 0, diskQueue: 0};
           for (var r = 0, rlen = aaData.length; r < rlen; r++) {
               totals.totalPackets += parseInt(aaData[r].totalPackets.replace(/\,/g,''), 10);
               totals.totalK += parseInt(aaData[r].totalK.replace(/\,/g,''), 10);
@@ -293,6 +309,7 @@ block content
               totals.deltaBytesPerSec += parseInt(aaData[r].deltaBytesPerSec.replace(/\,/g,''), 10);
               totals.monitoring += parseInt(aaData[r].monitoring.replace(/\,/g,''), 10);
               totals.memory += parseInt(aaData[r].memory.replace(/\,/g,''), 10);
+              totals.cpu += parseFloat(aaData[r].cpu.replace(/\,/g,''), 10)*100;
               totals.diskQueue += parseInt(aaData[r].diskQueue.replace(/\,/g,''), 10);
           }
 
@@ -301,22 +318,24 @@ block content
           totCells[5].innerHTML = numberWithCommas(totals.totalSessions);
           totCells[6].innerHTML = numberWithCommas(totals.monitoring);
           totCells[8].innerHTML = numberWithCommas(totals.memory);
-          totCells[9].innerHTML = numberWithCommas(totals.diskQueue);
-          totCells[10].innerHTML = numberWithCommas(totals.deltaPacketsPerSec);
-          totCells[11].innerHTML = numberWithCommas(totals.deltaBytesPerSec);
-          totCells[12].innerHTML = numberWithCommas(totals.deltaSessionsPerSec);
-          totCells[13].innerHTML = numberWithCommas(totals.deltaDroppedPerSec);
+          totCells[9].innerHTML = "" + (totals.cpu/100.0);
+          totCells[10].innerHTML = numberWithCommas(totals.diskQueue);
+          totCells[11].innerHTML = numberWithCommas(totals.deltaPacketsPerSec);
+          totCells[12].innerHTML = numberWithCommas(totals.deltaBytesPerSec);
+          totCells[13].innerHTML = numberWithCommas(totals.deltaSessionsPerSec);
+          totCells[14].innerHTML = numberWithCommas(totals.deltaDroppedPerSec);
 
           avgCells[3].innerHTML = numberWithCommas(Math.floor(totals.totalPackets/aaData.length));
           avgCells[4].innerHTML = numberWithCommas(Math.floor(totals.totalK/aaData.length));
           avgCells[5].innerHTML = numberWithCommas(Math.floor(totals.totalSessions/aaData.length));
           avgCells[6].innerHTML = numberWithCommas(Math.floor(totals.monitoring/aaData.length));
           avgCells[8].innerHTML = numberWithCommas(Math.floor(totals.memory/aaData.length));
-          avgCells[9].innerHTML = numberWithCommas(Math.floor(totals.diskQueue/aaData.length));
-          avgCells[10].innerHTML = numberWithCommas(Math.floor(totals.deltaPacketsPerSec/aaData.length));
-          avgCells[11].innerHTML = numberWithCommas(Math.floor(totals.deltaBytesPerSec/aaData.length));
-          avgCells[12].innerHTML = numberWithCommas(Math.floor(totals.deltaSessionsPerSec/aaData.length));
-          avgCells[13].innerHTML = numberWithCommas(Math.floor(totals.deltaDroppedPerSec/aaData.length));
+          avgCells[9].innerHTML = "" + Math.floor(totals.cpu/aaData.length)/100;
+          avgCells[10].innerHTML = numberWithCommas(Math.floor(totals.diskQueue/aaData.length));
+          avgCells[11].innerHTML = numberWithCommas(Math.floor(totals.deltaPacketsPerSec/aaData.length));
+          avgCells[12].innerHTML = numberWithCommas(Math.floor(totals.deltaBytesPerSec/aaData.length));
+          avgCells[13].innerHTML = numberWithCommas(Math.floor(totals.deltaSessionsPerSec/aaData.length));
+          avgCells[14].innerHTML = numberWithCommas(Math.floor(totals.deltaDroppedPerSec/aaData.length));
         }
       } );
 

diff --git a/viewer/views/statsDetail.jade b/viewer/views/statsDetail.jade
@@ -19,6 +19,11 @@ script(type='text/javascript').
                       "&interval=60" +
                       "&name=" + mname,
         "success": function(data) {
+          if (mname === "cpu") {
+            for (i = 0; i < data.length; i++) {
+              data[i] = data[i]/100.0;
+            }
+          }
           callback(null, data);
         }
       });
@@ -32,7 +37,7 @@ script(type='text/javascript').
         .call(dcontext.axis().orient("top"));
 
     div.selectAll(".horizon")
-        .data([dmetric("Packets/Min", "deltaPackets"), dmetric("Bytes/Min", "deltaBytes"), dmetric("Sessions/Min", "deltaSessions"), dmetric("Dropped/Min", "deltaDropped"), dmetric("Active Sessions", "monitoring"), dmetric("Free Space", "freeSpaceM"), dmetric("Memory", "memory"), dmetric("Disk Queue", "diskQueue")])
+        .data([dmetric("Packets/Min", "deltaPackets"), dmetric("Bytes/Min", "deltaBytes"), dmetric("Sessions/Min", "deltaSessions"), dmetric("Dropped/Min", "deltaDropped"), dmetric("Active Sessions", "monitoring"), dmetric("Free Space", "freeSpaceM"), dmetric("Memory", "memory"), dmetric("CPU", "cpu"), dmetric("Disk Queue", "diskQueue")])
       .enter().append("div")
         .attr("class", "horizon")
         .call(dcontext.horizon());