Added RRD support in ntopng

git-svn-id: https://svn.ntop.org/svn/ntop/trunk/ntopng@6260 a02cd0c1-8e76-42e1-a119-56f9641475e2

HTTPserver.h

@@ -35,8 +35,8 @@ class HTTPserver {
   HTTPserver(u_int16_t _port, const char *_docs_dir, const char *_scripts_dir);
   ~HTTPserver();
 
-  inline char* get_docs_dir()    { return(docs_dir);    };
-  inline char* get_scripts_dir() { return(scripts_dir); };
+  inline char* get_docs_dir()      { return(docs_dir);    };
+  inline char* get_scripts_dir()   { return(scripts_dir); };
 };
 
 extern int page_not_found(struct MHD_Connection *connection, const char *url);

Host.cpp

@@ -68,10 +68,21 @@ void Host::initialize(u_int8_t mac[6], bool init_all) {
 
     if(ntop->getRedis()->getAddress(host, rsp, sizeof(rsp)) == 0)
       symbolic_name = strdup(rsp);
-    else {
+    else
       ntop->getRedis()->queueHostToResolve(host);
-    }
-  }
+
+    updateLocal();
+  } else
+    localHost = false;
+}
+
+/* *************************************** */
+
+void Host::updateLocal() {
+  /* We need to check if this is a local host */
+
+  localHost = true; // FIX
+
 }
 
 /* *************************************** */
@@ -93,6 +104,7 @@ void Host::lua(lua_State* vm, bool host_details, bool returnHost) {
 
   if(host_details) {
     lua_newtable(vm);
+    lua_push_bool_table_entry(vm, "localhost", isLocalHost());
     lua_push_str_table_entry(vm, "ip", ip->print(buf, sizeof(buf)));
     lua_push_str_table_entry(vm, "mac", get_mac(buf, sizeof(buf)));
     lua_push_str_table_entry(vm, "name", get_name(buf, sizeof(buf)));

Host.h

@@ -34,7 +34,9 @@ class Host : public HashEntry {
   TrafficStats sent, rcvd;
   bool name_resolved;
   Mutex *m;
+  bool localHost;
 
+  void updateLocal();
   void initialize(u_int8_t mac[6], bool init_all);
 
  public:
@@ -48,6 +50,7 @@ class Host : public HashEntry {
   inline u_int32_t key()                       { return(ip->key());   }
   inline IpAddress* get_ip()                   { return(ip);          }
   inline u_int8_t*  get_mac()                  { return(mac_address); }
+  inline bool isLocalHost()                    { return(localHost);   }
   char* get_mac(char *buf, u_int buf_len);
   char*  get_name(char *buf, u_int buf_len);
 

Lua.cpp

@@ -21,6 +21,11 @@
 
 #include "ntop_includes.h"
 
+#define _GETOPT_H
+#define LIB_VERSION "1.4.7"
+
+#include "third-party/rrdtool-1.4.7/bindings/lua/rrdlua.c"
+
 /* ******************************* */
 
 Lua::Lua() {
@@ -142,39 +147,51 @@ static int ntop_get_interface_hosts_info(lua_State* vm) {
 
 /* ****************************************** */
 
-static int ntop_get_interface_host_info(lua_State* vm) {
-  NetworkInterface *ntop_interface;
-  char *host_ip;
+static int ntop_get_file_dir_exists(lua_State* vm) {
+  char *path;
+  struct stat buf;
 
   if(ntop_lua_check(vm, __FUNCTION__, 1, LUA_TSTRING)) return(0);
-  host_ip = (char*)lua_tostring(vm, 1);
+  path = (char*)lua_tostring(vm, 1);
+
+  return((stat(path, &buf) != 0) ? 0 : 1);
+}
+
+/* ****************************************** */
+
+static int ntop_get_interface_flows_info(lua_State* vm) {
+  NetworkInterface *ntop_interface;
 
   lua_getglobal(vm, "ntop_interface");
   if((ntop_interface = (NetworkInterface*)lua_touserdata(vm, lua_gettop(vm))) == NULL) {
     ntop->getTrace()->traceEvent(TRACE_ERROR, "INTERNAL ERROR: null interface");
     return(0);
   }
 
-  if(!ntop_interface->getHostInfo(vm, host_ip))
-    return(0);
-  else
-    return(1);
+  ntop_interface->getActiveFlowsList(vm);
+
+  return(1);
 }
 
 /* ****************************************** */
 
-static int ntop_get_interface_flows_info(lua_State* vm) {
+static int ntop_get_interface_host_info(lua_State* vm) {
   NetworkInterface *ntop_interface;
+  char *host_ip;
+
+  if(ntop_lua_check(vm, __FUNCTION__, 1, LUA_TSTRING)) return(0);
+  host_ip = (char*)lua_tostring(vm, 1);
 
   lua_getglobal(vm, "ntop_interface");
   if((ntop_interface = (NetworkInterface*)lua_touserdata(vm, lua_gettop(vm))) == NULL) {
     ntop->getTrace()->traceEvent(TRACE_ERROR, "INTERNAL ERROR: null interface");
     return(0);
   }
 
-  ntop_interface->getActiveFlowsList(vm);
-
-  return(1);
+  if(!ntop_interface->getHostInfo(vm, host_ip))
+    return(0);
+  else
+    return(1);
 }
 
 /* ****************************************** */
@@ -203,6 +220,14 @@ void lua_push_str_table_entry(lua_State *L, const char *key, char *value) {
 
 /* ****************************************** */
 
+void lua_push_bool_table_entry(lua_State *L, const char *key, bool value) {
+  lua_pushstring(L, key);
+  lua_pushboolean(L, value ? 1 : 0);
+  lua_settable(L, -3);
+}
+
+/* ****************************************** */
+
 void lua_push_int_table_entry(lua_State *L, const char *key, u_int32_t value) {
   lua_pushstring(L, key);
   lua_pushinteger(L, value);
@@ -258,6 +283,57 @@ static int ntop_get_resolved_address(lua_State* vm) {
   return(1);
 }
 
+/* ******************************************* */
+
+static void revertSlashIfWIN32(char *str, int mode) {
+#ifdef WIN32
+  int i;
+
+  for(i=0; str[i] != '\0'; i++)
+    switch(mode) {
+    case 0:
+      if(str[i] == '/') str[i] = '\\';
+      //else if(str[i] == ' ') str[i] = '_';
+      break;
+    case 1:
+      if(str[i] == '\\') str[i] = '/';
+      break;
+    }
+#endif
+}
+
+/* ****************************************** */
+
+static int ntop_mkdir_tree(lua_State* vm) {
+  char *dir, path[256];
+  int permission = 0777, i, rc;
+
+  if(ntop_lua_check(vm, __FUNCTION__, 1, LUA_TSTRING)) return(0);
+  if((dir = (char*)lua_tostring(vm, 1)) == NULL)       return(-1);
+  if(dir[0] == '\0')                                   return(1); /* Nothing to do */
+
+  snprintf(path, sizeof(path), "%s", dir);
+
+  revertSlashIfWIN32(path, 0);
+
+  /* Start at 1 to skip the root */
+  for(i=1; path[i] != '\0'; i++)
+    if(path[i] == CONST_PATH_SEP) {
+#ifdef WIN32
+      /* Do not create devices directory */
+      if((i > 1) && (path[i-1] == ':')) continue;
+#endif
+
+      path[i] = '\0';
+      rc = ntop_mkdir(path, permission);
+      path[i] = CONST_PATH_SEP;
+    }
+
+  rc = ntop_mkdir(path, permission);
+
+  return(rc == 0 ? 1 : -1);
+}
+
 /* ****************************************** */
 
 static int ntop_get_redis(lua_State* vm) {
@@ -275,6 +351,13 @@ static int ntop_get_redis(lua_State* vm) {
 
 /* ****************************************** */
 
+static int ntop_get_datadir(lua_State* vm) {
+  lua_pushfstring(vm, "%s", ntop->get_data_dir());
+  return(1);
+}
+
+/* ****************************************** */
+
 static int ntop_set_redis(lua_State* vm) {
   char *key, *value;
   Redis *redis = ntop->getRedis();
@@ -377,9 +460,12 @@ static const luaL_Reg ntop_interface_reg[] = {
 static const luaL_Reg ntop_reg[] = {
   { "getInfo",     ntop_get_info },
   { "dumpFile",    ntop_dump_file },
+  { "getDataDir",  ntop_get_datadir },
   { "getCache",    ntop_get_redis },
   { "setCache",    ntop_set_redis },
   { "getResolvedAddress",    ntop_get_resolved_address },
+  { "mkdir",       ntop_mkdir_tree },
+  { "exists",      ntop_get_file_dir_exists },
   { NULL,          NULL}
 };
 
@@ -427,6 +513,9 @@ void Lua::lua_register_classes(lua_State *L, bool http_mode) {
     lua_register(L, "print", ntop_lua_http_print);
   } else
     lua_register(L, "print", ntop_lua_cli_print);
+
+  /* Register RRD bindings */
+  luaopen_rrd(L);
 }
 
 /* ****************************************** */

Lua.h

@@ -49,5 +49,6 @@ class Lua {
 
 extern void lua_push_str_table_entry(lua_State *L, const char *key, char *value);
 extern void lua_push_int_table_entry(lua_State *L, const char *key, u_int32_t value);
+extern void lua_push_bool_table_entry(lua_State *L, const char *key, bool value);
 
 #endif /* _LUA_H_ */

Makefile.in

@@ -19,7 +19,7 @@ LUAJIT_LIB=$(LUAJIT_HOME)/src/libluajit.a
 ######
 LIBRRDTOOL_HOME=third-party/rrdtool-1.4.7
 LIBRRDTOOL_INC=-I$(LIBRRDTOOL_HOME)/src/
-LIBRRDTOOL=$(LIBRRDTOOL_HOME)/src/.libs/librrd_th.a
+LIBRRDTOOL_LIB=$(LIBRRDTOOL_HOME)/src/.libs/librrd_th.a -lxml2
 
 ######
 TARGET = ntopng

ntop_defines.h

@@ -58,4 +58,12 @@
 #define HOST_PURGE_FREQUENCY     5 /* sec */
 #define HOST_MAX_IDLE           60 /* sec */
 
+#ifdef WIN32
+#define ntop_mkdir(a, b) _mkdir(a)
+#define CONST_PATH_SEP                    '\\'
+#else
+#define ntop_mkdir(a, b) mkdir(a, b)
+#define CONST_PATH_SEP                    '/'
+#endif
+
 #endif /* _NTOP_DEFINES_H_ */

scripts/callbacks/minute.lua

@@ -2,4 +2,70 @@
 -- (C) 2013 - ntop.org
 --
 
-print ("Hello")
+-- Run RRD update every 5 minutes
+-- Use 30 just to avoid rounding issues
+diff = os.time() % 300
+
+-- io.write('Diff: '..diff..'\n')
+
+if(diff > 30) then
+   return
+end
+
+debug = 1
+interface.find("any")
+hosts_stats = interface.getHostsInfo()
+for key, value in pairs(hosts_stats) do
+   -- print ("[" .. key .. "][" .. (hosts_stats[key]["bytes.sent"]+hosts_stats[key]["bytes.rcvd"]) .. "]\n")
+
+   basedir = ntop.getDataDir() .. "/rrd/" .. key
+
+   if(not(ntop.exists(basedir))) then
+      if(debug) then io.write('Creating base directory', basedir, '\n') end
+      ntop.mkdir(basedir)
+   end
+
+   -- Traffic stats
+   name = basedir .. "/bytes.rrd"
+   if(not(ntop.exists(name))) then
+      if(debug) then io.write('Creating RRD ', name, '\n') end
+      rrd.create(
+	 name,
+	 '--start', 'now',
+	 '--step', '300',
+	 'DS:sent:COUNTER:600:U:U',
+	 'DS:rcvd:COUNTER:600:U:U',
+	 'RRA:AVERAGE:0.5:1:50400',  -- raw: 7 days = 7 * 24 = 168 * 300 sec = 50400
+	 'RRA:AVERAGE:0.5:12:2400',  -- 1h resolution (12 points)   2400 hours = 100 days
+	 'RRA:AVERAGE:0.5:288:365',  -- 1d resolution (288 points)  365 days
+	 'RRA:HWPREDICT:1440:0.1:0.0035:20')
+
+   end
+   rrd.update(name, "N:"..hosts_stats[key]["bytes.sent"] .. ":" .. hosts_stats[key]["bytes.rcvd"])
+   if(debug) then io.write('Updating RRD '..name..'\n') end
+
+   -- nDPI Protocols
+   host = interface.getHostInfo(key)
+   if((host ~= nil) and host.localhost) then
+      for k in pairs(host["ndpi"]) do
+	 name = basedir .. "/".. k .. ".rrd"
+	 if(not(ntop.exists(name))) then
+	    if(debug) then io.write('Creating RRD ', name, '\n') end
+	    rrd.create(
+	       name,
+	       '--start', 'now',
+	       '--step', '300',
+	       'DS:sent:COUNTER:600:U:U',
+	       'DS:rcvd:COUNTER:600:U:U',
+	       'RRA:AVERAGE:0.5:1:50400',  -- raw: 7 days = 7 * 24 = 168 * 300 sec = 50400
+	       'RRA:AVERAGE:0.5:12:2400',  -- 1h resolution (12 points)   2400 hours = 100 days
+	       'RRA:AVERAGE:0.5:288:365',  -- 1d resolution (288 points)  365 days
+	       'RRA:HWPREDICT:1440:0.1:0.0035:20')
+
+	 end
+	 rrd.update(name, "N:".. host["ndpi"][k]["sent"] .. ":" .. host["ndpi"][k]["rcvd"])
+	 if(debug) then io.write('Updating RRD '..name..'\n') end
+      end
+   end
+end
+