Start of 0.11.2, Cron Queries feature, regex improvements

  - New experimental "Cron Queries" feature
    * ONE and ONLY one viewer should have "cronQueries=true"
    * New [moloch-clusters] config section to send sessions
      from one cluster to another
  - Doubled the number of sockets from viewer to ES, now 20
  - Regex and wildcard support for file expression
  - Regex is stricter about back slashing (issue arkime#281)
  - Cache user lookups for 5 seconds

CHANGELOG

@@ -1,5 +1,20 @@
+0.11.2 2014/08/xx
+  - NOTICE: ES 1.1.x, 1.2.x, 1.3.x are supported by this version. 
+            ES 0.90.12 still works but will no longer be supported.
+            Restart viewer AFTER upgrading ES versions
+  - NOTICE: Requires running db.pl upgrade
+  - NOTICE: Requires running npm update for ES 1.3 support
+  - New experimental "Cron Queries" feature
+    * ONE and ONLY one viewer should have "cronQueries=true"
+    * New [moloch-clusters] config section to send sessions 
+      from one cluster to another
+  - Doubled the number of sockets from viewer to ES, now 20
+  - Regex and wildcard support for file expression
+  - Regex is stricter about back slashing (issue #281)
+  - Cache user lookups for 5 seconds
+
 0.11.1 2014/08/07
-  - NOTICE: ES 0.90.12+, 1.1.x, 1.2.0 are support by this version. 
+  - NOTICE: ES 0.90.12+, 1.1.x, 1.2.0 are supported by this version. 
             ES 1.0 is NOT supported.
             This is the LAST version to support 0.90.x
             Restart viewer AFTER upgrading ES versions

capture/dll.h

@@ -45,6 +45,15 @@
      (head)->name##count-- \
     )
 
+#define DLL_MOVE_TAIL(name,head,element) \
+    ((element)->name##prev->name##next = (element)->name##next, \
+     (element)->name##next->name##prev = (element)->name##prev, \
+     (element)->name##next             = (void *)(head), \
+     (element)->name##prev             = (head)->name##prev, \
+     (head)->name##prev->name##next    = (element), \
+     (head)->name##prev                = (element) \
+    )
+
 #define DLL_POP_HEAD(name, head, element) \
     ((head)->name##count == 0 ? ((element) = NULL, 0) : ((element) = (head)->name##next, DLL_REMOVE(name, (head), (element)), 1))
 

capture/field.c

@@ -74,7 +74,6 @@ int moloch_field_define(char *group, char *kind, char *expression, char *friendl
     char friendlyName2[1000];
     char help2[1000];
     char rawField[100];
-    int  pos = -1;
 
     MolochFieldInfo_t *minfo = 0;
     HASH_FIND(f_, fields, dbField, minfo);
@@ -109,11 +108,10 @@ int moloch_field_define(char *group, char *kind, char *expression, char *friendl
 
     if ((flags & MOLOCH_FIELD_FLAG_FAKE) == 0) {
         if (minfo->pos == -1) {
-            pos = config.maxField++;
-            minfo->pos      = pos;
+            minfo->pos = config.maxField++;
         }
 
-        config.fields[pos] = minfo;
+        config.fields[minfo->pos] = minfo;
 
         // Change leading part to dbGroup
         char *firstdot = strchr(minfo->dbField, '.');
@@ -135,7 +133,7 @@ int moloch_field_define(char *group, char *kind, char *expression, char *friendl
     }
 
     if (flags & MOLOCH_FIELD_FLAG_NODB)
-        return pos;
+        return minfo->pos;
 
     MolochFieldInfo_t *info = 0;
     if (flags & MOLOCH_FIELD_FLAG_CNT) {
@@ -239,7 +237,7 @@ int moloch_field_define(char *group, char *kind, char *expression, char *friendl
             moloch_db_add_field(group, "uptermfield", expression2, friendlyName2, dbField2, help2, NULL);
         }
     }
-    return pos;
+    return minfo->pos;
 }
 /******************************************************************************/
 int moloch_field_by_db(char *dbField)

capture/nids.c

@@ -285,8 +285,7 @@ void moloch_nids_mid_save_session(MolochSession_t *session)
     session->lastFileNum = 0;
 
     if (session->tcp_next) {
-        DLL_REMOVE(tcp_, &tcpWriteQ, session);
-        DLL_PUSH_TAIL(tcp_, &tcpWriteQ, session);
+        DLL_MOVE_TAIL(tcp_, &tcpWriteQ, session);
     }
 
     session->lastSave = nids_last_pcap_header->ts.tv_sec;
@@ -703,8 +702,7 @@ void moloch_nids_cb_ip(struct ip *packet, int len)
         if (pluginsCbs & MOLOCH_PLUGIN_NEW)
             moloch_plugins_cb_new(session);
     } else {
-        DLL_REMOVE(q_, sessionsQ, session);
-        DLL_PUSH_TAIL(q_, sessionsQ, session);
+        DLL_MOVE_TAIL(q_, sessionsQ, session);
     }
 
     switch (packet->ip_p) {
@@ -723,6 +721,10 @@ void moloch_nids_cb_ip(struct ip *packet, int len)
                           session->port2 == ntohs(tcphdr->th_dport))?0:1;
         session->tcp_flags |= *((char*)packet + 4 * packet->ip_hl+12);
         break;
+    case IPPROTO_ICMP:
+        session->which = (session->addr1 == packet->ip_src.s_addr &&
+                          session->addr2 == packet->ip_dst.s_addr)?0:1;
+        break;
     }
 
     session->bytes[session->which] += nids_last_pcap_header->caplen;
@@ -793,8 +795,7 @@ void moloch_nids_cb_ip(struct ip *packet, int len)
             //LOG("Saving because of at head %s", moloch_friendly_session_id(headSession->protocol, headSession->addr1, headSession->port1, headSession->addr2, headSession->port2));
             headSession->lastPacket.tv_sec = nids_last_pcap_header->ts.tv_sec;
 
-            DLL_REMOVE(q_, sessionsQ, headSession);
-            DLL_PUSH_TAIL(q_, sessionsQ, headSession);
+            DLL_MOVE_TAIL(q_, sessionsQ, headSession);
 
             moloch_nids_mid_save_session(headSession);
         } else

capture/parsers.c

@@ -200,7 +200,18 @@ void moloch_parsers_init()
         0,  MOLOCH_FIELD_FLAG_FAKE, 
         NULL);
 
-    cookie = magic_open(MAGIC_MIME);
+    int flags = MAGIC_MIME;
+#ifdef MAGIC_NO_CHECK_COMPRESS
+    flags |= MAGIC_NO_CHECK_COMPRESS |
+             MAGIC_NO_CHECK_TAR      |
+             MAGIC_NO_CHECK_APPTYPE  |
+             MAGIC_NO_CHECK_ELF      |
+             MAGIC_NO_CHECK_TOKENS;
+#endif
+#ifdef MAGIC_NO_CHECK_CDF
+    flags |= MAGIC_NO_CHECK_CDF;
+#endif
+    cookie = magic_open(flags);
     if (!cookie) {
         LOG("Error with libmagic %s", magic_error(cookie));
     } else {

config.ini

@@ -262,6 +262,9 @@ nodeClass = class1
 # Might use a different elasticsearch node
 elasticsearch=elasticsearchhost1
 
+# Uncomment if this node should process the cron queries, only ONE node should process cron queries 
+# cronQueries = true
+
 [node2]
 nodeClass = class2
 # Might use a different elasticsearch node
@@ -302,3 +305,13 @@ interface = eth4
 # headers-email is used to configure email headers to index
 #[headers-email]
 #x-priority=type:integer
+
+
+##############################################################################
+# If you have multiple clusters and you want the ability to send sessions
+# from one cluster to another either manually or with the cron feature fill out
+# this section
+
+#[moloch-clusters]
+#forensics=url:https://viewer1.host.domain:8005;passwordSecret:password4moloch;name:Forensics Cluster
+#shortname2=url:http://viewer2.host.domain:8123;passwordSecret:password4moloch;name:Testing Cluster

configure

@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for moloch 0.11.1
+# Generated by GNU Autoconf 2.69 for moloch 0.11.2-GIT.
 #
 #
 # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
@@ -576,8 +576,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='moloch'
 PACKAGE_TARNAME='moloch'
-PACKAGE_VERSION='0.11.1'
-PACKAGE_STRING='moloch 0.11.1'
+PACKAGE_VERSION='0.11.2-GIT'
+PACKAGE_STRING='moloch 0.11.2-GIT'
 PACKAGE_BUGREPORT=''
 PACKAGE_URL=''
 
@@ -1248,7 +1248,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures moloch 0.11.1 to adapt to many kinds of systems.
+\`configure' configures moloch 0.11.2-GIT to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1318,7 +1318,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of moloch 0.11.1:";;
+     short | recursive ) echo "Configuration of moloch 0.11.2-GIT:";;
    esac
   cat <<\_ACEOF
 
@@ -1418,7 +1418,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-moloch configure 0.11.1
+moloch configure 0.11.2-GIT
 generated by GNU Autoconf 2.69
 
 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -1557,7 +1557,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by moloch $as_me 0.11.1, which was
+It was created by moloch $as_me 0.11.2-GIT, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   $ $0 $@
@@ -2223,7 +2223,7 @@ fi
 
 # Define the identity of the package.
  PACKAGE='moloch'
- VERSION='0.11.1'
+ VERSION='0.11.2-GIT'
 
 
 cat >>confdefs.h <<_ACEOF
@@ -5079,7 +5079,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by moloch $as_me 0.11.1, which was
+This file was extended by moloch $as_me 0.11.2-GIT, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -5145,7 +5145,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-moloch config.status 0.11.1
+moloch config.status 0.11.2-GIT
 configured by $0, generated by GNU Autoconf 2.69,
   with options \\"\$ac_cs_config\\"
 

configure.ac

@@ -1,4 +1,4 @@
-AC_INIT([moloch], [0.11.1])
+AC_INIT([moloch], [0.11.2-GIT])
 AM_INIT_AUTOMAKE([-Wall -Werror foreign])
 AC_PROG_CC
 AC_PROG_CXX