New segments and length fields and tests, https ES support for viewer

- added session.segments and session.length (issue arkime#254) - support elasticsearch=http:// or https:// format (issue arkime#249) - Only libmagic the first 50 bytes - users tab can now sort various tabs - Turn of bloom filter for previous indexes if using db.pl expire - Set threadpool search queue size to unlimited - stats page works again with dynamic scripts disabled
baubaubau · Jun 2, 2014 · 65421c9 · 65421c9
1 parent 20171d0
commit 65421c9
Show file tree

Hide file tree

Showing 63 changed files with 731 additions and 334 deletions.
diff --git a/CHANGELOG b/CHANGELOG
@@ -8,6 +8,13 @@
   - Parsers can register for session save events (issue #248)
   - Fix compressES check with ES 1.1.1 (issue #255)
   - Show error for ip queries with regex or wildcard (issue #252)
+  - added session.segments and session.length (issue #254)
+  - support elasticsearch=http:// or https:// format (issue #249)
+  - Only libmagic the first 50 bytes
+  - users tab can now sort various tabs
+  - Turn of bloom filter for previous indexes if using db.pl expire
+  - Set threadpool search queue size to unlimited
+  - stats page works again with dynamic scripts disabled
 
 0.11.0 2014/05/08
   - BREAKING: elasticsearch 0.90.7 or newer required, recommend 0.90.12+, 

diff --git a/capture/db.c b/capture/db.c
@@ -227,6 +227,7 @@ void moloch_db_save_session(MolochSession_t *session, int final)
         return;
 
     totalSessions++;
+    session->segments++;
 
     static char     prefix[100];
     static time_t   prefix_time = 0;
@@ -285,6 +286,9 @@ void moloch_db_save_session(MolochSession_t *session, int final)
         BSB_INIT(jbsb, sJson, size);
     }
 
+    uint32_t timediff = (session->lastPacket.tv_sec - session->firstPacket.tv_sec)*1000 +
+                        (session->lastPacket.tv_usec - session->firstPacket.tv_usec)/1000;
+
     startPtr = BSB_WORK_PTR(jbsb);
     BSB_EXPORT_sprintf(jbsb, "{\"index\": {\"_index\": \"sessions-%s\", \"_type\": \"session\", \"_id\": \"%s\"}}\n", prefix, id);
 
@@ -294,6 +298,7 @@ void moloch_db_save_session(MolochSession_t *session, int final)
                       "\"lp\":%u,"
                       "\"fpd\":%" PRIu64 ","
                       "\"lpd\":%" PRIu64 ","
+                      "\"sl\":%u,"
                       "\"a1\":%u,"
                       "\"p1\":%u,"
                       "\"a2\":%u,"
@@ -303,6 +308,7 @@ void moloch_db_save_session(MolochSession_t *session, int final)
                       (uint32_t)session->lastPacket.tv_sec,
                       ((uint64_t)session->firstPacket.tv_sec)*1000 + ((uint64_t)session->firstPacket.tv_usec)/1000,
                       ((uint64_t)session->lastPacket.tv_sec)*1000 + ((uint64_t)session->lastPacket.tv_usec)/1000,
+                      timediff,
                       htonl(session->addr1),
                       session->port1,
                       htonl(session->addr2),
@@ -403,6 +409,7 @@ void moloch_db_save_session(MolochSession_t *session, int final)
                       "\"db\":%" PRIu64 ","
                       "\"db1\":%" PRIu64 ","
                       "\"db2\":%" PRIu64 ","
+                      "\"ss\":%u,"
                       "\"no\":\"%s\",",
                       session->packets[0] + session->packets[1],
                       session->packets[0],
@@ -413,6 +420,7 @@ void moloch_db_save_session(MolochSession_t *session, int final)
                       session->databytes[0] + session->databytes[1],
                       session->databytes[0],
                       session->databytes[1],
+                      session->segments,
                       config.nodeName);
 
     if (session->rootId) {
@@ -1658,10 +1666,13 @@ void moloch_db_add_field(char *group, char *kind, char *expression, char *friend
     if (ap) {
         while (1) {
             field = va_arg(ap, char *);
+            if (!field)
+                break;
+
             value = va_arg(ap, char *);
-            if (!field || !value) {
+            if (!value)
                 break;
-            }
+
             BSB_EXPORT_sprintf(bsb, ", \"%s\": ", field);
             if (*value == '{' || *value == '[')
                 BSB_EXPORT_sprintf(bsb, "%s", value);

diff --git a/capture/field.c b/capture/field.c
@@ -587,3 +587,34 @@ void moloch_field_certsinfo_free (MolochCertsInfo_t *certs)
 
     MOLOCH_TYPE_FREE(MolochCertsInfo_t, certs);
 }
+/******************************************************************************/
+int moloch_field_count(int pos, MolochSession_t *session)
+{
+    MolochField_t         *field;
+
+    if (!session->fields[pos])
+        return 0;
+
+    field = session->fields[pos];
+
+    switch (config.fields[pos]->type) {
+    case MOLOCH_FIELD_TYPE_INT:
+    case MOLOCH_FIELD_TYPE_STR:
+    case MOLOCH_FIELD_TYPE_IP:
+        return 1;
+    case MOLOCH_FIELD_TYPE_STR_ARRAY:
+        return field->sarray->len;
+    case MOLOCH_FIELD_TYPE_INT_ARRAY:
+        return field->iarray->len;
+    case MOLOCH_FIELD_TYPE_STR_HASH:
+        return HASH_COUNT(s_, *(field->shash));
+    case MOLOCH_FIELD_TYPE_INT_HASH:
+    case MOLOCH_FIELD_TYPE_IP_HASH:
+        return HASH_COUNT(s_, *(field->ihash));
+    case MOLOCH_FIELD_TYPE_CERTSINFO:
+        return HASH_COUNT(s_, *(field->cihash));
+    default:
+        LOG("ERROR - Unknown field type for counting %s %d", config.fields[pos]->dbField, config.fields[pos]->type);
+        exit (1);
+    }
+}
diff --git a/capture/http.c b/capture/http.c
@@ -563,7 +563,14 @@ void *moloch_http_create_server(char *hostname, int defaultPort, int maxConns, i
     DLL_INIT(r_, &server->requestQ[0]);
     DLL_INIT(r_, &server->requestQ[1]);
     DLL_INIT(e_, &server->connQ);
-    server->name = strdup(hostname);
+    if (strncmp(hostname, "http://", 7) == 0) {
+        server->name = strdup(hostname+7);
+    } else if (strncmp(hostname, "https://", 8) == 0) {
+        LOG("https not supported yet %s", hostname);
+        exit(0);
+    } else {
+        server->name = strdup(hostname);
+    }
     server->port = defaultPort;
     server->maxConns = maxConns;
     server->maxOutstandingRequests = maxOutstandingRequests;

diff --git a/capture/moloch.h b/capture/moloch.h
@@ -332,6 +332,7 @@ typedef struct moloch_session {
     uint16_t               port2;
     uint16_t               offsets[2];
     uint16_t               outstandingQueries;
+    uint16_t               segments;
 
     uint8_t                consumed[2];
     uint8_t                protocol;
@@ -635,6 +636,7 @@ int moloch_field_by_exp(char *exp);
 gboolean moloch_field_string_add(int pos, MolochSession_t *session, const char *string, int len, gboolean copy);
 gboolean moloch_field_int_add(int pos, MolochSession_t *session, int i);
 gboolean moloch_field_certsinfo_add(int pos, MolochSession_t *session, MolochCertsInfo_t *info, int len);
+int  moloch_field_count(int pos, MolochSession_t *session);
 void moloch_field_certsinfo_free (MolochCertsInfo_t *certs);
 void moloch_field_free(MolochSession_t *session);
 void moloch_field_exit();

diff --git a/capture/parsers.c b/capture/parsers.c
@@ -37,7 +37,10 @@ static magic_t               cookie;
 /******************************************************************************/
 void moloch_parsers_magic_tag(MolochSession_t *session, int field, const char *base, const char *data, int len)
 {
-    const char *m = magic_buffer(cookie, data, len);
+    if (len < 3)
+        return;
+
+    const char *m = magic_buffer(cookie, data, MIN(len,50));
     if (m) {
         char tmp[500];
         snprintf(tmp, sizeof(tmp), "%s:%s", base, m);
@@ -185,6 +188,18 @@ void moloch_parsers_init()
         MOLOCH_FIELD_TYPE_STR_HASH,  MOLOCH_FIELD_FLAG_CNT | MOLOCH_FIELD_FLAG_LINKED_SESSIONS,
         NULL);
 
+    moloch_field_define("general", "integer",
+        "session.segments", "Session Segments", "ss", 
+        "Number of segments in session so far",
+        0,  MOLOCH_FIELD_FLAG_FAKE, 
+        NULL);
+
+    moloch_field_define("general", "integer",
+        "session.length", "Session Length", "sl", 
+        "Session Length in milliseconds so far",
+        0,  MOLOCH_FIELD_FLAG_FAKE, 
+        NULL);
+
     cookie = magic_open(MAGIC_MIME);
     if (!cookie) {
         LOG("Error with libmagic %s", magic_error(cookie));

diff --git a/capture/parsers/http.c b/capture/parsers/http.c
@@ -292,13 +292,11 @@ http_add_value(MolochSession_t *session, HTTPInfo_t *http)
     case MOLOCH_FIELD_TYPE_INT_ARRAY:
     case MOLOCH_FIELD_TYPE_INT_HASH:
         moloch_field_int_add(pos, session, atoi(s));
-        g_string_free(http->valueString[session->which], TRUE);
         break;
     case MOLOCH_FIELD_TYPE_STR:
     case MOLOCH_FIELD_TYPE_STR_ARRAY:
     case MOLOCH_FIELD_TYPE_STR_HASH:
         moloch_field_string_add(pos, session, s, l, TRUE);
-        g_string_free(http->valueString[session->which], TRUE);
         break;
     case MOLOCH_FIELD_TYPE_IP_HASH:
     {
@@ -321,13 +319,12 @@ http_add_value(MolochSession_t *session, HTTPInfo_t *http)
         }
 
         g_strfreev(parts);
-        g_string_free(http->valueString[session->which], TRUE);
         break;
     }
     } /* SWITCH */
 
 
-    http->valueString[session->which] = 0;
+    g_string_truncate(http->valueString[session->which], 0);
     http->pos[session->which] = 0;
 }
 /******************************************************************************/