More tls/cert fields, date relative expressions

- New cert.notbefore, cert.notafter, cert.validfor fields (issue arkime#329)
- New starttime, stoptime, view fields (issue arkime#307)
- New tls.sessionid.dst, tls.sessionid.src, tls.sessionid fields (issue arkime#326)
- Use ELS doc_values for some fields to reduce ES memory

CHANGELOG

@@ -1,7 +1,8 @@
 0.11.3 2014/xx/xx
   - NOTICE: Only 1.[234].x are supported by this version. 
             Restart viewer AFTER upgrading ES versions
-  - NOTICE: Requires running npm update for ES 1.3 support
+  - NOTICE: Requires running npm update for ES 1.3 and moment support
+  - NOTICE: Requires running db.pl host:port upgrade
   - For NEW installs can now set a prefix= config variable and 
     db.pl --prefix option that will prefix all ES tables.  This makes 
     it easier for Moloch to share ES with other services OR multiple 
@@ -41,6 +42,12 @@
   - monitor + recursive should monitor new directories (issue #305)
   - Fixed addUser.js error with when mulitple es nodes are listed in config.ini (issue #322)
   - WISE - Tagger files can have views defined with #view:
+  - New cert.notbefore, cert.notafter, cert.validfor fields (issue #329)
+  - New starttime, stoptime, view fields (issue #307)
+  - New tls.sessionid.dst, tls.sessionid.src, tls.sessionid fields (issue #326) 
+  - Use ELS doc_values for some fields to reduce ES memory
+  - Added cert.cnt back
+
 
 
 0.11.2 2014/10/16

capture/config.c

@@ -425,8 +425,10 @@ void moloch_config_load_header(char *section, char *group, char *helpBase, char
         int count  = 0;
         char *kind = 0;
         for (v = 0; v < values_len; v++) {
-            if (strcmp(values[v], "type:integer") == 0) {
+            if (strcmp(values[v], "type:integer") == 0 ||
+                strcmp(values[v], "type:seconds") == 0) {
                 type = 1;
+                kind = values[v] + 5;
             } else if (strcmp(values[v], "type:ip") == 0) {
                 type = 2;
             } else if (strcmp(values[v], "unique:false") == 0) {
@@ -435,7 +437,6 @@ void moloch_config_load_header(char *section, char *group, char *helpBase, char
                 count = 1;
             }
         }
-        g_strfreev(values);
 
         int f = flags;
 
@@ -451,7 +452,6 @@ void moloch_config_load_header(char *section, char *group, char *helpBase, char
                 t = MOLOCH_FIELD_TYPE_STR_ARRAY;
             break;
         case 1:
-            kind = "integer";
             if (unique)
                 t = MOLOCH_FIELD_TYPE_INT_HASH;
             else
@@ -470,6 +470,7 @@ void moloch_config_load_header(char *section, char *group, char *helpBase, char
         HASH_FIND(s_, *hash, keys[k], hstring);
         if (hstring) {
             LOG("WARNING - ignoring field %s for %s", keys[k], section);
+            g_strfreev(values);
             continue;
         }
 
@@ -504,6 +505,7 @@ void moloch_config_load_header(char *section, char *group, char *helpBase, char
                     t, f, NULL);
         }
         moloch_config_add_header(hash, g_strdup(keys[k]), pos);
+        g_strfreev(values);
     }
     g_strfreev(keys);
 }

capture/db.c

@@ -29,7 +29,7 @@
 #include "patricia.h"
 #include "GeoIP.h"
 
-#define MOLOCH_MIN_DB_VERSION 18
+#define MOLOCH_MIN_DB_VERSION 21
 
 extern uint64_t         totalPackets;
 extern uint64_t         totalBytes;
@@ -807,6 +807,10 @@ void moloch_db_save_session(MolochSession_t *session, int final)
                     BSB_EXPORT_u08(jbsb, ',');
                 }
 
+                BSB_EXPORT_sprintf(jbsb, "\"notBefore\": %" PRId64 ",", certs->notBefore);
+                BSB_EXPORT_sprintf(jbsb, "\"notAfter\": %" PRId64 ",", certs->notAfter);
+                BSB_EXPORT_sprintf(jbsb, "\"diffDays\": %" PRId64 ",", (certs->notAfter - certs->notBefore)/(60*60*24));
+
                 BSB_EXPORT_rewind(jbsb, 1); // Remove last comma
 
                 moloch_field_certsinfo_free(certs);

capture/field.c

@@ -161,7 +161,8 @@ int moloch_field_define_text(char *text, int *shortcut)
         help = field;
 
     int type, flags = 0;
-    if (strcmp(kind, "integer") == 0)
+    if (strcmp(kind, "integer") == 0 ||
+        strcmp(kind, "seconds") == 0)
         type = MOLOCH_FIELD_TYPE_INT_HASH;
     else if (strcmp(kind, "ip") == 0)
         type = MOLOCH_FIELD_TYPE_IP_HASH;

capture/moloch.h

@@ -97,6 +97,8 @@ typedef struct {
 typedef struct moloch_tlsinfo{
     struct moloch_tlsinfo *t_next, *t_prev;
     uint32_t               t_hash;
+    uint64_t               notBefore;
+    uint64_t               notAfter;
     MolochCertInfo_t       issuer;
     MolochCertInfo_t       subject;
     MolochStringHead_t     alt;

capture/parsers/dns.c

@@ -373,6 +373,7 @@ void moloch_parser_init()
     DNS_CLASSIFY("\x01\x00");
     DNS_CLASSIFY("\x01\x10");
     DNS_CLASSIFY("\x01\x82");
+    DNS_CLASSIFY("\x81\x00");
     DNS_CLASSIFY("\x81\x80");
     DNS_CLASSIFY("\x81\x82");
     DNS_CLASSIFY("\x81\x90");

capture/parsers/tls.c

@@ -23,13 +23,17 @@ static int                   certsField;
 static int                   hostField;
 static int                   verField;
 static int                   cipherField;
+static int                   srcIdField;
+static int                   dstIdField;
 
 typedef struct {
     unsigned char       buf[8192];
     uint16_t            len;
     char                which;
 } TLSInfo_t;
 
+extern unsigned char    moloch_char_to_hexstr[256][3];
+
 /******************************************************************************/
 void
 tls_certinfo_process(MolochCertInfo_t *ci, BSB *bsb)
@@ -124,8 +128,21 @@ void tls_process_server_hello(MolochSession_t *session, const unsigned char *dat
 
     int skiplen = 0;
     BSB_IMPORT_u08(bsb, skiplen);   // Session Id Length
+    if (skiplen > 0 && BSB_REMAINING(bsb) > skiplen) {
+        unsigned char *ptr = BSB_WORK_PTR(bsb);
+        char sessionId[513];
+        int  i;
+
+        for(i=0; i < skiplen; i++) {
+            sessionId[i*2] = moloch_char_to_hexstr[ptr[i]][0];
+            sessionId[i*2+1] = moloch_char_to_hexstr[ptr[i]][1];
+        }
+        sessionId[skiplen*2] = 0;
+        moloch_field_string_add(dstIdField, session, sessionId, skiplen*2, TRUE);
+    }
     BSB_IMPORT_skip(bsb, skiplen);  // Session Id
 
+
     unsigned char *cipher;
     BSB_IMPORT_ptr(bsb, cipher, 2);
 
@@ -164,6 +181,80 @@ void tls_process_server_hello(MolochSession_t *session, const unsigned char *dat
         }
     }
 }
+
+#define char2num(ch) (isdigit(ch)?((ch) - '0'):0)
+#define str2num(str) (char2num((str)[0]) * 10 + char2num((str)[1]))
+#define str4num(str) (char2num((str)[0]) * 1000 + char2num((str)[1]) * 100 + char2num((str)[2]) * 10 + char2num((str)[3]))
+
+/******************************************************************************/
+uint64_t tls_parse_time(int tag, unsigned char* value, int len)
+{
+    int        offset = 0;
+    int        pos = 0;
+    struct tm  tm;
+
+    //UTCTime
+    if (tag == 23 && len > 12) {
+        if (len > 17 && value[12] != 'Z')
+            offset = str2num(value+13) * 60 + str2num(value+15);
+
+        if (value[12] == '-')
+            offset = -offset;
+
+        tm.tm_year = str2num(value+0);
+        tm.tm_mon  = str2num(value+2) - 1;
+        tm.tm_mday = str2num(value+4);
+        tm.tm_hour = str2num(value+6);
+        tm.tm_min  = str2num(value+8);
+        tm.tm_sec  = str2num(value+10);
+
+        if (tm.tm_year < 50)
+            tm.tm_year += 100;
+
+        return timegm(&tm) + offset;
+    }
+    //GeneralizedTime
+    else if (tag == 24 && len >= 10) {
+        memset(&tm, 0, sizeof(tm));
+        tm.tm_year = str4num(value+0) - 1900;
+        tm.tm_mon  = str2num(value+4) - 1;
+        tm.tm_mday = str2num(value+6);
+        tm.tm_hour = str2num(value+8);
+        if (len < 10 || value[10] == 'Z' || value[10] == '+' || value[10] == '-') {
+            pos = 10;
+            goto gtdone;
+        }
+        tm.tm_min  = str2num(value+10);
+        if (len < 12 || value[12] == 'Z' || value[12] == '+' || value[12] == '-') {
+            pos = 12;
+            goto gtdone;
+        }
+        tm.tm_sec  = str2num(value+12);
+        if (len < 14 || value[14] == 'Z' || value[14] == '+' || value[14] == '-') {
+            pos = 14;
+            goto gtdone;
+        }
+        if (value[14] == '.') {
+            pos = 18;
+        } else {
+            pos = 14;
+        }
+    gtdone:
+        if (pos == len) {
+            return mktime(&tm);
+        }
+
+        if (pos + 5 < len && (value[pos] == '+' || value[pos] == '-')) {
+            offset = str2num(value+pos+1) * 60 +  str2num(value+pos+3);
+
+            if (value[pos] == '-')
+                offset = -offset;
+        }
+
+        return timegm(&tm) + offset;
+    }
+    return 0;
+}
 /******************************************************************************/
 void tls_process_server_certificate(MolochSession_t *session, const unsigned char *data, int len)
 {
@@ -226,6 +317,15 @@ void tls_process_server_certificate(MolochSession_t *session, const unsigned cha
         if (!(value = moloch_parsers_asn_get_tlv(&bsb, &apc, &atag, &alen)))
             {badreason = 7; goto bad_cert;}
 
+        BSB_INIT(tbsb, value, alen);
+        if (!(value = moloch_parsers_asn_get_tlv(&tbsb, &apc, &atag, &alen)))
+            {badreason = 7; goto bad_cert;}
+        certs->notBefore = tls_parse_time(atag, value, alen);
+
+        if (!(value = moloch_parsers_asn_get_tlv(&tbsb, &apc, &atag, &alen)))
+            {badreason = 7; goto bad_cert;}
+        certs->notAfter = tls_parse_time(atag, value, alen);
+
         /* subject */
         if (!(value = moloch_parsers_asn_get_tlv(&bsb, &apc, &atag, &alen)))
             {badreason = 8; goto bad_cert;}
@@ -319,6 +419,18 @@ tls_process_client(MolochSession_t *session, const unsigned char *data, int len)
 
                 int skiplen = 0;
                 BSB_IMPORT_u08(cbsb, skiplen);   // Session Id Length
+                if (skiplen > 0 && BSB_REMAINING(cbsb) > skiplen) {
+                    unsigned char *ptr = BSB_WORK_PTR(cbsb);
+                    char sessionId[513];
+                    int  i;
+
+                    for(i=0; i < skiplen; i++) {
+                        sessionId[i*2] = moloch_char_to_hexstr[ptr[i]][0];
+                        sessionId[i*2+1] = moloch_char_to_hexstr[ptr[i]][1];
+                    }
+                    sessionId[skiplen*2] = 0;
+                    moloch_field_string_add(srcIdField, session, sessionId, skiplen*2, TRUE);
+                }
                 BSB_IMPORT_skip(cbsb, skiplen);  // Session Id
 
                 BSB_IMPORT_u16(cbsb, skiplen);   // Ciper Suites Length
@@ -474,10 +586,16 @@ void moloch_parser_init()
 {
     certsField = moloch_field_define("cert", "notreal",
         "cert", "tls", "tls",
-        "TLS Info",
+        "CERT Info",
         MOLOCH_FIELD_TYPE_CERTSINFO,  MOLOCH_FIELD_FLAG_CNT | MOLOCH_FIELD_FLAG_NODB,
         NULL);
 
+    moloch_field_define("cert", "integer",
+        "cert.cnt", "Cert Cnt", "tlscnt",
+        "Count of certificates",
+        0, MOLOCH_FIELD_FLAG_FAKE,
+        NULL);
+
     moloch_field_define("cert", "lotermfield",
         "cert.alt", "Alt Name", "tls.alt",
         "Certificate alternative names",
@@ -516,6 +634,24 @@ void moloch_parser_init()
         "rawField", "rawsOn",
         NULL);
 
+    moloch_field_define("cert", "seconds",
+        "cert.notbefore", "Not Before", "tls.notBefore",
+        "Certificate is not valid before this date",
+        0, MOLOCH_FIELD_FLAG_FAKE,
+        NULL);
+
+    moloch_field_define("cert", "seconds",
+        "cert.notafter", "Not After", "tls.notAfter",
+        "Certificate is not valid after this date",
+        0, MOLOCH_FIELD_FLAG_FAKE,
+        NULL);
+
+    moloch_field_define("cert", "integer",
+        "cert.validfor", "Days Valid For", "tls.diffDays",
+        "Certificate is valid for this may days",
+        0, MOLOCH_FIELD_FLAG_FAKE,
+        NULL);
+
     hostField = moloch_field_define("http", "lotermfield",
         "host.http", "Hostname", "ho", 
         "HTTP host header field", 
@@ -534,6 +670,25 @@ void moloch_parser_init()
         MOLOCH_FIELD_TYPE_STR_HASH,  MOLOCH_FIELD_FLAG_CNT, 
         NULL);
 
+    dstIdField = moloch_field_define("tls", "lotermfield",
+        "tls.sessionid.dst", "Dst Session Id", "tlsdstid-term", 
+        "SSL/TLS Dst Session Id", 
+        MOLOCH_FIELD_TYPE_STR_HASH,  0, 
+        NULL);
+
+    srcIdField = moloch_field_define("tls", "lotermfield",
+        "tls.sessionid.src", "Src Session Id", "tlssrcid-term", 
+        "SSL/TLS Src Session Id", 
+        MOLOCH_FIELD_TYPE_STR_HASH,  0, 
+        NULL);
+
+    moloch_field_define("general", "lotermfield",
+        "tls.sessionid", "Src or Dst Session Id", "tlsidall",
+        "Shorthand for tls.sessionid.src or tls.sessionid.dst",
+        0,  MOLOCH_FIELD_FLAG_FAKE,
+        "regex", "^tls\\\\.sessionid\\\\.(?:(?!\\\\.cnt$).)*$",
+        NULL);
+
     moloch_parsers_classifier_register_tcp("tls", 0, (unsigned char*)"\x16\x03", 2, tls_classify);
 }
 

capture/parsers/tls.detail.jade

@@ -3,13 +3,21 @@ if (session.tls || session["tlsver-term"])
   dl.sessionDetailMeta
     +arrayList(session, "tlsver-term", "Version", "tls.version")
     +arrayList(session, "tlscipher-term", "Cipher", "tls.cipher")
+    +arrayList(session, "tlssrcid-term", "Src Session Id", "tls.sessionid.src")
+    +arrayList(session, "tlsdstid-term", "Dst Session Id", "tls.sessionid.dst")
     if (session.tls)
       each cert in session.tls
         dt Certificate
         dd
           - if (cert.sn)
             | Serial: 
             a(href='#', onclick='return addExpression("cert.serial == #{cert.sn}");') #{cert.sn}
+          - if (cert.notBefore)
+            |  Not Before: 
+            a.formatSeconds(href='#', onclick='return addExpressionSeconds("cert.notbefore", #{cert.notBefore});') #{cert.notBefore}
+          - if (cert.notAfter)
+            |  Not After: 
+            a.formatSeconds(href='#', onclick='return addExpressionSeconds("cert.notafter", #{cert.notBefore});') #{cert.notBefore}
           - if (cert.iCn && Array.isArray(cert.iCn))
             |  Issuer Common: 
             each cn,i in cert.iCn