Fix smtp subject empty encoded fix, ES 5 min timeout

* Empty smtp subject encodings caused glib warnings or crashes
* Increased ES query timeout from 30s to 5 minutes
* email.subject is a case sensitive field

CHANGELOG

@@ -25,6 +25,8 @@
   - Start of viewer regression testing
   - Fix reverse http header parsing
   - simple mysql parser
+  - Fix smtp subject empty encoded sections
+  - Increase ES query timeout to 5 minutes
 
 0.10.1 2014/03/30
   - Status code not being set when . after mime data

capture/parsers/smtp.c

@@ -237,7 +237,9 @@ smtp_email_add_encoded(MolochSession_t *session, int pos, char *string, int len)
         /* question+1               = encoding */
         /* question+3 - endquestion = encoded-text */
 
-        if (*(question+1) == 'B' || *(question+1) == 'b') {
+        if (question+3 == endquestion) {
+            // The encoded text is empty
+        } else if (*(question+1) == 'B' || *(question+1) == 'b') {
             *question = 0;
             *endquestion = 0;
 
@@ -852,7 +854,7 @@ void moloch_parser_init()
         "requiredRight", "emailSearch",
         NULL);
 
-    subField = moloch_field_define("email", "lotextfield",
+    subField = moloch_field_define("email", "textfield",
         "email.subject", "Subject", "esub",
         "Email subject header",
         MOLOCH_FIELD_TYPE_STR_HASH,  MOLOCH_FIELD_FLAG_CNT | MOLOCH_FIELD_FLAG_FORCE_UTF8,

db/db.pl

@@ -811,6 +811,7 @@ sub fieldsUpdate
 
     esPost("/fields/field/dns.status/_update", '{doc: {type: "uptermfield"}}', 1);
     esPost("/fields/field/http.hasheader/_update", '{doc: {regex: "^http.hasheader\\\\.(?:(?!\\\\.cnt$).)*$"}}', 1);
+    esPost("/fields/field/email.subject/_update", '{doc: {type: "textfield"}}', 1);
 }
 
 

tests/smtp-subject-encoded-empty.test

@@ -0,0 +1,129 @@
+{
+   "packets" : [
+      {
+         "body" : {
+            "edstcnt" : 1,
+            "test" : {
+               "number" : [
+                  33554442
+               ],
+               "ip" : [
+                  167772161
+               ],
+               "ip-asn" : [
+                  "AS0000 This is neat"
+               ],
+               "string" : [
+                  "16777226:62855,33554442:25"
+               ],
+               "ip-geo" : [
+                  "RUS"
+               ],
+               "ip-rir" : [
+                  ""
+               ]
+            },
+            "db2" : 756,
+            "db" : 1106,
+            "esrc" : [
+               "[email protected]"
+            ],
+            "no" : "test",
+            "lp" : 1398431453,
+            "a2" : "10.0.0.2",
+            "esub" : [
+               "Can this 10 Second Trick Help Prevent YOUR Heart Attack?"
+            ],
+            "ta" : [
+               "dstip",
+               "node:test",
+               "protocol:smtp",
+               "smtp:statuscode:250",
+               "srcip",
+               "tcp"
+            ],
+            "pa1" : 19,
+            "fpd" : 1398431422481,
+            "fs" : [],
+            "by2" : 1688,
+            "g1" : "RUS",
+            "pa2" : 14,
+            "p1" : 62855,
+            "by" : 3304,
+            "as1" : "AS0000 This is neat",
+            "ehh" : [
+               "from",
+               "to",
+               "subject"
+            ],
+            "ehhcnt" : 3,
+            "g2" : "CAN",
+            "pr" : 6,
+            "ps" : [
+               24,
+               118,
+               208,
+               290,
+               681,
+               763,
+               873,
+               955,
+               1245,
+               1327,
+               1436,
+               1558,
+               1640,
+               1747,
+               1872,
+               1954,
+               2042,
+               2174,
+               2256,
+               2359,
+               2441,
+               2757,
+               2839,
+               2924,
+               3006,
+               3144,
+               3226,
+               3314,
+               3446,
+               3528,
+               3610,
+               3692,
+               3774
+            ],
+            "prot-term-cnt" : 2,
+            "lpd" : 1398431453159,
+            "fp" : 1398431422,
+            "as2" : "AS0001 Cool Beans!",
+            "pa" : 33,
+            "tacnt" : 6,
+            "fb1" : "45484c4f20787878",
+            "edst" : [
+               "[email protected]"
+            ],
+            "esubcnt" : 1,
+            "a1" : "10.0.0.1",
+            "fb2" : "3232302d78787878",
+            "db1" : 350,
+            "by1" : 1616,
+            "esrccnt" : 1,
+            "p2" : 25,
+            "rir2" : "TEST",
+            "prot-term" : [
+               "smtp",
+               "tcp"
+            ]
+         },
+         "header" : {
+            "index" : {
+               "_index" : "sessions-140425",
+               "_type" : "session"
+            }
+         }
+      }
+   ]
+}
+

tests/tests.pl

@@ -141,7 +141,7 @@ sub countTest {
 sub doViewer {
 my ($cmd) = @_;
 
-    plan tests => 486;
+    plan tests => 498;
 
     die "Must run in tests directory" if (! -f "../db/db.pl");
 
@@ -472,6 +472,8 @@ sub doViewer {
     $main::userAgent->get("http://localhost:9200/_refresh");
     countTest(0, "date=-1&expression=" . uri_escape("tags==COPYTEST1"));
 
+    unlink("copytest.pcap");
+
     if ($cmd eq "--viewer") {
         $main::userAgent->post("http://localhost:8123/shutdown");
     }

viewer/db.js

@@ -42,7 +42,8 @@ exports.initialize = function (info) {
 
   internals.elasticSearchClient = new ESC.Client({
     host: info.host + ":" + info.port,
-    apiVersion: "0.90"
+    apiVersion: "0.90",
+    requestTimeout: 300000
   });
 
   // Replace tag implementation