Initial bigendian support in viewer (issue arkime#259)

baubaubau · Jun 17, 2014 · 36860a5 · 36860a5
1 parent 68adc35
commit 36860a5
Show file tree

Hide file tree

Showing 6 changed files with 88 additions and 9 deletions.
diff --git a/CHANGELOG b/CHANGELOG
@@ -22,6 +22,7 @@
   - Netflow plugin timestamp fixes (issue #241)
   - Comma seperate list of elasticsearch hosts (issue #176)
   - New includes directive (issue #144)
+  - Initial bigendian support in viewer (issue #259)
 
 0.11.0 2014/05/08
   - BREAKING: elasticsearch 0.90.7 or newer required, recommend 0.90.12+, 

diff --git a/tests/README b/tests/README
@@ -2,3 +2,9 @@ This directory contains simple pcap regression tests.
 By running the command "./tests.pl" each pcap file is run thru moloch capture with the results compared against the matching .test file.
 The pcap file have been mostly anonymized.  mostly...  If you see data that shouldn't be there please let us know.
 If you have simple sample pcap files that we can use please share them!
+
+Running the command "./tests.pl --viewer" does basic viewer regression.
+
+
+Files with known non moloch source:
+bigendian.pcap - https://bugs.wireshark.org/bugzilla/show_bug.cgi?id=7221
diff --git a/tests/bigendian.pcap b/tests/bigendian.pcap
diff --git a/tests/bigendian.test b/tests/bigendian.test
@@ -0,0 +1,51 @@
+{
+   "packets" : [
+      {
+         "body" : {
+            "db2" : 0,
+            "db" : 0,
+            "lpd" : 1335958317529,
+            "fp" : 1335958313,
+            "no" : "test",
+            "lp" : 1335958317,
+            "pa" : 2,
+            "tacnt" : 2,
+            "sl" : 4376,
+            "a2" : "10.64.11.49",
+            "ss" : 1,
+            "ta" : [
+               "ICMP",
+               "node:test"
+            ],
+            "pa1" : 2,
+            "fpd" : 1335958313152,
+            "fs" : [],
+            "by2" : 0,
+            "a1" : "192.168.177.160",
+            "db1" : 0,
+            "pa2" : 0,
+            "p1" : 0,
+            "by1" : 196,
+            "by" : 196,
+            "p2" : 0,
+            "rir1" : "ARIN",
+            "prot-term" : [
+               "icmp"
+            ],
+            "pr" : 1,
+            "ps" : [
+               24,
+               138
+            ],
+            "prot-term-cnt" : 1
+         },
+         "header" : {
+            "index" : {
+               "_index" : "sessions-120502",
+               "_type" : "session"
+            }
+         }
+      }
+   ]
+}
+
diff --git a/tests/tests.pl b/tests/tests.pl
@@ -150,7 +150,7 @@ sub countTest {
 sub doViewer {
 my ($cmd) = @_;
 
-    plan tests => 736;
+    plan tests => 738;
 
     die "Must run in tests directory" if (! -f "../db/db.pl");
 
@@ -585,6 +585,12 @@ sub doViewer {
     countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/long-session.pcap||file=$pwd/socks5-reverse.pcap)&&session.length<908493"));
     countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/long-session.pcap||file=$pwd/socks5-reverse.pcap)&&session.length=[908493,908494]"));
 
+# bigendian tests
+    my $json = viewerGet("/sessions.json?date=-1&expression=" . uri_escape("file=$pwd/bigendian.pcap"));
+    is ($json->{iTotalDisplayRecords}, 1, "bigendian iTotalDisplayRecords");
+    my $response = $main::userAgent->get("http://localhost:8123/test/raw/" . $json->{aaData}->[0]->{id} . "?type=src");
+    is (unpack("H*", $response->content), "4fa11b290002538d08090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f30313233343536374fa11b2d0008129108090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f3031323334353637", "Correct bigendian tcpdump data");
+
 # adding/removing tags test expression
     countTest(3, "date=-1&expression=" . uri_escape("file=$pwd/copytest.pcap"));
     countTest(0, "date=-1&expression=" . uri_escape("tags==COPYTEST1"));

diff --git a/viewer/pcap.js b/viewer/pcap.js
@@ -66,6 +66,7 @@ Pcap.prototype.open = function(filename) {
   }
   this.filename = filename;
   this.fd = fs.openSync(filename, "r");
+  this.readHeader();
 };
 
 Pcap.prototype.openReadWrite = function(filename) {
@@ -112,7 +113,12 @@ Pcap.prototype.readHeader = function(cb) {
 
   this.headBuffer = new Buffer(24);
   fs.readSync(this.fd, this.headBuffer, 0, 24, 0);
-  this.linkType =  this.headBuffer.readUInt32LE(20);
+  this.bigEndian  = this.headBuffer.readUInt32LE(0) === 0xd4c3b2a1;
+  if (this.bigEndian) {
+    this.linkType   = this.headBuffer.readUInt32BE(20);
+  } else {
+    this.linkType   = this.headBuffer.readUInt32LE(20);
+  }
 
   if (cb) {
     cb(this.headBuffer);
@@ -137,7 +143,7 @@ Pcap.prototype.readPacket = function(pos, cb) {
       if (bytesRead < 16) {
         return cb(null);
       }
-      var len = buffer.readInt32LE(8);
+      var len = (self.bigEndian?buffer.readUInt32BE(8):buffer.readUInt32LE(8));
 
       if (len < 0 || len > 0xffff) {
         return cb(undefined);
@@ -342,12 +348,21 @@ Pcap.prototype.ether = function (buffer, obj, pos) {
 
 
 Pcap.prototype.pcap = function (buffer, obj) {
-  obj.pcap = {
-    ts_sec:   buffer.readUInt32LE(0),
-    ts_usec:  buffer.readUInt32LE(4),
-    incl_len: buffer.readUInt32LE(8),
-    orig_len: buffer.readUInt32LE(12)
-  };
+  if (this.bigEndian) {
+    obj.pcap = {
+      ts_sec:   buffer.readUInt32BE(0),
+      ts_usec:  buffer.readUInt32BE(4),
+      incl_len: buffer.readUInt32BE(8),
+      orig_len: buffer.readUInt32BE(12)
+    };
+  } else {
+    obj.pcap = {
+      ts_sec:   buffer.readUInt32LE(0),
+      ts_usec:  buffer.readUInt32LE(4),
+      incl_len: buffer.readUInt32LE(8),
+      orig_len: buffer.readUInt32LE(12)
+    };
+  }
 
   switch(this.linkType) {
   case 0: // NULL