From 0da00537ff73ee7ab24ce431265c7cf31015a70f Mon Sep 17 00:00:00 2001 From: Andy Wick Date: Wed, 8 Apr 2015 14:37:28 -0400 Subject: [PATCH] new spigraph tests and new multi tests --- tests/api-connections.t | 27 +++++++- tests/api-fresh.t | 24 ++++++- tests/api-sessions.t | 52 ++++++++++++-- tests/api-spigraph.t | 98 +++++++++++++++++++++++++++ tests/api-spiview.t | 145 +++++++++++++++++++++++++++++++++++++--- 5 files changed, 327 insertions(+), 19 deletions(-) create mode 100644 tests/api-spigraph.t diff --git a/tests/api-connections.t b/tests/api-connections.t index 2c5a72f5e6..9d540a1939 100644 --- a/tests/api-connections.t +++ b/tests/api-connections.t @@ -1,4 +1,4 @@ -use Test::More tests => 4; +use Test::More tests => 8; use Cwd; use URI::Escape; use MolochTest; @@ -11,20 +11,41 @@ my $pwd = getcwd() . "/pcap"; my $files = "(file=$pwd/socks-http-example.pcap||file=$pwd/socks-http-pass.pcap||file=$pwd/socks-https-example.pcap||file=$pwd/socks5-http-302.pcap||file=$pwd/socks5-rdp.pcap||file=$pwd/socks5-reverse.pcap||file=$pwd/socks5-smtp-503.pcap)"; - my $json = viewerGet("/connections.json?date=-1&expression=" . uri_escape("$files")); +my $json; +# a1 to a2 + $json = viewerGet("/connections.json?date=-1&expression=" . uri_escape("$files")); delete $json->{health}; eq_or_diff($json, from_json('{ "nodes": [ { "id": "10.0.0.1", "db": 26034, "by": 30979, "pa": 86, "cnt": 2, "sessions": 3, "type": 3, "pos": 0 }, { "id": "10.0.0.2", "db": 26119, "by": 31647, "pa": 96, "cnt": 3, "sessions": 4, "type": 3, "pos": 1 }, { "id": "10.180.156.185", "db": 33866, "by": 46190, "pa": 184, "cnt": 1, "sessions": 9, "type": 1, "pos": 2 }, { "id": "10.180.156.249", "db": 33866, "by": 46190, "pa": 184, "cnt": 1, "sessions": 9, "type": 2, "pos": 3 }, { "id": "10.0.0.3", "db": 85, "by": 668, "pa": 10, "cnt": 1, "sessions": 1, "type": 1, "pos": 4 } ], "links": [ { "value": 2, "source": 0, "target": 1, "by": 29487, "db": 25707, "pa": 66, "no": { "test": 1 } }, { "value": 1, "source": 1, "target": 0, "by": 1492, "db": 327, "pa": 20, "no": { "test": 1 } }, { "value": 9, "source": 2, "target": 3, "by": 46190, "db": 33866, "pa": 184, "no": { "test": 1 } }, { "value": 1, "source": 4, "target": 1, "by": 668, "db": 85, "pa": 10, "no": { "test": 1 } } ], "iTotalDisplayRecords": 13 }', {relaxed => 1}), "a1 to a2", { context => 3 }); + $json = multiGet("/connections.json?date=-1&expression=" . uri_escape("$files")); + delete $json->{health}; + eq_or_diff($json, from_json('{ "nodes": [ { "id": "10.0.0.1", "db": 26034, "by": 30979, "pa": 86, "cnt": 2, "sessions": 3, "type": 3, "pos": 0 }, { "id": "10.0.0.2", "db": 26119, "by": 31647, "pa": 96, "cnt": 3, "sessions": 4, "type": 3, "pos": 1 }, { "id": "10.180.156.185", "db": 33866, "by": 46190, "pa": 184, "cnt": 1, "sessions": 9, "type": 1, "pos": 2 }, { "id": "10.180.156.249", "db": 33866, "by": 46190, "pa": 184, "cnt": 1, "sessions": 9, "type": 2, "pos": 3 }, { "id": "10.0.0.3", "db": 85, "by": 668, "pa": 10, "cnt": 1, "sessions": 1, "type": 1, "pos": 4 } ], "links": [ { "value": 2, "source": 0, "target": 1, "by": 29487, "db": 25707, "pa": 66, "no": { "test": 1 } }, { "value": 1, "source": 1, "target": 0, "by": 1492, "db": 327, "pa": 20, "no": { "test": 1 } }, { "value": 9, "source": 2, "target": 3, "by": 46190, "db": 33866, "pa": 184, "no": { "test": 1 } }, { "value": 1, "source": 4, "target": 1, "by": 668, "db": 85, "pa": 10, "no": { "test": 1 } } ], "iTotalDisplayRecords": 13 }', {relaxed => 1}), "multi a1 to a2", { context => 3 }); + +# a1 to ip.dst $json = viewerGet("/connections.json?date=-1&dstField=ip.dst:port&expression=" . uri_escape("$files")); delete $json->{health}; eq_or_diff($json, from_json('{ "nodes": [ { "id": "10.0.0.1", "db": 25707, "by": 29487, "pa": 66, "cnt": 2, "sessions": 2, "type": 1, "pos": 0 }, { "id": "10.0.0.2:21477", "db": 1361, "by": 2176, "pa": 14, "cnt": 1, "sessions": 1, "type": 2, "pos": 1 }, { "id": "10.0.0.2", "db": 327, "by": 1492, "pa": 20, "cnt": 1, "sessions": 1, "type": 1, "pos": 2 }, { "id": "10.0.0.1:1080", "db": 327, "by": 1492, "pa": 20, "cnt": 1, "sessions": 1, "type": 2, "pos": 3 }, { "id": "10.180.156.185", "db": 33866, "by": 46190, "pa": 184, "cnt": 1, "sessions": 9, "type": 1, "pos": 4 }, { "id": "10.180.156.249:1080", "db": 33866, "by": 46190, "pa": 184, "cnt": 1, "sessions": 9, "type": 2, "pos": 5 }, { "id": "10.0.0.3", "db": 85, "by": 668, "pa": 10, "cnt": 1, "sessions": 1, "type": 1, "pos": 6 }, { "id": "10.0.0.2:42356", "db": 85, "by": 668, "pa": 10, "cnt": 1, "sessions": 1, "type": 2, "pos": 7 }, { "id": "10.0.0.2:8855", "db": 24346, "by": 27311, "pa": 52, "cnt": 1, "sessions": 1, "type": 2, "pos": 8 } ], "links": [ { "value": 1, "source": 0, "target": 1, "by": 2176, "db": 1361, "pa": 14, "no": { "test": 1 } }, { "value": 1, "source": 2, "target": 3, "by": 1492, "db": 327, "pa": 20, "no": { "test": 1 } }, { "value": 9, "source": 4, "target": 5, "by": 46190, "db": 33866, "pa": 184, "no": { "test": 1 } }, { "value": 1, "source": 6, "target": 7, "by": 668, "db": 85, "pa": 10, "no": { "test": 1 } }, { "value": 1, "source": 0, "target": 8, "by": 27311, "db": 24346, "pa": 52, "no": { "test": 1 } } ], "iTotalDisplayRecords": 13 }', {relaxed => 1}), "a1 to ip.dst", { context => 3 }); + $json = multiGet("/connections.json?date=-1&dstField=ip.dst:port&expression=" . uri_escape("$files")); + delete $json->{health}; + eq_or_diff($json, from_json('{ "nodes": [ { "id": "10.0.0.1", "db": 25707, "by": 29487, "pa": 66, "cnt": 2, "sessions": 2, "type": 1, "pos": 0 }, { "id": "10.0.0.2:21477", "db": 1361, "by": 2176, "pa": 14, "cnt": 1, "sessions": 1, "type": 2, "pos": 1 }, { "id": "10.0.0.2", "db": 327, "by": 1492, "pa": 20, "cnt": 1, "sessions": 1, "type": 1, "pos": 2 }, { "id": "10.0.0.1:1080", "db": 327, "by": 1492, "pa": 20, "cnt": 1, "sessions": 1, "type": 2, "pos": 3 }, { "id": "10.180.156.185", "db": 33866, "by": 46190, "pa": 184, "cnt": 1, "sessions": 9, "type": 1, "pos": 4 }, { "id": "10.180.156.249:1080", "db": 33866, "by": 46190, "pa": 184, "cnt": 1, "sessions": 9, "type": 2, "pos": 5 }, { "id": "10.0.0.3", "db": 85, "by": 668, "pa": 10, "cnt": 1, "sessions": 1, "type": 1, "pos": 6 }, { "id": "10.0.0.2:42356", "db": 85, "by": 668, "pa": 10, "cnt": 1, "sessions": 1, "type": 2, "pos": 7 }, { "id": "10.0.0.2:8855", "db": 24346, "by": 27311, "pa": 52, "cnt": 1, "sessions": 1, "type": 2, "pos": 8 } ], "links": [ { "value": 1, "source": 0, "target": 1, "by": 2176, "db": 1361, "pa": 14, "no": { "test": 1 } }, { "value": 1, "source": 2, "target": 3, "by": 1492, "db": 327, "pa": 20, "no": { "test": 1 } }, { "value": 9, "source": 4, "target": 5, "by": 46190, "db": 33866, "pa": 184, "no": { "test": 1 } }, { "value": 1, "source": 6, "target": 7, "by": 668, "db": 85, "pa": 10, "no": { "test": 1 } }, { "value": 1, "source": 0, "target": 8, "by": 27311, "db": 24346, "pa": 52, "no": { "test": 1 } } ], "iTotalDisplayRecords": 13 }', {relaxed => 1}), "multi a1 to ip.dst", { context => 3 }); + +# a1 to tls.notAfter $json = viewerGet("/connections.json?date=-1&dstField=tls.notAfter&expression=" . uri_escape("$files")); delete $json->{health}; eq_or_diff($json, from_json('{ "nodes": [ { "id": "1418212800", "db": 26760, "by": 32958, "pa": 93, "cnt": 1, "sessions": 3, "type": 2, "pos": 0 }, { "id": "1648944000", "db": 26760, "by": 32958, "pa": 93, "cnt": 1, "sessions": 3, "type": 2, "pos": 1 }, { "id": "10.180.156.185", "db": 53520, "by": 65916, "pa": 186, "cnt": 2, "sessions": 6, "type": 1, "pos": 2 } ], "links": [ { "value": 3, "source": 2, "target": 0, "by": 32958, "db": 26760, "pa": 93, "no": { "test": 1 } }, { "value": 3, "source": 2, "target": 1, "by": 32958, "db": 26760, "pa": 93, "no": { "test": 1 } } ], "iTotalDisplayRecords": 3 }', {relaxed => 1}), "a1 to tls.notAfter", { context => 3 }); - my $json = viewerGet("/connections.json?date=-1&expression=" . uri_escape("$files&&ip.protocol==blah")); + $json = multiGet("/connections.json?date=-1&dstField=tls.notAfter&expression=" . uri_escape("$files")); + delete $json->{health}; + eq_or_diff($json, from_json('{ "nodes": [ { "id": "1418212800", "db": 26760, "by": 32958, "pa": 93, "cnt": 1, "sessions": 3, "type": 2, "pos": 0 }, { "id": "1648944000", "db": 26760, "by": 32958, "pa": 93, "cnt": 1, "sessions": 3, "type": 2, "pos": 1 }, { "id": "10.180.156.185", "db": 53520, "by": 65916, "pa": 186, "cnt": 2, "sessions": 6, "type": 1, "pos": 2 } ], "links": [ { "value": 3, "source": 2, "target": 0, "by": 32958, "db": 26760, "pa": 93, "no": { "test": 1 } }, { "value": 3, "source": 2, "target": 1, "by": 32958, "db": 26760, "pa": 93, "no": { "test": 1 } } ], "iTotalDisplayRecords": 3 }', {relaxed => 1}), "multi a1 to tls.notAfter", { context => 3 }); + +# ip.protocol unknown + $json = viewerGet("/connections.json?date=-1&expression=" . uri_escape("$files&&ip.protocol==blah")); delete $json->{health}; eq_or_diff($json, from_json('{ "bsqErr": "Unknown protocol string blah" }', {relaxed => 1}), "ip.protocol==blah", { context => 3 }); + + $json = multiGet("/connections.json?date=-1&expression=" . uri_escape("$files&&ip.protocol==blah")); + delete $json->{health}; + eq_or_diff($json, from_json('{ "bsqErr": "Unknown protocol string blah" }', {relaxed => 1}), "multi ip.protocol==blah", { context => 3 }); diff --git a/tests/api-fresh.t b/tests/api-fresh.t index f923342d32..83d8202697 100644 --- a/tests/api-fresh.t +++ b/tests/api-fresh.t @@ -1,5 +1,5 @@ # Tests on a fresh install -use Test::More tests => 21; +use Test::More tests => 39; use Cwd; use URI::Escape; use MolochTest; @@ -38,24 +38,42 @@ my $json; is ($json->{iTotalRecords}, 0, "Correct sessions.json iTotalRecords"); is ($json->{health}->{number_of_data_nodes}, 1, "Correct sessions.json health number_of_data_nodes"); is ($json->{graph}->{interval}, 60, "Correct sessions.json graph interval"); + is (scalar @{$json->{graph}->{lpHisto}}, 0, "Correct sessions.json graph lpHisto"); + is (scalar @{$json->{graph}->{paHisto}}, 0, "Correct sessions.json graph paHisto"); + is (scalar @{$json->{graph}->{lpHisto}}, 0, "Correct sessions.json graph lpHisto"); + is (scalar keys %{$json->{map}}, 0, "Correct sessions.json map"); $json = viewerGet2("/spigraph.json"); is ($json->{iTotalRecords}, 0, "Correct spigraph.json iTotalRecords"); is ($json->{health}->{number_of_data_nodes}, 1, "Correct spigraph.json health number_of_data_nodes"); is ($json->{graph}->{interval}, 60, "Correct spigraph.json graph interval"); + is (scalar @{$json->{graph}->{lpHisto}}, 0, "Correct spigraph.json graph lpHisto"); + is (scalar @{$json->{graph}->{paHisto}}, 0, "Correct spigraph.json graph paHisto"); + is (scalar @{$json->{graph}->{lpHisto}}, 0, "Correct spigraph.json graph lpHisto"); + is (scalar keys %{$json->{map}}, 0, "Correct spigraph.json map"); $json = viewerGet2("/spiview.json"); is (scalar keys %{$json->{spi}}, 0, "Empty spiview.json spi"); is ($json->{iTotalRecords}, 0, "Correct spiview.json iTotalRecords"); + is (!exists $json->{graph}, 1, "Shouldn't have spiview.json graph"); + is (!exists $json->{map}, 1, "Shouldn't have spiview.json map"); + is (!exists $json->{health}, 1, "Shouldn't have spiview.json health"); - $json = viewerGet2("/spiview.json?spi=ta"); - is (scalar keys %{$json->{spi}}, 0, "Empty spiview.json spi"); + $json = viewerGet2("/spiview.json?spi=ta&facets=1"); + is (scalar keys %{$json->{spi}}, 1, "one spiview.json spi"); + is (scalar keys %{$json->{spi}->{ta}}, 5, "Five spiview.json ta"); is ($json->{iTotalRecords}, 0, "Correct spiview.json iTotalRecords"); is ($json->{health}->{number_of_data_nodes}, 1, "Correct spiview.json health number_of_data_nodes"); + is (scalar @{$json->{graph}->{lpHisto}}, 0, "Correct spiview.json graph lpHisto"); + is (scalar @{$json->{graph}->{paHisto}}, 0, "Correct spiview.json graph paHisto"); + is (scalar @{$json->{graph}->{lpHisto}}, 0, "Correct spiview.json graph lpHisto"); + is (scalar keys %{$json->{map}}, 0, "Correct spiview.json map"); $json = viewerGet2("/connections.json"); is ($json->{iTotalDisplayRecords}, 0, "Correct connections.json iTotalDisplayRecords"); is ($json->{health}->{number_of_data_nodes}, 1, "Correct connections.json health number_of_data_nodes"); + is (!exists $json->{graph}, 1, "Shouldn't have connections.json graph"); + is (!exists $json->{map}, 1, "Shouldn't have connections.json map"); $json = viewerGet2("/uniqueValue.json?type=tags"); is (scalar @{$json}, 0, "Empty uniqueValue"); diff --git a/tests/api-sessions.t b/tests/api-sessions.t index f3635c0b7a..4652e46cfd 100644 --- a/tests/api-sessions.t +++ b/tests/api-sessions.t @@ -1,4 +1,4 @@ -use Test::More tests => 27; +use Test::More tests => 52; use Cwd; use URI::Escape; use MolochTest; @@ -15,10 +15,16 @@ my $pwd = getcwd() . "/pcap"; my $response = $MolochTest::userAgent->get("http://$MolochTest::host:8123/test/raw/" . $json->{aaData}->[0]->{id} . "?type=src"); is (unpack("H*", $response->content), "4fa11b290002538d08090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f30313233343536374fa11b2d0008129108090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f3031323334353637", "Correct bigendian tcpdump data"); +# multi bigendian pcap file tests + my $json = multiGet("/sessions.json?date=-1&expression=" . uri_escape("file=$pwd/bigendian.pcap")); + is ($json->{iTotalDisplayRecords}, 1, "multi bigendian iTotalDisplayRecords"); + my $response = $MolochTest::userAgent->get("http://$MolochTest::host:8123/test/raw/" . $json->{aaData}->[0]->{id} . "?type=src"); + is (unpack("H*", $response->content), "4fa11b290002538d08090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f30313233343536374fa11b2d0008129108090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f3031323334353637", "multi Correct bigendian tcpdump data"); + # Check facets short $json = viewerGet("/sessions.json?startTime=1386004308&stopTime=1386004400&facets=1&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); - eq_or_diff($json->{map}, from_json('{"USA": 6}'), "map short"); + eq_or_diff($json->{map}, from_json('{"dst":{"USA": 3}, "src":{"USA": 3}}'), "map short"); eq_or_diff($json->{graph}->{lpHisto}, from_json('[["1386004309000", 1], ["1386004312000", 1], [1386004317000, 1]]'), "lpHisto short"); eq_or_diff($json->{graph}->{paHisto}, from_json('[["1386004309000", 14], ["1386004312000", 15], [1386004317000, 17]]'), "paHisto short"); eq_or_diff($json->{graph}->{dbHisto}, from_json('[["1386004309000", 1754], ["1386004312000", 1770], [1386004317000, 1763]]'), "dbHisto short"); @@ -27,10 +33,22 @@ my $pwd = getcwd() . "/pcap"; is ($json->{graph}->{xmax}, 1386004400000, "correct xmax short"); is ($json->{graph}->{xmin}, 1386004308000, "correct xmin short"); +# multi Check facets short + $json = multiGet("/sessions.json?startTime=1386004308&stopTime=1386004400&facets=1&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); + + eq_or_diff($json->{map}, from_json('{"dst":{"USA": 3}, "src":{"USA": 3}}'), "multi map short"); + eq_or_diff($json->{graph}->{lpHisto}, from_json('[["1386004309000", 1], ["1386004312000", 1], [1386004317000, 1]]'), "multi lpHisto short"); + eq_or_diff($json->{graph}->{paHisto}, from_json('[["1386004309000", 14], ["1386004312000", 15], [1386004317000, 17]]'), "multi paHisto short"); + eq_or_diff($json->{graph}->{dbHisto}, from_json('[["1386004309000", 1754], ["1386004312000", 1770], [1386004317000, 1763]]'), "multi dbHisto short"); + is ($json->{iTotalDisplayRecords}, 3, "multi records short"); + is ($json->{graph}->{interval}, 1, "multi correct interval short"); + is ($json->{graph}->{xmax}, 1386004400000, "multi correct xmax short"); + is ($json->{graph}->{xmin}, 1386004308000, "multi correct xmin short"); + # Check facets medium $json = viewerGet("/sessions.json?startTime=1386004308&stopTime=1386349908&facets=1&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); - eq_or_diff($json->{map}, from_json('{"USA": 6}'), "map medium"); + eq_or_diff($json->{map}, from_json('{"dst":{"USA": 3}, "src":{"USA": 3}}'), "map medium"); eq_or_diff($json->{graph}->{lpHisto}, from_json('[["1386004260000", 3]]'), "lpHisto medium"); eq_or_diff($json->{graph}->{paHisto}, from_json('[["1386004260000", 46]]'), "paHisto medium"); eq_or_diff($json->{graph}->{dbHisto}, from_json('[["1386004260000", 5287]]'), "dbHisto medium"); @@ -39,20 +57,46 @@ my $pwd = getcwd() . "/pcap"; is ($json->{graph}->{xmax}, 1386349908000, "correct xmax medium"); is ($json->{graph}->{xmin}, 1386004308000, "correct xmin medium"); +# mutli Check facets medium + $json = multiGet("/sessions.json?startTime=1386004308&stopTime=1386349908&facets=1&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); + + eq_or_diff($json->{map}, from_json('{"dst":{"USA": 3}, "src":{"USA": 3}}'), "multi map medium"); + eq_or_diff($json->{graph}->{lpHisto}, from_json('[["1386004260000", 3]]'), "multi lpHisto medium"); + eq_or_diff($json->{graph}->{paHisto}, from_json('[["1386004260000", 46]]'), "multi paHisto medium"); + eq_or_diff($json->{graph}->{dbHisto}, from_json('[["1386004260000", 5287]]'), "multi dbHisto medium"); + is ($json->{iTotalDisplayRecords}, 3, "multi records medium"); + is ($json->{graph}->{interval}, 60, "multi correct interval medium"); + is ($json->{graph}->{xmax}, 1386349908000, "multi correct xmax medium"); + is ($json->{graph}->{xmin}, 1386004308000, "multi correct xmin medium"); + # Check facets ALL $json = viewerGet("/sessions.json?date=-1&facets=1&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); - eq_or_diff($json->{map}, from_json('{"RUS": 1, "USA": 6, "CAN": 1}'), "map ALL"); + eq_or_diff($json->{map}, from_json('{"dst":{"USA": 3, "CAN": 1}, "src":{"USA": 3, "RUS":1}}'), "map ALL"); eq_or_diff($json->{graph}->{lpHisto}, from_json('[["1335956400000", 1], ["1386003600000", 3], [1387742400000, 1]]'), "lpHisto ALL"); eq_or_diff($json->{graph}->{paHisto}, from_json('[["1335956400000", 2], ["1386003600000", 46], [1387742400000, 4]]'), "paHisto ALL"); eq_or_diff($json->{graph}->{dbHisto}, from_json('[["1335956400000", 0], ["1386003600000", 5287], [1387742400000, 68]]'), "dbHisto ALL"); is ($json->{iTotalDisplayRecords}, 5, "records ALL"); is ($json->{graph}->{interval}, 3600, "correct interval ALL"); +# multi Check facets ALL + $json = multiGet("/sessions.json?date=-1&facets=1&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); + + eq_or_diff($json->{map}, from_json('{"dst":{"USA": 3, "CAN": 1}, "src":{"USA": 3, "RUS":1}}'), "multi map ALL"); + eq_or_diff($json->{graph}->{lpHisto}, from_json('[["1335956400000", 1], ["1386003600000", 3], [1387742400000, 1]]'), "multi lpHisto ALL"); + eq_or_diff($json->{graph}->{paHisto}, from_json('[["1335956400000", 2], ["1386003600000", 46], [1387742400000, 4]]'), "multi paHisto ALL"); + eq_or_diff($json->{graph}->{dbHisto}, from_json('[["1335956400000", 0], ["1386003600000", 5287], [1387742400000, 68]]'), "multi dbHisto ALL"); + is ($json->{iTotalDisplayRecords}, 5, "multi records ALL"); + is ($json->{graph}->{interval}, 3600, "multi correct interval ALL"); + # Check ip.protocol=blah my $json = viewerGet("/sessions.json?date=-1&&spi=a1&expression=" . uri_escape("file=$pwd/bigendian.pcap&&ip.protocol==blah")); is($json->{bsqErr}, "Unknown protocol string blah", "ip.protocol==blah"); +# multi Check ip.protocol=blah + my $json = multiGet("/sessions.json?date=-1&&spi=a1&expression=" . uri_escape("file=$pwd/bigendian.pcap&&ip.protocol==blah")); + is($json->{bsqErr}, "Unknown protocol string blah", "multi ip.protocol==blah"); + # csv my $csv = $MolochTest::userAgent->get("http://$MolochTest::host:8123/sessions.csv?date=-1&expression=" . uri_escape("file=$pwd/socks-http-example.pcap"))->content; $csv =~ s/\r//g; diff --git a/tests/api-spigraph.t b/tests/api-spigraph.t new file mode 100644 index 0000000000..91bd6ccf3a --- /dev/null +++ b/tests/api-spigraph.t @@ -0,0 +1,98 @@ +use Test::More tests => 43; +use Cwd; +use URI::Escape; +use MolochTest; +use JSON; +use Test::Differences; +use Data::Dumper; +use strict; + +my $pwd = getcwd() . "/pcap"; + +my $json; + +#node + $json = viewerGet("/spigraph.json?date=-1&field=no&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); + eq_or_diff($json->{map}, from_json('{"dst":{"USA": 3, "CAN": 1}, "src":{"USA": 3, "RUS":1}}'), "map field: no"); + eq_or_diff($json->{graph}->{lpHisto}, from_json('[["1335956400000", 1], ["1386003600000", 3], [1387742400000, 1]]'), "lpHisto field: no"); + eq_or_diff($json->{graph}->{paHisto}, from_json('[["1335956400000", 2], ["1386003600000", 46], [1387742400000, 4]]'), "paHisto field: no"); + eq_or_diff($json->{graph}->{dbHisto}, from_json('[["1335956400000", 0], ["1386003600000", 5287], [1387742400000, 68]]'), "dbHisto field: no"); + eq_or_diff($json->{items}, from_json('[{ "name": "test", "count": 5, "graph": { "lpHisto": [ [ 1335956400000, 1 ], [ 1386003600000, 3 ], [ 1387742400000, 1 ] ], "dbHisto": [ [ 1335956400000, 0 ], [ 1386003600000, 5287 ], [ 1387742400000, 68 ] ], "paHisto": [ [ 1335956400000, 2 ], [ 1386003600000, 46 ], [ 1387742400000, 4 ] ], "xmin": 1335956400000, "xmax": 1387742400000, "interval": 3600 }, "map": {"dst":{"USA": 3, "CAN": 1}, "src":{"USA": 3, "RUS":1}} } ]'), "items field: no"); + +#node multi + $json = multiGet("/spigraph.json?date=-1&field=no&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); + eq_or_diff($json->{map}, from_json('{"dst":{"USA": 3, "CAN": 1}, "src":{"USA": 3, "RUS":1}}'), "multi map field: no"); + eq_or_diff($json->{graph}->{lpHisto}, from_json('[["1335956400000", 1], ["1386003600000", 3], [1387742400000, 1]]'), "multi lpHisto field: no"); + eq_or_diff($json->{graph}->{paHisto}, from_json('[["1335956400000", 2], ["1386003600000", 46], [1387742400000, 4]]'), "multi paHisto field: no"); + eq_or_diff($json->{graph}->{dbHisto}, from_json('[["1335956400000", 0], ["1386003600000", 5287], [1387742400000, 68]]'), "multi dbHisto field: no"); + eq_or_diff($json->{items}, from_json('[{ "name": "test", "count": 5, "graph": { "lpHisto": [ [ 1335956400000, 1 ], [ 1386003600000, 3 ], [ 1387742400000, 1 ] ], "dbHisto": [ [ 1335956400000, 0 ], [ 1386003600000, 5287 ], [ 1387742400000, 68 ] ], "paHisto": [ [ 1335956400000, 2 ], [ 1386003600000, 46 ], [ 1387742400000, 4 ] ], "xmin": 1335956400000, "xmax": 1387742400000, "interval": 3600 }, "map": {"dst":{"USA": 3, "CAN": 1}, "src":{"USA": 3, "RUS":1}} } ]'), "multi items field: no"); + eq_or_diff($json->{map}, from_json('{"dst":{"USA": 3, "CAN": 1}, "src":{"USA": 3, "RUS":1}}'), "multi map ALL"); + + + +#ta + $json = viewerGet("/spigraph.json?date=-1&field=ta&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); + eq_or_diff($json->{map}, from_json('{"dst":{"USA": 3, "CAN": 1}, "src":{"USA": 3, "RUS":1}}'), "map field: ta"); + eq_or_diff($json->{graph}->{lpHisto}, from_json('[["1335956400000", 1], ["1386003600000", 3], [1387742400000, 1]]'), "lpHisto field: ta"); + eq_or_diff($json->{graph}->{paHisto}, from_json('[["1335956400000", 2], ["1386003600000", 46], [1387742400000, 4]]'), "paHisto field: ta"); + eq_or_diff($json->{graph}->{dbHisto}, from_json('[["1335956400000", 0], ["1386003600000", 5287], [1387742400000, 68]]'), "dbHisto field: ta"); + + my @items = sort({$a->{name} cmp $b->{name}} @{$json->{items}}); + eq_or_diff(\@items, from_json('[{"map":{"src":{},"dst":{}},"count":1,"name":"ICMP","graph":{"lpHisto":[[1335956400000,1]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1335956400000,2]],"dbHisto":[[1335956400000,0]]}},{"map":{"src":{"USA":3},"dst":{"USA":3}},"count":3,"name":"byhost2","graph":{"lpHisto":[[1386003600000,3]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1386003600000,46]],"dbHisto":[[1386003600000,5287]]}},{"map":{"src":{},"dst":{}},"count":1,"name":"byip2","graph":{"lpHisto":[[1335956400000,1]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1335956400000,2]],"dbHisto":[[1335956400000,0]]}},{"map":{"src":{"USA":3},"dst":{"USA":3}},"count":3,"name":"domainwise","graph":{"lpHisto":[[1386003600000,3]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1386003600000,46]],"dbHisto":[[1386003600000,5287]]}},{"map":{"src":{"RUS":1},"dst":{"CAN":1}},"count":1,"name":"dstip","graph":{"lpHisto":[[1387742400000,1]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1387742400000,4]],"dbHisto":[[1387742400000,68]]}},{"map":{"src":{"USA":3},"dst":{"USA":3}},"count":3,"name":"hosttaggertest1","graph":{"lpHisto":[[1386003600000,3]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1386003600000,46]],"dbHisto":[[1386003600000,5287]]}},{"map":{"src":{"USA":3},"dst":{"USA":3}},"count":3,"name":"hosttaggertest2","graph":{"lpHisto":[[1386003600000,3]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1386003600000,46]],"dbHisto":[[1386003600000,5287]]}},{"map":{"src":{"USA":3},"dst":{"USA":3}},"count":3,"name":"http:content:text/html","graph":{"lpHisto":[[1386003600000,3]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1386003600000,46]],"dbHisto":[[1386003600000,5287]]}},{"map":{"src":{"USA":3},"dst":{"USA":3}},"count":3,"name":"http:method:GET","graph":{"lpHisto":[[1386003600000,3]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1386003600000,46]],"dbHisto":[[1386003600000,5287]]}},{"map":{"src":{"USA":3},"dst":{"USA":3}},"count":3,"name":"http:statuscode:200","graph":{"lpHisto":[[1386003600000,3]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1386003600000,46]],"dbHisto":[[1386003600000,5287]]}},{"map":{"src":{},"dst":{}},"count":1,"name":"iptaggertest1","graph":{"lpHisto":[[1335956400000,1]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1335956400000,2]],"dbHisto":[[1335956400000,0]]}},{"map":{"src":{},"dst":{}},"count":1,"name":"iptaggertest2","graph":{"lpHisto":[[1335956400000,1]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1335956400000,2]],"dbHisto":[[1335956400000,0]]}},{"map":{"src":{"RUS":1},"dst":{"CAN":1}},"count":1,"name":"ipwisecsv","graph":{"lpHisto":[[1387742400000,1]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1387742400000,4]],"dbHisto":[[1387742400000,68]]}},{"map":{"src":{"RUS":1,"USA":3},"dst":{"USA":3,"CAN":1}},"count":5,"name":"node:test","graph":{"lpHisto":[[1335956400000,1],[1386003600000,3],[1387742400000,1]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1335956400000,2],[1386003600000,46],[1387742400000,4]],"dbHisto":[[1335956400000,0],[1386003600000,5287],[1387742400000,68]]}},{"map":{"src":{"RUS":1},"dst":{"CAN":1}},"count":1,"name":"protocol:bittorrent","graph":{"lpHisto":[[1387742400000,1]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1387742400000,4]],"dbHisto":[[1387742400000,68]]}},{"map":{"src":{"USA":3},"dst":{"USA":3}},"count":3,"name":"protocol:http","graph":{"lpHisto":[[1386003600000,3]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1386003600000,46]],"dbHisto":[[1386003600000,5287]]}},{"map":{"src":{"USA":3},"dst":{"USA":3}},"count":3,"name":"protocol:socks","graph":{"lpHisto":[[1386003600000,3]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1386003600000,46]],"dbHisto":[[1386003600000,5287]]}},{"map":{"src":{"RUS":1},"dst":{"CAN":1}},"count":1,"name":"srcip","graph":{"lpHisto":[[1387742400000,1]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1387742400000,4]],"dbHisto":[[1387742400000,68]]}},{"map":{"src":{"RUS":1,"USA":3},"dst":{"USA":3,"CAN":1}},"count":4,"name":"tcp","graph":{"lpHisto":[[1386003600000,3],[1387742400000,1]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1386003600000,46],[1387742400000,4]],"dbHisto":[[1386003600000,5287],[1387742400000,68]]}},{"map":{"src":{"USA":3},"dst":{"USA":3}},"count":3,"name":"wisebyhost2","graph":{"lpHisto":[[1386003600000,3]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1386003600000,46]],"dbHisto":[[1386003600000,5287]]}}]'), "items field: ta"); + +#ta multi + $json = multiGet("/spigraph.json?date=-1&field=ta&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); + eq_or_diff($json->{map}, from_json('{"dst":{"USA": 3, "CAN": 1}, "src":{"USA": 3, "RUS":1}}'), "multi map field: ta"); + eq_or_diff($json->{graph}->{lpHisto}, from_json('[["1335956400000", 1], ["1386003600000", 3], [1387742400000, 1]]'), "multi lpHisto field: ta"); + eq_or_diff($json->{graph}->{paHisto}, from_json('[["1335956400000", 2], ["1386003600000", 46], [1387742400000, 4]]'), "multi paHisto field: ta"); + eq_or_diff($json->{graph}->{dbHisto}, from_json('[["1335956400000", 0], ["1386003600000", 5287], [1387742400000, 68]]'), "multi dbHisto field: ta"); + + my @items = sort({$a->{name} cmp $b->{name}} @{$json->{items}}); + eq_or_diff(\@items, from_json('[{"map":{"src":{},"dst":{}},"count":1,"name":"ICMP","graph":{"lpHisto":[[1335956400000,1]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1335956400000,2]],"dbHisto":[[1335956400000,0]]}},{"map":{"src":{"USA":3},"dst":{"USA":3}},"count":3,"name":"byhost2","graph":{"lpHisto":[[1386003600000,3]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1386003600000,46]],"dbHisto":[[1386003600000,5287]]}},{"map":{"src":{},"dst":{}},"count":1,"name":"byip2","graph":{"lpHisto":[[1335956400000,1]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1335956400000,2]],"dbHisto":[[1335956400000,0]]}},{"map":{"src":{"USA":3},"dst":{"USA":3}},"count":3,"name":"domainwise","graph":{"lpHisto":[[1386003600000,3]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1386003600000,46]],"dbHisto":[[1386003600000,5287]]}},{"map":{"src":{"RUS":1},"dst":{"CAN":1}},"count":1,"name":"dstip","graph":{"lpHisto":[[1387742400000,1]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1387742400000,4]],"dbHisto":[[1387742400000,68]]}},{"map":{"src":{"USA":3},"dst":{"USA":3}},"count":3,"name":"hosttaggertest1","graph":{"lpHisto":[[1386003600000,3]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1386003600000,46]],"dbHisto":[[1386003600000,5287]]}},{"map":{"src":{"USA":3},"dst":{"USA":3}},"count":3,"name":"hosttaggertest2","graph":{"lpHisto":[[1386003600000,3]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1386003600000,46]],"dbHisto":[[1386003600000,5287]]}},{"map":{"src":{"USA":3},"dst":{"USA":3}},"count":3,"name":"http:content:text/html","graph":{"lpHisto":[[1386003600000,3]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1386003600000,46]],"dbHisto":[[1386003600000,5287]]}},{"map":{"src":{"USA":3},"dst":{"USA":3}},"count":3,"name":"http:method:GET","graph":{"lpHisto":[[1386003600000,3]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1386003600000,46]],"dbHisto":[[1386003600000,5287]]}},{"map":{"src":{"USA":3},"dst":{"USA":3}},"count":3,"name":"http:statuscode:200","graph":{"lpHisto":[[1386003600000,3]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1386003600000,46]],"dbHisto":[[1386003600000,5287]]}},{"map":{"src":{},"dst":{}},"count":1,"name":"iptaggertest1","graph":{"lpHisto":[[1335956400000,1]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1335956400000,2]],"dbHisto":[[1335956400000,0]]}},{"map":{"src":{},"dst":{}},"count":1,"name":"iptaggertest2","graph":{"lpHisto":[[1335956400000,1]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1335956400000,2]],"dbHisto":[[1335956400000,0]]}},{"map":{"src":{"RUS":1},"dst":{"CAN":1}},"count":1,"name":"ipwisecsv","graph":{"lpHisto":[[1387742400000,1]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1387742400000,4]],"dbHisto":[[1387742400000,68]]}},{"map":{"src":{"RUS":1,"USA":3},"dst":{"USA":3,"CAN":1}},"count":5,"name":"node:test","graph":{"lpHisto":[[1335956400000,1],[1386003600000,3],[1387742400000,1]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1335956400000,2],[1386003600000,46],[1387742400000,4]],"dbHisto":[[1335956400000,0],[1386003600000,5287],[1387742400000,68]]}},{"map":{"src":{"RUS":1},"dst":{"CAN":1}},"count":1,"name":"protocol:bittorrent","graph":{"lpHisto":[[1387742400000,1]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1387742400000,4]],"dbHisto":[[1387742400000,68]]}},{"map":{"src":{"USA":3},"dst":{"USA":3}},"count":3,"name":"protocol:http","graph":{"lpHisto":[[1386003600000,3]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1386003600000,46]],"dbHisto":[[1386003600000,5287]]}},{"map":{"src":{"USA":3},"dst":{"USA":3}},"count":3,"name":"protocol:socks","graph":{"lpHisto":[[1386003600000,3]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1386003600000,46]],"dbHisto":[[1386003600000,5287]]}},{"map":{"src":{"RUS":1},"dst":{"CAN":1}},"count":1,"name":"srcip","graph":{"lpHisto":[[1387742400000,1]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1387742400000,4]],"dbHisto":[[1387742400000,68]]}},{"map":{"src":{"RUS":1,"USA":3},"dst":{"USA":3,"CAN":1}},"count":4,"name":"tcp","graph":{"lpHisto":[[1386003600000,3],[1387742400000,1]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1386003600000,46],[1387742400000,4]],"dbHisto":[[1386003600000,5287],[1387742400000,68]]}},{"map":{"src":{"USA":3},"dst":{"USA":3}},"count":3,"name":"wisebyhost2","graph":{"lpHisto":[[1386003600000,3]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1386003600000,46]],"dbHisto":[[1386003600000,5287]]}}]'), "multi items field: ta"); + + +#a1 + $json = viewerGet("/spigraph.json?date=-1&field=a1&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); + eq_or_diff($json->{map}, from_json('{"dst":{"USA": 3, "CAN": 1}, "src":{"USA": 3, "RUS":1}}'), "map field: a1"); + eq_or_diff($json->{graph}->{lpHisto}, from_json('[["1335956400000", 1], ["1386003600000", 3], [1387742400000, 1]]'), "lpHisto field: a1"); + eq_or_diff($json->{graph}->{paHisto}, from_json('[["1335956400000", 2], ["1386003600000", 46], [1387742400000, 4]]'), "paHisto field: a1"); + eq_or_diff($json->{graph}->{dbHisto}, from_json('[["1335956400000", 0], ["1386003600000", 5287], [1387742400000, 68]]'), "dbHisto field: a1"); + my @items = sort({$a->{name} cmp $b->{name}} @{$json->{items}}); + eq_or_diff(\@items, from_json('[{"map":{"src":{"RUS":1},"dst":{"CAN":1}},"count":1,"name":"10.0.0.1","graph":{"lpHisto":[[1387742400000,1]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1387742400000,4]],"dbHisto":[[1387742400000,68]]}},{"map":{"src":{"USA":3},"dst":{"USA":3}},"count":3,"name":"10.180.156.185","graph":{"lpHisto":[[1386003600000,3]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1386003600000,46]],"dbHisto":[[1386003600000,5287]]}},{"map":{"src":{},"dst":{}},"count":1,"name":"192.168.177.160","graph":{"lpHisto":[[1335956400000,1]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1335956400000,2]],"dbHisto":[[1335956400000,0]]}}]'), "items field: a1"); + +#a1 multi + $json = multiGet("/spigraph.json?date=-1&field=a1&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); + eq_or_diff($json->{map}, from_json('{"dst":{"USA": 3, "CAN": 1}, "src":{"USA": 3, "RUS":1}}'), "multi map field: a1"); + eq_or_diff($json->{graph}->{lpHisto}, from_json('[["1335956400000", 1], ["1386003600000", 3], [1387742400000, 1]]'), "multi lpHisto field: a1"); + eq_or_diff($json->{graph}->{paHisto}, from_json('[["1335956400000", 2], ["1386003600000", 46], [1387742400000, 4]]'), "multi paHisto field: a1"); + eq_or_diff($json->{graph}->{dbHisto}, from_json('[["1335956400000", 0], ["1386003600000", 5287], [1387742400000, 68]]'), "multi dbHisto field: a1"); + my @items = sort({$a->{name} cmp $b->{name}} @{$json->{items}}); + eq_or_diff(\@items, from_json('[{"map":{"src":{"RUS":1},"dst":{"CAN":1}},"count":1,"name":"10.0.0.1","graph":{"lpHisto":[[1387742400000,1]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1387742400000,4]],"dbHisto":[[1387742400000,68]]}},{"map":{"src":{"USA":3},"dst":{"USA":3}},"count":3,"name":"10.180.156.185","graph":{"lpHisto":[[1386003600000,3]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1386003600000,46]],"dbHisto":[[1386003600000,5287]]}},{"map":{"src":{},"dst":{}},"count":1,"name":"192.168.177.160","graph":{"lpHisto":[[1335956400000,1]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1335956400000,2]],"dbHisto":[[1335956400000,0]]}}]'), "multi items field: a1"); + +#hh1 + $json = viewerGet("/spigraph.json?date=-1&field=hh1&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); + eq_or_diff($json->{map}, from_json('{"dst":{"USA": 3, "CAN": 1}, "src":{"USA": 3, "RUS":1}}'), "map field: hh1"); + eq_or_diff($json->{graph}->{lpHisto}, from_json('[["1335956400000", 1], ["1386003600000", 3], [1387742400000, 1]]'), "lpHisto field: hh1"); + eq_or_diff($json->{graph}->{paHisto}, from_json('[["1335956400000", 2], ["1386003600000", 46], [1387742400000, 4]]'), "paHisto field: hh1"); + eq_or_diff($json->{graph}->{dbHisto}, from_json('[["1335956400000", 0], ["1386003600000", 5287], [1387742400000, 68]]'), "dbHisto field: hh1"); + my @items = sort({$a->{name} cmp $b->{name}} @{$json->{items}}); + eq_or_diff(\@items, from_json('[{"map":{"src":{"USA":3},"dst":{"USA":3}},"count":3,"name":"accept","graph":{"lpHisto":[[1386003600000,3]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1386003600000,46]],"dbHisto":[[1386003600000,5287]]}},{"map":{"src":{"USA":3},"dst":{"USA":3}},"count":3,"name":"host","graph":{"lpHisto":[[1386003600000,3]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1386003600000,46]],"dbHisto":[[1386003600000,5287]]}},{"map":{"src":{"USA":3},"dst":{"USA":3}},"count":3,"name":"user-agent","graph":{"lpHisto":[[1386003600000,3]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1386003600000,46]],"dbHisto":[[1386003600000,5287]]}}]'), "items field: hh1"); + +#hh1 multi + $json = multiGet("/spigraph.json?date=-1&field=hh1&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); + eq_or_diff($json->{map}, from_json('{"dst":{"USA": 3, "CAN": 1}, "src":{"USA": 3, "RUS":1}}'), "multi map field: hh1"); + eq_or_diff($json->{graph}->{lpHisto}, from_json('[["1335956400000", 1], ["1386003600000", 3], [1387742400000, 1]]'), "multi lpHisto field: hh1"); + eq_or_diff($json->{graph}->{paHisto}, from_json('[["1335956400000", 2], ["1386003600000", 46], [1387742400000, 4]]'), "multi paHisto field: hh1"); + eq_or_diff($json->{graph}->{dbHisto}, from_json('[["1335956400000", 0], ["1386003600000", 5287], [1387742400000, 68]]'), "multi dbHisto field: hh1"); + my @items = sort({$a->{name} cmp $b->{name}} @{$json->{items}}); + eq_or_diff(\@items, from_json('[{"map":{"src":{"USA":3},"dst":{"USA":3}},"count":3,"name":"accept","graph":{"lpHisto":[[1386003600000,3]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1386003600000,46]],"dbHisto":[[1386003600000,5287]]}},{"map":{"src":{"USA":3},"dst":{"USA":3}},"count":3,"name":"host","graph":{"lpHisto":[[1386003600000,3]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1386003600000,46]],"dbHisto":[[1386003600000,5287]]}},{"map":{"src":{"USA":3},"dst":{"USA":3}},"count":3,"name":"user-agent","graph":{"lpHisto":[[1386003600000,3]],"xmax":1387742400000,"interval":3600,"xmin":1335956400000,"paHisto":[[1386003600000,46]],"dbHisto":[[1386003600000,5287]]}}]'), "multi items field: hh1"); + +#rawua + $json = viewerGet("/spigraph.json?date=-1&field=rawua&expression=" . uri_escape("file=$pwd/socks5-reverse.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); + my @items = sort({$a->{name} cmp $b->{name}} @{$json->{items}}); + eq_or_diff(\@items, from_json('[{"map":{"src":{"RUS":1},"dst":{"CAN":1}},"count":1,"name":"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)","graph":{"lpHisto":[[1386788400000,1]],"xmax":1387742400000,"interval":3600,"xmin":1386003600000,"paHisto":[[1386788400000,52]],"dbHisto":[[1386788400000,24346]]}},{"map":{"src":{"USA":3},"dst":{"USA":3}},"count":3,"name":"curl/7.24.0 (x86_64-apple-darwin12.0) libcurl/7.24.0 OpenSSL/0.9.8y zlib/1.2.5","graph":{"lpHisto":[[1386003600000,3]],"xmax":1387742400000,"interval":3600,"xmin":1386003600000,"paHisto":[[1386003600000,46]],"dbHisto":[[1386003600000,5287]]}}]'), "items field: rawua"); + +#rawua multi + $json = multiGet("/spigraph.json?date=-1&field=rawua&expression=" . uri_escape("file=$pwd/socks5-reverse.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); + my @items = sort({$a->{name} cmp $b->{name}} @{$json->{items}}); + eq_or_diff(\@items, from_json('[{"map":{"src":{"RUS":1},"dst":{"CAN":1}},"count":1,"name":"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)","graph":{"lpHisto":[[1386788400000,1]],"xmax":1387742400000,"interval":3600,"xmin":1386003600000,"paHisto":[[1386788400000,52]],"dbHisto":[[1386788400000,24346]]}},{"map":{"src":{"USA":3},"dst":{"USA":3}},"count":3,"name":"curl/7.24.0 (x86_64-apple-darwin12.0) libcurl/7.24.0 OpenSSL/0.9.8y zlib/1.2.5","graph":{"lpHisto":[[1386003600000,3]],"xmax":1387742400000,"interval":3600,"xmin":1386003600000,"paHisto":[[1386003600000,46]],"dbHisto":[[1386003600000,5287]]}}]'), "multi items field: rawua"); diff --git a/tests/api-spiview.t b/tests/api-spiview.t index 3df3a5bf6a..5cacbea753 100644 --- a/tests/api-spiview.t +++ b/tests/api-spiview.t @@ -1,4 +1,4 @@ -use Test::More tests => 3; +use Test::More tests => 86; use Cwd; use URI::Escape; use MolochTest; @@ -8,15 +8,142 @@ use Data::Dumper; use strict; my $pwd = getcwd() . "/pcap"; -my $files = "(file=$pwd/socks-http-example.pcap||file=$pwd/socks-http-pass.pcap||file=$pwd/socks-https-example.pcap||file=$pwd/socks5-http-302.pcap||file=$pwd/socks5-rdp.pcap||file=$pwd/socks5-reverse.pcap||file=$pwd/socks5-smtp-503.pcap)"; +# bigendian pcap file tests + my $json = viewerGet("/spiview.json?date=-1&facets=1&spi=a1,a2,pr&expression=" . uri_escape("file=$pwd/bigendian.pcap")); + eq_or_diff($json->{map}, from_json('{"src": {}, "dst":{}}'), "map bigendian"); + eq_or_diff($json->{graph}, from_json('{"lpHisto": [[1335956400000, 1]], "dbHisto": [[1335956400000, 0]], "paHisto": [[1335956400000, 2]], "xmin": null, "xmax": null, "interval":3600}'), "graph bigendian"); + eq_or_diff($json->{spi}->{a1}, from_json('{"other": 0, "missing": 0, "_type":"terms", "total":1, "terms":[{"count":1, "term":3232280992}]}'), "bigendian a1"); + eq_or_diff($json->{spi}->{a2}, from_json('{"other": 0, "missing": 0, "_type":"terms", "total":1, "terms":[{"count":1, "term":171969329}]}'), "bigendian a2"); + eq_or_diff($json->{spi}->{pr}, from_json('{"other": 0, "missing": 0, "_type":"terms", "total":1, "terms":[{"count":1, "term":"icmp"}]}'), "bigendian pr"); + is ($json->{health}->{number_of_data_nodes}, 1, "Correct health number_of_data_nodes bigendian"); - my $json = viewerGet("/spiview.json?date=-1&spi=a1&expression=" . uri_escape("$files&&ip.protocol==tcp")); - delete $json->{health}; - eq_or_diff($json, from_json('{ "iTotalRecords": 141, "spi": { "a1": { "_type": "terms", "missing": 0, "total": 13, "other": 0, "terms": [ { "term": 179608761, "count": 9 }, { "term": 167772161, "count": 2 }, { "term": 167772163, "count": 1 }, { "term": 167772162, "count": 1 } ] } }, "iTotalDisplayRecords": 13, "bsqErr": null }', {relaxed => 1}), "a1 ip.protocol==tcp", { context => 3 }); +# multi bigendian pcap file tests + my $json = multiGet("/spiview.json?date=-1&facets=1&spi=a1,a2,pr&expression=" . uri_escape("file=$pwd/bigendian.pcap")); + eq_or_diff($json->{map}, from_json('{"src": {}, "dst":{}}'), "multi map bigendian"); + eq_or_diff($json->{graph}, from_json('{"lpHisto": [[1335956400000, 1]], "dbHisto": [[1335956400000, 0]], "paHisto": [[1335956400000, 2]], "xmin": null, "xmax": null, "interval":3600}'), "multi graph bigendian"); + eq_or_diff($json->{spi}->{a1}, from_json('{"other": 0, "missing": 0, "_type":"terms", "total":1, "terms":[{"count":1, "term":3232280992}]}'), "multi bigendian a1"); + eq_or_diff($json->{spi}->{a2}, from_json('{"other": 0, "missing": 0, "_type":"terms", "total":1, "terms":[{"count":1, "term":171969329}]}'), "multi bigendian a2"); + eq_or_diff($json->{spi}->{pr}, from_json('{"other": 0, "missing": 0, "_type":"terms", "total":1, "terms":[{"count":1, "term":"icmp"}]}'), "multi bigendian pr"); + is ($json->{health}->{number_of_data_nodes}, 2, "multi Correct health number_of_data_nodes bigendian"); - my $json = viewerGet("/spiview.json?date=-1&&spi=a1&expression=" . uri_escape("$files&&ip.protocol==blah")); - eq_or_diff($json, from_json('{ "spi": {}, "bsqErr": "Unknown protocol string blah" }', {relaxed => 1}), "a1 ip.protocol==blah", { context => 3 }); +# bigendian pcap file tests no facets + my $json = viewerGet("/spiview.json?date=-1&spi=a1,a2,pr&expression=" . uri_escape("file=$pwd/bigendian.pcap")); + is (!exists $json->{map}, 1, "map bigendian no facets"); + is (!exists $json->{graph}, 1, "graph bigendian no facets"); + eq_or_diff($json->{spi}->{a1}, from_json('{"other": 0, "missing": 0, "_type":"terms", "total":1, "terms":[{"count":1, "term":3232280992}]}'), "bigendian a1 no facets"); + eq_or_diff($json->{spi}->{a2}, from_json('{"other": 0, "missing": 0, "_type":"terms", "total":1, "terms":[{"count":1, "term":171969329}]}'), "bigendian a2 no facets"); + eq_or_diff($json->{spi}->{pr}, from_json('{"other": 0, "missing": 0, "_type":"terms", "total":1, "terms":[{"count":1, "term":"icmp"}]}'), "bigendian pr no facets"); + is ($json->{health}->{number_of_data_nodes}, 1, "Correct health number_of_data_nodes bigendian no facets"); - my $json = viewerGet("/spiview.json?date=-1&&spi=a1&expression=" . uri_escape("$files&&ip.protocol==[tcp,blah2]")); - eq_or_diff($json, from_json('{ "spi": {}, "bsqErr": "Unknown protocol string blah2" }', {relaxed => 1}), "a1 ip.protocol==[tcp,blah2]", { context => 3 }); +# multi bigendian pcap file tests no facets + my $json = multiGet("/spiview.json?date=-1&spi=a1,a2,pr&expression=" . uri_escape("file=$pwd/bigendian.pcap")); + is (!exists $json->{map}, 1, "multi map bigendian no facets"); + is (!exists $json->{graph}, 1, "multi graph bigendian no facets"); + eq_or_diff($json->{spi}->{a1}, from_json('{"other": 0, "missing": 0, "_type":"terms", "total":1, "terms":[{"count":1, "term":3232280992}]}'), "multi bigendian a1 no facets"); + eq_or_diff($json->{spi}->{a2}, from_json('{"other": 0, "missing": 0, "_type":"terms", "total":1, "terms":[{"count":1, "term":171969329}]}'), "multi bigendian a2 no facets"); + eq_or_diff($json->{spi}->{pr}, from_json('{"other": 0, "missing": 0, "_type":"terms", "total":1, "terms":[{"count":1, "term":"icmp"}]}'), "multi bigendian pr no facets"); + is ($json->{health}->{number_of_data_nodes}, 2, "multi Correct health number_of_data_nodes bigendian no facets"); + + +# Check facets short + $json = viewerGet("/spiview.json?startTime=1386004308&stopTime=1386004400&facets=1&spi=a1,a2,pr&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); + + eq_or_diff($json->{map}, from_json('{"src":{"USA": 3}, "dst":{"USA": 3}}'), "map short"); + eq_or_diff($json->{graph}->{lpHisto}, from_json('[["1386004309000", 1], ["1386004312000", 1], [1386004317000, 1]]'), "lpHisto short"); + eq_or_diff($json->{graph}->{paHisto}, from_json('[["1386004309000", 14], ["1386004312000", 15], [1386004317000, 17]]'), "paHisto short"); + eq_or_diff($json->{graph}->{dbHisto}, from_json('[["1386004309000", 1754], ["1386004312000", 1770], [1386004317000, 1763]]'), "dbHisto short"); + is ($json->{iTotalDisplayRecords}, 3, "records short"); + is ($json->{graph}->{interval}, 1, "correct interval short"); + is ($json->{graph}->{xmax}, 1386004400000, "correct xmax short"); + is ($json->{graph}->{xmin}, 1386004308000, "correct xmin short"); + eq_or_diff($json->{spi}->{a1}, from_json('{"other": 0, "missing": 0, "_type":"terms", "total":3, "terms":[{"count":3, "term":179608761}]}'), "short a1"); + eq_or_diff($json->{spi}->{a2}, from_json('{"other": 0, "missing": 0, "_type":"terms", "total":3, "terms":[{"count":3, "term":179608825}]}'), "short a2"); + eq_or_diff($json->{spi}->{pr}, from_json('{"other": 0, "missing": 0, "_type":"terms", "total":3, "terms":[{"count":3, "term":"tcp"}]}'), "short pr"); + +# multi Check facets short + $json = multiGet("/spiview.json?startTime=1386004308&stopTime=1386004400&facets=1&spi=a1,a2,pr&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); + + eq_or_diff($json->{map}, from_json('{"src":{"USA": 3}, "dst":{"USA": 3}}'), "multi map short"); + eq_or_diff($json->{graph}->{lpHisto}, from_json('[["1386004309000", 1], ["1386004312000", 1], [1386004317000, 1]]'), "multi lpHisto short"); + eq_or_diff($json->{graph}->{paHisto}, from_json('[["1386004309000", 14], ["1386004312000", 15], [1386004317000, 17]]'), "multi paHisto short"); + eq_or_diff($json->{graph}->{dbHisto}, from_json('[["1386004309000", 1754], ["1386004312000", 1770], [1386004317000, 1763]]'), "multi dbHisto short"); + is ($json->{iTotalDisplayRecords}, 3, "multi records short"); + is ($json->{graph}->{interval}, 1, "multi correct interval short"); + is ($json->{graph}->{xmax}, 1386004400000, "multi correct xmax short"); + is ($json->{graph}->{xmin}, 1386004308000, "multi correct xmin short"); + eq_or_diff($json->{spi}->{a1}, from_json('{"other": 0, "missing": 0, "_type":"terms", "total":3, "terms":[{"count":3, "term":179608761}]}'), "multi short a1"); + eq_or_diff($json->{spi}->{a2}, from_json('{"other": 0, "missing": 0, "_type":"terms", "total":3, "terms":[{"count":3, "term":179608825}]}'), "multi short a2"); + eq_or_diff($json->{spi}->{pr}, from_json('{"other": 0, "missing": 0, "_type":"terms", "total":3, "terms":[{"count":3, "term":"tcp"}]}'), "multi short pr"); + + +# Check facets medium + $json = viewerGet("/spiview.json?startTime=1386004308&stopTime=1386349908&facets=1&spi=a1,a2,pr&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); + + eq_or_diff($json->{map}, from_json('{"src":{"USA": 3}, "dst":{"USA": 3}}'), "map medium"); + eq_or_diff($json->{graph}->{lpHisto}, from_json('[["1386004260000", 3]]'), "lpHisto medium"); + eq_or_diff($json->{graph}->{paHisto}, from_json('[["1386004260000", 46]]'), "paHisto medium"); + eq_or_diff($json->{graph}->{dbHisto}, from_json('[["1386004260000", 5287]]'), "dbHisto medium"); + is ($json->{iTotalDisplayRecords}, 3, "records medium"); + is ($json->{graph}->{interval}, 60, "correct interval medium"); + is ($json->{graph}->{xmax}, 1386349908000, "correct xmax medium"); + is ($json->{graph}->{xmin}, 1386004308000, "correct xmin medium"); + eq_or_diff($json->{spi}->{a1}, from_json('{"other": 0, "missing": 0, "_type":"terms", "total":3, + "terms":[{"count":3, "term":179608761}]}'), "medium a1"); + eq_or_diff($json->{spi}->{a2}, from_json('{"other": 0, "missing": 0, "_type":"terms", "total":3, + "terms":[{"count":3, "term":179608825}]}'), "medium a2"); + eq_or_diff($json->{spi}->{pr}, from_json('{"other": 0, "missing": 0, "_type":"terms", "total":3, + "terms":[{"count":3, "term":"tcp"}]}'), "medium pr"); + +# multi Check facets medium + $json = multiGet("/spiview.json?startTime=1386004308&stopTime=1386349908&facets=1&spi=a1,a2,pr&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); + + eq_or_diff($json->{map}, from_json('{"src":{"USA": 3}, "dst":{"USA": 3}}'), "multi map medium"); + eq_or_diff($json->{graph}->{lpHisto}, from_json('[["1386004260000", 3]]'), "multi lpHisto medium"); + eq_or_diff($json->{graph}->{paHisto}, from_json('[["1386004260000", 46]]'), "multi paHisto medium"); + eq_or_diff($json->{graph}->{dbHisto}, from_json('[["1386004260000", 5287]]'), "multi dbHisto medium"); + is ($json->{iTotalDisplayRecords}, 3, "multi records medium"); + is ($json->{graph}->{interval}, 60, "multi correct interval medium"); + is ($json->{graph}->{xmax}, 1386349908000, "multi correct xmax medium"); + is ($json->{graph}->{xmin}, 1386004308000, "multi correct xmin medium"); + eq_or_diff($json->{spi}->{a1}, from_json('{"other": 0, "missing": 0, "_type":"terms", "total":3, + "terms":[{"count":3, "term":179608761}]}'), "multi medium a1"); + eq_or_diff($json->{spi}->{a2}, from_json('{"other": 0, "missing": 0, "_type":"terms", "total":3, + "terms":[{"count":3, "term":179608825}]}'), "multi medium a2"); + eq_or_diff($json->{spi}->{pr}, from_json('{"other": 0, "missing": 0, "_type":"terms", "total":3, + "terms":[{"count":3, "term":"tcp"}]}'), "multi medium pr"); + + +# Check facets ALL + $json = viewerGet("/spiview.json?date=-1&facets=1&spi=a1,a2,pr&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); + + eq_or_diff($json->{map}, from_json('{"dst":{"USA": 3, "CAN": 1}, "src":{"USA": 3, "RUS":1}}'), "map ALL"); + eq_or_diff($json->{graph}->{lpHisto}, from_json('[["1335956400000", 1], ["1386003600000", 3], [1387742400000, 1]]'), "lpHisto ALL"); + eq_or_diff($json->{graph}->{paHisto}, from_json('[["1335956400000", 2], ["1386003600000", 46], [1387742400000, 4]]'), "paHisto ALL"); + eq_or_diff($json->{graph}->{dbHisto}, from_json('[["1335956400000", 0], ["1386003600000", 5287], [1387742400000, 68]]'), "dbHisto ALL"); + is ($json->{iTotalDisplayRecords}, 5, "records ALL"); + is ($json->{graph}->{interval}, 3600, "correct interval ALL"); + + eq_or_diff($json->{spi}->{a1}, from_json('{"other": 0, "missing": 0, "_type":"terms", "total":5, + "terms":[{"count":3, "term":179608761},{"count":1, "term":3232280992}, {"count":1, "term":167772161}]}'), "ALL a1"); + eq_or_diff($json->{spi}->{a2}, from_json('{"other": 0, "missing": 0, "_type":"terms", "total":5, + "terms":[{"count":3, "term":179608825}, {"count":1, "term":171969329}, {"count":1, "term":167772162}]}'), "ALL a2"); + eq_or_diff($json->{spi}->{pr}, from_json('{"other": 0, "missing": 0, "_type":"terms", "total":5, + "terms":[{"count":4, "term":"tcp"}, {"count":1, "term":"icmp"}]}'), "ALL pr"); + +# multi Check facets ALL + $json = multiGet("/spiview.json?date=-1&facets=1&spi=a1,a2,pr&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); + + eq_or_diff($json->{map}, from_json('{"dst":{"USA": 3, "CAN": 1}, "src":{"USA": 3, "RUS":1}}'), "multi map ALL"); + eq_or_diff($json->{graph}->{lpHisto}, from_json('[["1335956400000", 1], ["1386003600000", 3], [1387742400000, 1]]'), "multi lpHisto ALL"); + eq_or_diff($json->{graph}->{paHisto}, from_json('[["1335956400000", 2], ["1386003600000", 46], [1387742400000, 4]]'), "multi paHisto ALL"); + eq_or_diff($json->{graph}->{dbHisto}, from_json('[["1335956400000", 0], ["1386003600000", 5287], [1387742400000, 68]]'), "multi dbHisto ALL"); + is ($json->{iTotalDisplayRecords}, 5, "multi records ALL"); + is ($json->{graph}->{interval}, 3600, "multi correct interval ALL"); + + eq_or_diff($json->{spi}->{a1}, from_json('{"other": 0, "missing": 0, "_type":"terms", "total":5, + "terms":[{"count":3, "term":179608761},{"count":1, "term":167772161}, {"count":1, "term":3232280992}]}'), "multi ALL a1"); + eq_or_diff($json->{spi}->{a2}, from_json('{"other": 0, "missing": 0, "_type":"terms", "total":5, + "terms":[{"count":3, "term":179608825}, {"count":1, "term":167772162}, {"count":1, "term":171969329}]}'), "multi ALL a2"); + eq_or_diff($json->{spi}->{pr}, from_json('{"other": 0, "missing": 0, "_type":"terms", "total":5, + "terms":[{"count":4, "term":"tcp"}, {"count":1, "term":"icmp"}]}'), "multi ALL pr");