Skip to content

Commit

Permalink
List queries can now have wildcard and regex items.
Browse files Browse the repository at this point in the history
  • Loading branch information
@awick
awick committed Jun 24, 2014
1 parent e8a834a commit 227cfbd
Show file tree
Hide file tree
Showing 5 changed files with 154 additions and 96 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
- Comma seperate list of elasticsearch hosts (issue #176)
- New includes directive (issue #144)
- Initial bigendian support in viewer (issue #259)
- List queries can now have wildcard and regex items.
example: http.uri = [term, w*ldcard, /.*regex/]

0.11.0 2014/05/08
- BREAKING: elasticsearch 0.90.7 or newer required, recommend 0.90.12+,
Expand Down
62 changes: 59 additions & 3 deletions tests/tests.pl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ sub doGeo {
system("wget http://www.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz; gunzip GeoIP.dat.gz");
}

if (! -f "plugins/test.so") {
if (! -f "plugins/test.so" || (stat('../capture/moloch.h'))[9] > (stat('plugins/test.so'))[9]) {
system("cd plugins ; make");
}
}
Expand Down Expand Up @@ -150,7 +150,7 @@ sub countTest {
sub doViewer {
my ($cmd) = @_;

plan tests => 738;
plan tests => 842;

die "Must run in tests directory" if (! -f "../db/db.pl");

Expand Down Expand Up @@ -178,6 +178,8 @@ sub doViewer {

if (!$main::debug) {
$cmd .= " 2>&1 1>/dev/null";
} else {
$cmd .= " --debug";
}

if ($main::valgrind) {
Expand Down Expand Up @@ -504,6 +506,60 @@ sub doViewer {
countTest(0, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/http-500-head.pcap)&&http.uri==config.dat"));
countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/http-500-head.pcap)&&http.uri==a.zip"));
countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/http-500-head.pcap)&&http.uri==/.*a.zip/"));
countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/http-500-head.pcap)&&http.uri==/.*a.zip/"));
# http.uri slash tests - tokeniezd field
countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap)&&http.uri==/js/xxxxxx.js"));
countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap)&&http.uri==//js/xxxxxx.js"));
countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap)&&http.uri==js/xxxxxx"));
countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap)&&http.uri==/.*\\/js\\/xxxxxx.js/"));
countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap)&&http.uri==/*/js/xxxxxx.j*"));
countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap)&&http.uri==[/js/xxxxxx.js]"));
countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap)&&http.uri==[//js/xxxxxx.js]"));
countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap)&&http.uri==[js/xxxxxx]"));
countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap)&&http.uri==[/.*js\\/xxxxxx.js/]"));
countTest(0, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap)&&http.uri==[\"/.*js\\/xxxxxx.js/\"]"));
countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap)&&http.uri==[/*/js/xxxxxx.j*]"));
countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap)&&http.uri==[\"/\\/js\\/xxxxxx.js/\"]"));
countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap||file=$pwd/http-content-zip.pcap)&&http.uri==[a.zip,/js/xxxxxx.js]"));
countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap||file=$pwd/http-content-zip.pcap)&&http.uri==[a.zip,//js/xxxxxx.js]"));
countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap||file=$pwd/http-content-zip.pcap)&&http.uri==[a.zip,js/xxxxxx]"));
countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap||file=$pwd/http-content-zip.pcap)&&http.uri==[a.zip,/.*js\\/xxxxxx.js/]"));
countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap||file=$pwd/http-content-zip.pcap)&&http.uri==[a.zip,/*/js/xxxxxx.j*]"));
countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap||file=$pwd/http-content-zip.pcap)&&http.uri==[a.zip , /js/xxxxxx.js]"));
countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap||file=$pwd/http-content-zip.pcap)&&http.uri==[a.zip , //js/xxxxxx.js]"));
countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap||file=$pwd/http-content-zip.pcap)&&http.uri==[a.zip , js/xxxxxx]"));
countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap||file=$pwd/http-content-zip.pcap)&&http.uri==[a.zip , /.*js\\/xxxxxx.js/]"));
countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap||file=$pwd/http-content-zip.pcap)&&http.uri==[a.zip , /*/js/xxxxxx.j*]"));
# http.uri.path slash tests - not tokenized
countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap)&&http.uri.path==/js/xxxxxx.js"));
countTest(0, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap)&&http.uri.path==//js/xxxxxx.js"));
countTest(0, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap)&&http.uri.path==js/xxxxxx"));
countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap)&&http.uri.path==/\\/js\\/xxxxxx.js/"));
countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap)&&http.uri.path==/js/xxxxxx.j*"));
countTest(0, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap)&&http.uri.path==//js/xxxxxx.j*"));
countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap)&&http.uri.path==[/js/xxxxxx.js]"));
countTest(0, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap)&&http.uri.path==[//js/xxxxxx.js]"));
countTest(0, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap)&&http.uri.path==[js/xxxxxx]"));
countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap)&&http.uri.path==[/\\/js\\/xxxxxx.js/]"));
countTest(0, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap)&&http.uri.path==[\"/\\/js\\/xxxxxx.js/\"]"));
countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap)&&http.uri.path==[/\\/js\\/.*.js/]"));
countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap)&&http.uri.path==[/js/xxxxxx.js*]"));
countTest(0, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap)&&http.uri.path==[//js/xxxxxx.js*]"));
countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap||file=$pwd/http-content-zip.pcap)&&http.uri.path==[/js/xxxxxx.js,a.zip]"));
countTest(0, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap||file=$pwd/http-content-zip.pcap)&&http.uri.path==[//js/xxxxxx.js,a.zip]"));
countTest(0, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap||file=$pwd/http-content-zip.pcap)&&http.uri.path==[js/xxxxxx,a.zip]"));
countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap||file=$pwd/http-content-zip.pcap)&&http.uri.path==[/\\/js\\/xxxxxx.js/,a.zip]"));
countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap||file=$pwd/http-content-zip.pcap)&&http.uri.path==[/\\/js\\/.*.js/,a.zip]"));
countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap||file=$pwd/http-content-zip.pcap)&&http.uri.path==[/js/xxxxxx.js*,a.zip]"));
countTest(0, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap||file=$pwd/http-content-zip.pcap)&&http.uri.path==[//js/xxxxxx.js*,a.zip]"));
countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap||file=$pwd/http-content-zip.pcap)&&http.uri.path==[/js/xxxxxx.js,*a.zip*]"));
countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap||file=$pwd/http-content-zip.pcap)&&http.uri.path==[//js/xxxxxx.js,*a.zip*]"));
countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap||file=$pwd/http-content-zip.pcap)&&http.uri.path==[js/xxxxxx,*a.zip*]"));
countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap||file=$pwd/http-content-zip.pcap)&&http.uri.path==[/\\/js\\/xxxxxx.js/,*a.zip*]"));
countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap||file=$pwd/http-content-zip.pcap)&&http.uri.path==[\"/\\/js\\/xxxxxx.js/\",*a.zip*]"));
countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap||file=$pwd/http-content-zip.pcap)&&http.uri.path==[/\\/js\\/.*.js/,*a.zip*]"));
countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap||file=$pwd/http-content-zip.pcap)&&http.uri.path==[/js/xxxxxx.js*,*a.zip*]"));
countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap||file=$pwd/http-content-zip.pcap)&&http.uri.path==[//js/xxxxxx.js*,*a.zip*]"));
# http.hasheader, http.hasheader.src, http.hasheader.dst tests
countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/socks5-reverse.pcap)&&http.hasheader==server"));
countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/socks5-reverse.pcap)&&http.hasheader.dst==server"));
Expand All @@ -525,7 +581,7 @@ sub doViewer {
countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/socks5-reverse.pcap)&&http.hasheader.src==*kIe"));
#TODO countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/socks5-reverse.pcap)&&http.hasheader==/.*VeR/"));
#TODO countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/socks5-reverse.pcap)&&http.hasheader.dst==/.*VeR/"));
#TODO tTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/socks5-reverse.pcap)&&http.hasheader.src==/.*kIe/"));
#TODO countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/socks5-reverse.pcap)&&http.hasheader.src==/.*kIe/"));
countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/socks5-reverse.pcap)&&http.hasheader==[SeRver]"));
countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/socks5-reverse.pcap)&&http.hasheader.dst==[SeRver]"));
countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/socks5-reverse.pcap)&&http.hasheader.src==[CooKie]"));
Expand Down
92 changes: 46 additions & 46 deletions viewer/molochparser.jison
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
\s+ /* skip whitespace */
[-a-zA-Z0-9_.@:*?/]+ return 'STR'
\"[^"\\]*(?:\\.[^"\\]*)*\" return 'QUOTEDSTR'
\/[^/\\]*(?:\\.[^/\\]*)*\/ return 'REGEXSTR'
\/[^\/\\]*(?:\\.[^\/\\]*)*\/ return 'REGEXSTR'
\[[^\]\\]*(?:\\.[^\]\\]*)*\] return 'LIST'
"EXISTS!" return "EXISTS"
"<=" return 'lte'
Expand Down Expand Up @@ -125,7 +125,7 @@ function parseIpPort(yy, field, ipPortStr) {
// We really have a list of them
if (ipPortStr[0] === "[" && ipPortStr[ipPortStr.length -1] === "]") {
obj = {bool: {should: []}};
CSVtoArray(ipPortStr).forEach(function(str) {
ListToArray(ipPortStr).forEach(function(str) {
obj.bool.should.push(parseIpPort(yy, field, str));
});
return obj;
Expand Down Expand Up @@ -341,24 +341,48 @@ function stringQuery(yy, field, str) {
str = str.substring(1, str.length-1).replace(/\\(.)/g, "$1");
quoted = true;
} else if (str[0] === "[" && str[str.length -1] === "]") {
strs = CSVtoArray(str);
var rawField = field2Raw(yy, field);
strs = ListToArray(str);
if (info.transform) {
for (var i = 0; i < strs.length; i++) {
strs[i] = global.moloch[info.transform](strs[i]);
}
}
if (info.type.match(/termfield/)) {
obj = {terms: {}};
obj.terms[dbField] = strs;
} else if (info.type.match(/textfield/)) {
var obj = {query: {bool: {should: []}}};
strs.forEach(function(str) {
var should = {match: {}};
should.match[dbField] = {query: str, type: "phrase", operator: "and"}
var obj = {query: {bool: {should: []}}};
var terms = null;
strs.forEach(function(str) {
var should;
if (typeof str === "string" && str[0] === "/" && str[str.length -1] === "/") {
should = {regexp: {}};
should.regexp[rawField] = str.substring(1, str.length-1);
obj.query.bool.should.push(should);
});
}
} else if (typeof str === "string" && str.indexOf("*") !== -1) {
should = {wildcard: {}};
should.wildcard[rawField] = str;
obj.query.bool.should.push(should);
} else {
if (str[0] === "\"" && str[str.length -1] === "\"") {
str = str.substring(1, str.length-1).replace(/\\(.)/g, "$1");
}
if (info.type.match(/termfield/)) {
// Reuse same terms element
if (terms === null) {
terms = {terms: {}};
terms.terms[dbField] = [];
obj.query.bool.should.push(terms);
}
terms.terms[dbField].push(str);
} else {
should = {match: {}};
should.match[dbField] = {query: str, type: "phrase", operator: "and"}
obj.query.bool.should.push(should);
}
}
});
return obj;
}
Expand Down Expand Up @@ -408,48 +432,24 @@ global.moloch.ipProtocolLookup = function (text) {
}
};
// http://stackoverflow.com/a/8497474
// Return array of string values, or NULL if CSV string not well formed.
function CSVtoArray(text) {
function ListToArray(text) {
if (text[0] !== "[" || text[text.length -1] !== "]")
return text;
text = text.substring(1, text.length-1);
var re_valid = /^\s*(?:'[^'\\]*(?:\\[\S\s][^'\\]*)*'|"[^"\\]*(?:\\[\S\s][^"\\]*)*"|[^,'"\s\\]*(?:\s+[^,'"\s\\]+)*)\s*(?:,\s*(?:'[^'\\]*(?:\\[\S\s][^'\\]*)*'|"[^"\\]*(?:\\[\S\s][^"\\]*)*"|[^,'"\s\\]*(?:\s+[^,'"\s\\]+)*)\s*)*$/;
var re_value = /(?!\s*$)\s*(?:'([^'\\]*(?:\\[\S\s][^'\\]*)*)'|"([^"\\]*(?:\\[\S\s][^"\\]*)*)"|([^,'"\s\\]*(?:\s+[^,'"\s\\]+)*))\s*(?:,|$)/g;
// Return NULL if input string is not well formed CSV string.
if (!re_valid.test(text)) return null;
var a = []; // Initialize array to receive values.
text.replace(re_value, // "Walk" the string using replace with callback.
function(m0, m1, m2, m3) {
// Remove backslash from \' in single quoted values.
if (m1 !== undefined) a.push(m1.replace(/\\'/g, "'"));
// Remove backslash from \" in double quoted values.
else if (m2 !== undefined) a.push(m2.replace(/\\"/g, '"'));
else if (m3 !== undefined) a.push(m3);
return ''; // Return empty string.
});
// Handle special case of empty last value.
if (/,\s*$/.test(text)) a.push('');
return a;
};
function termOrTermsStr(dbField, str) {
var obj = {};
if (str[0] === "[" && str[str.length -1] === "]") {
obj = {terms: {}};
obj.terms[dbField] = CSVtoArray(str);
} else {
obj = {term: {}};
obj.term[dbField] = str;
// JS doesn't have negative look behind
text = text.substring(1, text.length-1);
strs = text.replace(/\\\\/g, "**BACKSLASH**").replace(/\\,/g, "**COMMA**").split(/\s*,\s*/);
for (var i = 0; i < strs.length; i++) {
strs[i] = strs[i].replace("**COMMA**", ",").replace("**BACKSLASH**", "\\");
}
return obj;
return strs;
}
function termOrTermsInt(dbField, str) {
var obj = {};
if (str[0] === "[" && str[str.length -1] === "]") {
obj = {terms: {}};
obj.terms[dbField] = CSVtoArray(str);
obj.terms[dbField] = ListToArray(str);
obj.terms[dbField].forEach(function(str) {
str = stripQuotes(str);
if (typeof str !== "integer" && str.match(/[^\d]+/))
Expand Down
Loading

0 comments on commit 227cfbd

Please sign in to comment.