Skip to content

Commit

Permalink
Parsers can register for session save events (issue arkime#248)
Browse files Browse the repository at this point in the history
  • Loading branch information
@awick
awick committed May 14, 2014
1 parent 6b4511b commit 50ed32c
Show file tree
Hide file tree
Showing 7 changed files with 211 additions and 4 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,10 @@
- NOTICE: ES 0.90.12+ & ES 1.1.1 are support by this version.
ES 1.0 is NOT supported.
This is the LAST version to support 0.90.x
Restart viewer AFTER upgrading ES versions
- NOTICE: When upgrading your runes.sh for 1.1.1 add a -d to the
command, ES no longer runs in background by default
- Parsers can register for session save events (issue #248)

0.11.0 2014/05/08
- BREAKING: elasticsearch 0.90.7 or newer required, recommend 0.90.12+,
Expand Down
7 changes: 5 additions & 2 deletions capture/moloch.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
#define UNUSED(x) x __attribute((unused))


#define MOLOCH_API_VERSION 9
#define MOLOCH_API_VERSION 10

/******************************************************************************/
/*
Expand Down Expand Up @@ -287,11 +287,13 @@ struct moloch_session;

typedef int (* MolochParserFunc) (struct moloch_session *session, void *uw, const unsigned char *data, int remaining);
typedef void (* MolochParserFreeFunc) (struct moloch_session *session, void *uw);
typedef void (* MolochParserSaveFunc) (struct moloch_session *session, void *uw, int final);

typedef struct {
MolochParserFunc parserFunc;
void *uw;
MolochParserFreeFunc parserFreeFunc;
MolochParserSaveFunc parserSaveFunc;

} MolochParserInfo_t;

Expand Down Expand Up @@ -470,7 +472,8 @@ void moloch_parsers_magic_tag(MolochSession_t *session, int field, const char *b
typedef void (* MolochClassifyFunc) (MolochSession_t *session, const unsigned char *data, int remaining);

void moloch_parsers_unregister(MolochSession_t *session, void *uw);
void moloch_parsers_register(MolochSession_t *session, MolochParserFunc func, void *uw, MolochParserFreeFunc ffunc);
void moloch_parsers_register2(MolochSession_t *session, MolochParserFunc func, void *uw, MolochParserFreeFunc ffunc, MolochParserSaveFunc sfunc);
#define moloch_parsers_register(session, func, uw, ffunc) moloch_parsers_register2(session, func, uw, ffunc, NULL)

void moloch_parsers_classifier_register_tcp_internal(const char *name, int offset, unsigned char *match, int matchlen, MolochClassifyFunc func, size_t sessionsize, int apiversion);
#define moloch_parsers_classifier_register_tcp(name, offset, match, matchlen, func) moloch_parsers_classifier_register_tcp_internal(name, offset, match, matchlen, func, sizeof(MolochSession_t), MOLOCH_API_VERSION)
Expand Down
16 changes: 16 additions & 0 deletions capture/nids.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -217,6 +217,14 @@ void moloch_session_id (char *buf, uint32_t addr1, uint16_t port1, uint32_t addr
/******************************************************************************/
void moloch_nids_save_session(MolochSession_t *session)
{
if (session->parserInfo) {
int i;
for (i = 0; i < session->parserNum; i++) {
if (session->parserInfo[i].parserSaveFunc)
session->parserInfo[i].parserSaveFunc(session, session->parserInfo[i].uw, TRUE);
}
}

if (pluginsCbs & MOLOCH_PLUGIN_PRE_SAVE)
moloch_plugins_cb_pre_save(session, TRUE);

Expand Down Expand Up @@ -251,6 +259,14 @@ void moloch_nids_save_session(MolochSession_t *session)
/******************************************************************************/
void moloch_nids_mid_save_session(MolochSession_t *session)
{
if (session->parserInfo) {
int i;
for (i = 0; i < session->parserNum; i++) {
if (session->parserInfo[i].parserSaveFunc)
session->parserInfo[i].parserSaveFunc(session, session->parserInfo[i].uw, FALSE);
}
}

if (pluginsCbs & MOLOCH_PLUGIN_PRE_SAVE)
moloch_plugins_cb_pre_save(session, FALSE);

Expand Down
4 changes: 3 additions & 1 deletion capture/parsers.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -308,7 +308,7 @@ void moloch_print_hex_string(unsigned char* data, unsigned int length)
printf("\n");
}
/******************************************************************************/
void moloch_parsers_register(MolochSession_t *session, MolochParserFunc func, void *uw, MolochParserFreeFunc ffunc)
void moloch_parsers_register2(MolochSession_t *session, MolochParserFunc func, void *uw, MolochParserFreeFunc ffunc, MolochParserSaveFunc sfunc)
{
if (session->parserLen == 0) {
session->parserLen = 2;
Expand All @@ -320,6 +320,7 @@ void moloch_parsers_register(MolochSession_t *session, MolochParserFunc func, v
session->parserInfo[session->parserNum].parserFunc = func;
session->parserInfo[session->parserNum].uw = uw;
session->parserInfo[session->parserNum].parserFreeFunc = ffunc;
session->parserInfo[session->parserNum].parserSaveFunc = sfunc;

session->parserNum++;
}
Expand All @@ -334,6 +335,7 @@ void moloch_parsers_unregister(MolochSession_t *session, void *uw)
session->parserInfo[i].parserFreeFunc = 0;
}

session->parserInfo[i].parserSaveFunc = 0;
session->parserInfo[i].parserFunc = 0;
session->parserInfo[i].uw = 0;
break;
Expand Down
22 changes: 21 additions & 1 deletion capture/parsers/http.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -475,6 +475,26 @@ int http_parse(MolochSession_t *session, void *uw, const unsigned char *data, in
return 0;
}
/******************************************************************************/
void http_save(MolochSession_t UNUSED(*session), void *uw, int final)
{
if (!final)
return;

HTTPInfo_t *http = uw;

#ifdef HTTPDEBUG
LOG("Save callback %d", final);
#endif
if (http->wParsers & 0x1) {
http_parser_execute(&http->parsers[0], &parserSettings, 0, 0);
}

if (http->wParsers & 0x2) {
http_parser_execute(&http->parsers[1], &parserSettings, 0, 0);
}

}
/******************************************************************************/
void http_free(MolochSession_t UNUSED(*session), void *uw)
{
HTTPInfo_t *http = uw;
Expand Down Expand Up @@ -515,7 +535,7 @@ void http_classify(MolochSession_t *session, const unsigned char *UNUSED(data),

http->session = session;

moloch_parsers_register(session, http_parse, http, http_free);
moloch_parsers_register2(session, http_parse, http, http_free, http_save);
}
/******************************************************************************/
void moloch_parser_init()
Expand Down
Binary file added tests/http-no-length.pcap
View file
Binary file not shown.
164 changes: 164 additions & 0 deletions tests/http-no-length.test
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,164 @@
{
"packets" : [
{
"body" : {
"ua" : [
"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.3) Gecko/20100401 Firefox/3.6.3"
],
"test" : {
"number" : [
33554442
],
"ip" : [
167772161
],
"ip-asn" : [
"AS0000 This is neat"
],
"string" : [
"16777226:50384,33554442:80"
],
"ip-geo" : [
"RUS"
],
"ip-rir" : [
""
]
},
"db2" : 744,
"db" : 1196,
"no" : "test",
"ho" : [
"xxxxxxx.xxxxxx.xx"
],
"lp" : 1273479693,
"a2" : "10.0.0.2",
"http" : {
"method-term-cnt" : 1,
"method-term" : [
"GET"
],
"statuscode" : [
200
],
"statuscode-cnt" : 1,
"bodymagic-term" : [
"text/plain"
],
"bodymagic-term-cnt" : 1
},
"hsvercnt" : 1,
"ta" : [
"dstip",
"http:content:text/plain",
"http:method:GET",
"http:statuscode:200",
"node:test",
"protocol:http",
"srcip",
"tcp"
],
"hpath" : [
"/js/xxxxxx.js"
],
"pa1" : 5,
"fpd" : 1273479692982,
"fs" : [],
"by2" : 1026,
"g1" : "RUS",
"hsver" : [
"1.1"
],
"pa2" : 5,
"uscnt" : 1,
"hocnt" : 1,
"p1" : 50384,
"by" : 1760,
"as1" : "AS0000 This is neat",
"g2" : "CAN",
"pr" : 6,
"ps" : [
24,
106,
188,
258,
780,
850,
937,
1734,
1804,
1874
],
"prot-term-cnt" : 2,
"hpathcnt" : 1,
"hh2" : [
"http:header:cache-control",
"http:header:connection",
"http:header:content-type",
"http:header:date",
"http:header:expires",
"http:header:last-modified",
"http:header:p3p",
"http:header:pragma",
"http:header:server",
"http:header:set-cookie"
],
"lpd" : 1273479693105,
"fp" : 1273479692,
"as2" : "AS0001 Cool Beans!",
"hmd5cnt" : 1,
"hh2cnt" : 10,
"pa" : 10,
"tacnt" : 8,
"fb1" : "474554202f6a732f",
"us" : [
"//xxxxxxx.xxxxxx.xx/js/xxxxxx.js"
],
"hh1" : [
"http:header:accept",
"http:header:accept-charset",
"http:header:accept-encoding",
"http:header:accept-language",
"http:header:connection",
"http:header:cookie",
"http:header:host",
"http:header:keep-alive",
"http:header:referer",
"http:header:user-agent"
],
"a1" : "10.0.0.1",
"fb2" : "485454502f312e30",
"db1" : 452,
"hmd5" : [
"9fb54a2726ca3cf54a82804d0e66d08a"
],
"hdrs" : {
"hreq-referercnt" : 1,
"hreq-referer" : [
"http://www.xxxxxxxx.com/xxxxxxxxxxxxxxx/xxxxxxxxxxxxxxxx.html"
]
},
"by1" : 734,
"hh1cnt" : 10,
"p2" : 80,
"rir2" : "TEST",
"hdvercnt" : 1,
"prot-term" : [
"http",
"tcp"
],
"hdver" : [
"1.0"
],
"uacnt" : 1
},
"header" : {
"index" : {
"_index" : "sessions-100510",
"_type" : "session"
}
}
}
]
}

0 comments on commit 50ed32c

Please sign in to comment.