Skip to content

Commit

Permalink
http.cookie.key, large SSL/TLS cert support, fields in tagger files
Browse files Browse the repository at this point in the history 
- New http.cookie.key expression
- Larger SSL/TLS certificates will now be parsed correctly
- New fields can be definied in tagger input files, examples coming soon
  • Loading branch information
@awick
awick committed Oct 10, 2014
1 parent 7d265b5 commit 1bb2502
Show file tree
Hide file tree
Showing 40 changed files with 987 additions and 430 deletions.
3 changes: 3 additions & 0 deletions CHANGELOG
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,9 @@
It is now possible to have a different tag per match
- Tagger now supports matching email and uri paths
- Sort session sections
- New http.cookie.key expression
- Handle larger SSL/TLS certificates
- New fields can be defined in tagger input files

0.11.1 2014/08/07
- NOTICE: ES 0.90.12+, 1.1.x, 1.2.0 are supported by this version.
Expand Down
102 changes: 102 additions & 0 deletions capture/field.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,108 @@ void moloch_field_define_json(unsigned char *expression, int expression_len, uns
return;
}
/******************************************************************************/
int moloch_field_define_text(char *text, int *shortcut)
{
int count = 0;
char *field = 0;
char *kind = 0;
char *help = 0;
char *db = 0;
char *group = 0;
char *friendly = 0;

if (config.debug)
LOG("Parsing %s", text);
char **elements = g_strsplit(text, ";", 0);
int e;
for (e = 0; elements[e]; e++) {
char *colon = strchr(elements[e], ':');
if (!colon)
continue;
*colon = 0;
colon++;
if (strcmp(elements[e], "field") == 0)
field = colon;
else if (strcmp(elements[e], "kind") == 0)
kind = colon;
else if (strcmp(elements[e], "group") == 0)
group = colon;
else if (strcmp(elements[e], "count") == 0)
count = strcmp(colon, "true") == 0;
else if (strcmp(elements[e], "friendly") == 0)
friendly = colon;
else if (strcmp(elements[e], "db") == 0)
db = colon;
else if (strcmp(elements[e], "help") == 0)
help = colon;
else if (strcmp(elements[e], "shortcut") == 0) {
if (shortcut)
*shortcut = atoi(colon);
}

}

if (!field) {
LOG("Didn't find field 'field:'");
g_strfreev(elements);
return -1;
}

if (!db) {
int pos = moloch_field_by_exp(field);
g_strfreev(elements);
if (pos != -1)
return pos;

LOG("Didn't find field 'db:'");
return -1;
}

if (!kind) {
LOG("Didn't find field 'kind:'");
g_strfreev(elements);
return -1;
}

if (strstr(kind, "termfield") != 0 && strstr(db, "-term") == 0) {
LOG("ERROR - db field %s for %s should end with -term", kind, db);
exit(1);
}

char groupbuf[100];
if (!group) {
char *dot = strchr(field, '.');
if (dot) {
memcpy(groupbuf, field, MIN(100, dot-field));
groupbuf[dot-field] = 0;
group = groupbuf;
} else {
group = "general";
}
}

if (!friendly)
friendly = field;

if (!help)
help = field;

int type, flags = 0;
if (strcmp(kind, "integer") == 0)
type = MOLOCH_FIELD_TYPE_INT_HASH;
else if (strcmp(kind, "ip") == 0)
type = MOLOCH_FIELD_TYPE_IP_HASH;
else
type = MOLOCH_FIELD_TYPE_STR_HASH;

if (count)
flags |= MOLOCH_FIELD_FLAG_COUNT;

int pos = moloch_field_define(group, kind, field, friendly, db, help, type, flags, NULL);
g_strfreev(elements);
return pos;
}
/******************************************************************************/
/* Changes ... to va_list */
static void moloch_nids_add_field_proxy(char *group, char *kind, char *expression, char *friendlyName, char *dbField, char *help, ...)
{
Expand Down
28 changes: 18 additions & 10 deletions capture/hash.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,10 +51,17 @@ typedef int (* HASH_CMP_FUNC)(const void *key, const void *element);

#define HASH_ADD(name, varname, key, element) \
do { \
element->name##hash = (varname).hash(key); \
element->name##bucket = element->name##hash % (varname).size; \
DLL_PUSH_TAIL(name, &((varname).buckets[element->name##bucket]), element); \
(varname).count++; \
const uint32_t _hh = element->name##hash = (varname).hash(key); \
const int _b = element->name##bucket = element->name##hash % (varname).size; \
const void *_end = (void*)&((varname).buckets[_b]); \
for (element->name##next = (varname).buckets[_b].name##next; element->name##next != _end; element->name##next = element->name##next->name##next) { \
if (_hh > element->name##next->name##hash) break; \
}\
element->name##prev = element->name##next->name##prev; \
element->name##prev->name##next = element; \
element->name##next->name##prev = element; \
(varname).buckets[_b].name##count++;\
(varname).count++; \
} while(0)

#define HASH_REMOVE(name, varname, element) \
Expand All @@ -65,13 +72,14 @@ typedef int (* HASH_CMP_FUNC)(const void *key, const void *element);

#define HASH_FIND_HASH(name, varname, h, key, element) \
do { \
uint32_t hh = h; \
int b = hh % (varname).size; \
for (element = (varname).buckets[b].name##next; element != (void*)&((varname).buckets[b]); element = element->name##next) { \
if (hh == element->name##hash && (varname).cmp(key, element)) \
break; \
const uint32_t _hh = h; \
const int _b = _hh % (varname).size; \
const void *_end = (void*)&((varname).buckets[_b]); \
for (element = (varname).buckets[_b].name##next; element != _end; element = element->name##next) { \
if (_hh == element->name##hash && (varname).cmp(key, element)) break; \
if (_hh > element->name##hash) {element = 0; break;} \
} \
if (element == (void *)&((varname).buckets[b])) element = 0; \
if (element == _end) element = 0; \
} while(0)

#define HASH_FIND_INT(name, varname, key, element) HASH_FIND_HASH(name, varname, (uint32_t)key, (void*)(long)key, element)
Expand Down
27 changes: 17 additions & 10 deletions capture/moloch.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
#define UNUSED(x) x __attribute((unused))


#define MOLOCH_API_VERSION 10
#define MOLOCH_API_VERSION 11

/******************************************************************************/
/*
Expand Down Expand Up @@ -292,12 +292,12 @@ typedef struct {

/******************************************************************************/
/*
* SPI Data Storage
* Parser
*/

struct moloch_session;

typedef int (* MolochParserFunc) (struct moloch_session *session, void *uw, const unsigned char *data, int remaining);
typedef int (* MolochParserFunc) (struct moloch_session *session, void *uw, const unsigned char *data, int remaining, int which);
typedef void (* MolochParserFreeFunc) (struct moloch_session *session, void *uw);
typedef void (* MolochParserSaveFunc) (struct moloch_session *session, void *uw, int final);

Expand All @@ -309,13 +309,20 @@ typedef struct {

} MolochParserInfo_t;

/******************************************************************************/
/*
* SPI Data Storage
*/
#define MOLOCH_SESSIONID_LEN 12
typedef struct moloch_session {
struct moloch_session *tcp_next, *tcp_prev;
struct moloch_session *q_next, *q_prev;
struct moloch_session *h_next, *h_prev;
int h_bucket;
uint32_t h_hash;

uint64_t sessionIda;
uint32_t sessionIdb;
MolochField_t **fields;

void **pluginData;
Expand Down Expand Up @@ -357,7 +364,6 @@ typedef struct moloch_session {

uint16_t haveNidsTcp:1;
uint16_t needSave:1;
uint16_t which:1;
} MolochSession_t;

typedef struct moloch_session_head {
Expand Down Expand Up @@ -476,14 +482,14 @@ void moloch_db_exit();
void moloch_parsers_init();
void moloch_parsers_initial_tag(MolochSession_t *session);
unsigned char *moloch_parsers_asn_get_tlv(BSB *bsb, int *apc, int *atag, int *alen);
char *moloch_parsers_asn_decode_oid(unsigned char *oid, int len);
void moloch_parsers_classify_tcp(MolochSession_t *session, const unsigned char *data, int remaining);
void moloch_parsers_classify_udp(MolochSession_t *session, const unsigned char *data, int remaining);
void moloch_parsers_asn_decode_oid(char *buf, int bufsz, unsigned char *oid, int len);
void moloch_parsers_classify_tcp(MolochSession_t *session, const unsigned char *data, int remaining, int which);
void moloch_parsers_classify_udp(MolochSession_t *session, const unsigned char *data, int remaining, int which);
void moloch_parsers_exit();

void moloch_parsers_magic_tag(MolochSession_t *session, int field, const char *base, const char *data, int len);

typedef void (* MolochClassifyFunc) (MolochSession_t *session, const unsigned char *data, int remaining);
typedef void (* MolochClassifyFunc) (MolochSession_t *session, const unsigned char *data, int remaining, int which);

void moloch_parsers_unregister(MolochSession_t *session, void *uw);
void moloch_parsers_register2(MolochSession_t *session, MolochParserFunc func, void *uw, MolochParserFreeFunc ffunc, MolochParserSaveFunc sfunc);
Expand Down Expand Up @@ -643,9 +649,10 @@ void moloch_yara_exit();

void moloch_field_init();
void moloch_field_define_json(unsigned char *expression, int expression_len, unsigned char *data, int data_len);
int moloch_field_define_text(char *text, int *shortcut);
int moloch_field_define(char *group, char *kind, char *expression, char *friendlyName, char *dbField, char *help, int type, int flags, ...);
int moloch_field_by_db(char *dbField);
int moloch_field_by_exp(char *exp);
int moloch_field_by_db(char *dbField);
int moloch_field_by_exp(char *exp);
gboolean moloch_field_string_add(int pos, MolochSession_t *session, const char *string, int len, gboolean copy);
gboolean moloch_field_int_add(int pos, MolochSession_t *session, int i);
gboolean moloch_field_certsinfo_add(int pos, MolochSession_t *session, MolochCertsInfo_t *info, int len);
Expand Down
Loading

0 comments on commit 1bb2502

Please sign in to comment.