Skip to content

Commit

Permalink
ES > 1.2, ES index prefix support, SMB1 & TLS improvements, Viewer Te…
Browse files Browse the repository at this point in the history 
…st Suite overhall

- NOTICE: Only 1.2.x and 1.3.x are supported by this version.
          Restart viewer AFTER upgrading ES versions
- NOTICE: Requires running npm update for ES 1.3 support
- For NEW installs can now set a prefix= config variable and
  db.pl --prefix option that will prefix all ES tables.  This makes
  it easier for Moloch to share ES with other services OR multiple
  Moloch clusters can use 1 ES cluster with different prefixes.
- New usersElasticsearch= and usersPrefix= config that make it possible
  for multiple Moloch clusters to share a single users table.
- viewer: removal of pre 1.2 ES things
- Some cron efficiency improvements
- Check more often if files need to be expired
- More SMB1 parsing
- More TLS cipherso
- Major viewer test suite restructure and improvements
  • Loading branch information
@awick
awick committed Oct 29, 2014
1 parent 1904492 commit 755759c
Show file tree
Hide file tree
Showing 160 changed files with 3,010 additions and 1,239 deletions.
18 changes: 18 additions & 0 deletions CHANGELOG
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,21 @@
0.11.3 2014/xx/xx
- NOTICE: Only 1.2.x and 1.3.x are supported by this version.
Restart viewer AFTER upgrading ES versions
- NOTICE: Requires running npm update for ES 1.3 support
- For NEW installs can now set a prefix= config variable and
db.pl --prefix option that will prefix all ES tables. This makes
it easier for Moloch to share ES with other services OR multiple
Moloch clusters can use 1 ES cluster with different prefixes.
- New usersElasticsearch= and usersPrefix= config that make it possible
for multiple Moloch clusters to share a single users table.
- viewer: removal of pre 1.2 ES things
- Some cron efficiency improvements
- Check more often if files need to be expired
- More SMB1 parsing
- More TLS cipherso
- Major viewer test suite restructure and improvements


0.11.2 2014/10/16
- NOTICE: ES 1.1.x, 1.2.x, 1.3.x are supported by this version.
ES 0.90.12 still works but will no longer be supported.
Expand Down
13 changes: 13 additions & 0 deletions capture/config.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -273,6 +273,18 @@ void moloch_config_load()
}
}

config.prefix = moloch_config_str(keyfile, "prefix", "");
int len = strlen(config.prefix);
if (len > 0 && config.prefix[len - 1] != '_') {
char *tmp = malloc(len + 2);
memcpy(tmp, config.prefix, len);
tmp[len] = '_';
tmp[len+1] = 0;
g_free(config.prefix);
config.prefix = tmp;
}


config.elasticsearch = moloch_config_str(keyfile, "elasticsearch", "localhost:9200");
config.interface = moloch_config_str(keyfile, "interface", NULL);
config.pcapDir = moloch_config_str_list(keyfile, "pcapDir", NULL);
Expand Down Expand Up @@ -502,6 +514,7 @@ void moloch_config_init()
if (config.debug) {
LOG("nodeClass: %s", config.nodeClass);
LOG("elasticsearch: %s", config.elasticsearch);
LOG("prefix: %s", config.prefix);
LOG("interface: %s", config.interface);
if (config.pcapDir) {
str = g_strjoinv(";", config.pcapDir);
Expand Down
55 changes: 23 additions & 32 deletions capture/db.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -300,7 +300,7 @@ void moloch_db_save_session(MolochSession_t *session, int final)
(session->lastPacket.tv_usec - session->firstPacket.tv_usec)/1000;

startPtr = BSB_WORK_PTR(jbsb);
BSB_EXPORT_sprintf(jbsb, "{\"index\": {\"_index\": \"sessions-%s\", \"_type\": \"session\", \"_id\": \"%s\"}}\n", prefix, id);
BSB_EXPORT_sprintf(jbsb, "{\"index\": {\"_index\": \"%ssessions-%s\", \"_type\": \"session\", \"_id\": \"%s\"}}\n", config.prefix, prefix, id);

dataPtr = BSB_WORK_PTR(jbsb);
BSB_EXPORT_sprintf(jbsb,
Expand Down Expand Up @@ -886,7 +886,7 @@ void moloch_db_load_stats()
uint32_t source_len;
unsigned char *source = 0;

stats_key_len = snprintf(stats_key, sizeof(stats_key), "/stats/stat/%s", config.nodeName);
stats_key_len = snprintf(stats_key, sizeof(stats_key), "/%sstats/stat/%s", config.prefix, config.nodeName);

unsigned char *data = moloch_http_get(esServer, stats_key, stats_key_len, &data_len);

Expand Down Expand Up @@ -993,7 +993,7 @@ void moloch_db_update_dstats(int n)

gettimeofday(&currentTime, NULL);

key_len = snprintf(key, sizeof(key), "/dstats/dstat/%s-%d-%d", config.nodeName, (int)(currentTime.tv_sec/intervals[n])%1440, intervals[n]);
key_len = snprintf(key, sizeof(key), "/%sdstats/dstat/%s-%d-%d", config.prefix, config.nodeName, (int)(currentTime.tv_sec/intervals[n])%1440, intervals[n]);
if (lastPackets[n] == 0) {
lastTime[n] = startTime;
}
Expand Down Expand Up @@ -1115,7 +1115,7 @@ void moloch_db_get_sequence_number(char *name, MolochSeqNum_cb func, gpointer uw
r->func = func;
r->uw = uw;

key_len = snprintf(key, sizeof(key), "/sequence/sequence/%s", name);
key_len = snprintf(key, sizeof(key), "/%ssequence/sequence/%s", config.prefix, name);
int json_len = snprintf(json, MOLOCH_HTTP_BUFFER_SIZE, "{}");
moloch_http_set(esServer, key, key_len, json, json_len, moloch_db_get_sequence_number_cb, r);
}
Expand All @@ -1129,7 +1129,7 @@ uint32_t moloch_db_get_sequence_number_sync(char *name)
unsigned char *version;
uint32_t version_len;

key_len = snprintf(key, sizeof(key), "/sequence/sequence/%s", name);
key_len = snprintf(key, sizeof(key), "/%ssequence/sequence/%s", config.prefix, name);

data = moloch_http_send_sync(esServer, "POST", key, key_len, "{}", 2, &data_len);
version = moloch_js0n_get(data, data_len, "_version", &version_len);
Expand Down Expand Up @@ -1158,30 +1158,21 @@ void moloch_db_load_file_num()
unsigned char *value;
uint32_t source_len;
unsigned char *source = 0;
uint32_t exists_len;
unsigned char *exists = 0;
uint32_t found_len;
unsigned char *found = 0;

/* First see if we have the new style number or not */
key_len = snprintf(key, sizeof(key), "/sequence/sequence/fn-%s", config.nodeName);
key_len = snprintf(key, sizeof(key), "/%ssequence/sequence/fn-%s", config.prefix, config.nodeName);
data = moloch_http_get(esServer, key, key_len, &data_len);

// 0.90
exists = moloch_js0n_get(data, data_len, "exists", &exists_len);
if (exists && memcmp("true", exists, 4) == 0) {
found = moloch_js0n_get(data, data_len, "found", &found_len);
if (found && memcmp("true", found, 4) == 0) {
goto fetch_file_num;
return;
}

// 1.1.1
exists = moloch_js0n_get(data, data_len, "found", &exists_len);
if (exists && memcmp("true", exists, 4) == 0) {
goto fetch_file_num;
return;
}


/* Don't have new style numbers, go create them */
key_len = snprintf(key, sizeof(key), "/files/file/_search?size=1&sort=num:desc&q=node:%s", config.nodeName);
key_len = snprintf(key, sizeof(key), "/%sfiles/file/_search?size=1&sort=num:desc&q=node:%s", config.prefix, config.nodeName);

data = moloch_http_get(esServer, key, key_len, &data_len);

Expand Down Expand Up @@ -1212,7 +1203,7 @@ void moloch_db_load_file_num()
}

/* Now create the new style */
key_len = snprintf(key, sizeof(key), "/sequence/sequence/fn-%s?version_type=external&version=%d", config.nodeName, fileNum + 100);
key_len = snprintf(key, sizeof(key), "/%ssequence/sequence/fn-%s?version_type=external&version=%d", config.prefix, config.nodeName, fileNum + 100);
moloch_http_send_sync(esServer, "POST", key, key_len, "{}", 2, NULL);

fetch_file_num:
Expand Down Expand Up @@ -1248,7 +1239,7 @@ char *moloch_db_create_file(time_t firstPacket, char *name, uint64_t size, uint3

if (name) {
json_len = snprintf(json, MOLOCH_HTTP_BUFFER_SIZE, "{\"num\":%d, \"name\":\"%s\", \"first\":%" PRIu64 ", \"node\":\"%s\", \"filesize\":%" PRIu64 ", \"locked\":1}", num, name, fp, config.nodeName, size);
key_len = snprintf(key, sizeof(key), "/files/file/%s-%d?refresh=true", config.nodeName,num);
key_len = snprintf(key, sizeof(key), "/%sfiles/file/%s-%d?refresh=true", config.prefix, config.nodeName,num);
} else {
tmp = localtime(&firstPacket);

Expand All @@ -1261,7 +1252,7 @@ char *moloch_db_create_file(time_t firstPacket, char *name, uint64_t size, uint3
snprintf(filename+strlen(filename), sizeof(filename) - strlen(filename), "%s-%02d%02d%02d-%08d.pcap", config.nodeName, tmp->tm_year%100, tmp->tm_mon+1, tmp->tm_mday, num);

json_len = snprintf(json, MOLOCH_HTTP_BUFFER_SIZE, "{\"num\":%d, \"name\":\"%s\", \"first\":%" PRIu64 ", \"node\":\"%s\", \"locked\":0}", num, filename, fp, config.nodeName);
key_len = snprintf(key, sizeof(key), "/files/file/%s-%d?refresh=true", config.nodeName,num);
key_len = snprintf(key, sizeof(key), "/%sfiles/file/%s-%d?refresh=true", config.prefix, config.nodeName,num);
}

moloch_http_set(esServer, key, key_len, json, json_len, NULL, NULL);
Expand All @@ -1284,7 +1275,7 @@ void moloch_db_check()
int key_len;
unsigned char *data;

key_len = snprintf(key, sizeof(key), "/dstats/version/version/_source");
key_len = snprintf(key, sizeof(key), "/%sdstats/version/version/_source", config.prefix);
data = moloch_http_get(esServer, key, key_len, &data_len);

if (!data || data_len == 0) {
Expand Down Expand Up @@ -1319,7 +1310,7 @@ void moloch_db_load_tags()
char key[100];
int key_len;

key_len = snprintf(key, sizeof(key), "/tags/tag/_search?size=3000&fields=n");
key_len = snprintf(key, sizeof(key), "/%stags/tag/_search?size=3000&fields=n", config.prefix);
unsigned char *data = moloch_http_get(esServer, key, key_len, &data_len);

if (!data) {
Expand Down Expand Up @@ -1420,7 +1411,7 @@ void moloch_db_free_tag_request(MolochTagRequest_t *r)
continue;
}

key_len = snprintf(key, sizeof(key), "/tags/tag/%s?fields=n", r->escaped);
key_len = snprintf(key, sizeof(key), "/%stags/tag/%s?fields=n", config.prefix, r->escaped);
moloch_http_send(esServer, "GET", key, key_len, NULL, 0, FALSE, moloch_db_tag_cb, r);
outstandingTagRequests++;
break;
Expand All @@ -1435,7 +1426,7 @@ void moloch_db_tag_create_cb(unsigned char *data, int UNUSED(data_len), gpointer
int key_len;

if (strstr((char *)data, "{\"error\":") != 0) {
key_len = snprintf(key, sizeof(key), "/tags/tag/%s?fields=n", r->escaped);
key_len = snprintf(key, sizeof(key), "/%stags/tag/%s?fields=n", config.prefix, r->escaped);
moloch_http_send(esServer, "GET", key, key_len, NULL, 0, FALSE, moloch_db_tag_cb, r);
return;
}
Expand All @@ -1459,7 +1450,7 @@ void moloch_db_tag_seq_cb(uint32_t newSeq, gpointer uw)

r->newSeq = newSeq;

key_len = snprintf(key, sizeof(key), "/tags/tag/%s?op_type=create", r->escaped);
key_len = snprintf(key, sizeof(key), "/%stags/tag/%s?op_type=create", config.prefix, r->escaped);
int json_len = snprintf(json, MOLOCH_HTTP_BUFFER_SIZE, "{\"n\":%u}", newSeq);

moloch_http_set(esServer, key, key_len, json, json_len, moloch_db_tag_create_cb, r);
Expand Down Expand Up @@ -1548,7 +1539,7 @@ void moloch_db_get_tag(void *uw, int tagtype, const char *tagname, MolochTag_cb
char key[500];
int key_len;

key_len = snprintf(key, sizeof(key), "/tags/tag/%s?fields=n", r->escaped);
key_len = snprintf(key, sizeof(key), "/%stags/tag/%s?fields=n", config.prefix, r->escaped);
moloch_http_send(esServer, "GET", key, key_len, NULL, 0, FALSE, moloch_db_tag_cb, r);
outstandingTagRequests++;
} else {
Expand Down Expand Up @@ -1610,7 +1601,7 @@ void moloch_db_load_fields()
char key[100];
int key_len;

key_len = snprintf(key, sizeof(key), "/fields/field/_search?size=3000");
key_len = snprintf(key, sizeof(key), "/%sfields/field/_search?size=3000", config.prefix);
unsigned char *data = moloch_http_get(esServer, key, key_len, &data_len);

if (!data) {
Expand Down Expand Up @@ -1665,7 +1656,7 @@ void moloch_db_add_field(char *group, char *kind, char *expression, char *friend

BSB_INIT(bsb, json, 10000);

key_len = snprintf(key, sizeof(key), "/fields/field/%s", expression);
key_len = snprintf(key, sizeof(key), "/%sfields/field/%s", config.prefix, expression);

BSB_EXPORT_sprintf(bsb, "{\"friendlyName\": \"%s\", \"group\": \"%s\", \"help\": \"%s\", \"dbField\": \"%s\", \"type\": \"%s\"",
friendlyName,
Expand Down Expand Up @@ -1702,7 +1693,7 @@ gboolean moloch_db_file_exists(char *filename)
char key[2000];
int key_len;

key_len = snprintf(key, sizeof(key), "/files/file/_search?size=1&sort=num:desc&q=node:%s+AND+name:\"%s\"", config.nodeName, filename);
key_len = snprintf(key, sizeof(key), "/%sfiles/file/_search?size=1&sort=num:desc&q=node:%s+AND+name:\"%s\"", config.prefix, config.nodeName, filename);

LOG("query: %s", key);
unsigned char *data = moloch_http_get(esServer, key, key_len, &data_len);
Expand Down
1 change: 1 addition & 0 deletions capture/moloch.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -231,6 +231,7 @@ typedef struct moloch_config {

int numPlugins;

char *prefix;
char *nodeClass;
char *elasticsearch;
char *interface;
Expand Down
Loading

0 comments on commit 755759c

Please sign in to comment.