Skip to content

Commit

Permalink
Added vlan and mac address (issue arkime#284)
Browse files Browse the repository at this point in the history
  • Loading branch information
@awick
awick committed Sep 3, 2014
1 parent 183825f commit 96f174a
Show file tree
Hide file tree
Showing 57 changed files with 830 additions and 151 deletions.
1 change: 1 addition & 0 deletions CHANGELOG
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
- Allow multiple -r and -R options
- Fixed update vs upgrade message (issue #287)
- Fixed expression errors not displayed on connections tab (issue #288)
- Added vlan and mac.src, mac.dst, mac indexing/expressions (issue #284)

0.11.1 2014/08/07
- NOTICE: ES 0.90.12+, 1.1.x, 1.2.0 are supported by this version.
Expand Down
69 changes: 68 additions & 1 deletion capture/nids.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ static MolochSessionHead_t udpSessionQ;
static MolochSessionHead_t icmpSessionQ;
static MolochSessionHead_t tcpWriteQ;

static MolochIntHead_t freeOutputBufs;
static MolochIntHead_t freeOutputBufs;
static pthread_mutex_t freeOutputMutex = PTHREAD_MUTEX_INITIALIZER;

typedef struct moloch_output {
Expand Down Expand Up @@ -88,6 +88,9 @@ static char offlinePcapFilename[PATH_MAX+1];

static int tagsField;
static int protocolField;
static int mac1Field;
static int mac2Field;
static int vlanField;

uint64_t totalPackets = 0;
uint64_t totalBytes = 0;
Expand Down Expand Up @@ -730,6 +733,43 @@ void moloch_nids_cb_ip(struct ip *packet, int len)
break;
}

/* Handle MACs and vlans on first few packets in each direction */
if (pcapFileHeader.linktype == 1 && session->packets[session->which] <= 1) {
char str1[20];
char str2[20];
snprintf(str1, sizeof(str1), "%02x:%02x:%02x:%02x:%02x:%02x",
nids_last_pcap_data[0],
nids_last_pcap_data[1],
nids_last_pcap_data[2],
nids_last_pcap_data[3],
nids_last_pcap_data[4],
nids_last_pcap_data[5]);


snprintf(str2, sizeof(str2), "%02x:%02x:%02x:%02x:%02x:%02x",
nids_last_pcap_data[6],
nids_last_pcap_data[7],
nids_last_pcap_data[8],
nids_last_pcap_data[9],
nids_last_pcap_data[10],
nids_last_pcap_data[11]);

if (session->which == 1) {
moloch_field_string_add(mac1Field, session, str1, 17, TRUE);
moloch_field_string_add(mac2Field, session, str2, 17, TRUE);
} else {
moloch_field_string_add(mac1Field, session, str2, 17, TRUE);
moloch_field_string_add(mac2Field, session, str1, 17, TRUE);
}

int n = 12;
while (nids_last_pcap_data[n] == 0x81 && nids_last_pcap_data[n+1] == 0x00) {
uint16_t vlan = ((uint16_t)(nids_last_pcap_data[n+2] << 8 | nids_last_pcap_data[n+3])) & 0xfff;
moloch_field_int_add(vlanField, session, vlan);
n += 4;
}
}

session->bytes[session->which] += nids_last_pcap_header->caplen;
session->lastPacket = nids_last_pcap_header->ts;

Expand Down Expand Up @@ -1455,6 +1495,8 @@ void moloch_nids_root_init()
pcapFileHeader.snaplen = pcap_snapshot(nids_params.pcap_desc);
pcapFileHeader.sigfigs = 0;
pcapFileHeader.linktype = dlt_to_linktype(pcap_datalink(nids_params.pcap_desc)) | pcap_datalink_ext(nids_params.pcap_desc);
if (config.debug)
LOG("linktype %x", pcapFileHeader.linktype);

config.maxWriteBuffers = config.pcapReadOffline?10:2000;
}
Expand Down Expand Up @@ -1525,6 +1567,31 @@ void moloch_nids_init()
MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_COUNT | MOLOCH_FIELD_FLAG_LINKED_SESSIONS,
NULL);

mac1Field = moloch_field_define("general", "lotermfield",
"mac.src", "Src MAC", "mac1-term",
"Source ethernet mac addresses set for session",
MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_COUNT | MOLOCH_FIELD_FLAG_LINKED_SESSIONS,
NULL);

mac2Field = moloch_field_define("general", "lotermfield",
"mac.dst", "Dst MAC", "mac2-term",
"Destination ethernet mac addresses set for session",
MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_COUNT | MOLOCH_FIELD_FLAG_LINKED_SESSIONS,
NULL);

moloch_field_define("general", "lotermfield",
"mac", "Src or Dst MAC", "macall",
"Shorthand for mac.src or mac.dst",
0, MOLOCH_FIELD_FLAG_FAKE,
"regex", "^mac\\\\.(?:(?!\\\\.cnt$).)*$",
NULL);

vlanField = moloch_field_define("general", "integer",
"vlan", "VLan", "vlan",
"vlan value",
MOLOCH_FIELD_TYPE_INT_HASH, MOLOCH_FIELD_FLAG_COUNT | MOLOCH_FIELD_FLAG_LINKED_SESSIONS,
NULL);

tagsField = moloch_field_by_db("ta");
moloch_db_get_tag(NULL, tagsField, "tcp", NULL);
moloch_db_get_tag(NULL, tagsField, "udp", NULL);
Expand Down
34 changes: 21 additions & 13 deletions tests/bigendian.test
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,11 @@
"body" : {
"db2" : 0,
"db" : 0,
"lpd" : 1335958317529,
"fp" : 1335958313,
"mac2-term" : [
"00:00:5e:00:01:b1"
],
"no" : "test",
"lp" : 1335958317,
"pa" : 2,
"tacnt" : 2,
"sl" : 4376,
"a2" : "10.64.11.49",
"ss" : 1,
"ta" : [
Expand All @@ -21,23 +19,33 @@
"fpd" : 1335958313152,
"fs" : [],
"by2" : 0,
"a1" : "192.168.177.160",
"db1" : 0,
"pa2" : 0,
"p1" : 0,
"by1" : 196,
"by" : 196,
"p2" : 0,
"rir1" : "ARIN",
"prot-term" : [
"icmp"
],
"pr" : 1,
"ps" : [
24,
138
],
"prot-term-cnt" : 1
"prot-term-cnt" : 1,
"lpd" : 1335958317529,
"fp" : 1335958313,
"pa" : 2,
"sl" : 4376,
"tacnt" : 2,
"a1" : "192.168.177.160",
"db1" : 0,
"by1" : 196,
"mac2-term-cnt" : 1,
"p2" : 0,
"mac1-term-cnt" : 1,
"prot-term" : [
"icmp"
],
"mac1-term" : [
"00:21:28:05:29:ba"
]
},
"header" : {
"index" : {
Expand Down
9 changes: 9 additions & 0 deletions tests/bt-tcp.test
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,10 @@
},
"db2" : 0,
"db" : 68,
"mac2-term" : [
"00:00:5e:00:01:02",
"00:1d:b5:ce:ef:c0"
],
"no" : "test",
"lp" : 1387744084,
"a2" : "10.0.0.2",
Expand Down Expand Up @@ -62,12 +66,17 @@
"tacnt" : 5,
"a1" : "10.0.0.1",
"db1" : 68,
"mac2-term-cnt" : 2,
"by1" : 248,
"p2" : 26001,
"mac1-term-cnt" : 1,
"rir2" : "TEST",
"prot-term" : [
"tcp",
"bittorrent"
],
"mac1-term" : [
"00:0f:f7:76:82:80"
]
},
"header" : {
Expand Down
82 changes: 53 additions & 29 deletions tests/bt-udp.test
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@
"body" : {
"db2" : 0,
"db" : 137,
"mac2-term" : [
"00:10:db:ff:26:00"
],
"no" : "test",
"lp" : 1387253713,
"a2" : "10.0.0.1",
Expand Down Expand Up @@ -40,11 +43,16 @@
"tacnt" : 5,
"a1" : "10.0.0.2",
"db1" : 137,
"mac2-term-cnt" : 1,
"by1" : 145,
"p2" : 3207,
"mac1-term-cnt" : 1,
"prot-term" : [
"udp",
"bittorrent"
],
"mac1-term" : [
"78:fe:3d:11:21:f2"
]
},
"header" : {
Expand All @@ -58,15 +66,11 @@
"body" : {
"db2" : 0,
"db" : 321,
"lpd" : 1387253793904,
"fp" : 1387253793,
"mac2-term" : [
"00:00:5e:00:01:03"
],
"no" : "test",
"as2" : "AS0002 Hmm!@#$%^&*()",
"lp" : 1387253793,
"pa" : 1,
"tacnt" : 3,
"fb1" : "64313a7264323a69",
"sl" : 0,
"a2" : "10.0.0.3",
"ss" : 1,
"ta" : [
Expand All @@ -78,22 +82,34 @@
"fpd" : 1387253793904,
"fs" : [],
"by2" : 0,
"a1" : "10.0.0.4",
"db1" : 321,
"pa2" : 0,
"p1" : 44102,
"by1" : 329,
"by" : 329,
"pr" : 17,
"ps" : [
185
],
"prot-term-cnt" : 2,
"lpd" : 1387253793904,
"fp" : 1387253793,
"as2" : "AS0002 Hmm!@#$%^&*()",
"pa" : 1,
"sl" : 0,
"fb1" : "64313a7264323a69",
"tacnt" : 3,
"a1" : "10.0.0.4",
"db1" : 321,
"by1" : 329,
"mac2-term-cnt" : 1,
"p2" : 12074,
"mac1-term-cnt" : 1,
"prot-term" : [
"udp",
"bittorrent"
],
"pr" : 17,
"ps" : [
185
],
"prot-term-cnt" : 2
"mac1-term" : [
"00:10:db:ff:26:00"
]
},
"header" : {
"index" : {
Expand All @@ -106,14 +122,11 @@
"body" : {
"db2" : 0,
"db" : 328,
"lpd" : 1387257610963,
"fp" : 1387257610,
"mac2-term" : [
"00:00:5e:00:01:03"
],
"no" : "test",
"lp" : 1387257610,
"pa" : 1,
"tacnt" : 3,
"fb1" : "64313a71393a6669",
"sl" : 0,
"a2" : "10.0.0.5",
"ss" : 1,
"ta" : [
Expand All @@ -125,22 +138,33 @@
"fpd" : 1387257610963,
"fs" : [],
"by2" : 0,
"a1" : "10.0.0.6",
"db1" : 328,
"pa2" : 0,
"p1" : 47061,
"by1" : 336,
"by" : 336,
"pr" : 17,
"ps" : [
530
],
"prot-term-cnt" : 2,
"lpd" : 1387257610963,
"fp" : 1387257610,
"pa" : 1,
"sl" : 0,
"fb1" : "64313a71393a6669",
"tacnt" : 3,
"a1" : "10.0.0.6",
"db1" : 328,
"by1" : 336,
"mac2-term-cnt" : 1,
"p2" : 20551,
"mac1-term-cnt" : 1,
"prot-term" : [
"udp",
"bittorrent"
],
"pr" : 17,
"ps" : [
530
],
"prot-term-cnt" : 2
"mac1-term" : [
"00:10:db:ff:26:00"
]
},
"header" : {
"index" : {
Expand Down
12 changes: 12 additions & 0 deletions tests/dns-dnskey.test
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,10 @@
},
"db2" : 0,
"db" : 66,
"vlan-cnt" : 1,
"mac2-term" : [
"00:19:e2:ba:2f:c1"
],
"no" : "test",
"lp" : 1393428477,
"a2" : "8.8.8.8",
Expand Down Expand Up @@ -52,12 +56,20 @@
"a1" : "10.0.0.1",
"db1" : 66,
"dnshocnt" : 1,
"mac2-term-cnt" : 1,
"by1" : 74,
"vlan" : [
500
],
"p2" : 53,
"mac1-term-cnt" : 1,
"rir2" : "ARIN",
"prot-term" : [
"udp",
"dns"
],
"mac1-term" : [
"00:1a:e3:dc:2e:c0"
]
},
"header" : {
Expand Down
Loading

0 comments on commit 96f174a

Please sign in to comment.