From 3b0b0e87d062636b2189c359cb89f90b4b2b2de2 Mon Sep 17 00:00:00 2001 From: Andy Wick Date: Fri, 30 Mar 2018 09:11:01 -0400 Subject: [PATCH] 1.0.0 (#829) * wip * wip * wip * wip * wip * wip * progress * all tests pass * fix tests * wise renames, textfield remove * support ipv6 in most places * remove logs * dns v6 * dns v6 tests * ES 6, move version to _meta * tcp zero window counts * wip * wip * updated build * added -u * Update tests for geo changes * error if field can't be found * fix v6 tests and tls parsers being loaded before http * try again for v6 fixes * parsers loaded alphabetically now * fix v6 dns tests * v6 fixes * fix version number * error on -terms, change single ip datatype * use Socket6 for 6 stuff * Added Socket6 install * switch to libmaxminddb and 2 letter countries * fix geo * need to do apt-get update * debug * fix indentation * fix grep * see if we can do docker ES * right docker package * docker docker docker * give up again * node 8, test fix * fix tests * fix test * minor field renames and cleanup * put version in * use npm install * print correct index * added deleteExisting option * Set new geo files and defaults * alpha2 * switch to mmdb_aget_value * correct path * fix info column for new names * change to sessions2 * add synced flush * allow either ntp port to be 123 * save headers set where we have data for the header * add include/exclude to es stats tab * fix hover menu * calculate sha256 for http/email files * support sha256 wise lookups * sha256 test file * support ipv6 input of .port * /user/list uses promises now * /history/list uses promises * /file/list uses promises and new api-files tests * first file might not be aero * update for new geo files * Elyse said move catch to end, seems to work better! * fix number of US ips now * only send up 100MB at a time * remove ubuntu 18 builds for now * switch scroll size to 1000 by default, sample reindex2.local.json * remove ubuntu18 from slack * use zcat so .gz stays and old file overwritten * stats every 5 seconds, deleteOnDone, percent in stats * spelling and logic error * change to 10 seconds * support disabled fields and update tests * url is hash now * make url hash * Add disabled if missing * sah256 now controlled in config by supportSha256, default off * spigraph.json uses promises * /sessions.json /stats.json use promises, minor lint cleanup * more parallel cleanup * try high ram/cpu * no HIGH for now :( * ip rules improvements src/dst ip/port can be used to trigger rules now ip fields in rules can now be CIDR * moved int outside for * Need to remove old file before creating new otherwise running process will crash * some cleanup * separate v4/v6 patricia trees * more separate v4/v6 patricia trees * simple writer now flushes after 10 seconds (issue #777) simple writer now flushes after 10 seconds of no writting there still can be pagesize bytes unwritten (issue #777) pagesize is usually 512 or 4096 on most systems * Added GHASH support * Add OUI support, sort .test files so future updates will be smaller * some oui were swapped, show oui in open session, tests * Clean up ES version, make reindex2 exe * need to set tv_nsec * wrong wget order for args * rename thread * beta1 * decode some dhcp * fix viewer display of dhcp * dhcp id,mac,out fields; cleanup mac/oui adding * update tls-cipher.h, tag sessions we thing have cert:self-signed * lower dhcp host, only tag if single self signed (not perfect) * fix tests, use moloch_field_string_add_lower everywhere * fix valve spelling * batch cleanup * reload geo and oui files without restarting (issue #692) * use standard stat field names * when reloading oui/geo make sure file size isn't changing first * track session size we send to ES * fix exporting with ports, add ip:port tests, add v6 tests, fixes #811 * clean up file monitoring more, rir can be reloaded now * remove yara 1.x support, reload yara if using >= 3.4 * support a trailing . or : for ips with no port and just ignore * pug 2.0.0 * clean up unique.txt since doesn't need to be async anymore * cleanup ip seperator in unique.txt * search across all indices when not doing last bounding es is supposedly faster at this now and this will give accurate results. previously wrapped sessions might not show up. * cron jobs now use the timestamp time and not last packet time cron jobs now use the timestamp time and not last packet time when choosing sessions to look at this means delay is shorter, although when upgrading to 1.0 some sessions will be not looked at. * beta2 * more pug version issues :( * Added CVE-2018-6794 test * static => LOCAL, fix some multithread stats * -term dropped from field names in 1.0.0, updated quic.version field definition (#817) * test quic and fix for last commit * fix ip in esstats.json * some cleanups * fixed function mismatch * change old column names to new names for user * Vue - Stats Page (#816) * vue app scaffold node stats page implemented * add static folder * remove jquery from stats page * more stats page navbars fixed to top kill cubism context when accordion is closed round decimal numbers pass in url query parameters update url query parameters display graphs in theme colors cleanup stats when component is destroyed * use title config to construct app title * set and cancel request interval for stats * set initialized flag * Vue - Stats page fix updating url param clearing other url params update url param on stats tab change use url query param to update stats tab hide/show stats navbar inputs based on tab open update stats view when url params change (from browser forward/back buttons) * pagination component reusable pagination component add paging to stats page use query params for stats.json req * add response interceptor to determine if server is down * add reusable error component display error on stats page * reusable loading component add loading overlay to stats page * don't show loading if there's an error on the stats page * add footer component * Vue - stats page search node tab sort node stats table * Vue stats page watch for the user to leave/return to stop/start the cubism context * Vue - shards stats sort shards on server added bootstrap overrides css apply theme colors to paging styles * Vue - shards stats include/exclude columns set moloch cookie fix dropdown styles fix tab styles fix header styles for long header text * Vue - shards stats es shards tab column menu drops down on the right side es shards tab watches for data interval * add loading indicator to shards stats * Vue - ES Stats * Vue - ES Indices * Vue - ES Indices and ES Tasks * Vue - stats cleanup fix refresh data interval * Vue - stats admin can delete index * delete index api * Vue - stats restrict who can cancel tasks and delete indices to admin checkbox for admin to display only cancellable tasks * fix stats tests * Vue stats page integration with angular app remove /vueapp and use /stats redirect from angular app to vue app redirect from vue app to angular app add help link from stats page * put back angular 404 page * update changelog * make rebuild * fix some install stuff for vue * vue app fixes use .send not .end to send vue app fix estask/list endpoint sorting fix fontawesome reference fix typo in es stats template fix loading on shards page * vue - stats fixes fix value formats use old bootstrap fonts * vue - make stats prettier es tasks tooltip for checkbox striped table rows fixed position of es health in navbar hoverable columns and rows on shards table max height and tooltip for shards column heaers * Vue - more stats prettifying left align text in tables put the es stats item dropdown on the right don't stripe average and total rows * make navbars the same for vue and angular apps * es tasks - set correct colspan on no results row * beta3 * update viewer readme and npm scripts * add contributing file * some additions to CONTRIBUTING * lint all js files fix jshint errors * add sorting and searching to es shards fixes #814 * fix typo in shard column on es indices stat tab - fixes #821 * fix test * Toppackagejson (#819) Top level package.json and node_modules to reduce duplication and maybe speed up builds. * allow shards column headers to be more than 2 lines * rc1 * release changes * initialPacket needs to be global for now * change slices default and max * spiDataMaxIndices 4, work around ES 6 issue * initial parliament tests * more tests, don't send settings down * add slight pause to wait for wise to start up * tests run with older perl * need to actually start parliament in all cases, duh * fix test for new mmdb --- CHANGELOG | 70 +- CONTRIBUTING.md | 88 + Makefile.am | 7 +- Makefile.in | 30 +- README.rst | 51 +- capture/Makefile.in | 4 +- capture/config.c | 133 +- capture/db.c | 1102 ++-- capture/field.c | 492 +- capture/http.c | 34 +- capture/main.c | 29 +- capture/moloch.h | 107 +- capture/molochmagic.pl | 2 +- capture/packet.c | 205 +- capture/parsers.c | 82 +- capture/parsers/dhcp.c | 173 + capture/parsers/dhcp.detail.jade | 8 + capture/parsers/dns.c | 92 +- capture/parsers/dns.detail.jade | 14 +- capture/parsers/email.detail.jade | 35 +- capture/parsers/http.c | 220 +- capture/parsers/http.detail.jade | 44 +- capture/parsers/irc.c | 14 +- capture/parsers/irc.detail.jade | 6 +- capture/parsers/krb5.c | 34 +- capture/parsers/krb5.detail.jade | 6 +- capture/parsers/ldap.c | 22 +- capture/parsers/ldap.detail.jade | 4 +- capture/parsers/misc.c | 69 +- capture/parsers/mysql.c | 14 +- capture/parsers/mysql.detail.jade | 4 +- capture/parsers/oracle.c | 25 +- capture/parsers/oracle.detail.jade | 6 +- capture/parsers/postgresql.c | 18 +- capture/parsers/postgresql.detail.jade | 6 +- capture/parsers/quic.c | 24 +- capture/parsers/quic.detail.jade | 6 +- capture/parsers/radius.c | 31 +- capture/parsers/radius.detail.jade | 8 +- capture/parsers/smb.c | 48 +- capture/parsers/smb.detail.jade | 16 +- capture/parsers/smtp.c | 183 +- capture/parsers/socks.c | 40 +- capture/parsers/ssh.c | 20 +- capture/parsers/ssh.detail.jade | 6 +- capture/parsers/tds.c | 10 +- capture/parsers/tls-cipher.h | 268 +- capture/parsers/tls.c | 107 +- capture/parsers/tls.detail.jade | 113 +- capture/parsers/tls.detail.pug | 49 - capture/plugins/Makefile.in | 4 +- capture/plugins/daq/reader-daq.c | 6 +- capture/plugins/lua/data.c | 2 +- capture/plugins/lua/httpService.c | 14 +- capture/plugins/lua/molua.c | 2 +- capture/plugins/lua/session.c | 32 +- capture/plugins/netflow.c | 6 +- capture/plugins/pfring/reader-pfring.c | 2 +- capture/plugins/scrubspi.c | 10 +- capture/plugins/snf/reader-snf.c | 4 +- capture/plugins/tagger.c | 114 +- capture/plugins/taggerUpload.pl | 1 - capture/plugins/wise.c | 90 +- capture/plugins/wiseService/package.json | 16 +- capture/plugins/wiseService/simpleSource.js | 3 + .../plugins/wiseService/source.alienvault.js | 8 +- .../wiseService/source.emergingthreats.js | 4 +- capture/plugins/wiseService/source.opendns.js | 12 +- .../wiseService/source.passivetotal.js | 4 +- capture/plugins/wiseService/source.threatq.js | 12 +- .../wiseService/source.threatstream.js | 16 +- .../plugins/wiseService/source.virustotal.js | 8 +- capture/plugins/wiseService/wiseCache.js | 4 +- capture/plugins/wiseService/wiseService.js | 38 +- capture/plugins/wiseService/wiseSource.js | 2 +- capture/plugins/writer-s3.c | 32 +- capture/plugins/writer-s3/package.json | 3 +- capture/reader-libpcap-file.c | 30 +- capture/reader-libpcap.c | 9 +- capture/reader-tpacketv3.c | 4 +- capture/readers.c | 2 +- capture/rules.c | 232 +- capture/session.c | 88 +- capture/thirdparty/patricia.c | 97 +- capture/thirdparty/patricia.h | 7 +- capture/writer-disk.c | 57 +- capture/writer-inplace.c | 16 +- capture/writer-null.c | 12 +- capture/writer-simple.c | 99 +- capture/writers.c | 2 +- capture/yara.c | 196 +- configure | 86 +- configure.ac | 52 +- db/db.pl | 932 +--- easybutton-build.sh | 34 +- package.json | 41 + parliament/.angular-cli.json | 2 +- parliament/package.json | 6 - parliament/parliament.js | 26 +- parliament/src/styles.css | 2 +- release/Configure | 2 +- release/Vagrantfile | 8 +- release/build.yml | 19 +- release/config.ini.sample | 119 +- release/doit.sh | 8 +- release/moloch_update_geo.sh | 14 +- screwdriver.yaml | 92 +- tests/MolochTest.pm | 49 +- tests/README | 3 +- tests/api-connections.t | 56 +- tests/api-files.t | 71 + tests/api-fresh.t | 4 +- tests/api-history.t | 6 +- tests/api-multies.t | 27 +- tests/api-scrub.t | 2 +- tests/api-sessionDetail.t | 2 +- tests/api-sessions.t | 42 +- tests/api-spigraph.t | 225 +- tests/api-spiview.t | 110 +- tests/api-stats.t | 7 +- tests/api-tagging.t | 14 +- tests/api-unique.t | 142 +- tests/api-users.t | 52 +- tests/cert.t | 12 +- tests/config.test.ini | 18 +- tests/dhcp.t | 25 + tests/dns.t | 12 +- tests/email.t | 17 +- tests/email.tagger2.json | 6 +- tests/email.wise | 4 +- tests/general.t | 84 +- tests/http.t | 52 +- tests/irc.t | 2 +- tests/mysql.t | 2 +- tests/parliament.t | 112 + tests/pcap/CVE-2018-6794.pcap | Bin 0 -> 7413 bytes tests/pcap/CVE-2018-6794.test | 426 ++ tests/pcap/aerospike.test | 307 +- tests/pcap/bigendian.test | 104 +- tests/pcap/bt-tcp.test | 301 +- tests/pcap/bt-udp.test | 274 +- tests/pcap/cassandra1.test | 125 +- tests/pcap/dns-dnskey.test | 179 +- tests/pcap/dns-error.test | 187 +- tests/pcap/dns-flags0000.test | 161 +- tests/pcap/dns-flags0110.test | 189 +- tests/pcap/dns-mx.test | 141 +- tests/pcap/dns-notify.test | 145 +- tests/pcap/dns-tcp.test | 494 +- tests/pcap/dns-udp.test | 424 +- tests/pcap/dns-update.test | 196 +- tests/pcap/dns-wiresharkrepo.test | 1224 ++--- tests/pcap/fbzero-android.test | 214 +- tests/pcap/ftp.test | 140 +- tests/pcap/gre-sample.test | 924 ++-- tests/pcap/http-301-get.test | 229 +- tests/pcap/http-500-head.test | 209 +- tests/pcap/http-basicauth.test | 299 +- tests/pcap/http-content-gzip.test | 331 +- tests/pcap/http-content-zip.test | 295 +- tests/pcap/http-digestauth.test | 249 +- tests/pcap/http-no-length.test | 336 +- tests/pcap/http-simple-get.test | 238 +- tests/pcap/http-syn-ack.test | 258 +- tests/pcap/http-wrapped-header.test | 444 +- tests/pcap/http-xff.test | 333 +- tests/pcap/https-generalizedtime.test | 265 +- tests/pcap/https2-301-get.test | 241 +- tests/pcap/https3-301-get.test | 267 +- tests/pcap/imap-tag.test | 155 +- tests/pcap/ip-boundaries.test | 86 +- tests/pcap/irc-cap-req.test | 178 +- tests/pcap/irc.test | 147 +- tests/pcap/kafka.test | 120 +- tests/pcap/krb5-tcp.test | 134 +- tests/pcap/krb5-udp.test | 110 +- tests/pcap/ldap-and-search.test | 148 +- tests/pcap/ldap-simpleauth.test | 196 +- tests/pcap/ldap-ssl.test | 234 +- tests/pcap/long-session.test | 340 +- tests/pcap/mongo.test | 612 ++- tests/pcap/mpls-basic.test | 480 +- tests/pcap/mysql-allow.test | 140 +- tests/pcap/mysql-deny.test | 122 +- tests/pcap/mysql-tls.test | 172 +- tests/pcap/nflog.test | 236 +- tests/pcap/no-syn-ack.test | 371 +- tests/pcap/openssl-ssl3.test | 316 +- tests/pcap/openssl-tls1-tls1_2.test | 340 +- tests/pcap/openssl-tls1.test | 294 +- tests/pcap/openssl-tls1_1.test | 274 +- tests/pcap/openssl-tls1_2-tls1.test | 320 +- tests/pcap/openssl-tls1_2.test | 310 +- tests/pcap/oracle.test | 120 +- tests/pcap/pop3-tag.test | 150 +- tests/pcap/postgres-badpass.test | 197 +- tests/pcap/postgres-good.test | 158 +- tests/pcap/postgres-no-sslrequest.test | 124 +- tests/pcap/pppoe.test | 240 +- tests/pcap/quic24-wireshark.test | 131 +- tests/pcap/quic33-wireshark.test | 128 +- tests/pcap/quic34.test | 126 +- tests/pcap/radius.test | 198 +- tests/pcap/smb-port80.test | 145 +- tests/pcap/smb-smb1-ascii.test | 212 +- tests/pcap/smb-smbclient.test | 222 +- .../pcap/smbtorture-ntlmssp-moloch-crash.test | 159 +- tests/pcap/smbtorture-ntlmssp.test | 164 +- tests/pcap/smtp-data-250.test | 246 +- tests/pcap/smtp-data-521.test | 302 +- tests/pcap/smtp-moloch-bof.test | 197 +- tests/pcap/smtp-nospaces.test | 162 +- tests/pcap/smtp-originating.test | 311 +- tests/pcap/smtp-rcpt-553.test | 174 +- tests/pcap/smtp-starttls.test | 397 +- tests/pcap/smtp-subject-8859-b.test | 326 +- tests/pcap/smtp-subject-8859-multi.test | 250 +- tests/pcap/smtp-subject-8859-q.test | 248 +- tests/pcap/smtp-subject-encoded-empty.test | 232 +- tests/pcap/smtp-subject-gb2312-b.test | 334 +- tests/pcap/smtp-subject-multi-nospace.test | 213 +- tests/pcap/smtp-subject-utf8-mixed.test | 252 +- tests/pcap/smtp-subject-utf8-q.test | 301 +- tests/pcap/smtp-subject-windows.test | 239 +- tests/pcap/smtp-zip.test | 330 +- tests/pcap/socks-http-example.test | 730 +-- tests/pcap/socks-http-pass.test | 508 +- tests/pcap/socks-https-example.test | 894 +-- tests/pcap/socks4-https.test | 307 +- tests/pcap/socks5-http-302-frag.test | 305 +- tests/pcap/socks5-http-302.test | 305 +- tests/pcap/socks5-rdp.test | 143 +- tests/pcap/socks5-reverse.test | 453 +- tests/pcap/socks5-smtp-503.test | 160 +- tests/pcap/ssh2-moloch-crash.test | 182 +- tests/pcap/ssh2.test | 180 +- tests/pcap/ssl-selfsign.pcap | Bin 0 -> 8191 bytes tests/pcap/ssl-selfsign.test | 137 + tests/pcap/stun.test | 614 +-- tests/pcap/tds5.test | 150 +- tests/pcap/thrift.test | 290 +- tests/pcap/tls-client-sessionid.test | 170 +- tests/pcap/tls13.test | 154 +- tests/pcap/twovlan.test | 300 +- tests/pcap/v6-http.test | 815 ++- tests/pcap/v6.test | 4836 ++++++++--------- tests/pcap/wireshark-bdat.test | 160 +- tests/pcap/wireshark-dhcp.pcap | Bin 0 -> 1400 bytes tests/pcap/wireshark-dhcp.test | 165 + tests/plugins/test.c | 4 +- tests/postgresql.t | 2 +- tests/quic.t | 21 + tests/rules.yaml | 20 + tests/sha256.wise | 1 + tests/smb.t | 2 +- tests/socks.t | 14 +- tests/ssh.t | 2 +- tests/tagger.t | 2 +- tests/tests.pl | 61 +- tests/tls.t | 2 +- tests/wise.t | 7 +- viewer/.jshintrc | 4 +- viewer/Makefile.in | 9 +- viewer/README.md | 134 +- viewer/app/app.js | 5 - .../connections/connections.component.js | 8 +- viewer/app/modules/help/help.html | 8 +- .../search/components/expression.typeahead.js | 26 - .../modules/search/services/field.service.js | 36 - .../search/tests/field.service.test.js | 6 - .../session/components/custom.columns.json | 64 +- .../components/session.detail.component.js | 8 +- .../components/session.field.component.js | 17 +- .../components/session.list.component.js | 10 +- .../session/templates/session.detail.html | 8 +- .../session/templates/session.field.html | 2 +- .../session/templates/session.info.html | 42 +- .../session/templates/session.list.html | 4 +- .../session/templates/session.sticky.html | 14 +- .../tests/session.detail.component.test.js | 20 +- .../tests/session.field.component.test.js | 44 +- .../tests/session.list.component.test.js | 216 +- viewer/app/modules/settings/settings.js | 8 +- viewer/app/modules/spiview/spiview.js | 19 +- viewer/app/modules/stats/stats.css | 7 +- .../app/modules/stats/stats.es.component.js | 22 + viewer/app/modules/stats/stats.es.html | 31 +- viewer/app/modules/stats/stats.html | 2 + viewer/app/modules/stats/stats.shards.html | 4 +- viewer/components/navbar/navbar.component.js | 8 +- viewer/components/navbar/navbar.css | 21 +- viewer/components/util/util.js | 21 +- viewer/components/util/util.test.js | 24 +- viewer/db.js | 218 +- viewer/molochparser.jison | 232 +- viewer/molochparser.js | 230 +- viewer/multies.js | 330 +- viewer/package.json | 47 +- viewer/public/jquery-jvectormap-world-en.js | 2 +- viewer/reindex2.js | 1167 ++++ viewer/rules.yara | 1 + viewer/viewer.js | 2005 +++---- viewer/views/sessionDetail.pug | 163 +- viewer/views/sessionPackets.pug | 6 +- viewer/vueapp/.babelrc | 12 + viewer/vueapp/.editorconfig | 9 + viewer/vueapp/.eslintignore | 4 + viewer/vueapp/.eslintrc.js | 31 + viewer/vueapp/.gitignore | 14 + viewer/vueapp/.postcssrc.js | 10 + viewer/vueapp/build/build.js | 41 + viewer/vueapp/build/check-versions.js | 54 + viewer/vueapp/build/logo.png | Bin 0 -> 6849 bytes viewer/vueapp/build/utils.js | 101 + viewer/vueapp/build/vue-loader.conf.js | 22 + viewer/vueapp/build/webpack.base.conf.js | 92 + viewer/vueapp/build/webpack.dev.conf.js | 96 + viewer/vueapp/build/webpack.prod.conf.js | 145 + viewer/vueapp/config/dev.env.js | 7 + viewer/vueapp/config/index.js | 76 + viewer/vueapp/config/prod.env.js | 4 + viewer/vueapp/index.html | 20 + viewer/vueapp/package.json | 72 + viewer/vueapp/src/App.vue | 229 + viewer/vueapp/src/assets/logo.png | Bin 0 -> 7882 bytes viewer/vueapp/src/assets/watching.gif | Bin 0 -> 6012 bytes viewer/vueapp/src/components/UserService.js | 45 + .../vueapp/src/components/stats/EsIndices.vue | 261 + .../vueapp/src/components/stats/EsStats.vue | 313 ++ .../vueapp/src/components/stats/EsTasks.vue | 204 + .../vueapp/src/components/stats/NodeStats.vue | 613 +++ viewer/vueapp/src/components/stats/Shards.vue | 336 ++ viewer/vueapp/src/components/stats/Stats.vue | 280 + .../vueapp/src/components/utils/ESHealth.vue | 93 + viewer/vueapp/src/components/utils/Error.vue | 24 + viewer/vueapp/src/components/utils/Footer.vue | 32 + .../src/components/utils/HasPermission.vue | 19 + .../vueapp/src/components/utils/Loading.vue | 137 + viewer/vueapp/src/components/utils/Navbar.vue | 103 + .../src/components/utils/Pagination.vue | 144 + .../vueapp/src/components/utils/ToggleBtn.vue | 43 + viewer/vueapp/src/cubismoverrides.css | 70 + viewer/vueapp/src/filters.js | 80 + viewer/vueapp/src/interceptors.js | 19 + viewer/vueapp/src/main.js | 54 + viewer/vueapp/src/overrides.css | 204 + viewer/vueapp/src/router/index.js | 41 + viewer/vueapp/src/themes/blue.css | 46 + viewer/vueapp/src/themes/cotton-candy.css | 46 + viewer/vueapp/src/themes/dark-2.css | 56 + viewer/vueapp/src/themes/dark-3.css | 56 + viewer/vueapp/src/themes/default.css | 46 + viewer/vueapp/src/themes/green.css | 46 + viewer/vueapp/static/.gitkeep | 0 354 files changed, 29692 insertions(+), 23159 deletions(-) create mode 100644 CONTRIBUTING.md create mode 100644 capture/parsers/dhcp.c create mode 100644 capture/parsers/dhcp.detail.jade delete mode 100644 capture/parsers/tls.detail.pug create mode 100644 package.json create mode 100644 tests/api-files.t create mode 100644 tests/dhcp.t create mode 100644 tests/parliament.t create mode 100644 tests/pcap/CVE-2018-6794.pcap create mode 100644 tests/pcap/CVE-2018-6794.test create mode 100644 tests/pcap/ssl-selfsign.pcap create mode 100644 tests/pcap/ssl-selfsign.test create mode 100644 tests/pcap/wireshark-dhcp.pcap create mode 100644 tests/pcap/wireshark-dhcp.test create mode 100644 tests/quic.t create mode 100644 tests/sha256.wise create mode 100755 viewer/reindex2.js create mode 120000 viewer/rules.yara create mode 100644 viewer/vueapp/.babelrc create mode 100644 viewer/vueapp/.editorconfig create mode 100644 viewer/vueapp/.eslintignore create mode 100644 viewer/vueapp/.eslintrc.js create mode 100644 viewer/vueapp/.gitignore create mode 100644 viewer/vueapp/.postcssrc.js create mode 100644 viewer/vueapp/build/build.js create mode 100644 viewer/vueapp/build/check-versions.js create mode 100644 viewer/vueapp/build/logo.png create mode 100644 viewer/vueapp/build/utils.js create mode 100644 viewer/vueapp/build/vue-loader.conf.js create mode 100644 viewer/vueapp/build/webpack.base.conf.js create mode 100755 viewer/vueapp/build/webpack.dev.conf.js create mode 100644 viewer/vueapp/build/webpack.prod.conf.js create mode 100644 viewer/vueapp/config/dev.env.js create mode 100644 viewer/vueapp/config/index.js create mode 100644 viewer/vueapp/config/prod.env.js create mode 100644 viewer/vueapp/index.html create mode 100644 viewer/vueapp/package.json create mode 100644 viewer/vueapp/src/App.vue create mode 100644 viewer/vueapp/src/assets/logo.png create mode 100644 viewer/vueapp/src/assets/watching.gif create mode 100644 viewer/vueapp/src/components/UserService.js create mode 100644 viewer/vueapp/src/components/stats/EsIndices.vue create mode 100644 viewer/vueapp/src/components/stats/EsStats.vue create mode 100644 viewer/vueapp/src/components/stats/EsTasks.vue create mode 100644 viewer/vueapp/src/components/stats/NodeStats.vue create mode 100644 viewer/vueapp/src/components/stats/Shards.vue create mode 100644 viewer/vueapp/src/components/stats/Stats.vue create mode 100644 viewer/vueapp/src/components/utils/ESHealth.vue create mode 100644 viewer/vueapp/src/components/utils/Error.vue create mode 100644 viewer/vueapp/src/components/utils/Footer.vue create mode 100644 viewer/vueapp/src/components/utils/HasPermission.vue create mode 100644 viewer/vueapp/src/components/utils/Loading.vue create mode 100644 viewer/vueapp/src/components/utils/Navbar.vue create mode 100644 viewer/vueapp/src/components/utils/Pagination.vue create mode 100644 viewer/vueapp/src/components/utils/ToggleBtn.vue create mode 100644 viewer/vueapp/src/cubismoverrides.css create mode 100644 viewer/vueapp/src/filters.js create mode 100644 viewer/vueapp/src/interceptors.js create mode 100644 viewer/vueapp/src/main.js create mode 100644 viewer/vueapp/src/overrides.css create mode 100644 viewer/vueapp/src/router/index.js create mode 100644 viewer/vueapp/src/themes/blue.css create mode 100644 viewer/vueapp/src/themes/cotton-candy.css create mode 100644 viewer/vueapp/src/themes/dark-2.css create mode 100644 viewer/vueapp/src/themes/dark-3.css create mode 100644 viewer/vueapp/src/themes/default.css create mode 100644 viewer/vueapp/src/themes/green.css create mode 100644 viewer/vueapp/static/.gitkeep diff --git a/CHANGELOG b/CHANGELOG index a1b311beed..4799a28e20 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,16 +1,70 @@ -ES Support: - * Moloch 0.50.x supports ES >= 5.5.0, not 6.x - * Moloch 0.20.x supports ES 2.4.x or > 5.3.1 not 6.x +NOTICE: If upgrading read https://github.com/aol/moloch/wiki/FAQ#How_do_I_upgrade_to_Moloch_1 -Node Support: - * Moloch >= 0.20.x requires NodeJS 6.x - * Moloch >= 0.16.x requires NodeJS 4.x +ES Versions: + * Moloch >= 1.0.0 supports ES >= 5.5.0, kind of 6.x (no prod tested, only for new installs), not 7.x + * Moloch >= 0.50.0 supports ES >= 5.5.0, not 6.x + * Moloch >= 0.18.1 supports ES 2.4.x, >= 5.3,1 not 6.x + +Node Versions: + * Moloch >= 1.0.0 requires NodeJS 8.x + * Moloch >= 0.20.0 requires NodeJS 6.x + * Moloch >= 0.18.1 requires NodeJS 4.x NOTICE: Upgrading from versions prior to 0.18.0 or prior to ES 5 require an outage for db.pl upgrade see: https://github.com/aol/moloch/wiki/FAQ#How_do_I_upgrade_to_ES_5x NOTICE: Restart wiseService before capture when upgrading +1.0.0-rc2 2018/03/xx + - viewer - Change default spiDataMaxIndices to 4 everywhere + - viewer - work around for ES 6 issue https://github.com/elastic/elasticsearch/issues/27740 + - capture - fixed netflow plugin + - tests - initial parliament tests + +1.0.0-rc1 2018/03/20 + - viewer - minor stats page fixes + - release - new top level package.json/node_modules to make package smaller + +1.0.0-beta3 2018/03/15 + - viewer - stats page implemented in Vue instead of Angular + - capture - some code clean and better thread safe counters + - viewer - convert field names in saved column sets from pre 1.0 + +1.0.0-beta2 2018/03/08 + - capture - decode some dhcp + - capture - tag a tls session with cert:self-signed + - capture - reload geo, rir, yara and oui files without restarting (issue #692) + - capture - remove yara 1.x support + - viewer - cron jobs now use the timestamp time and not last packet time when choosing sessions to look at + this means delay is shorter, although when upgrading to 1.0 some sessions will be not looked at. + +1.0.0-beta1 2018/02/20 + - capture - calculate sha256 too (set supportSha256 tru) + - wise - support sha256 lookups + - capture - fix disable fields + - capture - src/dst ip/port can be used to trigger rules now + - capture - ip fields in rules can now be CIDR + - capture - simple writer now flushes after 10 seconds of no writting + there still can be pagesize bytes unwritten (issue #777) + +1.0.0-alpha2 2018/01/31 + - Read alpha1 below + - release - correct geo files + - capture - set default geo file path + +1.0.0-alpha1 2018/01/26 + - NOTICE: Supported ES Versions: 5.6.x, 6.x (for new installs) + - NOTICE: hasheader for email/http for old sessions will not be migrated + - all - rename all field names + - all - no more analyzed ES fields, everything is a keyword field + - all - full ipv6 support + - all - tags index removed, tags/hasheader stored as first class fields + - all - new reindex2 script to move from pre 1.0 to 1.0 + - capture - http uri field no longer starts with // (issue #732) + - capture - use maxminddb instead of geoip now (issue #771) + - all - Country codes are now 2 letters instead of 3 letters + - release - node 8.9.4 + 0.50.1 2017/03/29 - NOTICE: Supported ES Versions: >= 5.5.0, 6.x is NOT supported - release - upgrade curl, yara, glib @@ -129,7 +183,7 @@ NOTICE: Restart wiseService before capture when upgrading - viewer - handle corrupt theme - capture - fix quic parser crash - release - libyaml as a dependency - + 0.19.0 2017/07/11 - NOTICE: Supported ES Versions: 2.4.x, 5.1.2, 5.2.x, > 5.3.1, 5.4.x @@ -231,7 +285,7 @@ NOTICE: Restart wiseService before capture when upgrading - capture - tpacketv3 handles multiple interfaces correctly (issue #658) - easybutton - singlehost and config removed, build remains for now make install & make config should work - - capture - + - capture - 0.17.1 2017/01/30 - NOTICE: ES 2.4 or ES 5.1.2 required (ES 5.x isn't recommended for production yet) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000000..217c901f16 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,88 @@ +# Contributing to Moloch + +:sparkles: Glad to see you here! :sparkles: + +--- + +### Just have a question :question: + +* Visit our [FAQs](https://github.com/aol/moloch/wiki/FAQ) +* Or talk to us directly in the [Moloch-FPC Slack](https://slackinvite.molo.ch/) + +--- + +### Where do I start? :traffic_light: + +First, checkout the main [Moloch README](README.rst) for information on how to build and run Moloch. + +**Then, get some test data!** + +* Start Elasticsearch +* Move to the Moloch tests directory +* Run `./tests.pl --viewer` + +> **Note:** this will only work if viewer is not already running. + +You should now have test data loaded, so let's **start the web app**: + +* Move to the Moloch viewer directory +* Run `npm install` +* Move to the vueapp directory +* Run `npm install` +* Move back up to the viewer directory +* Run `npm run start:test` +* Now browse to the app at `http://localhost:8123` + +> :clock1: _On first load, you will likely see this message: "No results or none that match your search within your time range." This is because the data that was loaded is from all time ranges, so make sure you search for ALL times ranges._ + +For more information about running the Moloch Viewer web application, visit the [viewer README](viewer/README.md). + +--- + +### How do I contribute? + +#### Documentation! :page_with_curl: + +Documentation, READMEs, examples, and FAQs are important. Please help improve and add to them. + +#### Bugs :bug: :beetle: :ant: + +**Before submitting a bug report:** +* Ensure the bug was not already reported by searching for [existing issues in Moloch](https://github.com/aol/moloch/issues) + * If an issues is already open, make a comment that you are experiencing the same thing and provide any additional details +* Check the [FAQs](https://github.com/aol/moloch/wiki/FAQ) for a list of common questions and problems + +Bugs are tracked as [GitHub Issues](https://guides.github.com/features/issues/). +**Please follow these guidelines when submitting a bug:** +* Provide a clear and descriptive title +* Describe the exact steps to reproduce the problem +* Explain the expected behavior +* Fill out the [issue template](https://github.com/aol/moloch/issues/new) completely + +#### Feature Requests :sparkles: + +Feature requests include new features and minor improvements to existing functionality. + +Feature requests are tracked as [GitHub Issues](https://guides.github.com/features/issues/). +**Please follow these guidelines when submitting a feature request:** +* Provide a clear and descriptive title +* Describe the suggested feature in as much detail as possible +* Use examples to help us understand the use case of the feature +* If you are requesting a minor improvement, describe the current behavior and why it is not sufficient +* If possible, provide examples of where this feature exists elsewhere in other tools +* Follow the directions in the [issue template](https://github.com/aol/moloch/issues/new) + +#### Pull Requests :muscle: + +**We welcome all collaboration!** If you can fix it or implement it, please do! :hammer: + +**To better help us review your pull request, please follow these guidelines:** +* Provide a clear and descriptive title +* Clearly describe the problem and solution +* Include the relevant issue number(s) if applicable +* Ensure that all tests still pass by navigating to the `tests` directory and running `./tests.pl --viewer` +* If making changes to the client code, please run the unit tests by navigating to the `viewer` directory and running `npm test` + +--- + +### THANKS! :heart: diff --git a/Makefile.am b/Makefile.am index 3c1f9b699f..bc2b9fe0fa 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,3 +1,8 @@ -SUBDIRS = capture db viewer parliament release +SUBDIRS = . capture db viewer parliament release +install-exec-local: + npm install + @INSTALL@ -D package.json @prefix@/package.json + (cd @prefix@ ; npm install --production) + config: @prefix@/bin/Configure diff --git a/Makefile.in b/Makefile.in index 760b7cc49f..45354dc902 100644 --- a/Makefile.in +++ b/Makefile.in @@ -49,7 +49,7 @@ DIST_COMMON = $(am__configure_deps) $(srcdir)/Makefile.am \ $(top_srcdir)/tests/plugins/Makefile.in \ $(top_srcdir)/viewer/Makefile.in \ $(top_srcdir)/viewer/version.js.in compile config.guess \ - config.sub install-sh missing + config.sub depcomp install-sh missing ylwrap ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 am__aclocal_m4_deps = $(top_srcdir)/configure.ac am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ @@ -143,8 +143,6 @@ ECHO_N = @ECHO_N@ ECHO_T = @ECHO_T@ EGREP = @EGREP@ EXEEXT = @EXEEXT@ -GEOIP_CFLAGS = @GEOIP_CFLAGS@ -GEOIP_LIBS = @GEOIP_LIBS@ GIT = @GIT@ GLIB2_CFLAGS = @GLIB2_CFLAGS@ GLIB2_LIBS = @GLIB2_LIBS@ @@ -163,6 +161,8 @@ LUA_LIBS = @LUA_LIBS@ MAGIC_CFLAGS = @MAGIC_CFLAGS@ MAGIC_LIBS = @MAGIC_LIBS@ MAKEINFO = @MAKEINFO@ +MAXMINDDB_CFLAGS = @MAXMINDDB_CFLAGS@ +MAXMINDDB_LIBS = @MAXMINDDB_LIBS@ MKDIR_P = @MKDIR_P@ OBJEXT = @OBJEXT@ PACKAGE = @PACKAGE@ @@ -234,7 +234,7 @@ target_alias = @target_alias@ top_build_prefix = @top_build_prefix@ top_builddir = @top_builddir@ top_srcdir = @top_srcdir@ -SUBDIRS = capture db viewer parliament release +SUBDIRS = . capture db viewer parliament release all: all-recursive .SUFFIXES: @@ -571,7 +571,7 @@ distcheck: dist *.zip*) \ unzip $(distdir).zip ;;\ esac - chmod -R a-w $(distdir); chmod a+w $(distdir) + chmod -R a-w $(distdir); chmod u+w $(distdir) mkdir $(distdir)/_build mkdir $(distdir)/_inst chmod a-w $(distdir) @@ -683,7 +683,7 @@ install-dvi: install-dvi-recursive install-dvi-am: -install-exec-am: +install-exec-am: install-exec-local install-html: install-html-recursive @@ -736,12 +736,18 @@ uninstall-am: distcleancheck distdir distuninstallcheck dvi dvi-am html \ html-am info info-am install install-am install-data \ install-data-am install-dvi install-dvi-am install-exec \ - install-exec-am install-html install-html-am install-info \ - install-info-am install-man install-pdf install-pdf-am \ - install-ps install-ps-am install-strip installcheck \ - installcheck-am installdirs installdirs-am maintainer-clean \ - maintainer-clean-generic mostlyclean mostlyclean-generic pdf \ - pdf-am ps ps-am tags tags-recursive uninstall uninstall-am + install-exec-am install-exec-local install-html \ + install-html-am install-info install-info-am install-man \ + install-pdf install-pdf-am install-ps install-ps-am \ + install-strip installcheck installcheck-am installdirs \ + installdirs-am maintainer-clean maintainer-clean-generic \ + mostlyclean mostlyclean-generic pdf pdf-am ps ps-am tags \ + tags-recursive uninstall uninstall-am + +install-exec-local: + npm install + @INSTALL@ -D package.json @prefix@/package.json + (cd @prefix@ ; npm install --production) config: @prefix@/bin/Configure diff --git a/README.rst b/README.rst index b4ab0eff74..9bb835b8a9 100644 --- a/README.rst +++ b/README.rst @@ -70,9 +70,9 @@ Moloch is a complex system to build and install manually. The following are roug Installing Elasticsearch ------------------------ -Recommended version **5.5.x** for Moloch 0.18 and later. `Download elasticsearch `_. +Recommended version **5.6.x** for Moloch 0.18 and later. `Download elasticsearch `_. **Important:** At this time all development is done with `elasticsearch - 5.5.1 `_. + 5.6.7 `_. Inside the *installed* ``$MOLOCH_PREFIX/db`` directory run the ``db.pl http://A_ES_HOSTNAME:9200 init`` script. @@ -82,50 +82,7 @@ Inside the *installed* ``$MOLOCH_PREFIX/db`` directory run the Building Capture ---------------- -1. Install prerequisite standard packages. - - - CentOS:: - - yum install wget curl pcre pcre-devel pkgconfig flex bison gcc-c++ zlib-devel e2fsprogs-devel openssl-devel file-devel make gettext libuuid-devel perl-JSON bzip2-libs bzip2-devel perl-libwww-perl libpng-devel xz libffi-devel - - - Ubuntu:: - - apt-get install wget curl libpcre3-dev uuid-dev libmagic-dev pkg-config g++ flex bison zlib1g-dev libffi-dev gettext libgeoip-dev make libjson-perl libbz2-dev libwww-perl libpng-dev xz-utils libffi-dev - - - OS X:: - - port install yara libpcap openssl pcre flex bison zlib file gettext p5-JSON p5-libwww-perl libffi xz ossp-uuid libgeoip glib2 - ./configure --with-libpcap=/opt/local --with-yara=/opt/local --with-GeoIP=/opt/local LDFLAGS=-L/opt/local/lib --with-glib2=no GLIB2_CFLAGS="-I/opt/local/include/glib-2.0 -I/opt/local/lib/glib-2.0/include" GLIB2_LIBS="-L/opt/local/lib -lglib-2.0 -lgmodule-2.0 -lgobject-2.0 -lgio-2.0" - -2. Building ``capture`` can be a pain because of OS versions. - - - Try ``./easybutton-build.sh`` which will download all the following, compile them statically, and run the local configure script. - - Or if you want build yourself, or use some already installed packages then here are the pieces you need: - - + `glib-2 `_ version 2.40 or - higher (2.50.2 is recommended):: - - wget http://ftp.gnome.org/pub/gnome/sources/glib/2.50/glib-2.50.2.tar.xz - ./configure --disable-xattr --disable-shared --enable-static --disable-libelf --disable-selinux --disable-libmount --with-pcre=internal - - + `yara `_ version 1.6 or higher:: - - wget https://github.com/VirusTotal/yara/archive/v3.5.0.tar.gz -O yara-3.5.0.tar.gz - ./configure --enable-static - - + `MaxMind GeoIP `_ - The OS version may be recent enough:: - wget http://www.maxmind.com/download/geoip/api/c/GeoIP-1.6.9.tar.gz - libtoolize -f # Only some platforms need this - ./configure --enable-static - - + `libpcap `_ - version 1.3 or higher (most OS versions are older):: - - wget http://www.tcpdump.org/release/libpcap-1.7.4.tar.gz - ./configure --disable-dbus - -3. Run ``configure``. Optionally use the ``--with-`` directives to use static libraries from build directories. - -4. Run ``make``. +Use the ``./easybutton-build.sh`` script to download all thirdparty libraries and build moloch. .. _building-viewer: @@ -133,7 +90,7 @@ Building Viewer --------------- 1. Install `Node.js `_ version 6.x, currently 8.x is not supported. (Moloch versions before 0.18 required 4) -2. In the ``viewer`` directory run ``npm update``. +2. In the ``viewer`` directory run ``npm install``. .. _configuration: diff --git a/capture/Makefile.in b/capture/Makefile.in index 4a98648abe..23b853d250 100644 --- a/capture/Makefile.in +++ b/capture/Makefile.in @@ -9,7 +9,7 @@ INCLUDE_PCAP = @PCAP_CFLAGS@ INCLUDE_OTHER = -Ithirdparty \ @GLIB2_CFLAGS@ \ @YARA_CFLAGS@ \ - @GEOIP_CFLAGS@ \ + @MAXMINDDB_CFLAGS@ \ @MAGIC_CFLAGS@ \ @CURL_CFLAGS@ @@ -18,7 +18,7 @@ LIB_PCAP = @PCAP_LIBS@ LIB_SNF = /opt/snf/lib/*.so LIB_OTHER = @GLIB2_LIBS@ \ @YARA_LIBS@ \ - @GEOIP_LIBS@ \ + @MAXMINDDB_LIBS@ \ @CURL_LIBS@ \ @LIBS@ \ thirdparty/http_parser.o \ diff --git a/capture/config.c b/capture/config.c index b6bc085179..db247d1ec5 100644 --- a/capture/config.c +++ b/capture/config.c @@ -17,8 +17,11 @@ */ #include "moloch.h" +#include +#include #include #include +#include extern MolochConfig_t config; @@ -372,11 +375,10 @@ void moloch_config_load() config.bpf = moloch_config_str(keyfile, "bpf", NULL); config.yara = moloch_config_str(keyfile, "yara", NULL); config.emailYara = moloch_config_str(keyfile, "emailYara", NULL); - config.geoipFile = moloch_config_str(keyfile, "geoipFile", NULL); config.rirFile = moloch_config_str(keyfile, "rirFile", NULL); - config.geoipASNFile = moloch_config_str(keyfile, "geoipASNFile", NULL); - config.geoip6File = moloch_config_str(keyfile, "geoip6File", NULL); - config.geoipASN6File = moloch_config_str(keyfile, "geoipASN6File", NULL); + config.ouiFile = moloch_config_str(keyfile, "ouiFile", NULL); + config.geoLite2ASN = moloch_config_str(keyfile, "geoLite2ASN", "/data/moloch/etc/GeoLite2-ASN.mmdb"); + config.geoLite2Country = moloch_config_str(keyfile, "geoLite2Country", "/data/moloch/etc/GeoLite2-Country.mmdb"); config.dropUser = moloch_config_str(keyfile, "dropUser", NULL); config.dropGroup = moloch_config_str(keyfile, "dropGroup", NULL); config.pluginsDir = moloch_config_str_list(keyfile, "pluginsDir", NULL); @@ -438,6 +440,7 @@ void moloch_config_load() config.parseSMB = moloch_config_boolean(keyfile, "parseSMB", TRUE); config.parseQSValue = moloch_config_boolean(keyfile, "parseQSValue", FALSE); config.parseCookieValue = moloch_config_boolean(keyfile, "parseCookieValue", FALSE); + config.supportSha256 = moloch_config_boolean(keyfile, "supportSha256", FALSE); config.reqBodyOnlyUtf8 = moloch_config_boolean(keyfile, "reqBodyOnlyUtf8", TRUE); config.compressES = moloch_config_boolean(keyfile, "compressES", FALSE); config.antiSynDrop = moloch_config_boolean(keyfile, "antiSynDrop", TRUE); @@ -445,15 +448,6 @@ void moloch_config_load() } /******************************************************************************/ -void moloch_config_get_tag_cb(MolochIpInfo_t *ii, int UNUSED(tagtype), const char *tagName, uint32_t tag) -{ - if (ii->numtags >= 10) return; - - ii->tags[ii->numtags] = tag; - ii->tagsStr[ii->numtags] = strdup(tagName); - ii->numtags++; -} -/******************************************************************************/ void moloch_config_load_local_ips() { GError *error = 0; @@ -482,7 +476,10 @@ void moloch_config_load_local_ips() } else if (strncmp(values[v], "rir:", 4) == 0) { ii->rir = g_strdup(values[v]+4); } else if (strncmp(values[v], "tag:", 4) == 0) { - moloch_db_get_tag(ii, 0, values[v]+4, (MolochTag_cb)moloch_config_get_tag_cb); + if (ii->numtags < 10) { + ii->tagsStr[ii->numtags] = strdup(values[v]+4); + ii->numtags++; + } } else if (strncmp(values[v], "country:", 8) == 0) { ii->country = g_strdup(values[v]+8); } @@ -590,7 +587,7 @@ void moloch_config_load_header(char *section, char *group, char *helpBase, char switch (type) { case 0: - kind = "textfield"; + kind = "termfield"; if (unique) t = MOLOCH_FIELD_TYPE_STR_HASH; else @@ -621,39 +618,93 @@ void moloch_config_load_header(char *section, char *group, char *helpBase, char char expression[100]; char field[100]; - char rawfield[100]; char help[100]; - if (type == 0) { - snprintf(expression, sizeof(expression), "%s%s", expBase, name); - snprintf(field, sizeof(field), "%s%s.snow", dbBase, name); - snprintf(rawfield, sizeof(rawfield), "%s%s.raw", dbBase, name); - snprintf(help, sizeof(help), "%s%s", helpBase, name); - } else { - snprintf(expression, sizeof(expression), "%s%s", expBase, name); - snprintf(field, sizeof(field), "%s%s", dbBase, name); - rawfield[0] = 0; - snprintf(help, sizeof(help), "%s%s", helpBase, name); - } + snprintf(expression, sizeof(expression), "%s%s", expBase, name); + snprintf(field, sizeof(field), "%s%s", dbBase, name); + snprintf(help, sizeof(help), "%s%s", helpBase, name); int pos; - if (rawfield[0]) { - pos = moloch_field_define(group, kind, - expression, expression, field, - help, - t, f, - "rawField", rawfield, NULL); - } else { - pos = moloch_field_define(group, kind, - expression, expression, field, - help, - t, f, NULL); - } + pos = moloch_field_define(group, kind, + expression, expression, field, + help, + t, f, NULL); moloch_config_add_header(hash, g_strdup(keys[k]), pos); g_strfreev(values); } g_strfreev(keys); } + +/******************************************************************************/ +typedef struct { + char *desc; + char *name; + MolochFileChange_cb cb; + off_t size; + int64_t modify; + char freeOld; +} MolochFileChange_t; + +LOCAL int numFiles; +LOCAL MolochFileChange_t files[100]; +/******************************************************************************/ +void moloch_config_monitor_file(char *desc, char *name, MolochFileChange_cb cb) +{ + struct stat sb; + + if (numFiles >= 100) + LOGEXIT("Couldn't monitor anymore files %s %s", desc, name); + + if (stat(name, &sb) != 0) { + LOGEXIT("Couldn't stat %s file %s error %s", desc, name, strerror(errno)); + } + + files[numFiles].desc = g_strdup(desc); + files[numFiles].name = g_strdup(name); + files[numFiles].cb = cb; + files[numFiles].modify = sb.st_mtime; + numFiles++; + cb(name); +} +/******************************************************************************/ +gboolean moloch_config_reload_files (gpointer UNUSED(user_data)) +{ + int i; + struct stat sb; + + for (i = 0; i < numFiles; i++) { + if (files[i].freeOld) { + if (config.debug) + LOG("Free old %s %s", files[i].desc, files[i].name); + files[i].cb(NULL); + files[i].freeOld = 0; + } + + if (stat(files[i].name, &sb) != 0) { + LOG("Couldn't stat %s file %s error %s", files[i].desc, files[i].name, strerror(errno)); + continue; + } + + if (sb.st_size <= 1) { // Ignore tiny files for reloads + continue; + } + + if (sb.st_mtime > files[i].modify) { + if (files[i].size != sb.st_size) { + files[i].size = sb.st_size; + continue; + } + if (config.debug) + LOG("Load new %s %s", files[i].desc, files[i].name); + files[i].cb(files[i].name); + files[i].freeOld = 1; + files[i].size = 0; + files[i].modify = sb.st_mtime; + } + } + + return TRUE; +} /******************************************************************************/ void moloch_config_init() { @@ -679,6 +730,10 @@ void moloch_config_init() printf("Must set a pcapDir to save files to\n"); exit(1); } + + if (!config.dryRun) { + g_timeout_add_seconds( 10, moloch_config_reload_files, 0); + } } /******************************************************************************/ void moloch_config_exit() diff --git a/capture/db.c b/capture/db.c index 3f6790ca9f..4e8dc9a3a2 100644 --- a/capture/db.c +++ b/capture/db.c @@ -17,33 +17,38 @@ */ #include "moloch.h" #include "molochconfig.h" +#include +#include #include #include #include #include #include #include +#include #include "patricia.h" -#include "GeoIP.h" -#define MOLOCH_MIN_DB_VERSION 34 +#include "maxminddb.h" +MMDB_s *geoCountry; +MMDB_s *geoASN; + +#define MOLOCH_MIN_DB_VERSION 50 extern uint64_t totalPackets; -extern uint64_t totalBytes; -extern uint64_t totalSessions; -static uint16_t myPid; +LOCAL uint64_t totalSessions = 0; +LOCAL uint64_t totalSessionBytes; +LOCAL uint16_t myPid; extern uint32_t pluginsCbs; LOCAL struct timeval startTime; -LOCAL GeoIP *gi = 0; -LOCAL GeoIP *giASN = 0; -LOCAL GeoIP *gi6 = 0; -LOCAL GeoIP *giASN6 = 0; LOCAL char *rirs[256]; void * esServer = 0; -LOCAL patricia_tree_t *ipTree = 0; +LOCAL patricia_tree_t *ipTree4 = 0; +LOCAL patricia_tree_t *ipTree6 = 0; + +LOCAL patricia_tree_t *ouiTree = 0; extern char *moloch_char_to_hex; extern unsigned char moloch_char_to_hexstr[256][3]; @@ -55,26 +60,19 @@ LOCAL MOLOCH_LOCK_DEFINE(nextFileNum); /******************************************************************************/ extern MolochConfig_t config; -/******************************************************************************/ -typedef struct moloch_tag { - struct moloch_tag *tag_next, *tag_prev; - char *tagName; - uint32_t tag_hash; - int tagValue; - short tag_bucket; - short tag_count; -} MolochTag_t; - -HASH_VAR(tag_, tags, MolochTag_t, 19991); - /******************************************************************************/ void moloch_db_add_local_ip(char *str, MolochIpInfo_t *ii) { patricia_node_t *node; - if (!ipTree) { - ipTree = New_Patricia(128); + if (!ipTree4) { + ipTree4 = New_Patricia(32); + ipTree6 = New_Patricia(128); + } + if (strchr(str, '.') != 0) { + node = make_and_lookup(ipTree4, str); + } else { + node = make_and_lookup(ipTree6, str); } - node = make_and_lookup(ipTree, str); node->data = ii; } /******************************************************************************/ @@ -89,80 +87,31 @@ void moloch_db_free_local_ip(MolochIpInfo_t *ii) MOLOCH_TYPE_FREE(MolochIpInfo_t, ii); } /******************************************************************************/ -MolochIpInfo_t *moloch_db_get_local_ip6(MolochSession_t *session, struct in6_addr *ip) +LOCAL MolochIpInfo_t *moloch_db_get_local_ip6(MolochSession_t *session, struct in6_addr *ip) { - prefix_t prefix; patricia_node_t *node; if (IN6_IS_ADDR_V4MAPPED(ip)) { - prefix.family = AF_INET; - prefix.bitlen = 32; - prefix.add.sin.s_addr = ((uint32_t *)ip->s6_addr)[3]; + if ((node = patricia_search_best3 (ipTree4, ((u_char *)ip->s6_addr) + 12, 32)) == NULL) + return 0; } else { - prefix.family = AF_INET6; - prefix.bitlen = 128; - memcpy(&prefix.add.sin6.s6_addr, ip, 16); + if ((node = patricia_search_best3 (ipTree6, (u_char *)ip->s6_addr, 128)) == NULL) + return 0; } - if ((node = patricia_search_best2 (ipTree, &prefix, 1)) == NULL) - return 0; MolochIpInfo_t *ii = node->data; int t; for (t = 0; t < ii->numtags; t++) { - moloch_field_int_add(config.tagsField, session, ii->tags[t]); moloch_field_string_add(config.tagsStringField, session, ii->tagsStr[t], -1, TRUE); } return ii; } -/******************************************************************************/ -MolochIpInfo_t *moloch_db_get_local_ip4(MolochSession_t *session, uint32_t ip) -{ - prefix_t prefix; - patricia_node_t *node; - - prefix.family = AF_INET; - prefix.bitlen = 32; - prefix.add.sin.s_addr = ip; - - if ((node = patricia_search_best2 (ipTree, &prefix, 1)) == NULL) - return 0; - - MolochIpInfo_t *ii = node->data; - int t; - - for (t = 0; t < ii->numtags; t++) { - moloch_field_int_add(config.tagsField, session, ii->tags[t]); - moloch_field_string_add(config.tagsStringField, session, ii->tagsStr[t], -1, TRUE); - } - - return ii; -} -/******************************************************************************/ -uint32_t moloch_db_tag_hash(const void *key) -{ - char *p = (char *)key; - uint32_t n = 0; - while (*p) { - n = (n << 5) - n + *p; - p++; - } - return n; -} /******************************************************************************/ -int moloch_db_tag_cmp(const void *keyv, const void *elementv) -{ - char *key = (char*)keyv; - MolochTag_t *element = (MolochTag_t *)elementv; - - return strcmp(key, element->tagName) == 0; -} - -/******************************************************************************/ -void moloch_db_js0n_str(BSB *bsb, unsigned char *in, gboolean utf8) +LOCAL void moloch_db_js0n_str(BSB *bsb, unsigned char *in, gboolean utf8) { BSB_EXPORT_u08(*bsb, '"'); while (*in) { @@ -232,8 +181,11 @@ void moloch_db_geo_lookup6(MolochSession_t *session, struct in6_addr addr, char MolochIpInfo_t *ii = 0; *g = *as = *rir = 0; *asFree = 0; + static const char *countryPath[] = {"country", "iso_code", NULL}; + static const char *asoPath[] = {"autonomous_system_organization", NULL}; + static const char *asnPath[] = {"autonomous_system_number", NULL}; - if (ipTree) { + if (ipTree4) { if ((ii = moloch_db_get_local_ip6(session, &addr))) { *g = ii->country; *as = ii->asn; @@ -241,63 +193,54 @@ void moloch_db_geo_lookup6(MolochSession_t *session, struct in6_addr addr, char } } - if (IN6_IS_ADDR_V4MAPPED(&addr)) { - if (!*g && gi) { - *g = (char *)GeoIP_country_code3_by_ipnum(gi, htonl(MOLOCH_V6_TO_V4(addr))); - } + struct sockaddr *sa; + struct sockaddr_in sin; + struct sockaddr_in6 sin6; - if (!*as && giASN) { - *as = GeoIP_name_by_ipnum(giASN, htonl(MOLOCH_V6_TO_V4(addr))); - if (*as) { - *asFree = 1; - } - } + if (IN6_IS_ADDR_V4MAPPED(&addr)) { + sin.sin_family = AF_INET; + sin.sin_addr.s_addr = MOLOCH_V6_TO_V4(addr); + sa = (struct sockaddr *)&sin; if (!*rir) { *rir = rirs[MOLOCH_V6_TO_V4(addr) & 0xff]; } } else { - if (!*g && gi6) { - *g = (char *)GeoIP_country_code3_by_ipnum_v6(gi6, addr); - } + sin6.sin6_family = AF_INET6; + sin6.sin6_addr = addr; + sa = (struct sockaddr *)&sin6; + } - if (!*as && giASN6) { - *as = GeoIP_name_by_ipnum_v6(giASN6, addr); - if (*as) { - *asFree = 1; + + int error = 0; + if (!*g) { + MMDB_lookup_result_s result = MMDB_lookup_sockaddr(geoCountry, sa, &error); + if (error == MMDB_SUCCESS && result.found_entry) { + MMDB_entry_data_s entry_data; + int status = MMDB_aget_value(&result.entry, &entry_data, countryPath); + if (status == MMDB_SUCCESS) { + *g = (char *)entry_data.utf8_string; } } } -} -/******************************************************************************/ -void moloch_db_geo_lookup4(MolochSession_t *session, uint32_t addr, char **g, char **as, char **rir, int *asFree) -{ - MolochIpInfo_t *ii = 0; - *g = *as = *rir = 0; - *asFree = 0; - if (ipTree) { - if ((ii = moloch_db_get_local_ip4(session, addr))) { - *g = ii->country; - *as = ii->asn; - *rir = ii->rir; - } - } + if (!*as) { + MMDB_lookup_result_s result = MMDB_lookup_sockaddr(geoASN, sa, &error); + if (error == MMDB_SUCCESS && result.found_entry) { + MMDB_entry_data_s org; + MMDB_entry_data_s num; - if (!*g && gi) { - *g = (char *)GeoIP_country_code3_by_ipnum(gi, htonl(addr)); - } + int status = MMDB_aget_value(&result.entry, &org, asoPath); + status += MMDB_aget_value(&result.entry, &num, asnPath); - if (!*as && giASN) { - *as = GeoIP_name_by_ipnum(giASN, htonl(addr)); - if (*as) { - *asFree = 1; + if (status == MMDB_SUCCESS) { + char buf[1000]; + sprintf(buf, "AS%d %.*s", num.uint32, org.data_size, org.utf8_string); + *as = g_strdup(buf); + *asFree = 1; + } } } - - if (!*rir) { - *rir = rirs[addr & 0xff]; - } } /******************************************************************************/ LOCAL void moloch_db_send_bulk(char *json, int len) @@ -364,7 +307,7 @@ void moloch_db_save_session(MolochSession_t *session, int final) } } - totalSessions++; + MOLOCH_THREAD_INCR(totalSessions); session->segments++; const int thread = session->thread; @@ -435,29 +378,21 @@ void moloch_db_save_session(MolochSession_t *session, int final) BSB jbsb = dbInfo[thread].bsb; startPtr = BSB_WORK_PTR(jbsb); - BSB_EXPORT_sprintf(jbsb, "{\"index\": {\"_index\": \"%ssessions-%s\", \"_type\": \"session\", \"_id\": \"%s\"}}\n", config.prefix, dbInfo[thread].prefix, id); + BSB_EXPORT_sprintf(jbsb, "{\"index\": {\"_index\": \"%ssessions2-%s\", \"_type\": \"session\", \"_id\": \"%s\"}}\n", config.prefix, dbInfo[thread].prefix, id); dataPtr = BSB_WORK_PTR(jbsb); BSB_EXPORT_sprintf(jbsb, - "{\"fp\":%u," - "\"lp\":%u," - "\"fpd\":%" PRIu64 "," - "\"lpd\":%" PRIu64 "," - "\"sl\":%u," - "\"a1\":%u," - "\"p1\":%u," - "\"a2\":%u," - "\"p2\":%u," - "\"pr\":%u,", - (uint32_t)session->firstPacket.tv_sec, - (uint32_t)session->lastPacket.tv_sec, + "{\"firstPacket\":%" PRIu64 "," + "\"lastPacket\":%" PRIu64 "," + "\"length\":%u," + "\"srcPort\":%u," + "\"dstPort\":%u," + "\"ipProtocol\":%u,", ((uint64_t)session->firstPacket.tv_sec)*1000 + ((uint64_t)session->firstPacket.tv_usec)/1000, ((uint64_t)session->lastPacket.tv_sec)*1000 + ((uint64_t)session->lastPacket.tv_usec)/1000, timediff, - htonl(MOLOCH_V6_TO_V4(session->addr1)), session->port1, - htonl(MOLOCH_V6_TO_V4(session->addr2)), session->port2, session->protocol); @@ -470,7 +405,9 @@ void moloch_db_save_session(MolochSession_t *session, int final) "\"psh\": %d," "\"fin\": %d," "\"rst\": %d," - "\"urg\": %d" + "\"urg\": %d," + "\"srcZero\": %d," + "\"dstZero\": %d" "},", session->tcpFlagCnt[MOLOCH_TCPFLAG_SYN], session->tcpFlagCnt[MOLOCH_TCPFLAG_SYN_ACK], @@ -478,12 +415,15 @@ void moloch_db_save_session(MolochSession_t *session, int final) session->tcpFlagCnt[MOLOCH_TCPFLAG_PSH], session->tcpFlagCnt[MOLOCH_TCPFLAG_FIN], session->tcpFlagCnt[MOLOCH_TCPFLAG_RST], - session->tcpFlagCnt[MOLOCH_TCPFLAG_URG]); + session->tcpFlagCnt[MOLOCH_TCPFLAG_URG], + session->tcpFlagCnt[MOLOCH_TCPFLAG_SRC_ZERO], + session->tcpFlagCnt[MOLOCH_TCPFLAG_DST_ZERO] + ); } if (session->firstBytesLen[0] > 0) { int i; - BSB_EXPORT_cstr(jbsb, "\"fb1\":\""); + BSB_EXPORT_cstr(jbsb, "\"srcPayload8\":\""); for (i = 0; i < session->firstBytesLen[0]; i++) { BSB_EXPORT_ptr(jbsb, moloch_char_to_hexstr[(unsigned char)session->firstBytes[0][i]], 2); } @@ -491,7 +431,7 @@ void moloch_db_save_session(MolochSession_t *session, int final) } if (session->firstBytesLen[1] > 0) { - BSB_EXPORT_cstr(jbsb, "\"fb2\":\""); + BSB_EXPORT_cstr(jbsb, "\"dstPayload8\":\""); for (i = 0; i < session->firstBytesLen[1]; i++) { BSB_EXPORT_ptr(jbsb, moloch_char_to_hexstr[(unsigned char)session->firstBytes[1][i]], 2); } @@ -506,40 +446,16 @@ void moloch_db_save_session(MolochSession_t *session, int final) ip = MOLOCH_V6_TO_V4(session->addr2); snprintf(ipdst, sizeof(ipdst), "%d.%d.%d.%d", ip & 0xff, (ip >> 8) & 0xff, (ip >> 16) & 0xff, (ip >> 24) & 0xff); } else { - strcpy(ipsrc, "0.0.0.0"); - strcpy(ipdst, "0.0.0.0"); - /* - Requires ES 5 inet_ntop(AF_INET6, &session->addr1, ipsrc, sizeof(ipsrc)); inet_ntop(AF_INET6, &session->addr2, ipdst, sizeof(ipdst)); - */ - - BSB_EXPORT_cstr(jbsb, "\"tipv61-term\":\""); - for (i = 0; i < 16; i++) { - BSB_EXPORT_ptr(jbsb, moloch_char_to_hexstr[(unsigned char)session->addr1.s6_addr[i]], 2); - } - BSB_EXPORT_cstr(jbsb, "\","); - - BSB_EXPORT_cstr(jbsb, "\"tipv62-term\":\""); - for (i = 0; i < 16; i++) { - BSB_EXPORT_ptr(jbsb, moloch_char_to_hexstr[(unsigned char)session->addr2.s6_addr[i]], 2); - } - BSB_EXPORT_cstr(jbsb, "\","); } BSB_EXPORT_sprintf(jbsb, "\"timestamp\":%" PRIu64 "," - "\"firstPacket\":%" PRIu64 "," - "\"lastPacket\":%" PRIu64 "," - "\"ipSrc\":\"%s\"," - "\"portSrc\":%u," - "\"ipDst\":\"%s\"," - "\"portDst\":%u,", + "\"srcIp\":\"%s\"," + "\"dstIp\":\"%s\",", ((uint64_t)currentTime.tv_sec)*1000 + ((uint64_t)currentTime.tv_usec)/1000, - ((uint64_t)session->firstPacket.tv_sec)*1000 + ((uint64_t)session->firstPacket.tv_usec)/1000, - ((uint64_t)session->lastPacket.tv_sec)*1000 + ((uint64_t)session->lastPacket.tv_usec)/1000, ipsrc, - session->port1, - ipdst, - session->port2); + ipdst); char *g1, *g2, *as1, *as2, *rir1, *rir2; @@ -549,13 +465,13 @@ void moloch_db_save_session(MolochSession_t *session, int final) moloch_db_geo_lookup6(session, session->addr2, &g2, &as2, &rir2, &asFree2); if (g1) - BSB_EXPORT_sprintf(jbsb, "\"g1\":\"%s\",", g1); + BSB_EXPORT_sprintf(jbsb, "\"srcGEO\":\"%2.2s\",", g1); if (g2) - BSB_EXPORT_sprintf(jbsb, "\"g2\":\"%s\",", g2); + BSB_EXPORT_sprintf(jbsb, "\"dstGEO\":\"%2.2s\",", g2); if (as1) { - BSB_EXPORT_cstr(jbsb, "\"as1\":"); + BSB_EXPORT_cstr(jbsb, "\"srcASN\":"); moloch_db_js0n_str(&jbsb, (unsigned char*)as1, TRUE); BSB_EXPORT_u08(jbsb, ','); if (asFree1) @@ -563,7 +479,7 @@ void moloch_db_save_session(MolochSession_t *session, int final) } if (as2) { - BSB_EXPORT_cstr(jbsb, "\"as2\":"); + BSB_EXPORT_cstr(jbsb, "\"dstASN\":"); moloch_db_js0n_str(&jbsb, (unsigned char*)as2, TRUE); BSB_EXPORT_u08(jbsb, ','); if (asFree2) @@ -572,23 +488,23 @@ void moloch_db_save_session(MolochSession_t *session, int final) if (rir1) - BSB_EXPORT_sprintf(jbsb, "\"rir1\":\"%s\",", rir1); + BSB_EXPORT_sprintf(jbsb, "\"srcRIR\":\"%s\",", rir1); if (rir2) - BSB_EXPORT_sprintf(jbsb, "\"rir2\":\"%s\",", rir2); + BSB_EXPORT_sprintf(jbsb, "\"dstRIR\":\"%s\",", rir2); BSB_EXPORT_sprintf(jbsb, - "\"pa\":%u," - "\"pa1\":%u," - "\"pa2\":%u," - "\"by\":%" PRIu64 "," - "\"by1\":%" PRIu64 "," - "\"by2\":%" PRIu64 "," - "\"db\":%" PRIu64 "," - "\"db1\":%" PRIu64 "," - "\"db2\":%" PRIu64 "," - "\"ss\":%u," - "\"no\":\"%s\",", + "\"totPackets\":%u," + "\"srcPackets\":%u," + "\"dstPackets\":%u," + "\"totBytes\":%" PRIu64 "," + "\"srcBytes\":%" PRIu64 "," + "\"dstBytes\":%" PRIu64 "," + "\"totDataBytes\":%" PRIu64 "," + "\"srcDataBytes\":%" PRIu64 "," + "\"dstDataBytes\":%" PRIu64 "," + "\"segmentCnt\":%u," + "\"node\":\"%s\",", session->packets[0] + session->packets[1], session->packets[0], session->packets[1], @@ -604,9 +520,9 @@ void moloch_db_save_session(MolochSession_t *session, int final) if (session->rootId) { if (session->rootId[0] == 'R') session->rootId = g_strdup(id); - BSB_EXPORT_sprintf(jbsb, "\"ro\":\"%s\",", session->rootId); + BSB_EXPORT_sprintf(jbsb, "\"rootId\":\"%s\",", session->rootId); } - BSB_EXPORT_cstr(jbsb, "\"ps\":["); + BSB_EXPORT_cstr(jbsb, "\"packetPos\":["); for(i = 0; i < session->filePosArray->len; i++) { if (i != 0) BSB_EXPORT_u08(jbsb, ','); @@ -614,7 +530,7 @@ void moloch_db_save_session(MolochSession_t *session, int final) } BSB_EXPORT_cstr(jbsb, "],"); - BSB_EXPORT_cstr(jbsb, "\"psl\":["); + BSB_EXPORT_cstr(jbsb, "\"packetLen\":["); for(i = 0; i < session->fileLenArray->len; i++) { if (i != 0) BSB_EXPORT_u08(jbsb, ','); @@ -622,7 +538,7 @@ void moloch_db_save_session(MolochSession_t *session, int final) } BSB_EXPORT_cstr(jbsb, "],"); - BSB_EXPORT_cstr(jbsb, "\"fs\":["); + BSB_EXPORT_cstr(jbsb, "\"fileId\":["); for(i = 0; i < session->fileNumArray->len; i++) { if (i == 0) BSB_EXPORT_sprintf(jbsb, "%u", (uint32_t)g_array_index(session->fileNumArray, uint32_t, i)); @@ -668,9 +584,7 @@ void moloch_db_save_session(MolochSession_t *session, int final) break; case MOLOCH_FIELD_TYPE_STR_ARRAY: if (flags & MOLOCH_FIELD_FLAG_CNT) { - BSB_EXPORT_sprintf(jbsb, "\"%scnt\":%d,", config.fields[pos]->dbField, session->fields[pos]->sarray->len); - } else if (flags & MOLOCH_FIELD_FLAG_COUNT) { - BSB_EXPORT_sprintf(jbsb, "\"%s-cnt\":%d,", config.fields[pos]->dbField, session->fields[pos]->sarray->len); + BSB_EXPORT_sprintf(jbsb, "\"%sCnt\":%d,", config.fields[pos]->dbField, session->fields[pos]->sarray->len); } BSB_EXPORT_sprintf(jbsb, "\"%s\":[", config.fields[pos]->dbField); for(i = 0; i < session->fields[pos]->sarray->len; i++) { @@ -688,9 +602,7 @@ void moloch_db_save_session(MolochSession_t *session, int final) case MOLOCH_FIELD_TYPE_STR_HASH: shash = session->fields[pos]->shash; if (flags & MOLOCH_FIELD_FLAG_CNT) { - BSB_EXPORT_sprintf(jbsb, "\"%scnt\":%d,", config.fields[pos]->dbField, HASH_COUNT(s_, *shash)); - } else if (flags & MOLOCH_FIELD_FLAG_COUNT) { - BSB_EXPORT_sprintf(jbsb, "\"%s-cnt\":%d,", config.fields[pos]->dbField, HASH_COUNT(s_, *shash)); + BSB_EXPORT_sprintf(jbsb, "\"%sCnt\":%d,", config.fields[pos]->dbField, HASH_COUNT(s_, *shash)); } BSB_EXPORT_sprintf(jbsb, "\"%s\":[", config.fields[pos]->dbField); HASH_FORALL(s_, *shash, hstring, @@ -707,12 +619,28 @@ void moloch_db_save_session(MolochSession_t *session, int final) BSB_EXPORT_rewind(jbsb, 1); // Remove last comma BSB_EXPORT_cstr(jbsb, "],"); break; + case MOLOCH_FIELD_TYPE_STR_GHASH: + ghash = session->fields[pos]->ghash; + if (flags & MOLOCH_FIELD_FLAG_CNT) { + BSB_EXPORT_sprintf(jbsb, "\"%sCnt\": %d,", config.fields[pos]->dbField, g_hash_table_size(ghash)); + } + BSB_EXPORT_sprintf(jbsb, "\"%s\":[", config.fields[pos]->dbField); + g_hash_table_iter_init (&iter, ghash); + while (g_hash_table_iter_next (&iter, &ikey, NULL)) { + moloch_db_js0n_str(&jbsb, ikey, flags & MOLOCH_FIELD_FLAG_FORCE_UTF8); + BSB_EXPORT_u08(jbsb, ','); + } + + if (freeField) { + g_hash_table_destroy(ghash); + } + BSB_EXPORT_rewind(jbsb, 1); // Remove last comma + BSB_EXPORT_cstr(jbsb, "],"); + break; case MOLOCH_FIELD_TYPE_INT_HASH: ihash = session->fields[pos]->ihash; if (flags & MOLOCH_FIELD_FLAG_CNT) { - BSB_EXPORT_sprintf(jbsb, "\"%scnt\": %d,", config.fields[pos]->dbField, HASH_COUNT(i_, *ihash)); - } else if (flags & MOLOCH_FIELD_FLAG_COUNT) { - BSB_EXPORT_sprintf(jbsb, "\"%s-cnt\": %d,", config.fields[pos]->dbField, HASH_COUNT(i_, *ihash)); + BSB_EXPORT_sprintf(jbsb, "\"%sCnt\": %d,", config.fields[pos]->dbField, HASH_COUNT(i_, *ihash)); } BSB_EXPORT_sprintf(jbsb, "\"%s\":[", config.fields[pos]->dbField); HASH_FORALL(i_, *ihash, hint, @@ -731,9 +659,7 @@ void moloch_db_save_session(MolochSession_t *session, int final) case MOLOCH_FIELD_TYPE_INT_GHASH: ghash = session->fields[pos]->ghash; if (flags & MOLOCH_FIELD_FLAG_CNT) { - BSB_EXPORT_sprintf(jbsb, "\"%scnt\": %d,", config.fields[pos]->dbField, g_hash_table_size(ghash)); - } else if (flags & MOLOCH_FIELD_FLAG_COUNT) { - BSB_EXPORT_sprintf(jbsb, "\"%s-cnt\": %d,", config.fields[pos]->dbField, g_hash_table_size(ghash)); + BSB_EXPORT_sprintf(jbsb, "\"%sCnt\": %d,", config.fields[pos]->dbField, g_hash_table_size(ghash)); } BSB_EXPORT_sprintf(jbsb, "\"%s\":[", config.fields[pos]->dbField); g_hash_table_iter_init (&iter, ghash); @@ -749,26 +675,19 @@ void moloch_db_save_session(MolochSession_t *session, int final) BSB_EXPORT_cstr(jbsb, "],"); break; case MOLOCH_FIELD_TYPE_IP: { - const int value = session->fields[pos]->i; char *as; char *g; char *rir; int asFree; - const int post = (flags & MOLOCH_FIELD_FLAG_IPPRE) == 0; - moloch_db_geo_lookup4(session, value, &g, &as, &rir, &asFree); + ikey = session->fields[pos]->ip; + moloch_db_geo_lookup6(session, *(struct in6_addr *)ikey, &g, &as, &rir, &asFree); if (g) { - if (post) - BSB_EXPORT_sprintf(jbsb, "\"%s-geo\":\"%s\",", config.fields[pos]->dbField, g); - else - BSB_EXPORT_sprintf(jbsb, "\"g%s\":\"%s\",", config.fields[pos]->dbField, g); + BSB_EXPORT_sprintf(jbsb, "\"%.*sGEO\":\"%2.2s\",", config.fields[pos]->dbFieldLen-2, config.fields[pos]->dbField, g); } if (as) { - if (post) - BSB_EXPORT_sprintf(jbsb, "\"%s-asn\":", config.fields[pos]->dbField); - else - BSB_EXPORT_sprintf(jbsb, "\"as%s\":", config.fields[pos]->dbField); + BSB_EXPORT_sprintf(jbsb, "\"%.*sASN\":", config.fields[pos]->dbFieldLen-2, config.fields[pos]->dbField); moloch_db_js0n_str(&jbsb, (unsigned char*)as, TRUE); if (asFree) { free(as); @@ -777,106 +696,22 @@ void moloch_db_save_session(MolochSession_t *session, int final) } if (rir) { - if (post) - BSB_EXPORT_sprintf(jbsb, "\"%s-rir\":\"%s\",", config.fields[pos]->dbField, rir); - else - BSB_EXPORT_sprintf(jbsb, "\"rir%s\":\"%s\",", config.fields[pos]->dbField, rir); - } - - BSB_EXPORT_sprintf(jbsb, "\"%s\":%u,", config.fields[pos]->dbField, htonl(value)); - } - break; - case MOLOCH_FIELD_TYPE_IP_HASH: { - const int post = (flags & MOLOCH_FIELD_FLAG_IPPRE) == 0; - ihash = session->fields[pos]->ihash; - if (flags & MOLOCH_FIELD_FLAG_CNT) { - BSB_EXPORT_sprintf(jbsb, "\"%scnt\":%d,", config.fields[pos]->dbField, HASH_COUNT(i_, *ihash)); - } else if (flags & MOLOCH_FIELD_FLAG_COUNT) { - BSB_EXPORT_sprintf(jbsb, "\"%s-cnt\":%d,", config.fields[pos]->dbField, HASH_COUNT(i_, *ihash)); - } else if (flags & MOLOCH_FIELD_FLAG_SCNT) { - BSB_EXPORT_sprintf(jbsb, "\"%sscnt\":%d,", config.fields[pos]->dbField, HASH_COUNT(i_, *ihash)); - } - - char *as[MAX_IPS]; - char *g[MAX_IPS]; - char *rir[MAX_IPS]; - int asFree[MAX_IPS]; - int i; - int cnt = 0; - - BSB_EXPORT_sprintf(jbsb, "\"%s\":[", config.fields[pos]->dbField); - HASH_FORALL(i_, *ihash, hint, - moloch_db_geo_lookup4(session, hint->i_hash, &g[cnt], &as[cnt], &rir[cnt], &asFree[cnt]); - cnt++; - if (cnt >= MAX_IPS) - break; - BSB_EXPORT_sprintf(jbsb, "%u,", htonl(hint->i_hash)); - ); - BSB_EXPORT_rewind(jbsb, 1); // Remove last comma - BSB_EXPORT_cstr(jbsb, "],"); - - if (post) - BSB_EXPORT_sprintf(jbsb, "\"%s-geo\":[", config.fields[pos]->dbField); - else - BSB_EXPORT_sprintf(jbsb, "\"g%s\":[", config.fields[pos]->dbField); - for (i = 0; i < cnt; i++) { - if (g[i]) { - BSB_EXPORT_sprintf(jbsb, "\"%s\",", g[i]); - } else { - BSB_EXPORT_cstr(jbsb, "\"---\","); - } - } - BSB_EXPORT_rewind(jbsb, 1); // Remove last comma - BSB_EXPORT_cstr(jbsb, "],"); - - if (post) - BSB_EXPORT_sprintf(jbsb, "\"%s-asn\":[", config.fields[pos]->dbField); - else - BSB_EXPORT_sprintf(jbsb, "\"as%s\":[", config.fields[pos]->dbField); - for (i = 0; i < cnt; i++) { - if (as[i]) { - moloch_db_js0n_str(&jbsb, (unsigned char*)as[i], TRUE); - BSB_EXPORT_u08(jbsb, ','); - if(asFree[i]) - free(as[i]); - } else { - BSB_EXPORT_cstr(jbsb, "\"---\","); - } + BSB_EXPORT_sprintf(jbsb, "\"%.*sRIR\":\"%s\",", config.fields[pos]->dbFieldLen-2, config.fields[pos]->dbField, rir); } - BSB_EXPORT_rewind(jbsb, 1); // Remove last comma - BSB_EXPORT_cstr(jbsb, "],"); - if (post) - BSB_EXPORT_sprintf(jbsb, "\"%s-rir\":[", config.fields[pos]->dbField); - else - BSB_EXPORT_sprintf(jbsb, "\"rir%s\":[", config.fields[pos]->dbField); - for (i = 0; i < cnt; i++) { - if (rir[i]) { - BSB_EXPORT_sprintf(jbsb, "\"%s\",", rir[i]); - } else { - BSB_EXPORT_cstr(jbsb, "\"\","); - } + if (IN6_IS_ADDR_V4MAPPED((struct in6_addr *)ikey)) { + uint32_t ip = MOLOCH_V6_TO_V4(*(struct in6_addr *)ikey); + snprintf(ipsrc, sizeof(ipsrc), "%d.%d.%d.%d", ip & 0xff, (ip >> 8) & 0xff, (ip >> 16) & 0xff, (ip >> 24) & 0xff); + } else { + inet_ntop(AF_INET6, ikey, ipsrc, sizeof(ipsrc)); } - BSB_EXPORT_rewind(jbsb, 1); // Remove last comma - BSB_EXPORT_cstr(jbsb, "],"); - - if (freeField) { - HASH_FORALL_POP_HEAD(i_, *ihash, hint, - MOLOCH_TYPE_FREE(MolochInt_t, hint); - ); - MOLOCH_TYPE_FREE(MolochIntHashStd_t, ihash); + BSB_EXPORT_sprintf(jbsb, "\"%s\":\"%s\",", config.fields[pos]->dbField, ipsrc); } break; - } case MOLOCH_FIELD_TYPE_IP_GHASH: { - const int post = (flags & MOLOCH_FIELD_FLAG_IPPRE) == 0; ghash = session->fields[pos]->ghash; if (flags & MOLOCH_FIELD_FLAG_CNT) { - BSB_EXPORT_sprintf(jbsb, "\"%scnt\":%d,", config.fields[pos]->dbField, g_hash_table_size(ghash)); - } else if (flags & MOLOCH_FIELD_FLAG_COUNT) { - BSB_EXPORT_sprintf(jbsb, "\"%s-cnt\":%d,", config.fields[pos]->dbField, g_hash_table_size(ghash)); - } else if (flags & MOLOCH_FIELD_FLAG_SCNT) { - BSB_EXPORT_sprintf(jbsb, "\"%sscnt\":%d,", config.fields[pos]->dbField, g_hash_table_size(ghash)); + BSB_EXPORT_sprintf(jbsb, "\"%sCnt\":%d,", config.fields[pos]->dbField, g_hash_table_size(ghash)); } char *as[MAX_IPS]; @@ -889,23 +724,27 @@ void moloch_db_save_session(MolochSession_t *session, int final) BSB_EXPORT_sprintf(jbsb, "\"%s\":[", config.fields[pos]->dbField); g_hash_table_iter_init (&iter, ghash); while (g_hash_table_iter_next (&iter, &ikey, NULL)) { - moloch_db_geo_lookup4(session, (int)(long)ikey, &g[cnt], &as[cnt], &rir[cnt], &asFree[cnt]); + moloch_db_geo_lookup6(session, *(struct in6_addr *)ikey, &g[cnt], &as[cnt], &rir[cnt], &asFree[cnt]); cnt++; if (cnt >= MAX_IPS) break; - BSB_EXPORT_sprintf(jbsb, "%u,", htonl((int)(long)ikey)); + if (IN6_IS_ADDR_V4MAPPED((struct in6_addr *)ikey)) { + uint32_t ip = MOLOCH_V6_TO_V4(*(struct in6_addr *)ikey); + snprintf(ipsrc, sizeof(ipsrc), "%d.%d.%d.%d", ip & 0xff, (ip >> 8) & 0xff, (ip >> 16) & 0xff, (ip >> 24) & 0xff); + } else { + inet_ntop(AF_INET6, ikey, ipsrc, sizeof(ipsrc)); + } + + BSB_EXPORT_sprintf(jbsb, "\"%s\",", ipsrc); } BSB_EXPORT_rewind(jbsb, 1); // Remove last comma BSB_EXPORT_cstr(jbsb, "],"); - if (post) - BSB_EXPORT_sprintf(jbsb, "\"%s-geo\":[", config.fields[pos]->dbField); - else - BSB_EXPORT_sprintf(jbsb, "\"g%s\":[", config.fields[pos]->dbField); + BSB_EXPORT_sprintf(jbsb, "\"%.*sGEO\":[", config.fields[pos]->dbFieldLen-2, config.fields[pos]->dbField); for (i = 0; i < cnt; i++) { if (g[i]) { - BSB_EXPORT_sprintf(jbsb, "\"%s\",", g[i]); + BSB_EXPORT_sprintf(jbsb, "\"%2.2s\",", g[i]); } else { BSB_EXPORT_cstr(jbsb, "\"---\","); } @@ -913,10 +752,7 @@ void moloch_db_save_session(MolochSession_t *session, int final) BSB_EXPORT_rewind(jbsb, 1); // Remove last comma BSB_EXPORT_cstr(jbsb, "],"); - if (post) - BSB_EXPORT_sprintf(jbsb, "\"%s-asn\":[", config.fields[pos]->dbField); - else - BSB_EXPORT_sprintf(jbsb, "\"as%s\":[", config.fields[pos]->dbField); + BSB_EXPORT_sprintf(jbsb, "\"%.*sASN\":[", config.fields[pos]->dbFieldLen-2, config.fields[pos]->dbField); for (i = 0; i < cnt; i++) { if (as[i]) { moloch_db_js0n_str(&jbsb, (unsigned char*)as[i], TRUE); @@ -930,10 +766,7 @@ void moloch_db_save_session(MolochSession_t *session, int final) BSB_EXPORT_rewind(jbsb, 1); // Remove last comma BSB_EXPORT_cstr(jbsb, "],"); - if (post) - BSB_EXPORT_sprintf(jbsb, "\"%s-rir\":[", config.fields[pos]->dbField); - else - BSB_EXPORT_sprintf(jbsb, "\"rir%s\":[", config.fields[pos]->dbField); + BSB_EXPORT_sprintf(jbsb, "\"%.*sRIR\":[", config.fields[pos]->dbFieldLen-2, config.fields[pos]->dbField); for (i = 0; i < cnt; i++) { if (rir[i]) { BSB_EXPORT_sprintf(jbsb, "\"%s\",", rir[i]); @@ -953,8 +786,8 @@ void moloch_db_save_session(MolochSession_t *session, int final) case MOLOCH_FIELD_TYPE_CERTSINFO: { MolochCertsInfoHashStd_t *cihash = session->fields[pos]->cihash; - BSB_EXPORT_sprintf(jbsb, "\"tlscnt\":%d,", HASH_COUNT(t_, *cihash)); - BSB_EXPORT_cstr(jbsb, "\"tls\":["); + BSB_EXPORT_sprintf(jbsb, "\"certCnt\":%d,", HASH_COUNT(t_, *cihash)); + BSB_EXPORT_cstr(jbsb, "\"cert\":["); MolochCertsInfo_t *certs; MolochString_t *string; @@ -963,7 +796,7 @@ void moloch_db_save_session(MolochSession_t *session, int final) BSB_EXPORT_u08(jbsb, '{'); if (certs->issuer.commonName.s_count > 0) { - BSB_EXPORT_cstr(jbsb, "\"iCn\":["); + BSB_EXPORT_cstr(jbsb, "\"issuerCN\":["); while (certs->issuer.commonName.s_count > 0) { DLL_POP_HEAD(s_, &certs->issuer.commonName, string); moloch_db_js0n_str(&jbsb, (unsigned char *)string->str, string->utf8); @@ -979,13 +812,13 @@ void moloch_db_save_session(MolochSession_t *session, int final) BSB_EXPORT_sprintf(jbsb, "\"hash\":\"%s\",", certs->hash); if (certs->issuer.orgName) { - BSB_EXPORT_cstr(jbsb, "\"iOn\":"); + BSB_EXPORT_cstr(jbsb, "\"issuerON\":"); moloch_db_js0n_str(&jbsb, (unsigned char *)certs->issuer.orgName, certs->issuer.orgUtf8); BSB_EXPORT_u08(jbsb, ','); } if (certs->subject.commonName.s_count) { - BSB_EXPORT_cstr(jbsb, "\"sCn\":["); + BSB_EXPORT_cstr(jbsb, "\"subjectCN\":["); while (certs->subject.commonName.s_count > 0) { DLL_POP_HEAD(s_, &certs->subject.commonName, string); moloch_db_js0n_str(&jbsb, (unsigned char *)string->str, string->utf8); @@ -999,14 +832,14 @@ void moloch_db_save_session(MolochSession_t *session, int final) } if (certs->subject.orgName) { - BSB_EXPORT_cstr(jbsb, "\"sOn\":"); + BSB_EXPORT_cstr(jbsb, "\"subjectON\":"); moloch_db_js0n_str(&jbsb, (unsigned char *)certs->subject.orgName, certs->subject.orgUtf8); BSB_EXPORT_u08(jbsb, ','); } if (certs->serialNumber) { int k; - BSB_EXPORT_cstr(jbsb, "\"sn\":\""); + BSB_EXPORT_cstr(jbsb, "\"serial\":\""); for (k = 0; k < certs->serialNumberLen; k++) { BSB_EXPORT_sprintf(jbsb, "%02x", certs->serialNumber[k]); } @@ -1015,7 +848,7 @@ void moloch_db_save_session(MolochSession_t *session, int final) } if (certs->alt.s_count) { - BSB_EXPORT_sprintf(jbsb, "\"altcnt\":%d,", certs->alt.s_count); + BSB_EXPORT_sprintf(jbsb, "\"altCnt\":%d,", certs->alt.s_count); BSB_EXPORT_cstr(jbsb, "\"alt\":["); while (certs->alt.s_count > 0) { DLL_POP_HEAD(s_, &certs->alt, string); @@ -1029,10 +862,10 @@ void moloch_db_save_session(MolochSession_t *session, int final) BSB_EXPORT_u08(jbsb, ','); } - BSB_EXPORT_sprintf(jbsb, "\"notBefore\": %" PRId64 ",", certs->notBefore); - BSB_EXPORT_sprintf(jbsb, "\"notAfter\": %" PRId64 ",", certs->notAfter); + BSB_EXPORT_sprintf(jbsb, "\"notBefore\": %" PRId64 ",", certs->notBefore*1000); + BSB_EXPORT_sprintf(jbsb, "\"notAfter\": %" PRId64 ",", certs->notAfter*1000); if (certs->notAfter >= certs->notBefore) - BSB_EXPORT_sprintf(jbsb, "\"diffDays\": %" PRId64 ",", (certs->notAfter - certs->notBefore)/(60*60*24)); + BSB_EXPORT_sprintf(jbsb, "\"validDays\": %" PRId64 ",", (certs->notAfter - certs->notBefore)/(60*60*24)); BSB_EXPORT_rewind(jbsb, 1); // Remove last comma @@ -1067,6 +900,8 @@ void moloch_db_save_session(MolochSession_t *session, int final) goto cleanup; } + MOLOCH_THREAD_INCR_NUM(totalSessionBytes, (int)(BSB_WORK_PTR(jbsb)-dataPtr)); + if (config.dryRun) { if (config.tests) { static int outputed; @@ -1099,7 +934,7 @@ void moloch_db_save_session(MolochSession_t *session, int final) MOLOCH_UNLOCK(dbInfo[thread].lock); } /******************************************************************************/ -long long zero_atoll(char *v) { +LOCAL uint64_t zero_atoll(char *v) { if (v) return atoll(v); return 0; @@ -1107,15 +942,15 @@ long long zero_atoll(char *v) { /******************************************************************************/ #define NUMBER_OF_STATS 4 -static uint64_t dbTotalPackets[NUMBER_OF_STATS]; -static uint64_t dbTotalK[NUMBER_OF_STATS]; -static uint64_t dbTotalSessions[NUMBER_OF_STATS]; -static uint64_t dbTotalDropped[NUMBER_OF_STATS]; +LOCAL uint64_t dbTotalPackets[NUMBER_OF_STATS]; +LOCAL uint64_t dbTotalK[NUMBER_OF_STATS]; +LOCAL uint64_t dbTotalSessions[NUMBER_OF_STATS]; +LOCAL uint64_t dbTotalDropped[NUMBER_OF_STATS]; -static char stats_key[200]; -static int stats_key_len = 0; +LOCAL char stats_key[200]; +LOCAL int stats_key_len = 0; -void moloch_db_load_stats() +LOCAL void moloch_db_load_stats() { size_t data_len; uint32_t len; @@ -1145,14 +980,14 @@ void moloch_db_load_stats() } /******************************************************************************/ #if defined(__APPLE__) && defined(__MACH__) -uint64_t moloch_db_memory_size() +LOCAL uint64_t moloch_db_memory_size() { struct rusage usage; getrusage(RUSAGE_SELF, &usage); return usage.ru_maxrss; } #elif defined(__linux__) -uint64_t moloch_db_memory_size() +LOCAL uint64_t moloch_db_memory_size() { int fd = open("/proc/self/statm", O_RDONLY, 0); if (fd == -1) @@ -1180,7 +1015,7 @@ uint64_t moloch_db_memory_size() return getpagesize() * size; } #else -uint64_t moloch_db_memory_size() +LOCAL uint64_t moloch_db_memory_size() { struct rusage usage; getrusage(RUSAGE_SELF, &usage); @@ -1188,17 +1023,18 @@ uint64_t moloch_db_memory_size() } #endif /******************************************************************************/ -uint64_t moloch_db_memory_max() +LOCAL uint64_t moloch_db_memory_max() { return (uint64_t)sysconf (_SC_PHYS_PAGES) * (uint64_t)sysconf (_SC_PAGESIZE); } /******************************************************************************/ -void moloch_db_update_stats(int n, gboolean sync) +LOCAL void moloch_db_update_stats(int n, gboolean sync) { static uint64_t lastPackets[NUMBER_OF_STATS]; static uint64_t lastBytes[NUMBER_OF_STATS]; static uint64_t lastSessions[NUMBER_OF_STATS]; + static uint64_t lastSessionBytes[NUMBER_OF_STATS]; static uint64_t lastDropped[NUMBER_OF_STATS]; static uint64_t lastFragsDropped[NUMBER_OF_STATS]; static uint64_t lastOverloadDropped[NUMBER_OF_STATS]; @@ -1225,6 +1061,7 @@ void moloch_db_update_stats(int n, gboolean sync) uint64_t totalDropped = moloch_packet_dropped_packets(); uint64_t fragsDropped = moloch_packet_dropped_frags(); uint64_t esDropped = moloch_http_dropped_count(esServer); + uint64_t totalBytes = moloch_packet_total_bytes(); for (i = 0; config.pcapDir[i]; i++) { struct statvfs vfs; @@ -1292,6 +1129,7 @@ void moloch_db_update_stats(int n, gboolean sync) "\"deltaPackets\": %" PRIu64 ", " "\"deltaBytes\": %" PRIu64 ", " "\"deltaSessions\": %" PRIu64 ", " + "\"deltaSessionBytes\": %" PRIu64 ", " "\"deltaDropped\": %" PRIu64 ", " "\"deltaFragsDropped\": %" PRIu64 ", " "\"deltaOverloadDropped\": %" PRIu64 ", " @@ -1326,6 +1164,7 @@ void moloch_db_update_stats(int n, gboolean sync) (totalPackets - lastPackets[n]), (totalBytes - lastBytes[n]), (totalSessions - lastSessions[n]), + (totalSessionBytes - lastSessionBytes[n]), (totalDropped - lastDropped[n]), (fragsDropped - lastFragsDropped[n]), (overloadDropped - lastOverloadDropped[n]), @@ -1336,6 +1175,7 @@ void moloch_db_update_stats(int n, gboolean sync) lastBytes[n] = totalBytes; lastPackets[n] = totalPackets; lastSessions[n] = totalSessions; + lastSessionBytes[n] = totalSessionBytes; lastDropped[n] = totalDropped; lastFragsDropped[n] = fragsDropped; lastOverloadDropped[n] = overloadDropped; @@ -1353,7 +1193,7 @@ void moloch_db_update_stats(int n, gboolean sync) } } /******************************************************************************/ -gboolean moloch_db_update_stats_gfunc (gpointer user_data) +LOCAL gboolean moloch_db_update_stats_gfunc (gpointer user_data) { moloch_db_update_stats((long)user_data, 0); @@ -1361,7 +1201,7 @@ gboolean moloch_db_update_stats_gfunc (gpointer user_data) } /******************************************************************************/ // Runs on main thread -gboolean moloch_db_flush_gfunc (gpointer user_data ) +LOCAL gboolean moloch_db_flush_gfunc (gpointer user_data ) { int thread; struct timeval currentTime; @@ -1396,7 +1236,7 @@ typedef struct moloch_seq_request { } MolochSeqRequest_t; void moloch_db_get_sequence_number(char *name, MolochSeqNum_cb func, gpointer uw); -void moloch_db_get_sequence_number_cb(int UNUSED(code), unsigned char *data, int data_len, gpointer uw) +LOCAL void moloch_db_get_sequence_number_cb(int UNUSED(code), unsigned char *data, int data_len, gpointer uw) { MolochSeqRequest_t *r = uw; uint32_t version_len; @@ -1456,14 +1296,14 @@ uint32_t moloch_db_get_sequence_number_sync(char *name) } } /******************************************************************************/ -void moloch_db_fn_seq_cb(uint32_t newSeq, gpointer UNUSED(uw)) +LOCAL void moloch_db_fn_seq_cb(uint32_t newSeq, gpointer UNUSED(uw)) { MOLOCH_LOCK(nextFileNum); nextFileNum = newSeq; MOLOCH_UNLOCK(nextFileNum); } /******************************************************************************/ -void moloch_db_load_file_num() +LOCAL void moloch_db_load_file_num() { char key[200]; int key_len; @@ -1535,7 +1375,7 @@ void moloch_db_load_file_num() /******************************************************************************/ // Modified From https://github.com/phaag/nfdump/blob/master/bin/flist.c // Copyright (c) 2014, Peter Haag -void moloch_db_mkpath(char *path) +LOCAL void moloch_db_mkpath(char *path) { struct stat sb; char *slash = path; @@ -1724,352 +1564,279 @@ char *moloch_db_create_file(time_t firstPacket, char *name, uint64_t size, int l return moloch_db_create_file_full(firstPacket, name, size, locked, id, NULL); } /******************************************************************************/ -void moloch_db_check() +LOCAL void moloch_db_check() { size_t data_len; - char key[100]; + char key[1000]; int key_len; + char tname[100]; unsigned char *data; - key_len = snprintf(key, sizeof(key), "/%sdstats/version/version/_source", config.prefix); + snprintf(tname, sizeof(tname), "%ssessions2_template", config.prefix); + + key_len = snprintf(key, sizeof(key), "/_template/%s?filter_path=**._meta", tname); data = moloch_http_get(esServer, key, key_len, &data_len); if (!data || data_len == 0) { LOGEXIT("ERROR - Couldn't load version information, database might be down or out of date. Run \"db/db.pl host:port upgrade\""); } - uint32_t version_len; - unsigned char *version = 0; + uint32_t template_len; + unsigned char *template = 0; - version = moloch_js0n_get(data, data_len, "version", &version_len); - - if (!version || atoi((char*)version) < MOLOCH_MIN_DB_VERSION) { - LOGEXIT("ERROR - Database version '%.*s' is too old, needs to be at least (%d), run \"db/db.pl host:port upgrade\"", version_len, version, MOLOCH_MIN_DB_VERSION); + template = moloch_js0n_get(data, data_len, tname, &template_len); + if(!template || template_len == 0) { + LOGEXIT("ERROR - Couldn't load version information, database might be down or out of date. Run \"db/db.pl host:port upgrade\""); } - free(data); -} - -/******************************************************************************/ -void moloch_db_load_tags() -{ - size_t data_len; - char key[100]; - int key_len; - key_len = snprintf(key, sizeof(key), "/%stags/tag/_search?size=3000", config.prefix); - unsigned char *data = moloch_http_get(esServer, key, key_len, &data_len); - - if (!data) { - return; - } + uint32_t mappings_len; + unsigned char *mappings = 0; - uint32_t hits_len; - unsigned char *hits = 0; - hits = moloch_js0n_get(data, data_len, "hits", &hits_len); - if (!hits) { - free(data); - return; + mappings = moloch_js0n_get(template, template_len, "mappings", &mappings_len); + if(!mappings || mappings_len == 0) { + LOGEXIT("ERROR - Couldn't load version information, database might be down or out of date. Run \"db/db.pl host:port upgrade\""); } - uint32_t ahits_len; - unsigned char *ahits = 0; - ahits = moloch_js0n_get(hits, hits_len, "hits", &ahits_len); + uint32_t session_len; + unsigned char *session = 0; - if (!ahits) { - free(data); - return; + session = moloch_js0n_get(mappings, mappings_len, "session", &session_len); + if(!session || session_len == 0) { + LOGEXIT("ERROR - Couldn't load version information, database might be down or out of date. Run \"db/db.pl host:port upgrade\""); } - uint32_t out[2*8000]; - memset(out, 0, sizeof(out)); - js0n(ahits, ahits_len, out); - int i; - for (i = 0; out[i]; i+= 2) { - uint32_t id_len; - unsigned char *id = 0; - id = moloch_js0n_get(ahits+out[i], out[i+1], "_id", &id_len); + uint32_t meta_len; + unsigned char *meta = 0; - uint32_t source_len; - unsigned char *source = 0; - source = moloch_js0n_get(ahits+out[i], out[i+1], "_source", &source_len); - if (!source) { - continue; - } + meta = moloch_js0n_get(session, session_len, "_meta", &meta_len); + if(!meta || meta_len == 0) { + LOGEXIT("ERROR - Couldn't load version information, database might be down or out of date. Run \"db/db.pl host:port upgrade\""); + } - uint32_t n_len; - unsigned char *n = 0; - n = moloch_js0n_get(source, source_len, "n", &n_len); + uint32_t version_len; + unsigned char *version = 0; + version = moloch_js0n_get(meta, meta_len, "molochDbVersion", &version_len); - if (id && n) { - MolochTag_t *tag = MOLOCH_TYPE_ALLOC(MolochTag_t); - tag->tagName = g_strndup((char*)id, (int)id_len); - if (*n == '[') - tag->tagValue = atol((char*)n+1); - else - tag->tagValue = atol((char*)n); - HASH_ADD(tag_, tags, tag->tagName, tag); - } else { - LOG ("ERROR - Could not load %.*s", out[i+1], ahits+out[i]); - } + if (!version || atoi((char*)version) < MOLOCH_MIN_DB_VERSION) { + LOGEXIT("ERROR - Database version '%.*s' is too old, needs to be at least (%d), run \"db/db.pl host:port upgrade\"", version_len, version, MOLOCH_MIN_DB_VERSION); } free(data); } -typedef struct moloch_tag_request { - struct moloch_tag_request *t_next, *t_prev; - int t_count; - void *uw; - MolochTag_cb func; - int tagtype; - char *tag; - char *escaped; - uint32_t newSeq; -} MolochTagRequest_t; - -LOCAL int outstandingTagRequests = 0; -LOCAL MolochTagRequest_t tagRequests; -LOCAL MOLOCH_LOCK_DEFINE(tagRequests); - -void moloch_db_tag_cb(int code, unsigned char *data, int data_len, gpointer uw); - -/******************************************************************************/ -int moloch_db_tags_loading() { - return outstandingTagRequests + tagRequests.t_count; -} /******************************************************************************/ -void moloch_db_free_tag_request(MolochTagRequest_t *r) +LOCAL void moloch_db_load_geo_country(char *name) { - g_free(r->escaped); - free(r->tag); - MOLOCH_TYPE_FREE(MolochTagRequest_t, r); - outstandingTagRequests--; - - while (tagRequests.t_count > 0) { - char key[500]; - int key_len; - - MOLOCH_LOCK(tagRequests); - DLL_POP_HEAD(t_, &tagRequests, r); - MOLOCH_UNLOCK(tagRequests); - - MolochTag_t *tag; - HASH_FIND(tag_, tags, r->tag, tag); - - if (tag) { - if (r->func) - r->func(r->uw, r->tagtype, r->tag, tag->tagValue, TRUE); - g_free(r->escaped); - free(r->tag); - MOLOCH_TYPE_FREE(MolochTagRequest_t, r); - continue; - } + static MMDB_s *countryOld; - key_len = snprintf(key, sizeof(key), "/%stags/tag/%s", config.prefix, r->escaped); - moloch_http_send(esServer, "GET", key, key_len, NULL, 0, NULL, FALSE, moloch_db_tag_cb, r); - outstandingTagRequests++; - break; - } -} -/******************************************************************************/ -void moloch_db_tag_seq_cb(uint32_t newSeq, gpointer uw); -void moloch_db_tag_create_cb(int code, unsigned char *data, int UNUSED(data_len), gpointer uw) -{ - MolochTagRequest_t *r = uw; - char key[500]; - int key_len; - - // Try again on error - if (code == 0) { - moloch_db_tag_seq_cb(r->newSeq, uw); + // Reload country + if (!name) { + MMDB_close(countryOld); + g_free(countryOld); + countryOld = NULL; return; } - if (strstr((char *)data, "{\"error\":") != 0) { - key_len = snprintf(key, sizeof(key), "/%stags/tag/%s", config.prefix, r->escaped); - moloch_http_send(esServer, "GET", key, key_len, NULL, 0, NULL, FALSE, moloch_db_tag_cb, r); - return; - } + MMDB_s *country = malloc(sizeof(MMDB_s)); + int status = MMDB_open(name, MMDB_MODE_MMAP, country); + if (MMDB_SUCCESS != status) { + LOGEXIT("Couldn't initialize Country file %s error %s", name, MMDB_strerror(status)); - MolochTag_t *tag = MOLOCH_TYPE_ALLOC(MolochTag_t); - tag->tagName = g_strdup(r->tag); - tag->tagValue = r->newSeq; - HASH_ADD(tag_, tags, tag->tagName, tag); + } + if (geoCountry) + LOG("Loading new version of country file"); - if (r->func) - r->func(r->uw, r->tagtype, r->tag, r->newSeq, TRUE); - moloch_db_free_tag_request(r); + countryOld = geoCountry; + geoCountry = country; } /******************************************************************************/ -void moloch_db_tag_seq_cb(uint32_t newSeq, gpointer uw) +LOCAL void moloch_db_load_geo_asn(char *name) { - MolochTagRequest_t *r = uw; - char key[500]; - int key_len; - char *json = moloch_http_get_buffer(MOLOCH_HTTP_BUFFER_SIZE); + static MMDB_s *asnOld; + + // Reload asn + if (!name) { + MMDB_close(asnOld); + g_free(asnOld); + asnOld = NULL; + return; + } - r->newSeq = newSeq; + MMDB_s *asn = malloc(sizeof(MMDB_s)); + int status = MMDB_open(name, MMDB_MODE_MMAP, asn); + if (MMDB_SUCCESS != status) { + LOGEXIT("Couldn't initialize ASN file %s error %s", name, MMDB_strerror(status)); - key_len = snprintf(key, sizeof(key), "/%stags/tag/%s?op_type=create", config.prefix, r->escaped); - int json_len = snprintf(json, MOLOCH_HTTP_BUFFER_SIZE, "{\"n\":%u}", newSeq); + } + if (geoASN) + LOG("Loading new version of asn file"); - moloch_http_set(esServer, key, key_len, json, json_len, moloch_db_tag_create_cb, r); + asnOld = geoASN; + geoASN = asn; } /******************************************************************************/ -void moloch_db_tag_cb(int UNUSED(code), unsigned char *data, int data_len, gpointer uw) +LOCAL void moloch_db_load_rir(char *name) { - MolochTagRequest_t *r = uw; + static char *oldRirs[256]; - if (!data) { - if (r->func) - r->func(r->uw, r->tagtype, r->tag, 0, TRUE); - moloch_db_free_tag_request(r); + if (!name) { + int i; + for (i = 0; i < 256; i++) { + if (oldRirs[i]) { + g_free(oldRirs[i]); + oldRirs[i] = NULL; + } + } return; } - uint32_t source_len; - unsigned char *source = 0; - source = moloch_js0n_get(data, data_len, "_source", &source_len); + FILE *fp; + char line[1000]; + if (!(fp = fopen(name, "r"))) { + printf("Couldn't open RIR from %s", name); + exit(1); + } - if (source) { - uint32_t n_len; - unsigned char *n = 0; - n = moloch_js0n_get(source, source_len, "n", &n_len); - - MolochTag_t *tag = MOLOCH_TYPE_ALLOC(MolochTag_t); - tag->tagName = g_strdup(r->tag); - if (*n == '[') - tag->tagValue = atol((char*)n+1); - else - tag->tagValue = atol((char*)n); - HASH_ADD(tag_, tags, tag->tagName, tag); + while(fgets(line, sizeof(line), fp)) { + int cnt = 0, quote = 0, num = 0; + char *ptr, *start; - if (r->func) - r->func(r->uw, r->tagtype, r->tag, tag->tagValue, TRUE); - moloch_db_free_tag_request(r); - return; - } + for (start = ptr = line; *ptr != 0; ptr++) { + if (*ptr == '"') { + quote = !quote; + continue; + } - moloch_db_get_sequence_number("tags", moloch_db_tag_seq_cb, r) ; -} -/******************************************************************************/ -uint32_t moloch_db_peek_tag(const char *tagname) -{ - MolochTag_t *tag; - HASH_FIND(tag_, tags, tagname, tag); + if (quote || *ptr != ',') + continue; - if (!tag) - return 0; - return tag->tagValue; + // We have comma outside of quotes + *ptr = 0; + if (cnt == 0) { + num = atoi(start); + if (num > 255) + break; + } else if (*start && cnt == 3) { + gchar **parts = g_strsplit(start, ".", 0); + if (parts[1] && *parts[1]) { + oldRirs[num] = rirs[num]; + rirs[num] = g_ascii_strup(parts[1], -1); + } + g_strfreev(parts); + + break; + } + + cnt++; + start = ptr+1; + } + } + fclose(fp); } /******************************************************************************/ -void moloch_db_get_tag(void *uw, int tagtype, const char *tagname, MolochTag_cb func) +/* Only called in main thread. Check if the file changed, if so reload. + * Don't free old version until called again incase other threads are using. + */ +LOCAL void moloch_db_load_oui(char *name) { - MolochTag_t *tag; - HASH_FIND(tag_, tags, tagname, tag); + static patricia_tree_t *ouiOld; - if (tag) { - if (func) - func(uw, tagtype, tagname, tag->tagValue, FALSE); + // Clean up old elements + if (!name) { + Destroy_Patricia(ouiOld, g_free); + ouiOld = NULL; return; } - if (config.dryRun) { - static int tagNum = 1; - MOLOCH_LOCK(tagRequests); - - HASH_FIND(tag_, tags, tagname, tag); - - if (tag) { - MOLOCH_UNLOCK(tagRequests); - if (func) - func(uw, tagtype, tagname, tag->tagValue, FALSE); - return; + if (ouiTree) + LOG("Loading new version of oui file"); + + // Load the data + patricia_tree_t *oui = New_Patricia(48); // 48 - Ethernet Size + FILE *fp; + char line[2000]; + if (!(fp = fopen(config.ouiFile, "r"))) { + printf("Couldn't open OUI from %s", config.ouiFile); + exit(1); + } + + while(fgets(line, sizeof(line), fp)) { + char *hash = strchr(line, '#'); + if (hash) + *hash = 0; + + // Trim + int len = strlen(line); + if (len < 4) continue; + while (len > 0 && isspace(line[len-1]) ) + len--; + line[len] = 0; + + // Break into pieces + gchar **parts = g_strsplit(line, "\t", 0); + char *str; + if (parts[2]) { + if (parts[2][0]) + str = parts[2]; + else if (parts[3]) // The file sometimes has 2 tabs in a row :( + str = parts[3]; + } else { + str = parts[1]; } - MolochTag_t *tag = MOLOCH_TYPE_ALLOC(MolochTag_t); - tag->tagName = g_strdup(tagname); - tag->tagValue = tagNum++; - HASH_ADD(tag_, tags, tag->tagName, tag); - MOLOCH_UNLOCK(tagRequests); - - if (func) - func(uw, tagtype, tagname, tag->tagValue, FALSE); - return; - } + // Remove separators and get bitlen + int i = 0, j = 0, bitlen = 24; + for (i = 0; parts[0][i]; i++) { + if (parts[0][i] == ':' || parts[0][i] == '-' || parts[0][i] == '.') + continue; + if (parts[0][i] == '/') { + bitlen = atoi(parts[0] + i + 1); + break; + } - MolochTagRequest_t *r = MOLOCH_TYPE_ALLOC(MolochTagRequest_t); - r->uw = uw; - r->func = func; - r->tag = strdup(tagname); - r->tagtype = tagtype; - r->escaped = g_uri_escape_string (tagname, G_URI_RESERVED_CHARS_ALLOWED_IN_PATH_ELEMENT, 0); + parts[0][j] = parts[0][i]; + j++; + } + parts[0][j] = 0; + // Convert to binary + unsigned char buf[16]; + for (i=0, j=0; i < len && j < 8; i += 2, j++) { + buf[j] = moloch_hex_to_char[(int)parts[0][i]][(int)parts[0][i+1]]; + } + // Create node + prefix_t *prefix; + patricia_node_t *node; - if (outstandingTagRequests == 0) { - char key[500]; - int key_len; + prefix = New_Prefix2(AF_INET6, buf, bitlen, NULL); + node = patricia_lookup(oui, prefix); + Deref_Prefix(prefix); + node->data = g_strdup(str); - key_len = snprintf(key, sizeof(key), "/%stags/tag/%s", config.prefix, r->escaped); - moloch_http_send(esServer, "GET", key, key_len, NULL, 0, NULL, FALSE, moloch_db_tag_cb, r); - outstandingTagRequests++; - } else { - MOLOCH_LOCK(tagRequests); - DLL_PUSH_TAIL(t_, &tagRequests, r); - MOLOCH_UNLOCK(tagRequests); + g_strfreev(parts); } + fclose(fp); + + // Save old tree to free later and flip to new tree + ouiOld = ouiTree; + ouiTree = oui; } /******************************************************************************/ -void moloch_db_load_rir() +void moloch_db_oui_lookup(int field, MolochSession_t *session, const uint8_t *mac) { - memset(rirs, 0, sizeof(rirs)); - if (config.rirFile) { - FILE *fp; - char line[1000]; - if (!(fp = fopen(config.rirFile, "r"))) { - printf("Couldn't open RIR from %s", config.rirFile); - exit(1); - } - - while(fgets(line, sizeof(line), fp)) { - int cnt = 0, quote = 0, num = 0; - char *ptr, *start; - - for (start = ptr = line; *ptr != 0; ptr++) { - if (*ptr == '"') { - quote = !quote; - continue; - } + patricia_node_t *node; - if (quote || *ptr != ',') - continue; - - // We have comma outside of quotes - *ptr = 0; - if (cnt == 0) { - num = atoi(start); - if (num >= 255) - break; - } else if (*start && cnt == 3) { - gchar **parts = g_strsplit(start, ".", 0); - if (parts[1] && *parts[1]) { - rirs[num] = g_ascii_strup(parts[1], -1); - } - g_strfreev(parts); + if (!ouiTree) + return; - break; - } + if ((node = patricia_search_best3 (ouiTree, mac, 48)) == NULL) + return; - cnt++; - start = ptr+1; - } - } - fclose(fp); - } + moloch_field_string_add(field, session, node->data, -1, TRUE); } /******************************************************************************/ -void moloch_db_load_fields() +LOCAL void moloch_db_load_fields() { size_t data_len; char key[100]; @@ -2136,7 +1903,7 @@ void moloch_db_add_field(char *group, char *kind, char *expression, char *friend key_len = snprintf(key, sizeof(key), "/%sfields/field/%s", config.prefix, expression); - BSB_EXPORT_sprintf(bsb, "{\"friendlyName\": \"%s\", \"group\": \"%s\", \"help\": \"%s\", \"dbField\": \"%s\", \"type\": \"%s\"", + BSB_EXPORT_sprintf(bsb, "{\"friendlyName\": \"%s\", \"group\": \"%s\", \"help\": \"%s\", \"dbField2\": \"%s\", \"type\": \"%s\"", friendlyName, group, help, @@ -2245,18 +2012,6 @@ gboolean moloch_db_file_exists(char *filename) /******************************************************************************/ int moloch_db_can_quit() { - if (outstandingTagRequests > 0) { - if (config.debug) - LOG ("Can't quit, outstandingTagRequests %d", outstandingTagRequests); - return 1; - } - - if (tagRequests.t_count > 0) { - if (config.debug) - LOG ("Can't quit, tagRequests %d", tagRequests.t_count); - return 1; - } - int thread; for (thread = 0; thread < config.packetThreads; thread++) { if (dbInfo[thread].json && BSB_LENGTH(dbInfo[thread].bsb) > 0) { @@ -2276,11 +2031,11 @@ int moloch_db_can_quit() return 0; } /******************************************************************************/ -static guint timers[10]; +LOCAL guint timers[10]; void moloch_db_init() { if (config.tests) { - fprintf(stderr, "{\"sessions\": [\n"); + fprintf(stderr, "{\"sessions2\": [\n"); } if (!config.dryRun) { esServer = moloch_http_create_server(config.elasticsearch, config.maxESConns, config.maxESRequests, config.compressES); @@ -2289,58 +2044,23 @@ void moloch_db_init() headers[1] = NULL; moloch_http_set_headers(esServer, headers); } - DLL_INIT(t_, &tagRequests); - HASH_INIT(tag_, tags, moloch_db_tag_hash, moloch_db_tag_cmp); myPid = getpid(); gettimeofday(&startTime, NULL); if (!config.dryRun) { moloch_db_check(); moloch_db_load_file_num(); - if (!config.noLoadTags) - moloch_db_load_tags(); moloch_db_load_stats(); moloch_db_load_fields(); } moloch_add_can_quit(moloch_db_can_quit, "DB"); - if (config.geoipFile) { - gi = GeoIP_open(config.geoipFile, GEOIP_MEMORY_CACHE); - if (!gi) { - printf("Couldn't initialize GeoIP %s from %s", strerror(errno), config.geoipFile); - exit(1); - } - GeoIP_set_charset(gi, GEOIP_CHARSET_UTF8); - } - - if (config.geoip6File) { - gi6 = GeoIP_open(config.geoip6File, GEOIP_MEMORY_CACHE); - if (!gi6) { - printf("Couldn't initialize GeoIP %s from %s", strerror(errno), config.geoip6File); - exit(1); - } - GeoIP_set_charset(gi6, GEOIP_CHARSET_UTF8); - } - - if (config.geoipASNFile) { - giASN = GeoIP_open(config.geoipASNFile, GEOIP_MEMORY_CACHE); - if (!giASN) { - printf("Couldn't initialize GeoIP ASN %s from %s", strerror(errno), config.geoipASNFile); - exit(1); - } - GeoIP_set_charset(giASN, GEOIP_CHARSET_UTF8); - } - - if (config.geoipASN6File) { - giASN6 = GeoIP_open(config.geoipASN6File, GEOIP_MEMORY_CACHE); - if (!giASN6) { - printf("Couldn't initialize GeoIP ASN 6 %s from %s", strerror(errno), config.geoipASN6File); - exit(1); - } - GeoIP_set_charset(giASN6, GEOIP_CHARSET_UTF8); - } - - moloch_db_load_rir(); + moloch_config_monitor_file("country file", config.geoLite2Country, moloch_db_load_geo_country); + moloch_config_monitor_file("asn file", config.geoLite2ASN, moloch_db_load_geo_asn); + if (config.ouiFile) + moloch_config_monitor_file("oui file", config.ouiFile, moloch_db_load_oui); + if (config.rirFile) + moloch_config_monitor_file("rir file", config.rirFile, moloch_db_load_rir); if (!config.dryRun) { timers[0] = g_timeout_add_seconds( 2, moloch_db_update_stats_gfunc, 0); @@ -2370,29 +2090,13 @@ void moloch_db_exit() } if (config.tests) { - int comma = 0; - MolochTag_t *tag; - fprintf(stderr, "], \"tags\": {\n"); - HASH_FORALL(tag_, tags, tag, - if (comma) - fprintf(stderr, ",\n"); - else { - comma = 1; - fprintf(stderr, "\n"); - } - fprintf(stderr, " \"%d\": \"%s\"", tag->tagValue, tag->tagName); - ); - fprintf(stderr, "\n}}\n"); + fprintf(stderr, "], \"tags\": {}}\n"); } - if (ipTree) { - Destroy_Patricia(ipTree, moloch_db_free_local_ip); - ipTree = 0; + if (ipTree4) { + Destroy_Patricia(ipTree4, moloch_db_free_local_ip); + Destroy_Patricia(ipTree6, moloch_db_free_local_ip); + ipTree4 = 0; + ipTree6 = 0; } - - MolochTag_t *tag; - HASH_FORALL_POP_HEAD(tag_, tags, tag, - g_free(tag->tagName); - MOLOCH_TYPE_FREE(MolochTag_t, tag); - ); } diff --git a/capture/field.c b/capture/field.c index 90c2d1d866..3b88a9d430 100644 --- a/capture/field.c +++ b/capture/field.c @@ -20,8 +20,8 @@ #include #include "patricia.h" -extern patricia_tree_t *ipTree; extern MolochConfig_t config; + HASH_VAR(d_, fieldsByDb, MolochFieldInfo_t, 13); HASH_VAR(e_, fieldsByExp, MolochFieldInfo_t, 13); @@ -30,10 +30,12 @@ HASH_VAR(e_, fieldsByExp, MolochFieldInfo_t, 13); #define MOLOCH_FIELD_SPECIAL_STOP_PCAP -3 #define MOLOCH_FIELD_SPECIAL_MIN_SAVE -4 +#define MOLOCH_FIELD_EXSPECIAL_START (MOLOCH_FIELDS_MAX-4) + LOCAL va_list empty_va_list; /******************************************************************************/ -int moloch_field_exp_cmp(const void *keyv, const void *elementv) +LOCAL int moloch_field_exp_cmp(const void *keyv, const void *elementv) { char *key = (char*)keyv; MolochFieldInfo_t *element = (MolochFieldInfo_t *)elementv; @@ -46,6 +48,7 @@ void moloch_field_define_json(unsigned char *expression, int expression_len, uns MolochFieldInfo_t *info = MOLOCH_TYPE_ALLOC0(MolochFieldInfo_t); int i; uint32_t out[4*100]; // Can have up to 100 elements at any level + int disabled = 0; memset(out, 0, sizeof(out)); if (js0n(data, data_len, out) != 0) { @@ -56,7 +59,7 @@ void moloch_field_define_json(unsigned char *expression, int expression_len, uns for (i = 0; out[i]; i += 4) { if (strncmp("group", (char*)data + out[i], 5) == 0) { info->group = g_strndup((char*)data + out[i+2], out[i+3]); - } else if (strncmp("dbField", (char*)data + out[i], 7) == 0) { + } else if (strncmp("dbField2", (char*)data + out[i], 7) == 0) { info->dbFieldFull = info->dbField = g_strndup((char*)data + out[i+2], out[i+3]); info->dbFieldLen = out[i+3]; } else if (strncmp("type", (char*)data + out[i], 4) == 0) { @@ -65,11 +68,16 @@ void moloch_field_define_json(unsigned char *expression, int expression_len, uns info->category = g_strndup((char*)data + out[i+2], out[i+3]); } else if (strncmp("disabled", (char*)data + out[i], 8) == 0) { if (strncmp((char *)data + out[i+2], "true", 4) == 0) { - info->flags |= MOLOCH_FIELD_FLAG_DISABLED; + disabled = 1; } } } + if (disabled) + info->flags |= MOLOCH_FIELD_FLAG_DISABLED; + else + info->flags &= ~MOLOCH_FIELD_FLAG_DISABLED; + info->pos = -1; HASH_ADD(d_, fieldsByDb, info->dbField, info); HASH_ADD(e_, fieldsByExp, info->expression, info); @@ -143,8 +151,8 @@ int moloch_field_define_text(char *text, int *shortcut) return -1; } - if (strstr(kind, "termfield") != 0 && strstr(db, "-term") == 0) { - LOGEXIT("ERROR - db field %s for %s should end with -term in '%s'", kind, db, text); + if (strstr(kind, "termfield") != 0 && strstr(db, "-term") != 0) { + LOGEXIT("ERROR - db field %s for %s should NOT end with -term in '%s' with Moloch 1.0", kind, db, text); } char groupbuf[100]; @@ -177,7 +185,7 @@ int moloch_field_define_text(char *text, int *shortcut) type = MOLOCH_FIELD_TYPE_STR_HASH; if (count) - flags |= MOLOCH_FIELD_FLAG_COUNT; + flags |= MOLOCH_FIELD_FLAG_CNT; int pos = moloch_field_define(group, kind, field, friendly, db, help, type, flags, "category", category, NULL); g_strfreev(elements); @@ -185,13 +193,13 @@ int moloch_field_define_text(char *text, int *shortcut) } /******************************************************************************/ /* Changes ... to va_list */ -static void moloch_session_add_field_proxy(char *group, char *kind, char *expression, char *friendlyName, char *dbField, char *help, ...) +/*static void moloch_session_add_field_proxy(char *group, char *kind, char *expression, char *friendlyName, char *dbField, char *help, ...) { va_list args; va_start(args, help); moloch_db_add_field(group, kind, expression, friendlyName, dbField, help, TRUE, args); va_end(args); -} +}*/ /******************************************************************************/ int moloch_field_define(char *group, char *kind, char *expression, char *friendlyName, char *dbField, char *help, int type, int flags, ...) { @@ -199,7 +207,6 @@ int moloch_field_define(char *group, char *kind, char *expression, char *friendl char expression2[1000]; char friendlyName2[1000]; char help2[1000]; - char rawField[100]; MolochFieldInfo_t *minfo = 0; HASH_FIND(d_, fieldsByDb, dbField, minfo); @@ -223,6 +230,8 @@ int moloch_field_define(char *group, char *kind, char *expression, char *friendl va_end(args); } } else { + flags |= (minfo->flags & MOLOCH_FIELD_FLAG_DISABLED); + char *category = NULL; if (strcmp(kind, minfo->kind) != 0) { LOG("WARNING - Field kind in db %s doesn't match field kind %s in capture for field %s", minfo->kind, kind, expression); @@ -244,19 +253,12 @@ int moloch_field_define(char *group, char *kind, char *expression, char *friendl } } - // Hack to remove trailing .snow on capture side - int dbLen = strlen(minfo->dbField); - if (dbLen > 5 && memcmp(".snow", minfo->dbField+dbLen-5, 5) == 0) { - minfo->dbField[dbLen-5] = 0; - minfo->dbFieldLen -= 5; - } - minfo->type = type; minfo->flags = flags; if ((flags & MOLOCH_FIELD_FLAG_FAKE) == 0) { if (minfo->pos == -1) { - minfo->pos = config.maxField++; + minfo->pos = MOLOCH_THREAD_INCROLD(config.maxField); if (config.maxField > 255) { LOGEXIT("ERROR - Max Fields is too large %d", config.maxField); } @@ -272,8 +274,7 @@ int moloch_field_define(char *group, char *kind, char *expression, char *friendl if (memcmp(minfo->dbField, lastGroup, (firstdot-minfo->dbField)+1) == 0) { minfo->dbGroupNum = groupNum; } else { - groupNum++; - minfo->dbGroupNum = groupNum; + minfo->dbGroupNum = MOLOCH_THREAD_INCRNEW(groupNum); memcpy(lastGroup, minfo->dbField, (firstdot-minfo->dbField)+1); } minfo->dbGroup = minfo->dbField; @@ -288,29 +289,7 @@ int moloch_field_define(char *group, char *kind, char *expression, char *friendl MolochFieldInfo_t *info = 0; if (flags & MOLOCH_FIELD_FLAG_CNT) { - snprintf(dbField2, sizeof(dbField2), "%scnt", dbField); - HASH_FIND(d_, fieldsByDb, dbField2, info); - if (!info) { - snprintf(expression2, sizeof(expression2), "%s.cnt", expression); - snprintf(friendlyName2, sizeof(friendlyName2), "%s Cnt", friendlyName); - snprintf(help2, sizeof(help2), "Unique number of %s", help); - moloch_db_add_field(group, "integer", expression2, friendlyName2, dbField2, help2, FALSE, empty_va_list); - } - } - - if (flags & MOLOCH_FIELD_FLAG_SCNT) { - snprintf(dbField2, sizeof(dbField2), "%sscnt", dbField); - HASH_FIND(d_, fieldsByDb, dbField2, info); - if (!info) { - snprintf(expression2, sizeof(expression2), "%s.cnt", expression); - snprintf(friendlyName2, sizeof(friendlyName2), "%s Cnt", friendlyName); - snprintf(help2, sizeof(help2), "Unique number of %s", help); - moloch_db_add_field(group, "integer", expression2, friendlyName2, dbField2, help2, FALSE, empty_va_list); - } - } - - if (flags & MOLOCH_FIELD_FLAG_COUNT) { - snprintf(dbField2, sizeof(dbField2), "%s-cnt", dbField); + snprintf(dbField2, sizeof(dbField2), "%sCnt", dbField); HASH_FIND(d_, fieldsByDb, dbField2, info); if (!info) { snprintf(expression2, sizeof(expression2), "%s.cnt", expression); @@ -332,8 +311,9 @@ int moloch_field_define(char *group, char *kind, char *expression, char *friendl } if (flags & MOLOCH_FIELD_FLAG_IPPRE) { + int l = strlen(dbField)-2; int fnlen = strlen(friendlyName); - snprintf(dbField2, sizeof(dbField2), "g%s", dbField); + snprintf(dbField2, sizeof(dbField2), "%.*sGEO", l, dbField); HASH_FIND(d_, fieldsByDb, dbField2, info); if (!info) { snprintf(expression2, sizeof(expression2), "country.%s", expression+3); @@ -342,17 +322,16 @@ int moloch_field_define(char *group, char *kind, char *expression, char *friendl moloch_db_add_field(group, "uptermfield", expression2, friendlyName2, dbField2, help2, FALSE, empty_va_list); } - snprintf(dbField2, sizeof(dbField2), "as%s", dbField); + snprintf(dbField2, sizeof(dbField2), "%.*sASN", l, dbField); HASH_FIND(d_, fieldsByDb, dbField2, info); if (!info) { snprintf(expression2, sizeof(expression2), "asn.%s", expression+3); snprintf(friendlyName2, sizeof(friendlyName2), "%.*s ASN", fnlen-2, friendlyName); snprintf(help2, sizeof(help2), "GeoIP ASN string calculated from the %s", help); - snprintf(rawField, sizeof(rawField), "raw%s", dbField2); - moloch_session_add_field_proxy(group, "textfield", expression2, friendlyName2, dbField2, help2, "rawField", rawField, NULL); + moloch_db_add_field(group, "termfield", expression2, friendlyName2, dbField2, help2, FALSE, empty_va_list); } - snprintf(dbField2, sizeof(dbField2), "rir%s", dbField); + snprintf(dbField2, sizeof(dbField2), "%.*sRIR", l, dbField); HASH_FIND(d_, fieldsByDb, dbField2, info); if (!info) { snprintf(expression2, sizeof(expression2), "rir.%s", expression+3); @@ -360,8 +339,9 @@ int moloch_field_define(char *group, char *kind, char *expression, char *friendl snprintf(help2, sizeof(help2), "Regional Internet Registry string calculated from %s", help); moloch_db_add_field(group, "uptermfield", expression2, friendlyName2, dbField2, help2, FALSE, empty_va_list); } - } else if (type == MOLOCH_FIELD_TYPE_IP || type == MOLOCH_FIELD_TYPE_IP_HASH || type == MOLOCH_FIELD_TYPE_IP_GHASH) { - snprintf(dbField2, sizeof(dbField2), "%s-geo", dbField); + } else if (type == MOLOCH_FIELD_TYPE_IP || type == MOLOCH_FIELD_TYPE_IP_GHASH) { + int l = strlen(dbField)-2; + snprintf(dbField2, sizeof(dbField2), "%.*sGEO", l, dbField); HASH_FIND(d_, fieldsByDb, dbField2, info); if (!info) { snprintf(expression2, sizeof(expression2), "%s.country", expression); @@ -370,18 +350,16 @@ int moloch_field_define(char *group, char *kind, char *expression, char *friendl moloch_db_add_field(group, "uptermfield", expression2, friendlyName2, dbField2, help2, FALSE, empty_va_list); } - snprintf(dbField2, sizeof(dbField2), "%s-asn.snow", dbField); + snprintf(dbField2, sizeof(dbField2), "%.*sASN", l, dbField); HASH_FIND(d_, fieldsByDb, dbField2, info); if (!info) { - snprintf(dbField2, sizeof(dbField2), "%s-asn.snow", dbField); snprintf(expression2, sizeof(expression2), "%s.asn", expression); snprintf(friendlyName2, sizeof(friendlyName2), "%s ASN", friendlyName); - snprintf(rawField, sizeof(rawField), "%s-asn.raw", dbField); snprintf(help2, sizeof(help2), "GeoIP ASN string calculated from the %s", help); - moloch_session_add_field_proxy(group, "textfield", expression2, friendlyName2, dbField2, help2, "rawField", rawField, NULL); + moloch_db_add_field(group, "termfield", expression2, friendlyName2, dbField2, help2, FALSE, empty_va_list); } - snprintf(dbField2, sizeof(dbField2), "%s-rir", dbField); + snprintf(dbField2, sizeof(dbField2), "%.*sRIR", l, dbField); HASH_FIND(d_, fieldsByDb, dbField2, info); if (!info) { snprintf(expression2, sizeof(expression2), "%s.rir", expression); @@ -397,9 +375,10 @@ int moloch_field_by_db(const char *dbField) { MolochFieldInfo_t *info = 0; HASH_FIND(d_, fieldsByDb, dbField, info); - if (info) - return info->pos; - return -1; + if (!info || info->pos == -1) + LOGEXIT("dbField %s wasn't defined", dbField); + + return info->pos; } /******************************************************************************/ int moloch_field_by_exp(const char *exp) @@ -418,11 +397,12 @@ int moloch_field_by_exp(const char *exp) } else { info->type = MOLOCH_FIELD_TYPE_STR_HASH; } - info->pos = config.maxField++; + info->pos = MOLOCH_THREAD_INCROLD(config.maxField); config.fields[info->pos] = info; return info->pos; } - return -1; + + LOGEXIT("expr %s wasn't defined", exp); } /******************************************************************************/ void moloch_field_by_exp_add_special(char *exp, int pos) @@ -433,35 +413,14 @@ void moloch_field_by_exp_add_special(char *exp, int pos) HASH_ADD(e_, fieldsByExp, info->expression, info); } /******************************************************************************/ -void moloch_field_init() -{ - config.maxField = 0; - HASH_INIT(d_, fieldsByDb, moloch_string_hash, moloch_string_cmp); - HASH_INIT(e_, fieldsByExp, moloch_string_hash, moloch_field_exp_cmp); - - moloch_field_by_exp_add_special("dontSaveSPI", MOLOCH_FIELD_SPECIAL_STOP_SPI); - moloch_field_by_exp_add_special("_dontSaveSPI", MOLOCH_FIELD_SPECIAL_STOP_SPI); - moloch_field_by_exp_add_special("_maxPacketsToSave", MOLOCH_FIELD_SPECIAL_STOP_PCAP); - moloch_field_by_exp_add_special("_minPacketsBeforeSavingSPI", MOLOCH_FIELD_SPECIAL_MIN_SAVE); -} -/******************************************************************************/ -void moloch_field_exit() +void moloch_field_by_exp_add_exspecial(char *exp, int pos, int type) { - MolochFieldInfo_t *info = 0; - - HASH_FORALL_POP_HEAD(d_, fieldsByDb, info, - if (info->dbFieldFull) - g_free(info->dbFieldFull); - if (info->expression) - g_free(info->expression); - if (info->group) - g_free(info->group); - if (info->kind) - g_free(info->kind); - if (info->category) - g_free(info->category); - MOLOCH_TYPE_FREE(MolochFieldInfo_t, info); - ); + MolochFieldInfo_t *info = MOLOCH_TYPE_ALLOC0(MolochFieldInfo_t); + info->expression = exp; + info->pos = pos; + info->type = type; + config.fields[pos] = info; + HASH_ADD(e_, fieldsByExp, info->expression, info); } /******************************************************************************/ const char *moloch_field_string_add(int pos, MolochSession_t *session, const char *string, int len, gboolean copy) @@ -499,6 +458,10 @@ const char *moloch_field_string_add(int pos, MolochSession_t *session, const cha hstring->utf8 = 0; HASH_ADD(s_, *hash, hstring->str, hstring); goto added; + case MOLOCH_FIELD_TYPE_STR_GHASH: + field->ghash = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL); + g_hash_table_add(field->ghash, (gpointer)string); + goto added; default: LOGEXIT("Not a string %s", config.fields[pos]->dbField); } @@ -530,7 +493,7 @@ const char *moloch_field_string_add(int pos, MolochSession_t *session, const cha if (hstring) { field->jsonSize -= (6 + 2*len); - return FALSE; + return NULL; } hstring = MOLOCH_TYPE_ALLOC(MolochString_t); if (copy) { @@ -544,6 +507,15 @@ const char *moloch_field_string_add(int pos, MolochSession_t *session, const cha } HASH_ADD(s_, *(field->shash), hstring->str, hstring); goto added; + case MOLOCH_FIELD_TYPE_STR_GHASH: + if (g_hash_table_lookup(field->ghash, string)) { + field->jsonSize -= (6 + 2*len); + return NULL; + } + if (copy) + string = g_strndup(string, len); + g_hash_table_add(field->ghash, (gpointer)string); + goto added; default: LOGEXIT("Not a string %s", config.fields[pos]->dbField); } @@ -653,8 +625,6 @@ gboolean moloch_field_int_add(int pos, MolochSession_t *session, int i) session->fields[pos] = field; field->jsonSize = 3 + config.fields[pos]->dbFieldLen + 10; switch (config.fields[pos]->type) { - case MOLOCH_FIELD_TYPE_IP: - field->jsonSize += 100; case MOLOCH_FIELD_TYPE_INT: field->i = i; goto added; @@ -662,8 +632,6 @@ gboolean moloch_field_int_add(int pos, MolochSession_t *session, int i) field->iarray = g_array_new(FALSE, FALSE, 4); g_array_append_val(field->iarray, i); goto added; - case MOLOCH_FIELD_TYPE_IP_HASH: - field->jsonSize += 100; case MOLOCH_FIELD_TYPE_INT_HASH: hash = MOLOCH_TYPE_ALLOC(MolochIntHashStd_t); HASH_INIT(i_, *hash, moloch_int_hash, moloch_int_cmp); @@ -671,8 +639,6 @@ gboolean moloch_field_int_add(int pos, MolochSession_t *session, int i) hint = MOLOCH_TYPE_ALLOC(MolochInt_t); HASH_ADD(i_, *hash, (void *)(long)i, hint); goto added; - case MOLOCH_FIELD_TYPE_IP_GHASH: - field->jsonSize += 100; case MOLOCH_FIELD_TYPE_INT_GHASH: field->ghash = g_hash_table_new(NULL, NULL); g_hash_table_add(field->ghash, (void *)(long)i); @@ -685,16 +651,12 @@ gboolean moloch_field_int_add(int pos, MolochSession_t *session, int i) field = session->fields[pos]; field->jsonSize += (3 + 10); switch (config.fields[pos]->type) { - case MOLOCH_FIELD_TYPE_IP: - field->jsonSize += 100; case MOLOCH_FIELD_TYPE_INT: field->i = i; goto added; case MOLOCH_FIELD_TYPE_INT_ARRAY: g_array_append_val(field->iarray, i); goto added; - case MOLOCH_FIELD_TYPE_IP_HASH: - field->jsonSize += 100; case MOLOCH_FIELD_TYPE_INT_HASH: HASH_FIND_INT(i_, *(field->ihash), i, hint); if (hint) { @@ -704,27 +666,234 @@ gboolean moloch_field_int_add(int pos, MolochSession_t *session, int i) hint = MOLOCH_TYPE_ALLOC(MolochInt_t); HASH_ADD(i_, *(field->ihash), (void *)(long)i, hint); goto added; - case MOLOCH_FIELD_TYPE_IP_GHASH: + case MOLOCH_FIELD_TYPE_INT_GHASH: if (!g_hash_table_add(field->ghash, (void *)(long)i)) { field->jsonSize -= 13; return FALSE; + } + goto added; + default: + LOGEXIT("Not a int %s", config.fields[pos]->dbField); + } + +added: + if (config.fields[pos]->ruleEnabled) + moloch_rules_run_field_set(session, pos, (gpointer)(long)i); + + return TRUE; +} +/******************************************************************************/ +gboolean moloch_field_ip_equal (gconstpointer v1, gconstpointer v2) +{ + return memcmp (v1, v2, 16) == 0; +} +/******************************************************************************/ +guint moloch_field_ip_hash (gconstpointer v) +{ + const signed char *p; + guint32 h = 5381; + int i; + + for (i = 0, p = v; i < 16; i++, p++) { + h = (h << 5) + h + *p; + } + + return h; +} + +/******************************************************************************/ +void *moloch_field_parse_ip(const char *str) { + + struct in6_addr *v = g_malloc(sizeof(struct in6_addr)); + + if (memchr(str, '.', 4)) { + struct in_addr addr; + if (inet_aton(str, &addr) == 0) { + g_free(v); + return NULL; + } + + ((uint32_t *)v->s6_addr)[0] = 0; + ((uint32_t *)v->s6_addr)[1] = 0; + ((uint32_t *)v->s6_addr)[2] = htonl(0xffff); + ((uint32_t *)v->s6_addr)[3] = addr.s_addr; + } else { + if (inet_pton(AF_INET6, str, v) == 0) { + g_free(v); + return NULL; + } + } + + return v; +} +/******************************************************************************/ +gboolean moloch_field_ip_add_str(int pos, MolochSession_t *session, char *str) +{ + MolochField_t *field; + + if (config.fields[pos]->flags & MOLOCH_FIELD_FLAG_DISABLED || pos >= session->maxFields) + return FALSE; + + struct in6_addr *v = moloch_field_parse_ip(str); + + if (!v) { + return FALSE; + } + + if (!session->fields[pos]) { + field = MOLOCH_TYPE_ALLOC(MolochField_t); + session->fields[pos] = field; + field->jsonSize = 3 + config.fields[pos]->dbFieldLen + 10 + 100; + switch (config.fields[pos]->type) { + case MOLOCH_FIELD_TYPE_IP: + field->ip = v; + goto added; + case MOLOCH_FIELD_TYPE_IP_GHASH: + field->ghash = g_hash_table_new_full(moloch_field_ip_hash, moloch_field_ip_equal, g_free, NULL); + + if (!g_hash_table_add(field->ghash, v)) { + g_free(v); + } + goto added; + default: + LOGEXIT("Not a ip %s", config.fields[pos]->dbField); + } + } + + field = session->fields[pos]; + field->jsonSize += (3 + 10 + 100); + switch (config.fields[pos]->type) { + case MOLOCH_FIELD_TYPE_IP: + g_free(field->ip); + field->ip = v; + goto added; + case MOLOCH_FIELD_TYPE_IP_GHASH: + if (!g_hash_table_add(field->ghash, v)) { + field->jsonSize -= 3 + 10 + 100; + return FALSE; } else { - field->jsonSize += 100; goto added; } - case MOLOCH_FIELD_TYPE_INT_GHASH: - if (!g_hash_table_add(field->ghash, (void *)(long)i)) { - field->jsonSize -= 13; + default: + LOGEXIT("Not a ip %s", config.fields[pos]->dbField); + } + +added: + if (config.fields[pos]->ruleEnabled) + moloch_rules_run_field_set(session, pos, v); + + return TRUE; +} +/******************************************************************************/ +gboolean moloch_field_ip4_add(int pos, MolochSession_t *session, int i) +{ + MolochField_t *field; + + if (config.fields[pos]->flags & MOLOCH_FIELD_FLAG_DISABLED || pos >= session->maxFields) + return FALSE; + + struct in6_addr *v = g_malloc(sizeof(struct in6_addr)); + + ((uint32_t *)v->s6_addr)[0] = 0; + ((uint32_t *)v->s6_addr)[1] = 0; + ((uint32_t *)v->s6_addr)[2] = htonl(0xffff); + ((uint32_t *)v->s6_addr)[3] = i; + + if (!session->fields[pos]) { + field = MOLOCH_TYPE_ALLOC(MolochField_t); + session->fields[pos] = field; + field->jsonSize = 3 + config.fields[pos]->dbFieldLen + 10 + 100; + switch (config.fields[pos]->type) { + case MOLOCH_FIELD_TYPE_IP: + field->ip = v; + goto added; + case MOLOCH_FIELD_TYPE_IP_GHASH: + field->ghash = g_hash_table_new_full(moloch_field_ip_hash, moloch_field_ip_equal, g_free, NULL); + + if (!g_hash_table_add(field->ghash, v)) { + g_free(v); + } + goto added; + default: + LOGEXIT("Not a ip %s", config.fields[pos]->dbField); + } + } + + field = session->fields[pos]; + field->jsonSize += (3 + 10 + 100); + switch (config.fields[pos]->type) { + case MOLOCH_FIELD_TYPE_IP: + g_free(field->ip); + field->ip = v; + goto added; + case MOLOCH_FIELD_TYPE_IP_GHASH: + if (!g_hash_table_add(field->ghash, v)) { + field->jsonSize -= 3 + 10 + 100; return FALSE; + } else { + goto added; } + default: + LOGEXIT("Not a ip %s", config.fields[pos]->dbField); + } + +added: + if (config.fields[pos]->ruleEnabled) + moloch_rules_run_field_set(session, pos, v); + + return TRUE; +} +/******************************************************************************/ +gboolean moloch_field_ip6_add(int pos, MolochSession_t *session, const uint8_t *val) +{ + MolochField_t *field; + + if (config.fields[pos]->flags & MOLOCH_FIELD_FLAG_DISABLED || pos >= session->maxFields) + return FALSE; + + struct in6_addr *v = g_memdup(val, sizeof(struct in6_addr)); + + if (!session->fields[pos]) { + field = MOLOCH_TYPE_ALLOC(MolochField_t); + session->fields[pos] = field; + field->jsonSize = 3 + config.fields[pos]->dbFieldLen + 10 + 100; + switch (config.fields[pos]->type) { + case MOLOCH_FIELD_TYPE_IP: + field->ip = v; + goto added; + case MOLOCH_FIELD_TYPE_IP_GHASH: + field->ghash = g_hash_table_new_full(moloch_field_ip_hash, moloch_field_ip_equal, g_free, NULL); + + if (!g_hash_table_add(field->ghash, v)) { + g_free(v); + } + goto added; + default: + LOGEXIT("Not a ip %s", config.fields[pos]->dbField); + } + } + + field = session->fields[pos]; + field->jsonSize += (3 + 10 + 100); + switch (config.fields[pos]->type) { + case MOLOCH_FIELD_TYPE_IP: + g_free(field->ip); + field->ip = v; goto added; + case MOLOCH_FIELD_TYPE_IP_GHASH: + if (!g_hash_table_add(field->ghash, v)) { + field->jsonSize -= 3 + 10 + 100; + return FALSE; + } else { + goto added; + } default: - LOGEXIT("Not a int %s", config.fields[pos]->dbField); + LOGEXIT("Not a ip %s", config.fields[pos]->dbField); } added: if (config.fields[pos]->ruleEnabled) - moloch_rules_run_field_set(session, pos, (gpointer)(long)i); + moloch_rules_run_field_set(session, pos, v); return TRUE; } @@ -815,6 +984,22 @@ gboolean moloch_field_certsinfo_add(int pos, MolochSession_t *session, MolochCer } } /******************************************************************************/ +void moloch_field_macoui_add(MolochSession_t *session, int macField, int ouiField, const uint8_t *mac) +{ + char str[20]; + + snprintf(str, sizeof(str), "%02x:%02x:%02x:%02x:%02x:%02x", + mac[0], + mac[1], + mac[2], + mac[3], + mac[4], + mac[5]); + + if (moloch_field_string_add(macField, session, str, 17, TRUE)) + moloch_db_oui_lookup(ouiField, session, mac); +} +/******************************************************************************/ void moloch_field_free(MolochSession_t *session) { int pos; @@ -850,7 +1035,6 @@ void moloch_field_free(MolochSession_t *session) case MOLOCH_FIELD_TYPE_INT_ARRAY: g_array_free(field->iarray, TRUE); break; - case MOLOCH_FIELD_TYPE_IP_HASH: case MOLOCH_FIELD_TYPE_INT_HASH: ihash = session->fields[pos]->ihash; HASH_FORALL_POP_HEAD(i_, *ihash, hint, @@ -858,8 +1042,12 @@ void moloch_field_free(MolochSession_t *session) ); MOLOCH_TYPE_FREE(MolochIntHashStd_t, ihash); break; + case MOLOCH_FIELD_TYPE_IP: + g_free(session->fields[pos]->ip); + break; case MOLOCH_FIELD_TYPE_IP_GHASH: case MOLOCH_FIELD_TYPE_INT_GHASH: + case MOLOCH_FIELD_TYPE_STR_GHASH: g_hash_table_destroy(session->fields[pos]->ghash); break; case MOLOCH_FIELD_TYPE_CERTSINFO: @@ -926,10 +1114,10 @@ int moloch_field_count(int pos, MolochSession_t *session) case MOLOCH_FIELD_TYPE_STR_HASH: return HASH_COUNT(s_, *(field->shash)); case MOLOCH_FIELD_TYPE_INT_HASH: - case MOLOCH_FIELD_TYPE_IP_HASH: return HASH_COUNT(s_, *(field->ihash)); - case MOLOCH_FIELD_TYPE_INT_GHASH: case MOLOCH_FIELD_TYPE_IP_GHASH: + case MOLOCH_FIELD_TYPE_INT_GHASH: + case MOLOCH_FIELD_TYPE_STR_GHASH: return g_hash_table_size(field->ghash); case MOLOCH_FIELD_TYPE_CERTSINFO: return HASH_COUNT(s_, *(field->cihash)); @@ -960,25 +1148,33 @@ void moloch_field_ops_run(MolochSession_t *session, MolochFieldOps_t *ops) } continue; } + // Exspecial Fields + if (op->fieldPos >= MOLOCH_FIELD_EXSPECIAL_START) { + switch (op->fieldPos) { + case MOLOCH_FIELD_EXSPECIAL_SRC_IP: + case MOLOCH_FIELD_EXSPECIAL_SRC_PORT: + case MOLOCH_FIELD_EXSPECIAL_DST_IP: + case MOLOCH_FIELD_EXSPECIAL_DST_PORT: + break; + } + continue; + } switch (config.fields[op->fieldPos]->type) { case MOLOCH_FIELD_TYPE_INT_HASH: case MOLOCH_FIELD_TYPE_INT_GHASH: - if (op->fieldPos == config.tagsField) { - moloch_session_add_tag(session, op->str); - continue; - } - // Fall Thru case MOLOCH_FIELD_TYPE_INT: case MOLOCH_FIELD_TYPE_INT_ARRAY: + moloch_field_int_add(op->fieldPos, session, op->strLenOrInt); + break; case MOLOCH_FIELD_TYPE_IP: - case MOLOCH_FIELD_TYPE_IP_HASH: case MOLOCH_FIELD_TYPE_IP_GHASH: - moloch_field_int_add(op->fieldPos, session, op->strLenOrInt); + moloch_field_ip_add_str(op->fieldPos, session, op->str); break; case MOLOCH_FIELD_TYPE_STR: case MOLOCH_FIELD_TYPE_STR_ARRAY: case MOLOCH_FIELD_TYPE_STR_HASH: + case MOLOCH_FIELD_TYPE_STR_GHASH: moloch_field_string_add(op->fieldPos, session, op->str, op->strLenOrInt, TRUE); break; } @@ -1038,20 +1234,19 @@ void moloch_field_ops_add(MolochFieldOps_t *ops, int fieldPos, char *value, int LOG("WARNING - Unknown special field pos %d", fieldPos); break; } + } else if (fieldPos >= MOLOCH_FIELD_EXSPECIAL_START) { + switch (op->fieldPos) { + case MOLOCH_FIELD_EXSPECIAL_SRC_IP: + case MOLOCH_FIELD_EXSPECIAL_SRC_PORT: + case MOLOCH_FIELD_EXSPECIAL_DST_IP: + case MOLOCH_FIELD_EXSPECIAL_DST_PORT: + LOG("Warning - not allow to set src/dst ip/port: %s", op->str); + break; + } } else { switch (config.fields[fieldPos]->type) { case MOLOCH_FIELD_TYPE_INT_HASH: case MOLOCH_FIELD_TYPE_INT_GHASH: - if (fieldPos == config.tagsField) { - moloch_db_get_tag(NULL, config.tagsField, value, NULL); // Preload the tagname -> tag mapping - if (ops->flags & MOLOCH_FIELD_OPS_FLAGS_COPY) - op->str = g_strndup(value, valuelen); - else - op->str = value; - op->strLenOrInt = valuelen; - break; - } - // Fall thru case MOLOCH_FIELD_TYPE_INT: case MOLOCH_FIELD_TYPE_INT_ARRAY: op->str = 0; @@ -1060,18 +1255,15 @@ void moloch_field_ops_add(MolochFieldOps_t *ops, int fieldPos, char *value, int case MOLOCH_FIELD_TYPE_STR: case MOLOCH_FIELD_TYPE_STR_ARRAY: case MOLOCH_FIELD_TYPE_STR_HASH: + case MOLOCH_FIELD_TYPE_STR_GHASH: + case MOLOCH_FIELD_TYPE_IP: + case MOLOCH_FIELD_TYPE_IP_GHASH: if (ops->flags & MOLOCH_FIELD_OPS_FLAGS_COPY) op->str = g_strndup(value, valuelen); else op->str = value; op->strLenOrInt = valuelen; break; - case MOLOCH_FIELD_TYPE_IP: - case MOLOCH_FIELD_TYPE_IP_HASH: - case MOLOCH_FIELD_TYPE_IP_GHASH: - op->str = 0; - op->strLenOrInt = inet_addr(value); - break; default: LOG("WARNING - Unsupported expression type %d for %s", config.fields[fieldPos]->type, value); return; @@ -1080,3 +1272,39 @@ void moloch_field_ops_add(MolochFieldOps_t *ops, int fieldPos, char *value, int ops->num++; } /******************************************************************************/ +void moloch_field_init() +{ + config.maxField = 0; + HASH_INIT(d_, fieldsByDb, moloch_string_hash, moloch_string_cmp); + HASH_INIT(e_, fieldsByExp, moloch_string_hash, moloch_field_exp_cmp); + + moloch_field_by_exp_add_special("dontSaveSPI", MOLOCH_FIELD_SPECIAL_STOP_SPI); + moloch_field_by_exp_add_special("_dontSaveSPI", MOLOCH_FIELD_SPECIAL_STOP_SPI); + moloch_field_by_exp_add_special("_maxPacketsToSave", MOLOCH_FIELD_SPECIAL_STOP_PCAP); + moloch_field_by_exp_add_special("_minPacketsBeforeSavingSPI", MOLOCH_FIELD_SPECIAL_MIN_SAVE); + + moloch_field_by_exp_add_exspecial("ip.src", MOLOCH_FIELD_EXSPECIAL_SRC_IP, MOLOCH_FIELD_TYPE_IP); + moloch_field_by_exp_add_exspecial("port.src", MOLOCH_FIELD_EXSPECIAL_SRC_PORT, MOLOCH_FIELD_TYPE_INT); + moloch_field_by_exp_add_exspecial("ip.dst", MOLOCH_FIELD_EXSPECIAL_DST_IP, MOLOCH_FIELD_TYPE_IP); + moloch_field_by_exp_add_exspecial("port.dst", MOLOCH_FIELD_EXSPECIAL_DST_PORT, MOLOCH_FIELD_TYPE_INT); +} +/******************************************************************************/ +void moloch_field_exit() +{ + MolochFieldInfo_t *info = 0; + + HASH_FORALL_POP_HEAD(d_, fieldsByDb, info, + if (info->dbFieldFull) + g_free(info->dbFieldFull); + if (info->expression) + g_free(info->expression); + if (info->group) + g_free(info->group); + if (info->kind) + g_free(info->kind); + if (info->category) + g_free(info->category); + MOLOCH_TYPE_FREE(MolochFieldInfo_t, info); + ); +} +/******************************************************************************/ diff --git a/capture/http.c b/capture/http.c index cbadd39f5b..f3760e54f3 100644 --- a/capture/http.c +++ b/capture/http.c @@ -77,14 +77,14 @@ typedef struct molochhttpconnhead_t { } MolochHttpConnHead_t; -static HASH_VAR(s_, connections, MolochHttpConnHead_t, 119); -static MOLOCH_LOCK_DEFINE(connections); +LOCAL HASH_VAR(s_, connections, MolochHttpConnHead_t, 119); +LOCAL MOLOCH_LOCK_DEFINE(connections); -static MolochHttpRequestHead_t requests; -static int requestsTimer; -static MOLOCH_LOCK_DEFINE(requests); +LOCAL MolochHttpRequestHead_t requests; +LOCAL int requestsTimer; +LOCAL MOLOCH_LOCK_DEFINE(requests); -uint64_t connectionsSet[2048]; +LOCAL uint64_t connectionsSet[2048]; typedef struct { MolochHttpServer_t *server; @@ -116,8 +116,8 @@ struct molochhttpserver_t { MolochHttpHeader_cb headerCb; }; -static z_stream z_strm; -static MOLOCH_LOCK_DEFINE(z_strm); +LOCAL z_stream z_strm; +LOCAL MOLOCH_LOCK_DEFINE(z_strm); LOCAL gboolean moloch_http_send_timer_callback(gpointer); LOCAL void moloch_http_add_request(MolochHttpServer_t *server, MolochHttpRequest_t *request, gboolean async); @@ -130,7 +130,7 @@ int moloch_http_conn_cmp(const void *keyv, const void *elementv) return memcmp(keyv, conn->sessionId, MIN(((uint8_t *)keyv)[0], conn->sessionId[0])) == 0; } /******************************************************************************/ -static size_t moloch_http_curl_write_callback(void *contents, size_t size, size_t nmemb, void *requestP) +LOCAL size_t moloch_http_curl_write_callback(void *contents, size_t size, size_t nmemb, void *requestP) { MolochHttpRequest_t *request = requestP; @@ -315,7 +315,7 @@ LOCAL void moloch_http_add_request(MolochHttpServer_t *server, MolochHttpRequest } } /******************************************************************************/ -static void moloch_http_curlm_check_multi_info(MolochHttpServer_t *server) +LOCAL void moloch_http_curlm_check_multi_info(MolochHttpServer_t *server) { char *eff_url; CURLMsg *msg; @@ -397,7 +397,7 @@ static void moloch_http_curlm_check_multi_info(MolochHttpServer_t *server) } } /******************************************************************************/ -static gboolean moloch_http_watch_callback(int fd, GIOCondition condition, gpointer serverV) +LOCAL gboolean moloch_http_watch_callback(int fd, GIOCondition condition, gpointer serverV) { MolochHttpServer_t *server = serverV; @@ -412,7 +412,7 @@ static gboolean moloch_http_watch_callback(int fd, GIOCondition condition, gpoin return TRUE; } /******************************************************************************/ -static int moloch_http_curlm_socket_callback(CURL *UNUSED(easy), curl_socket_t fd, int what, void *serverV, void *evP) +LOCAL int moloch_http_curlm_socket_callback(CURL *UNUSED(easy), curl_socket_t fd, int what, void *serverV, void *evP) { MolochHttpServer_t *server = serverV; long ev = (long)evP; @@ -435,7 +435,7 @@ static int moloch_http_curlm_socket_callback(CURL *UNUSED(easy), curl_socket_t f } /******************************************************************************/ /* Called by glib when our timeout expires */ -static gboolean moloch_http_timer_callback(gpointer serverV) +LOCAL gboolean moloch_http_timer_callback(gpointer serverV) { MolochHttpServer_t *server = serverV; @@ -445,7 +445,7 @@ static gboolean moloch_http_timer_callback(gpointer serverV) return G_SOURCE_CONTINUE; } /******************************************************************************/ -static int moloch_http_curlm_timeout_callback(CURLM *UNUSED(multi), long timeout_ms, void *serverV) +LOCAL int moloch_http_curlm_timeout_callback(CURLM *UNUSED(multi), long timeout_ms, void *serverV) { MolochHttpServer_t *server = serverV; @@ -491,7 +491,7 @@ size_t moloch_http_curlm_header_function(char *buffer, size_t size, size_t nitem return sz; } /******************************************************************************/ -static gboolean moloch_http_curl_watch_open_callback(int fd, GIOCondition condition, gpointer snameV) +LOCAL gboolean moloch_http_curl_watch_open_callback(int fd, GIOCondition condition, gpointer snameV) { MolochHttpServerName_t *sname = snameV; MolochHttpServer_t *server = sname->server; @@ -654,7 +654,7 @@ int moloch_http_curl_close_callback(void *snameV, curl_socket_t fd) return 0; } /******************************************************************************/ -static gboolean moloch_http_send_timer_callback(gpointer UNUSED(unused)) +LOCAL gboolean moloch_http_send_timer_callback(gpointer UNUSED(unused)) { MolochHttpRequest_t *request; @@ -688,7 +688,7 @@ gboolean moloch_http_send(void *serverV, const char *method, const char *key, ui // Are we overloaded if (dropable && !config.quitting && server->outstanding > server->maxOutstandingRequests) { LOG("ERROR - Dropping request %.*s of size %d queue %d is too big", key_len, key, data_len, server->outstanding); - server->dropped++; + MOLOCH_THREAD_INCR(server->dropped); if (data) { MOLOCH_SIZE_FREE(buffer, data); diff --git a/capture/main.c b/capture/main.c index 628838f07e..e9ec7f7b4f 100644 --- a/capture/main.c +++ b/capture/main.c @@ -47,7 +47,7 @@ extern MolochPcapFileHdr_t pcapFileHeader; MOLOCH_LOCK_DEFINE(LOG); /******************************************************************************/ -static gboolean showVersion = FALSE; +LOCAL gboolean showVersion = FALSE; /******************************************************************************/ gboolean moloch_debug_flag() @@ -57,7 +57,7 @@ gboolean moloch_debug_flag() return TRUE; } -static GOptionEntry entries[] = +LOCAL GOptionEntry entries[] = { { "config", 'c', 0, G_OPTION_ARG_FILENAME, &config.configFile, "Config file name, default '/data/moloch/etc/config.ini'", NULL }, { "pcapfile", 'r', 0, G_OPTION_ARG_FILENAME_ARRAY, &config.pcapReadFiles, "Offline pcap file", NULL }, @@ -107,7 +107,7 @@ void parse_args(int argc, char **argv) extern char *curl_version(void); extern char *pcre_version(void); - extern char *GeoIP_lib_version(void); + extern const char *MMDB_lib_version(void); context = g_option_context_new ("- capture"); g_option_context_add_main_entries (context, entries, NULL); @@ -132,7 +132,7 @@ void parse_args(int argc, char **argv) printf("pcre: %s\n", pcre_version()); //printf("magic: %d\n", magic_version()); printf("yara: %s\n", moloch_yara_version()); - printf("GeoIP: %s\n", GeoIP_lib_version()); + printf("maxminddb: %s\n", MMDB_lib_version()); exit(0); } @@ -382,12 +382,12 @@ typedef struct { } MolochWatchFd_t; /******************************************************************************/ -static void moloch_gio_destroy(gpointer data) +LOCAL void moloch_gio_destroy(gpointer data) { g_free(data); } /******************************************************************************/ -static gboolean moloch_gio_invoke(GIOChannel *source, GIOCondition condition, gpointer data) +LOCAL gboolean moloch_gio_invoke(GIOChannel *source, GIOCondition condition, gpointer data) { MolochWatchFd_t *watch = data; @@ -443,9 +443,9 @@ void moloch_drop_privileges() } /******************************************************************************/ -static MolochCanQuitFunc canQuitFuncs[20]; -static const char *canQuitNames[20]; -int canQuitFuncsNum; +LOCAL MolochCanQuitFunc canQuitFuncs[20]; +LOCAL const char *canQuitNames[20]; +LOCAL int canQuitFuncsNum; void moloch_add_can_quit (MolochCanQuitFunc func, const char *name) { @@ -465,8 +465,8 @@ void moloch_add_can_quit (MolochCanQuitFunc func, const char *name) */ gboolean moloch_quit_gfunc (gpointer UNUSED(user_data)) { -static gboolean readerExit = TRUE; -static gboolean writerExit = TRUE; +LOCAL gboolean readerExit = TRUE; +LOCAL gboolean writerExit = TRUE; // On the first run shutdown reader and sessions if (readerExit) { @@ -518,7 +518,7 @@ void moloch_quit() */ gboolean moloch_ready_gfunc (gpointer UNUSED(user_data)) { - if (moloch_db_tags_loading() || moloch_http_queue_length(esServer)) + if (moloch_http_queue_length(esServer)) return TRUE; if (config.debug) @@ -606,8 +606,6 @@ void moloch_mlockall_init() /******************************************************************************/ int main(int argc, char **argv) { - LOG("THREAD %p", (gpointer)pthread_self()); - signal(SIGHUP, reload); signal(SIGINT, controlc); signal(SIGUSR1, exit); @@ -616,6 +614,9 @@ int main(int argc, char **argv) mainLoop = g_main_loop_new(NULL, FALSE); parse_args(argc, argv); + if (config.debug) + LOG("THREAD %p", (gpointer)pthread_self()); + moloch_hex_init(); moloch_config_init(); moloch_writers_init(); diff --git a/capture/moloch.h b/capture/moloch.h index c4e7865b30..ee9cb6f010 100644 --- a/capture/moloch.h +++ b/capture/moloch.h @@ -37,7 +37,7 @@ #define UNUSED(x) x __attribute((unused)) -#define MOLOCH_API_VERSION 51 +#define MOLOCH_API_VERSION 100 #define MOLOCH_SESSIONID_LEN 37 @@ -103,28 +103,29 @@ typedef struct moloch_trie { */ typedef struct { - MolochStringHead_t commonName; //2.5.4.3 - char *orgName; // 2.5.4.10 + MolochStringHead_t commonName; // 2.5.4.3 + char *orgName; // 2.5.4.10 char orgUtf8; } MolochCertInfo_t; -typedef struct moloch_tlsinfo { - struct moloch_tlsinfo *t_next, *t_prev; - uint32_t t_hash; - uint64_t notBefore; - uint64_t notAfter; - MolochCertInfo_t issuer; - MolochCertInfo_t subject; - MolochStringHead_t alt; - unsigned char *serialNumber; - short serialNumberLen; - short t_bucket; - unsigned char hash[60]; +typedef struct moloch_certsinfo { + struct moloch_certsinfo *t_next, *t_prev; + uint32_t t_hash; + uint64_t notBefore; + uint64_t notAfter; + MolochCertInfo_t issuer; + MolochCertInfo_t subject; + MolochStringHead_t alt; + unsigned char *serialNumber; + short serialNumberLen; + short t_bucket; + unsigned char hash[60]; + char isCA; } MolochCertsInfo_t; typedef struct { - struct moloch_tlsinfo *t_next, *t_prev; - int t_count; + struct moloch_certsinfo *t_next, *t_prev; + int t_count; } MolochCertsInfoHead_t; typedef HASH_VAR(s_, MolochCertsInfoHash_t, MolochCertsInfoHead_t, 1); @@ -139,20 +140,18 @@ typedef HASH_VAR(s_, MolochCertsInfoHashStd_t, MolochCertsInfoHead_t, 5); #define MOLOCH_FIELD_TYPE_INT 0 #define MOLOCH_FIELD_TYPE_INT_ARRAY 1 #define MOLOCH_FIELD_TYPE_INT_HASH 2 -#define MOLOCH_FIELD_TYPE_STR 3 -#define MOLOCH_FIELD_TYPE_STR_ARRAY 4 -#define MOLOCH_FIELD_TYPE_STR_HASH 5 -#define MOLOCH_FIELD_TYPE_IP 6 -#define MOLOCH_FIELD_TYPE_IP_GHASH 7 -#define MOLOCH_FIELD_TYPE_CERTSINFO 8 -#define MOLOCH_FIELD_TYPE_INT_GHASH 9 -#define MOLOCH_FIELD_TYPE_IP_HASH 10 +#define MOLOCH_FIELD_TYPE_INT_GHASH 3 +#define MOLOCH_FIELD_TYPE_STR 4 +#define MOLOCH_FIELD_TYPE_STR_ARRAY 5 +#define MOLOCH_FIELD_TYPE_STR_HASH 6 +#define MOLOCH_FIELD_TYPE_STR_GHASH 7 +#define MOLOCH_FIELD_TYPE_IP 8 +#define MOLOCH_FIELD_TYPE_IP_GHASH 9 +#define MOLOCH_FIELD_TYPE_CERTSINFO 10 /* These are ones you should set */ /* Field should be set on all linked sessions */ #define MOLOCH_FIELD_FLAG_LINKED_SESSIONS 0x0001 -/* Create a XXX-cnt field with unique count */ -#define MOLOCH_FIELD_FLAG_COUNT 0x0002 /* Force the field to be utf8 */ #define MOLOCH_FIELD_FLAG_FORCE_UTF8 0x0004 /* Don't create in fields db table */ @@ -161,16 +160,13 @@ typedef HASH_VAR(s_, MolochCertsInfoHashStd_t, MolochCertsInfoHead_t, 5); #define MOLOCH_FIELD_FLAG_FAKE 0x0010 /* Don't create in capture list */ #define MOLOCH_FIELD_FLAG_DISABLED 0x0020 - -/* These are ones you shouldn't set, for old cruf before we were smarter */ -/* XXXcnt - dont use */ +/* Added Cnt */ #define MOLOCH_FIELD_FLAG_CNT 0x1000 -/* XXXscnt - dont use */ -#define MOLOCH_FIELD_FLAG_SCNT 0x2000 /* prepend ip stuff - dont use*/ #define MOLOCH_FIELD_FLAG_IPPRE 0x4000 + typedef struct moloch_field_info { struct moloch_field_info *d_next, *d_prev; /* Must be first */ char *dbFieldFull; /* Must be second - this is the full version example:mysql.user-term */ @@ -208,6 +204,7 @@ typedef struct { MolochIntHashStd_t *ihash; MolochCertsInfoHashStd_t *cihash; GHashTable *ghash; + struct in6_addr *ip; }; uint32_t jsonSize; } MolochField_t; @@ -241,6 +238,11 @@ typedef struct { #define MOLOCH_COND_BROADCAST(var) pthread_cond_broadcast(&var##_cond) #define MOLOCH_COND_SIGNAL(var) pthread_cond_signal(&var##_cond) +#define MOLOCH_THREAD_INCR(var) __sync_add_and_fetch(&var, 1); +#define MOLOCH_THREAD_INCRNEW(var) __sync_add_and_fetch(&var, 1); +#define MOLOCH_THREAD_INCROLD(var) __sync_fetch_and_add(&var, 1); +#define MOLOCH_THREAD_INCR_NUM(var, num) __sync_fetch_and_add(&var, num); + #define MOLOCH_MAX_PACKET_THREADS 24 #define MAX_INTERFACES 32 @@ -263,6 +265,11 @@ typedef struct { enum MolochRotate { MOLOCH_ROTATE_HOURLY, MOLOCH_ROTATE_HOURLY6, MOLOCH_ROTATE_DAILY, MOLOCH_ROTATE_WEEKLY, MOLOCH_ROTATE_MONTHLY }; #define MOLOCH_FIELDS_MAX 256 +#define MOLOCH_FIELD_EXSPECIAL_SRC_IP (MOLOCH_FIELDS_MAX-1) +#define MOLOCH_FIELD_EXSPECIAL_SRC_PORT (MOLOCH_FIELDS_MAX-2) +#define MOLOCH_FIELD_EXSPECIAL_DST_IP (MOLOCH_FIELDS_MAX-3) +#define MOLOCH_FIELD_EXSPECIAL_DST_PORT (MOLOCH_FIELDS_MAX-4) +#define MOLOCH_FIELD_EXSPECIAL_START (MOLOCH_FIELDS_MAX-4) typedef struct moloch_config { gboolean quitting; @@ -296,7 +303,6 @@ typedef struct moloch_config { HASH_VAR(s_, dontSaveTags, MolochStringHead_t, 11); MolochFieldInfo_t *fields[MOLOCH_FIELDS_MAX]; int maxField; - int tagsField; int tagsStringField; int numPlugins; @@ -312,11 +318,10 @@ typedef struct moloch_config { char *bpf; char *yara; char *emailYara; - char *geoipFile; - char *geoipASNFile; - char *geoip6File; - char *geoipASN6File; + char *geoLite2ASN; + char *geoLite2Country; char *rirFile; + char *ouiFile; char *dropUser; char *dropGroup; char **pluginsDir; @@ -358,6 +363,7 @@ typedef struct moloch_config { char parseSMB; char parseQSValue; char parseCookieValue; + char supportSha256; char reqBodyOnlyUtf8; char compressES; char antiSynDrop; @@ -370,7 +376,6 @@ typedef struct { char *asn; char *rir; int numtags; - int tags[10]; char *tagsStr[10]; } MolochIpInfo_t; @@ -463,6 +468,8 @@ typedef enum { MOLOCH_TCPFLAG_FIN, MOLOCH_TCPFLAG_RST, MOLOCH_TCPFLAG_URG, + MOLOCH_TCPFLAG_SRC_ZERO, + MOLOCH_TCPFLAG_DST_ZERO, MOLOCH_TCPFLAG_MAX } MolochSesTcpFlags; /******************************************************************************/ @@ -682,6 +689,9 @@ gchar **moloch_config_str_list(GKeyFile *keyfile, char *key, char *d); uint32_t moloch_config_int(GKeyFile *keyfile, char *key, uint32_t d, uint32_t min, uint32_t max); char moloch_config_boolean(GKeyFile *keyfile, char *key, char d); +typedef void (*MolochFileChange_cb)(char *name); +void moloch_config_monitor_file(char *desc, char *name, MolochFileChange_cb cb); + /******************************************************************************/ @@ -690,18 +700,17 @@ char moloch_config_boolean(GKeyFile *keyfile, char *key, char d); */ void moloch_db_init(); -int moloch_db_tags_loading(); char *moloch_db_create_file(time_t firstPacket, char *name, uint64_t size, int locked, uint32_t *id); char *moloch_db_create_file_full(time_t firstPacket, char *name, uint64_t size, int locked, uint32_t *id, ...); void moloch_db_save_session(MolochSession_t *session, int final); -void moloch_db_get_tag(void *uw, int tagtype, const char *tag, MolochTag_cb func); -uint32_t moloch_db_peek_tag(const char *tagname); void moloch_db_add_local_ip(char *str, MolochIpInfo_t *ii); void moloch_db_add_field(char *group, char *kind, char *expression, char *friendlyName, char *dbField, char *help, int haveap, va_list ap); void moloch_db_update_field(char *expression, char *name, char *value); void moloch_db_update_filesize(uint32_t fileid, uint64_t size); gboolean moloch_db_file_exists(char *filename); void moloch_db_exit(); +void moloch_db_oui_lookup(int field, MolochSession_t *session, const uint8_t *mac); + // Replace how SPI data is sent to ES. // The implementation must either call a moloch_http_free_buffer or another moloch_http routine that frees the buffer @@ -810,8 +819,6 @@ void moloch_session_exit(); void moloch_session_add_protocol(MolochSession_t *session, const char *protocol); gboolean moloch_session_has_protocol(MolochSession_t *session, const char *protocol); void moloch_session_add_tag(MolochSession_t *session, const char *tag); -void moloch_session_add_tag_type(MolochSession_t *session, int field, const char *tag); -gboolean moloch_session_has_tag(MolochSession_t *session, const char *tag); #define moloch_session_incr_outstanding(session) (session)->outstandingQueries++ gboolean moloch_session_decr_outstanding(MolochSession_t *session); @@ -834,12 +841,12 @@ int moloch_session_thread_outstanding(int thread); int moloch_session_cmd_outstanding(); typedef enum { - MOLOCH_SES_CMD_ADD_TAG, MOLOCH_SES_CMD_FUNC } MolochSesCmd; typedef void (*MolochCmd_func)(MolochSession_t *session, gpointer uw1, gpointer uw2); void moloch_session_add_cmd(MolochSession_t *session, MolochSesCmd cmd, gpointer uw1, gpointer uw2, MolochCmd_func func); +void moloch_session_add_cmd_thread(int thread, gpointer uw1, gpointer uw2, MolochCmd_func func); /******************************************************************************/ /* @@ -855,6 +862,7 @@ int moloch_packet_frags_outstanding(); int moloch_packet_frags_size(); uint64_t moloch_packet_dropped_frags(); uint64_t moloch_packet_dropped_overload(); +uint64_t moloch_packet_total_bytes(); void moloch_packet_thread_wake(int thread); void moloch_packet_flush(); void moloch_packet_process_data(MolochSession_t *session, const uint8_t *data, int len, int which); @@ -987,7 +995,12 @@ const char *moloch_field_string_add(int pos, MolochSession_t *session, const cha gboolean moloch_field_string_add_lower(int pos, MolochSession_t *session, const char *string, int len); const char *moloch_field_string_uw_add(int pos, MolochSession_t *session, const char *string, int len, gpointer uw, gboolean copy); gboolean moloch_field_int_add(int pos, MolochSession_t *session, int i); +gboolean moloch_field_ip4_add(int pos, MolochSession_t *session, int i); +gboolean moloch_field_ip6_add(int pos, MolochSession_t *session, const uint8_t *val); +gboolean moloch_field_ip_add_str(int pos, MolochSession_t *session, char *str); gboolean moloch_field_certsinfo_add(int pos, MolochSession_t *session, MolochCertsInfo_t *info, int len); +void moloch_field_macoui_add(MolochSession_t *session, int macField, int ouiField, const uint8_t *mac); + int moloch_field_count(int pos, MolochSession_t *session); void moloch_field_certsinfo_free (MolochCertsInfo_t *certs); void moloch_field_free(MolochSession_t *session); @@ -998,6 +1011,11 @@ void moloch_field_ops_free(MolochFieldOps_t *ops); void moloch_field_ops_add(MolochFieldOps_t *ops, int fieldPos, char *value, int valuelen); void moloch_field_ops_run(MolochSession_t *session, MolochFieldOps_t *ops); +void *moloch_field_parse_ip(const char *str); +gboolean moloch_field_ip_equal (gconstpointer v1, gconstpointer v2); +guint moloch_field_ip_hash (gconstpointer v); + + /******************************************************************************/ /* * writers.c @@ -1061,6 +1079,7 @@ void moloch_rules_init(); void moloch_rules_recompile(); void moloch_rules_run_field_set(MolochSession_t *session, int pos, const gpointer value); int moloch_rules_run_every_packet(MolochPacket_t *packet); +void moloch_rules_session_create(MolochSession_t *session); void moloch_rules_run_session_setup(MolochSession_t *session, MolochPacket_t *packet); void moloch_rules_run_after_classify(MolochSession_t *session); void moloch_rules_run_before_save(MolochSession_t *session, int final); diff --git a/capture/molochmagic.pl b/capture/molochmagic.pl index 1d78f3f264..c4975ca8ac 100755 --- a/capture/molochmagic.pl +++ b/capture/molochmagic.pl @@ -78,7 +78,7 @@ sub process { } ################################################################################ print <copied) { free(packet->pkt); @@ -147,7 +149,7 @@ void moloch_packet_process_data(MolochSession_t *session, const uint8_t *data, i } } /******************************************************************************/ -void moloch_packet_tcp_finish(MolochSession_t *session) +LOCAL void moloch_packet_tcp_finish(MolochSession_t *session) { MolochTcpData_t *ftd; MolochTcpData_t *next; @@ -199,7 +201,7 @@ void moloch_packet_tcp_finish(MolochSession_t *session) } /******************************************************************************/ -void moloch_packet_process_icmp(MolochSession_t * const UNUSED(session), MolochPacket_t * const UNUSED(packet)) +LOCAL void moloch_packet_process_icmp(MolochSession_t * const UNUSED(session), MolochPacket_t * const packet) { const uint8_t *data = packet->pkt + packet->payloadOffset; @@ -209,7 +211,7 @@ void moloch_packet_process_icmp(MolochSession_t * const UNUSED(session), MolochP } } /******************************************************************************/ -void moloch_packet_process_udp(MolochSession_t * const session, MolochPacket_t * const packet) +LOCAL void moloch_packet_process_udp(MolochSession_t * const session, MolochPacket_t * const packet) { const uint8_t *data = packet->pkt + packet->payloadOffset + 8; int len = packet->payloadLen - 8; @@ -236,7 +238,7 @@ void moloch_packet_process_udp(MolochSession_t * const session, MolochPacket_t * } } /******************************************************************************/ -int moloch_packet_process_tcp(MolochSession_t * const session, MolochPacket_t * const packet) +LOCAL int moloch_packet_process_tcp(MolochSession_t * const session, MolochPacket_t * const packet) { if (session->stopTCP) return 1; @@ -252,6 +254,10 @@ int moloch_packet_process_tcp(MolochSession_t * const session, MolochPacket_t * const uint32_t seq = ntohl(tcphdr->th_seq); + if (tcphdr->th_win == 0 && (tcphdr->th_flags & TH_RST) == 0) { + session->tcpFlagCnt[MOLOCH_TCPFLAG_SRC_ZERO + packet->direction]++; + } + if (len < 0) return 1; @@ -613,6 +619,10 @@ LOCAL void *moloch_packet_thread(void *threadp) break; } + if (isNew) { + moloch_rules_session_create(session); + } + /* Check if the stop saving bpf filters match */ if (session->packets[packet->direction] == 0 && session->stopSaving == 0) { moloch_rules_run_session_setup(session, packet); @@ -648,31 +658,13 @@ LOCAL void *moloch_packet_thread(void *threadp) if (pcapFileHeader.linktype == 1 && session->firstBytesLen[packet->direction] < 8 && session->packets[packet->direction] < 10) { const uint8_t *pcapData = packet->pkt; - char str1[20]; - char str2[20]; - snprintf(str1, sizeof(str1), "%02x:%02x:%02x:%02x:%02x:%02x", - pcapData[0], - pcapData[1], - pcapData[2], - pcapData[3], - pcapData[4], - pcapData[5]); - - - snprintf(str2, sizeof(str2), "%02x:%02x:%02x:%02x:%02x:%02x", - pcapData[6], - pcapData[7], - pcapData[8], - pcapData[9], - pcapData[10], - pcapData[11]); if (packet->direction == 1) { - moloch_field_string_add(mac1Field, session, str1, 17, TRUE); - moloch_field_string_add(mac2Field, session, str2, 17, TRUE); + moloch_field_macoui_add(session, mac1Field, oui1Field, pcapData+0); + moloch_field_macoui_add(session, mac2Field, oui2Field, pcapData+6); } else { - moloch_field_string_add(mac1Field, session, str2, 17, TRUE); - moloch_field_string_add(mac2Field, session, str1, 17, TRUE); + moloch_field_macoui_add(session, mac1Field, oui1Field, pcapData+6); + moloch_field_macoui_add(session, mac2Field, oui2Field, pcapData+0); } int n = 12; @@ -685,8 +677,8 @@ LOCAL void *moloch_packet_thread(void *threadp) switch(packet->vpnType) { case MOLOCH_PACKET_VPNTYPE_GRE: ip4 = (struct ip*)(packet->pkt + packet->vpnIpOffset); - moloch_field_int_add(greIpField, session, ip4->ip_src.s_addr); - moloch_field_int_add(greIpField, session, ip4->ip_dst.s_addr); + moloch_field_ip4_add(greIpField, session, ip4->ip_src.s_addr); + moloch_field_ip4_add(greIpField, session, ip4->ip_dst.s_addr); moloch_session_add_protocol(session, "gre"); break; case MOLOCH_PACKET_VPNTYPE_PPPOE: @@ -723,7 +715,7 @@ LOCAL void *moloch_packet_thread(void *threadp) } /******************************************************************************/ -int moloch_packet_gre4(MolochPacketBatch_t * batch, MolochPacket_t * const packet, const uint8_t *data, int len) +LOCAL int moloch_packet_gre4(MolochPacketBatch_t * batch, MolochPacket_t * const packet, const uint8_t *data, int len) { BSB bsb; @@ -786,7 +778,7 @@ void moloch_packet_frags_free(MolochFrags_t * const frags) MOLOCH_TYPE_FREE(MolochFrags_t, frags); } /******************************************************************************/ -gboolean moloch_packet_frags_process(MolochPacket_t * const packet) +LOCAL gboolean moloch_packet_frags_process(MolochPacket_t * const packet) { MolochPacket_t * fpacket; MolochFrags_t *frags; @@ -905,7 +897,7 @@ gboolean moloch_packet_frags_process(MolochPacket_t * const packet) return TRUE; } /******************************************************************************/ -void moloch_packet_frags4(MolochPacketBatch_t *batch, MolochPacket_t * const packet) +LOCAL void moloch_packet_frags4(MolochPacketBatch_t *batch, MolochPacket_t * const packet) { MolochFrags_t *frags; @@ -939,7 +931,7 @@ int moloch_packet_frags_outstanding() return 0; } /******************************************************************************/ -void moloch_packet_log(int ses) +LOCAL void moloch_packet_log(int ses) { MolochReaderStats_t stats; if (moloch_reader_stats(&stats)) { @@ -970,10 +962,8 @@ void moloch_packet_log(int ses) ); } /******************************************************************************/ -int moloch_packet_ip(MolochPacketBatch_t *batch, MolochPacket_t * const packet, const char * const sessionId) +LOCAL int moloch_packet_ip(MolochPacketBatch_t *batch, MolochPacket_t * const packet, const char * const sessionId) { - totalBytes += packet->pktlen; - if (totalPackets == 0) { MolochReaderStats_t stats; if (!moloch_reader_stats(&stats)) { @@ -984,7 +974,7 @@ int moloch_packet_ip(MolochPacketBatch_t *batch, MolochPacket_t * const packet, LOG("%" PRIu64 " Initial Dropped = %d", totalPackets, initialDropped); } - __sync_add_and_fetch(&totalPackets, 1); + MOLOCH_THREAD_INCR(totalPackets); if (totalPackets % config.logEveryXPackets == 0) { moloch_packet_log(packet->ses); } @@ -992,6 +982,8 @@ int moloch_packet_ip(MolochPacketBatch_t *batch, MolochPacket_t * const packet, packet->hash = moloch_session_hash(sessionId); uint32_t thread = packet->hash % config.packetThreads; + totalBytes[thread] += packet->pktlen; + if (DLL_COUNT(packet_, &packetQ[thread]) >= config.maxPacketsInQueue) { MOLOCH_LOCK(packetQ[thread].lock); overloadDrops[thread]++; @@ -1016,7 +1008,7 @@ int moloch_packet_ip(MolochPacketBatch_t *batch, MolochPacket_t * const packet, return MOLOCH_PACKET_SUCCESS; } /******************************************************************************/ -int moloch_packet_ip4(MolochPacketBatch_t * batch, MolochPacket_t * const packet, const uint8_t *data, int len) +LOCAL int moloch_packet_ip4(MolochPacketBatch_t *batch, MolochPacket_t * const packet, const uint8_t *data, int len) { struct ip *ip4 = (struct ip*)data; struct tcphdr *tcphdr = 0; @@ -1045,19 +1037,13 @@ int moloch_packet_ip4(MolochPacketBatch_t * batch, MolochPacket_t * const packet #endif return MOLOCH_PACKET_CORRUPT; } - if (ipTree) { - prefix_t prefix; + if (ipTree4) { patricia_node_t *node; - prefix.family = AF_INET; - prefix.bitlen = 32; - - prefix.add.sin= ip4->ip_src; - if ((node = patricia_search_best2 (ipTree, &prefix, 1)) && node->data == NULL) + if ((node = patricia_search_best3 (ipTree4, (u_char*)&ip4->ip_src, 32)) && node->data == NULL) return MOLOCH_PACKET_IP_DROPPED; - prefix.add.sin= ip4->ip_dst; - if ((node = patricia_search_best2 (ipTree, &prefix, 1)) && node->data == NULL) + if ((node = patricia_search_best3 (ipTree4, (u_char*)&ip4->ip_dst, 32)) && node->data == NULL) return MOLOCH_PACKET_IP_DROPPED; } @@ -1122,7 +1108,7 @@ int moloch_packet_ip4(MolochPacketBatch_t * batch, MolochPacket_t * const packet return moloch_packet_ip(batch, packet, sessionId); } /******************************************************************************/ -int moloch_packet_ip6(MolochPacketBatch_t * batch, MolochPacket_t * const packet, const uint8_t *data, int len) +LOCAL int moloch_packet_ip6(MolochPacketBatch_t * batch, MolochPacket_t * const packet, const uint8_t *data, int len) { struct ip6_hdr *ip6 = (struct ip6_hdr *)data; struct tcphdr *tcphdr = 0; @@ -1138,6 +1124,16 @@ int moloch_packet_ip6(MolochPacketBatch_t * batch, MolochPacket_t * const packet return MOLOCH_PACKET_CORRUPT; } + if (ipTree6) { + patricia_node_t *node; + + if ((node = patricia_search_best3 (ipTree6, (u_char*)&ip6->ip6_src, 128)) && node->data == NULL) + return MOLOCH_PACKET_IP_DROPPED; + + if ((node = patricia_search_best3 (ipTree6, (u_char*)&ip6->ip6_dst, 128)) && node->data == NULL) + return MOLOCH_PACKET_IP_DROPPED; + } + int ip_hdr_len = sizeof(struct ip6_hdr); packet->ipOffset = (uint8_t*)data - packet->pkt; @@ -1211,7 +1207,7 @@ int moloch_packet_ip6(MolochPacketBatch_t * batch, MolochPacket_t * const packet return moloch_packet_ip(batch, packet, sessionId); } /******************************************************************************/ -int moloch_packet_pppoe(MolochPacketBatch_t * batch, MolochPacket_t * const packet, const uint8_t *data, int len) +LOCAL int moloch_packet_pppoe(MolochPacketBatch_t * batch, MolochPacket_t * const packet, const uint8_t *data, int len) { if (len < 8 || data[0] != 0x11 || data[1] != 0) { #ifdef DEBUG_PACKET @@ -1239,7 +1235,7 @@ int moloch_packet_pppoe(MolochPacketBatch_t * batch, MolochPacket_t * const pack } } /******************************************************************************/ -int moloch_packet_mpls(MolochPacketBatch_t * batch, MolochPacket_t * const packet, const uint8_t *data, int len) +LOCAL int moloch_packet_mpls(MolochPacketBatch_t * batch, MolochPacket_t * const packet, const uint8_t *data, int len) { while (1) { if (len < 4 + (int)sizeof(struct ip)) { @@ -1272,7 +1268,7 @@ int moloch_packet_mpls(MolochPacketBatch_t * batch, MolochPacket_t * const packe return MOLOCH_PACKET_CORRUPT; } /******************************************************************************/ -int moloch_packet_ether(MolochPacketBatch_t * batch, MolochPacket_t * const packet, const uint8_t *data, int len) +LOCAL int moloch_packet_ether(MolochPacketBatch_t * batch, MolochPacket_t * const packet, const uint8_t *data, int len) { if (len < 14) { #ifdef DEBUG_PACKET @@ -1309,7 +1305,7 @@ int moloch_packet_ether(MolochPacketBatch_t * batch, MolochPacket_t * const pack return MOLOCH_PACKET_CORRUPT; } /******************************************************************************/ -int moloch_packet_sll(MolochPacketBatch_t * batch, MolochPacket_t * const packet, const uint8_t *data, int len) +LOCAL int moloch_packet_sll(MolochPacketBatch_t * batch, MolochPacket_t * const packet, const uint8_t *data, int len) { if (len < 16) { #ifdef DEBUG_PACKET @@ -1337,7 +1333,7 @@ int moloch_packet_sll(MolochPacketBatch_t * batch, MolochPacket_t * const packet return MOLOCH_PACKET_CORRUPT; } /******************************************************************************/ -int moloch_packet_nflog(MolochPacketBatch_t * batch, MolochPacket_t * const packet, const uint8_t *data, int len) +LOCAL int moloch_packet_nflog(MolochPacketBatch_t * batch, MolochPacket_t * const packet, const uint8_t *data, int len) { if (len < 14 || (data[0] != AF_INET && data[0] != AF_INET6) || @@ -1438,7 +1434,7 @@ void moloch_packet_batch(MolochPacketBatch_t * batch, MolochPacket_t * const pac default: LOGEXIT("ERROR - Unsupported pcap link type %d", pcapFileHeader.linktype); } - __sync_add_and_fetch(&packetStats[rc], 1); + MOLOCH_THREAD_INCR(packetStats[rc]); if (rc) { moloch_packet_free(packet); @@ -1457,7 +1453,7 @@ int moloch_packet_outstanding() return count; } /******************************************************************************/ -uint32_t moloch_packet_frag_hash(const void *key) +LOCAL uint32_t moloch_packet_frag_hash(const void *key) { int i; uint32_t n = 0; @@ -1467,7 +1463,7 @@ uint32_t moloch_packet_frag_hash(const void *key) return n; } /******************************************************************************/ -int moloch_packet_frag_cmp(const void *keyv, const void *elementv) +LOCAL int moloch_packet_frag_cmp(const void *keyv, const void *elementv) { MolochFrags_t *element = (MolochFrags_t *)elementv; @@ -1484,15 +1480,15 @@ void moloch_packet_init() pcapFileHeader.sigfigs = 0; mac1Field = moloch_field_define("general", "lotermfield", - "mac.src", "Src MAC", "mac1-term", + "mac.src", "Src MAC", "srcMac", "Source ethernet mac addresses set for session", - MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_COUNT | MOLOCH_FIELD_FLAG_LINKED_SESSIONS, + MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT | MOLOCH_FIELD_FLAG_LINKED_SESSIONS, NULL); mac2Field = moloch_field_define("general", "lotermfield", - "mac.dst", "Dst MAC", "mac2-term", + "mac.dst", "Dst MAC", "dstMac", "Destination ethernet mac addresses set for session", - MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_COUNT | MOLOCH_FIELD_FLAG_LINKED_SESSIONS, + MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT | MOLOCH_FIELD_FLAG_LINKED_SESSIONS, NULL); moloch_field_define("general", "lotermfield", @@ -1502,32 +1498,29 @@ void moloch_packet_init() "regex", "^mac\\\\.(?:(?!\\\\.cnt$).)*$", NULL); + oui1Field = moloch_field_define("general", "termfield", + "oui.src", "Src OUI", "srcOui", + "Source ethernet oui set for session", + MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT | MOLOCH_FIELD_FLAG_LINKED_SESSIONS, + NULL); + + oui2Field = moloch_field_define("general", "termfield", + "oui.dst", "Dst OUI", "dstOui", + "Destination ethernet oui set for session", + MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT | MOLOCH_FIELD_FLAG_LINKED_SESSIONS, + NULL); + + vlanField = moloch_field_define("general", "integer", "vlan", "VLan", "vlan", "vlan value", - MOLOCH_FIELD_TYPE_INT_GHASH, MOLOCH_FIELD_FLAG_COUNT | MOLOCH_FIELD_FLAG_LINKED_SESSIONS, + MOLOCH_FIELD_TYPE_INT_GHASH, MOLOCH_FIELD_FLAG_CNT | MOLOCH_FIELD_FLAG_LINKED_SESSIONS, NULL); greIpField = moloch_field_define("general", "ip", - "gre.ip", "GRE IP", "greip", + "gre.ip", "GRE IP", "greIp", "GRE ip addresses for session", - MOLOCH_FIELD_TYPE_IP_GHASH, MOLOCH_FIELD_FLAG_COUNT | MOLOCH_FIELD_FLAG_LINKED_SESSIONS, - NULL); - - moloch_field_define("general", "lotermfield", - "tipv6.src", "IPv6 Src", "tipv61-term", - "Temporary IPv6 Source", - 0, MOLOCH_FIELD_FLAG_FAKE, - "portField", "p1", - "transform", "ipv6ToHex", - NULL); - - moloch_field_define("general", "lotermfield", - "tipv6.dst", "IPv6 Dst", "tipv62-term", - "Temporary IPv6 Destination", - 0, MOLOCH_FIELD_FLAG_FAKE, - "portField", "p2", - "transform", "ipv6ToHex", + MOLOCH_FIELD_TYPE_IP_GHASH, MOLOCH_FIELD_FLAG_CNT | MOLOCH_FIELD_FLAG_LINKED_SESSIONS, NULL); moloch_field_define("general", "integer", @@ -1573,13 +1566,13 @@ void moloch_packet_init() NULL); icmpTypeField = moloch_field_define("general", "integer", - "icmp.type", "ICMP Type", "icmpType", + "icmp.type", "ICMP Type", "icmp.type", "ICMP type field values", MOLOCH_FIELD_TYPE_INT_GHASH, 0, NULL); icmpCodeField = moloch_field_define("general", "integer", - "icmp.code", "ICMP Code", "icmpCode", + "icmp.code", "ICMP Code", "icmp.code", "ICMP code field values", MOLOCH_FIELD_TYPE_INT_GHASH, 0, NULL); @@ -1627,13 +1620,30 @@ uint64_t moloch_packet_dropped_overload() return count; } /******************************************************************************/ +uint64_t moloch_packet_total_bytes() +{ + uint64_t count = 0; + + int t; + + for (t = 0; t < config.packetThreads; t++) { + count += totalBytes[t]; + } + return count; +} +/******************************************************************************/ void moloch_packet_add_packet_ip(char *ipstr, int mode) { patricia_node_t *node; - if (!ipTree) { - ipTree = New_Patricia(128); + if (strchr(ipstr, '.') != 0) { + if (!ipTree4) + ipTree4 = New_Patricia(32); + node = make_and_lookup(ipTree4, ipstr); + } else { + if (!ipTree6) + ipTree6 = New_Patricia(128); + node = make_and_lookup(ipTree6, ipstr); } - node = make_and_lookup(ipTree, ipstr); node->data = (void *)(long)mode; } /******************************************************************************/ @@ -1646,9 +1656,14 @@ void moloch_packet_set_linksnap(int linktype, int snaplen) /******************************************************************************/ void moloch_packet_exit() { - if (ipTree) { - Destroy_Patricia(ipTree, NULL); - ipTree = 0; + if (ipTree4) { + Destroy_Patricia(ipTree4, NULL); + ipTree4 = 0; + } + + if (ipTree6) { + Destroy_Patricia(ipTree6, NULL); + ipTree6 = 0; } moloch_packet_log(SESSION_TCP); } diff --git a/capture/parsers.c b/capture/parsers.c index dbfa7181b8..f6f85730b7 100644 --- a/capture/parsers.c +++ b/capture/parsers.c @@ -25,9 +25,9 @@ /******************************************************************************/ extern MolochConfig_t config; -static gchar classTag[100]; +LOCAL gchar classTag[100]; -static magic_t cookie[MOLOCH_MAX_PACKET_THREADS]; +LOCAL magic_t cookie[MOLOCH_MAX_PACKET_THREADS]; extern unsigned char moloch_char_to_hexstr[256][3]; @@ -611,25 +611,22 @@ void moloch_parsers_asn_decode_oid(char *buf, int bufsz, unsigned char *oid, int value = 0; } } - +/******************************************************************************/ +LOCAL int cstring_cmp(const void *a, const void *b) +{ + return strcmp(*(char **)a, *(char **)b); +} /******************************************************************************/ void moloch_parsers_init() { - moloch_field_define("general", "lotermfield", - "user", "User", "user", - "External user set for session", - MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT | MOLOCH_FIELD_FLAG_LINKED_SESSIONS, - "category", "user", - NULL); - moloch_field_define("general", "integer", - "session.segments", "Session Segments", "ss", + "session.segments", "Session Segments", "segmentCnt", "Number of segments in session so far", 0, MOLOCH_FIELD_FLAG_FAKE, NULL); moloch_field_define("general", "integer", - "session.length", "Session Length", "sl", + "session.length", "Session Length", "length", "Session Length in milliseconds so far", 0, MOLOCH_FIELD_FLAG_FAKE, NULL); @@ -707,13 +704,10 @@ void moloch_parsers_init() continue; const gchar *filename; - while (1) { - filename = g_dir_read_name(dir); - - // No more files, stop processing this directory - if (!filename) - break; + gchar *filenames[100]; + int flen = 0; + while ((filename = g_dir_read_name(dir))) { // Skip hidden files/directories if (filename[0] == '.') continue; @@ -731,11 +725,19 @@ void moloch_parsers_init() continue; /* Already loaded */ } - gchar *path = g_build_filename (config.parsersDir[d], filename, NULL); + filenames[flen] = g_strdup(filename); + flen++; + } + + qsort((void *)filenames, (size_t)flen, sizeof(char *), cstring_cmp); + + int i; + for (i = 0; i < flen; i++) { + gchar *path = g_build_filename (config.parsersDir[d], filenames[i], NULL); GModule *parser = g_module_open (path, 0); /*G_MODULE_BIND_LAZY | G_MODULE_BIND_LOCAL);*/ if (!parser) { - LOG("ERROR - Couldn't load parser %s from '%s'\n%s", filename, path, g_module_error()); + LOG("ERROR - Couldn't load parser %s from '%s'\n%s", filenames[i], path, g_module_error()); g_free (path); continue; } @@ -744,18 +746,23 @@ void moloch_parsers_init() MolochPluginInitFunc parser_init; if (!g_module_symbol(parser, "moloch_parser_init", (gpointer *)(char*)&parser_init) || parser_init == NULL) { - LOG("ERROR - Module %s doesn't have a moloch_parser_init", filename); + LOG("ERROR - Module %s doesn't have a moloch_parser_init", filenames[i]); + g_free(filenames[i]); continue; } + if (config.debug > 1) { + LOG("Loaded %s", path); + } + + parser_init(); hstring = MOLOCH_TYPE_ALLOC0(MolochString_t); - hstring->str = g_strdup(filename); - hstring->len = strlen(filename); + hstring->str = filenames[i]; + hstring->len = strlen(filenames[i]); HASH_ADD(s_, loaded, hstring->str, hstring); } - g_dir_close(dir); } @@ -765,35 +772,18 @@ void moloch_parsers_init() ); // Set tags field up AFTER loading plugins - config.tagsField = moloch_field_define("general", "termfield", - "tags", "Tags", "ta", + config.tagsStringField = moloch_field_define("general", "termfield", + "tags", "Tags", "tags", "Tags set for session", - MOLOCH_FIELD_TYPE_INT_GHASH, MOLOCH_FIELD_FLAG_CNT | MOLOCH_FIELD_FLAG_LINKED_SESSIONS, - NULL); - - config.tagsStringField = moloch_field_define("general", "notreal", - "tags", "Tags", "tags-term", - "Tags set for session", - MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_LINKED_SESSIONS | MOLOCH_FIELD_FLAG_NODB, + MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT | MOLOCH_FIELD_FLAG_LINKED_SESSIONS, NULL); moloch_field_define("general", "lotermfield", - "asset", "Asset", "asset-term", + "asset", "Asset", "asset", "Asset name", - MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_COUNT | MOLOCH_FIELD_FLAG_LINKED_SESSIONS, + MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT | MOLOCH_FIELD_FLAG_LINKED_SESSIONS, NULL); - if (config.nodeClass) { - snprintf(classTag, sizeof(classTag), "node:%s", config.nodeClass); - moloch_db_get_tag(NULL, config.tagsField, classTag, NULL); - } - - if (config.extraTags) { - int i; - for (i = 0; config.extraTags[i]; i++) { - moloch_db_get_tag(NULL, config.tagsField, config.extraTags[i], NULL); - } - } if (config.extraOps) { int i; diff --git a/capture/parsers/dhcp.c b/capture/parsers/dhcp.c new file mode 100644 index 0000000000..edfb2fe375 --- /dev/null +++ b/capture/parsers/dhcp.c @@ -0,0 +1,173 @@ +/* Copyright 2018, Oath Inc.. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this Software except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "moloch.h" +#include + +extern MolochConfig_t config; +LOCAL int typeField; +LOCAL int hostField; +LOCAL int macField; +LOCAL int ouiField; +LOCAL int idField; + +/******************************************************************************/ +LOCAL void dhcpv6_udp_classify(MolochSession_t *session, const unsigned char *data, int UNUSED(len), int UNUSED(which), void *UNUSED(uw)) +{ + if ((data[0] != 1 && data[0] != 11) || !MOLOCH_SESSION_v6(session)) + return; + moloch_session_add_protocol(session, "dhcpv6"); +} +/******************************************************************************/ +LOCAL int dhcp_udp_parser(MolochSession_t *session, void *UNUSED(uw), const unsigned char *data, int len, int UNUSED(which)) +{ + static char *names[] = { + "", + "DISCOVER", + "OFFER", + "REQUEST", + "DECLINE", + "ACK", + "NAK", + "RELEASE", + "INFORM", + "FORCERENEW", + "LEASEQUERY", + "LEASEUNASSIGNED", + "LEASEUNKNOWN", + "LEASEACTIVE", + "BULKLEASEQUERY", + "LEASEQUERYDONE", + "ACTIVELEASEQUERY", + "LEASEQUERYSTATUS", + "TLS"}; + + if (len < 256) + return 0; + + BSB bsb; + + BSB_INIT(bsb, data, len); + int hardwareType = data[1]; + + if (hardwareType == 1) { + moloch_field_macoui_add(session, macField, ouiField, data+28); + } + + char str[100]; + uint32_t id = 0; + BSB_IMPORT_skip(bsb, 4); + BSB_IMPORT_u32(bsb, id); + snprintf(str, sizeof(str), "%x", id); + moloch_field_string_add(idField, session, str, -1, TRUE); + + // 236 offset + magic len - 4 skip - u32 import + BSB_IMPORT_skip(bsb, 236 + 4 - 4 - 4); + while (BSB_REMAINING(bsb) >= 2) { + int t = 0; + int l = 0; + uint32_t value = 0; + unsigned char *valueStr = 0; + BSB_IMPORT_u08(bsb, t); + if (t == 255) // End Tag, no length + break; + BSB_IMPORT_u08(bsb, l); + if (BSB_IS_ERROR(bsb) || l > BSB_REMAINING(bsb)) + break; + switch(t) { + case 12: // Host Name + BSB_IMPORT_ptr(bsb, valueStr, l); + moloch_field_string_add_lower(hostField, session, (char *)valueStr, l); + break; + case 53: // Message Type + if (l == 1) { + BSB_IMPORT_u08(bsb, value); + moloch_field_string_add(typeField, session, names[value], -1, TRUE); + } else { + BSB_IMPORT_skip(bsb, l); + } + break; + case 61: // Client identifier + BSB_IMPORT_u08(bsb, value); + if (l == 7 && value == 1) { + BSB_IMPORT_ptr(bsb, valueStr, 6); + moloch_field_macoui_add(session, macField, ouiField, valueStr); + } else { + BSB_IMPORT_skip(bsb, l-1); + } + break; + case 81: // FQDN + BSB_IMPORT_u08(bsb, value); + BSB_IMPORT_skip(bsb, 2); + if (value != 0) // Don't support any encodings right now + BSB_IMPORT_skip(bsb, l - 1); + else { + BSB_IMPORT_ptr(bsb, valueStr, l-3); + moloch_field_string_add_lower(hostField, session, (char *)valueStr, l-3); + } + break; + + default: + BSB_IMPORT_skip(bsb, l); + } + } + return 0; +} +/******************************************************************************/ +LOCAL void dhcp_udp_classify(MolochSession_t *session, const unsigned char *data, int len, int UNUSED(which), void *UNUSED(uw)) +{ + + if (len < 256 || (data[0] != 1 && data[0] != 2) || MOLOCH_SESSION_v6(session) || memcmp(data+236, "\x63\x82\x53\x63", 4) != 0) + return; + + moloch_parsers_register(session, dhcp_udp_parser, 0, 0); + moloch_session_add_protocol(session, "dhcp"); +} +/******************************************************************************/ +void moloch_parser_init() +{ + typeField = moloch_field_define("dhcp", "uptermfield", + "dhcp.type", "Type", "dhcp.type", + "DHCP Type", + MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, + NULL); + + hostField = moloch_field_define("dhcp", "lotermfield", + "dhcp.host", "Host", "dhcp.host", + "DHCP Host", + MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, + NULL); + + macField = moloch_field_define("dhcp", "lotermfield", + "dhcp.mac", "Client MAC", "dhcp.mac", + "Client ethernet MAC ", + MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, + NULL); + + ouiField = moloch_field_define("dhcp", "termfield", + "dhcp.oui", "Client OUI", "dhcp.oui", + "Client ethernet OUI ", + MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, + NULL); + + idField = moloch_field_define("dhcp", "lotermfield", + "dhcp.id", "Transaction id", "dhcp.id", + "DHCP Transaction Id", + MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, + NULL); + + + moloch_parsers_classifier_register_port("dhcpv6", NULL, 547, MOLOCH_PARSERS_PORT_UDP, dhcpv6_udp_classify); + moloch_parsers_classifier_register_port("dhcp", NULL, 67, MOLOCH_PARSERS_PORT_UDP, dhcp_udp_classify); +} diff --git a/capture/parsers/dhcp.detail.jade b/capture/parsers/dhcp.detail.jade new file mode 100644 index 0000000000..8bbcc935f5 --- /dev/null +++ b/capture/parsers/dhcp.detail.jade @@ -0,0 +1,8 @@ +if (session.dhcp) + div.sessionDetailMeta.bold dhcp + dl.sessionDetailMeta + +arrayList(session.dhcp, "type", "Type", "dhcp.type") + +arrayList(session.dhcp, "host", "Host", "dhcp.host") + +arrayList(session.dhcp, "id", "Id", "dhcp.id") + +arrayList(session.dhcp, "mac", "Client MAC", "dhcp.mac") + +arrayList(session.dhcp, "oui", "Client OUI", "dhcp.oui") diff --git a/capture/parsers/dns.c b/capture/parsers/dns.c index 4a78c71e6f..19fe406c61 100644 --- a/capture/parsers/dns.c +++ b/capture/parsers/dns.c @@ -14,17 +14,17 @@ */ #include "moloch.h" -static char *qclasses[256]; -static char *qtypes[256]; -static char *statuses[16] = {"NOERROR", "FORMERR", "SERVFAIL", "NXDOMAIN", "NOTIMPL", "REFUSED", "YXDOMAIN", "YXRRSET", "NXRRSET", "NOTAUTH", "NOTZONE", "11", "12", "13", "14", "15"}; -static char *opcodes[16] = {"QUERY", "IQUERY", "STATUS", "3", "NOTIFY", "UPDATE", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15"}; - -static int ipField; -static int hostField; -static int queryTypeField; -static int queryClassField; -static int statusField; -static int opCodeField; +LOCAL char *qclasses[256]; +LOCAL char *qtypes[256]; +LOCAL char *statuses[16] = {"NOERROR", "FORMERR", "SERVFAIL", "NXDOMAIN", "NOTIMPL", "REFUSED", "YXDOMAIN", "YXRRSET", "NXRRSET", "NOTAUTH", "NOTZONE", "11", "12", "13", "14", "15"}; +LOCAL char *opcodes[16] = {"QUERY", "IQUERY", "STATUS", "3", "NOTIFY", "UPDATE", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15"}; + +LOCAL int ipField; +LOCAL int hostField; +LOCAL int queryTypeField; +LOCAL int queryClassField; +LOCAL int statusField; +LOCAL int opCodeField; typedef struct { unsigned char *data[2]; @@ -36,7 +36,7 @@ typedef struct { extern MolochConfig_t config; /******************************************************************************/ -void dns_free(MolochSession_t *UNUSED(session), void *uw) +LOCAL void dns_free(MolochSession_t *UNUSED(session), void *uw) { DNSInfo_t *info = uw; @@ -47,7 +47,7 @@ void dns_free(MolochSession_t *UNUSED(session), void *uw) MOLOCH_TYPE_FREE(DNSInfo_t, info); } /******************************************************************************/ -int dns_name_element(BSB *nbsb, BSB *bsb) +LOCAL int dns_name_element(BSB *nbsb, BSB *bsb) { int nlen = 0; BSB_IMPORT_u08(*bsb, nlen); @@ -77,7 +77,7 @@ int dns_name_element(BSB *nbsb, BSB *bsb) return 0; } /******************************************************************************/ -unsigned char *dns_name(const unsigned char *full, int fulllen, BSB *inbsb, unsigned char *name, int *namelen) +LOCAL unsigned char *dns_name(const unsigned char *full, int fulllen, BSB *inbsb, unsigned char *name, int *namelen) { BSB nbsb; int didPointer = 0; @@ -122,7 +122,7 @@ unsigned char *dns_name(const unsigned char *full, int fulllen, BSB *inbsb, unsi return name; } /******************************************************************************/ -void dns_parser(MolochSession_t *session, int kind, const unsigned char *data, int len) +LOCAL void dns_parser(MolochSession_t *session, int kind, const unsigned char *data, int len) { if (len < 17) @@ -167,8 +167,6 @@ void dns_parser(MolochSession_t *session, int kind, const unsigned char *data, i if (opcode == 5) continue; - char *lower = g_ascii_strdown((char*)name, namelen); - if (qclass <= 255 && qclasses[qclass]) { moloch_field_string_add(queryClassField, session, qclasses[qclass], -1, TRUE); } @@ -177,9 +175,8 @@ void dns_parser(MolochSession_t *session, int kind, const unsigned char *data, i moloch_field_string_add(queryTypeField, session, qtypes[qtype], -1, TRUE); } - if (lower && !moloch_field_string_add(hostField, session, lower, namelen, FALSE)) { - g_free(lower); - } + if (namelen > 0) + moloch_field_string_add_lower(hostField, session, (char *)name, namelen); } moloch_field_string_add(opCodeField, session, opcodes[opcode], -1, TRUE); switch(kind) { @@ -235,13 +232,10 @@ void dns_parser(MolochSession_t *session, int kind, const unsigned char *data, i unsigned char *ptr = BSB_WORK_PTR(bsb); in.s_addr = ptr[3] << 24 | ptr[2] << 16 | ptr[1] << 8 | ptr[0]; - moloch_field_int_add(ipField, session, in.s_addr); + moloch_field_ip4_add(ipField, session, in.s_addr); if (opcode == 5) { - char *lower = g_ascii_strdown((char*)name, namelen); - if (lower && !moloch_field_string_add(hostField, session, lower, namelen, FALSE)) { - g_free(lower); - } + moloch_field_string_add_lower(hostField, session, (char *)name, namelen); } break; } @@ -255,11 +249,7 @@ void dns_parser(MolochSession_t *session, int kind, const unsigned char *data, i if (!namelen || BSB_IS_ERROR(rdbsb) || !name) continue; - char *lower = g_ascii_strdown((char*)name, namelen); - - if (lower && !moloch_field_string_add(hostField, session, lower, namelen, FALSE)) { - g_free(lower); - } + moloch_field_string_add_lower(hostField, session, (char *)name, namelen); break; } case 15: { @@ -273,18 +263,26 @@ void dns_parser(MolochSession_t *session, int kind, const unsigned char *data, i if (!namelen || BSB_IS_ERROR(rdbsb) || !name) continue; - char *lower = g_ascii_strdown((char*)name, namelen); + moloch_field_string_add_lower(hostField, session, (char *)name, namelen); + } + case 28: { + if (rdlength != 16) + break; + unsigned char *ptr = BSB_WORK_PTR(bsb); + + moloch_field_ip6_add(ipField, session, ptr); - if (lower && !moloch_field_string_add(hostField, session, lower, namelen, FALSE)) { - g_free(lower); + if (opcode == 5) { + moloch_field_string_add_lower(hostField, session, (char *)name, namelen); } + break; } } /* switch */ BSB_IMPORT_skip(bsb, rdlength); } } /******************************************************************************/ -int dns_tcp_parser(MolochSession_t *session, void *uw, const unsigned char *data, int len, int which) +LOCAL int dns_tcp_parser(MolochSession_t *session, void *uw, const unsigned char *data, int len, int which) { DNSInfo_t *info = uw; while (len >= 2) { @@ -336,7 +334,7 @@ int dns_tcp_parser(MolochSession_t *session, void *uw, const unsigned char *data return 0; } /******************************************************************************/ -void dns_tcp_classify(MolochSession_t *session, const unsigned char *UNUSED(data), int UNUSED(len), int UNUSED(which), void *UNUSED(uw)) +LOCAL void dns_tcp_classify(MolochSession_t *session, const unsigned char *UNUSED(data), int UNUSED(len), int UNUSED(which), void *UNUSED(uw)) { if (/*which == 0 &&*/ session->port2 == 53 && !moloch_session_has_protocol(session, "dns")) { moloch_session_add_protocol(session, "dns"); @@ -345,7 +343,7 @@ void dns_tcp_classify(MolochSession_t *session, const unsigned char *UNUSED(data } } /******************************************************************************/ -int dns_udp_parser(MolochSession_t *session, void *uw, const unsigned char *data, int len, int UNUSED(which)) +LOCAL int dns_udp_parser(MolochSession_t *session, void *uw, const unsigned char *data, int len, int UNUSED(which)) { if (uw == 0 || (session->port1 != 53 && session->port2 != 53)) { dns_parser(session, (long)uw, data, len); @@ -353,7 +351,7 @@ int dns_udp_parser(MolochSession_t *session, void *uw, const unsigned char *data return 0; } /******************************************************************************/ -void dns_udp_classify(MolochSession_t *session, const unsigned char *UNUSED(data), int UNUSED(len), int UNUSED(which), void *UNUSED(uw)) +LOCAL void dns_udp_classify(MolochSession_t *session, const unsigned char *UNUSED(data), int UNUSED(len), int UNUSED(which), void *UNUSED(uw)) { moloch_parsers_register(session, dns_udp_parser, uw, 0); } @@ -361,7 +359,7 @@ void dns_udp_classify(MolochSession_t *session, const unsigned char *UNUSED(data void moloch_parser_init() { ipField = moloch_field_define("dns", "ip", - "ip.dns", "IP", "dnsip", + "ip.dns", "IP", "dns.ip", "IP from DNS result", MOLOCH_FIELD_TYPE_IP_GHASH, MOLOCH_FIELD_FLAG_CNT | MOLOCH_FIELD_FLAG_IPPRE, "aliases", "[\"dns.ip\"]", @@ -369,7 +367,7 @@ void moloch_parser_init() NULL); hostField = moloch_field_define("dns", "lotermfield", - "host.dns", "Host", "dnsho", + "host.dns", "Host", "dns.host", "DNS host looked up", MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, "aliases", "[\"dns.host\"]", @@ -377,27 +375,27 @@ void moloch_parser_init() NULL); statusField = moloch_field_define("dns", "uptermfield", - "dns.status", "Status Code", "dns.status-term", + "dns.status", "Status Code", "dns.status", "DNS lookup return code", - MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_COUNT, + MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, NULL); opCodeField = moloch_field_define("dns", "uptermfield", - "dns.opcode", "Op Code", "dns.opcode-term", + "dns.opcode", "Op Code", "dns.opcode", "DNS lookup op code", - MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_COUNT, + MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, NULL); queryTypeField = moloch_field_define("dns", "uptermfield", - "dns.query.type", "Query Type", "dns.qt-term", + "dns.query.type", "Query Type", "dns.qt", "DNS lookup query type", - MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_COUNT, + MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, NULL); queryClassField = moloch_field_define("dns", "uptermfield", - "dns.query.class", "Query Class", "dns.qc-term", + "dns.query.class", "Query Class", "dns.qc", "DNS lookup query class", - MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_COUNT, + MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, NULL); diff --git a/capture/parsers/dns.detail.jade b/capture/parsers/dns.detail.jade index ced1aaffdc..3fecee1359 100644 --- a/capture/parsers/dns.detail.jade +++ b/capture/parsers/dns.detail.jade @@ -1,9 +1,9 @@ -if (session.dnsip || session.dnsho) +if (session.dns) div.sessionDetailMeta.bold DNS dl.sessionDetailMeta(suffix="dns") - +ipArrayListPre(session, "dnsip", "IPs", "dns") - +arrayList(session, 'dnsho', "Hosts", "host.dns") - +arrayList(session.dns, 'opcode-term', "Op Code", "dns.opcode") - +arrayList(session.dns, 'status-term', "Status Code", "dns.status") - +arrayList(session.dns, 'qt-term', "Query Type", "dns.query.type") - +arrayList(session.dns, 'qc-term', "Query Class", "dns.query.class") + +ipArrayList(session.dns, "ip", "IPs", "dns") + +arrayList(session.dns, 'host', "Hosts", "host.dns") + +arrayList(session.dns, 'opcode', "Op Code", "dns.opcode") + +arrayList(session.dns, 'status', "Status Code", "dns.status") + +arrayList(session.dns, 'qt', "Query Type", "dns.query.type") + +arrayList(session.dns, 'qc', "Query Class", "dns.query.class") diff --git a/capture/parsers/email.detail.jade b/capture/parsers/email.detail.jade index 0a60e5728f..2dca6a6233 100644 --- a/capture/parsers/email.detail.jade +++ b/capture/parsers/email.detail.jade @@ -1,19 +1,20 @@ -if (user.emailSearch && (session.esub || session.esrc || session.edst || session.eua || session.eid || session.emv || session.ect || session.efn || session.emd5 || session.eip)) +if (user.emailSearch && session.email) div.sessionDetailMeta.bold Email dl.sessionDetailMeta(suffix="email") - +ipArrayListPre(session, "eip", "IPs", "email") - +arrayList(session, 'eho', "Hosts", "host.email") - +arrayList(session, 'ehh', "Headers", "email.has-header") - +arrayList(session, 'esub', "Subjects", "email.subject") - +arrayList(session, 'esrc', "Senders", "email.src") - +arrayList(session, 'edst', "Destinations", "email.dst") - +arrayList(session, 'eua', "User Agents", "email.x-mailer") - +arrayList(session, 'eid', "Message Ids", "email.message-id") - +arrayList(session, 'emv', "Mime Versions", "email.mime-version") - +arrayList(session, 'ect', "Content Types", "email.content-type") - +arrayList(session, 'efn', "Filenames", "email.fn") - +arrayList(session, 'emd5', "Attachment MD5s", "email.md5") - +arrayList(session.email, "bodymagic-term", "Attachment libfile", "email.bodymagic") - if (session.hdrs) - each value,i in emailFields - +arrayList(session.hdrs, "ehead-" + value.name, value.name + " Header", "email." + value.name) + +ipArrayList(session.email, "ip", "IPs", "email") + +arrayList(session.email, 'host', "Hosts", "host.email") + +arrayList(session.email, 'headers', "Headers", "email.has-header") + +arrayList(session.email, 'subject', "Subjects", "email.subject") + +arrayList(session.email, 'src', "Senders", "email.src") + +arrayList(session.email, 'dst', "Destinations", "email.dst") + +arrayList(session.email, 'useragent', "User Agents", "email.x-mailer") + +arrayList(session.email, 'id', "Message Ids", "email.message-id") + +arrayList(session.email, 'mimeVersion', "Mime Versions", "email.mime-version") + +arrayList(session.email, 'contentType', "Content Types", "email.content-type") + +arrayList(session.email, 'filename', "Filenames", "email.fn") + +arrayList(session.email, 'md5', "Attachment MD5s", "email.md5") + +arrayList(session.email, 'sha256', "Attachment SHA256s", "email.sha256") + +arrayList(session.email, "bodyMagic", "Attachment libfile", "email.bodymagic") + if (session.email.headers) + each value,i in session.email.headers + +arrayList(session.email, "header-" + value, value + " Header", "email." + value) diff --git a/capture/parsers/http.c b/capture/parsers/http.c index e52651e7cd..f7631d92c5 100644 --- a/capture/parsers/http.c +++ b/capture/parsers/http.c @@ -33,7 +33,7 @@ typedef struct { short pos[2]; http_parser parsers[2]; - GChecksum *checksum[2]; + GChecksum *checksum[4]; const char *magicString[2]; uint16_t wParsers:2; @@ -45,35 +45,35 @@ typedef struct { } HTTPInfo_t; extern MolochConfig_t config; -static http_parser_settings parserSettings; +LOCAL http_parser_settings parserSettings; extern uint32_t pluginsCbs; -static MolochStringHashStd_t httpReqHeaders; -static MolochStringHashStd_t httpResHeaders; - -static int cookieKeyField; -static int cookieValueField; -static int hostField; -static int userField; -static int atField; -static int urlsField; -static int xffField; -static int uaField; -static int tagsReqField; -static int tagsResField; -static int md5Field; -static int verReqField; -static int verResField; -static int pathField; -static int keyField; -static int valueField; -static int magicField; -static int statuscodeField; -static int methodField; -static int reqBodyField; +LOCAL MolochStringHashStd_t httpReqHeaders; +LOCAL MolochStringHashStd_t httpResHeaders; + +LOCAL int cookieKeyField; +LOCAL int cookieValueField; +LOCAL int hostField; +LOCAL int userField; +LOCAL int atField; +LOCAL int urlsField; +LOCAL int xffField; +LOCAL int uaField; +LOCAL int tagsReqField; +LOCAL int tagsResField; +LOCAL int md5Field; +LOCAL int sha256Field; +LOCAL int verReqField; +LOCAL int verResField; +LOCAL int pathField; +LOCAL int keyField; +LOCAL int valueField; +LOCAL int magicField; +LOCAL int statuscodeField; +LOCAL int methodField; +LOCAL int reqBodyField; /******************************************************************************/ -int -moloch_hp_cb_on_message_begin (http_parser *parser) +LOCAL int moloch_hp_cb_on_message_begin (http_parser *parser) { HTTPInfo_t *http = parser->data; MolochSession_t *session = http->session; @@ -87,6 +87,9 @@ moloch_hp_cb_on_message_begin (http_parser *parser) http->inValue &= ~(1 << http->which); http->inBody &= ~(1 << http->which); g_checksum_reset(http->checksum[http->which]); + if (config.supportSha256) { + g_checksum_reset(http->checksum[http->which+2]); + } if (pluginsCbs & MOLOCH_PLUGIN_HP_OMB) moloch_plugins_cb_hp_omb(session, parser); @@ -94,8 +97,7 @@ moloch_hp_cb_on_message_begin (http_parser *parser) return 0; } /******************************************************************************/ -int -moloch_hp_cb_on_url (http_parser *parser, const char *at, size_t length) +LOCAL int moloch_hp_cb_on_url (http_parser *parser, const char *at, size_t length) { HTTPInfo_t *http = parser->data; @@ -113,8 +115,7 @@ moloch_hp_cb_on_url (http_parser *parser, const char *at, size_t length) } /******************************************************************************/ -int -moloch_hp_cb_on_body (http_parser *parser, const char *at, size_t length) +LOCAL int moloch_hp_cb_on_body (http_parser *parser, const char *at, size_t length) { HTTPInfo_t *http = parser->data; MolochSession_t *session = http->session; @@ -144,6 +145,9 @@ moloch_hp_cb_on_body (http_parser *parser, const char *at, size_t length) } g_checksum_update(http->checksum[http->which], (guchar *)at, length); + if (config.supportSha256) { + g_checksum_update(http->checksum[http->which+2], (guchar *)at, length); + } if (pluginsCbs & MOLOCH_PLUGIN_HP_OB) moloch_plugins_cb_hp_ob(session, parser, at, length); @@ -152,8 +156,7 @@ moloch_hp_cb_on_body (http_parser *parser, const char *at, size_t length) } /******************************************************************************/ -void -moloch_http_parse_authorization(MolochSession_t *session, char *str) +LOCAL void moloch_http_parse_authorization(MolochSession_t *session, char *str) { gsize olen; @@ -164,10 +167,7 @@ moloch_http_parse_authorization(MolochSession_t *session, char *str) if (!space) return; - char *lower = g_ascii_strdown(str, space-str); - if (!moloch_field_string_add(atField, session, lower, space-str, FALSE)) { - g_free(lower); - } + moloch_field_string_add_lower(atField, session, str, space-str); if (strncasecmp("basic", str, 5) == 0) { str += 5; @@ -206,8 +206,7 @@ moloch_http_parse_authorization(MolochSession_t *session, char *str) } } /******************************************************************************/ -int -moloch_hp_cb_on_message_complete (http_parser *parser) +LOCAL int moloch_hp_cb_on_message_complete (http_parser *parser) { HTTPInfo_t *http = parser->data; MolochSession_t *session = http->session; @@ -222,14 +221,17 @@ moloch_hp_cb_on_message_complete (http_parser *parser) if (http->inBody & (1 << http->which)) { const char *md5 = g_checksum_get_string(http->checksum[http->which]); moloch_field_string_uw_add(md5Field, session, (char*)md5, 32, (gpointer)http->magicString[http->which], TRUE); + if (config.supportSha256) { + const char *sha256 = g_checksum_get_string(http->checksum[http->which+2]); + moloch_field_string_uw_add(sha256Field, session, (char*)sha256, 64, (gpointer)http->magicString[http->which], TRUE); + } } return 0; } /******************************************************************************/ -void -http_add_value(MolochSession_t *session, HTTPInfo_t *http) +LOCAL void http_add_value(MolochSession_t *session, HTTPInfo_t *http) { int pos = http->pos[http->which]; char *s = http->valueString[http->which]->str; @@ -251,28 +253,25 @@ http_add_value(MolochSession_t *session, HTTPInfo_t *http) case MOLOCH_FIELD_TYPE_STR: case MOLOCH_FIELD_TYPE_STR_ARRAY: case MOLOCH_FIELD_TYPE_STR_HASH: + case MOLOCH_FIELD_TYPE_STR_GHASH: moloch_field_string_add(pos, session, s, l, TRUE); break; - case MOLOCH_FIELD_TYPE_IP_HASH: case MOLOCH_FIELD_TYPE_IP_GHASH: { int i; gchar **parts = g_strsplit(http->valueString[http->which]->str, ",", 0); for (i = 0; parts[i]; i++) { - gchar *ip = parts[i]; - while (*ip == ' ') - ip++; + moloch_field_ip_add_str(pos, session, parts[i]); - in_addr_t ia = inet_addr(ip); + /* Add back maybe if (ia == 0 || ia == 0xffffffff) { moloch_session_add_tag(session, "http:bad-xff"); if (config.debug) LOG("INFO - Didn't understand ip: %s %s %d", http->valueString[http->which]->str, ip, ia); continue; } - - moloch_field_int_add(pos, session, ia); + */ } g_strfreev(parts); @@ -285,8 +284,7 @@ http_add_value(MolochSession_t *session, HTTPInfo_t *http) http->pos[http->which] = 0; } /******************************************************************************/ -int -moloch_hp_cb_on_header_field (http_parser *parser, const char *at, size_t length) +LOCAL int moloch_hp_cb_on_header_field (http_parser *parser, const char *at, size_t length) { HTTPInfo_t *http = parser->data; MolochSession_t *session = http->session; @@ -324,12 +322,10 @@ moloch_hp_cb_on_header_field (http_parser *parser, const char *at, size_t length } /******************************************************************************/ -int -moloch_hp_cb_on_header_value (http_parser *parser, const char *at, size_t length) +LOCAL int moloch_hp_cb_on_header_value (http_parser *parser, const char *at, size_t length) { HTTPInfo_t *http = parser->data; MolochSession_t *session = http->session; - char header[200]; MolochString_t *hstring = 0; #ifdef HTTPDEBUG @@ -349,12 +345,11 @@ moloch_hp_cb_on_header_value (http_parser *parser, const char *at, size_t length http->pos[http->which] = (long)(hstring?hstring->uw:0); - snprintf(header, sizeof(header), "http:header:%s", lower); - g_free(lower); if (http->which == http->urlWhich) - moloch_session_add_tag_type(session, tagsReqField, header); + moloch_field_string_add(tagsReqField, session, lower, -1, TRUE); else - moloch_session_add_tag_type(session, tagsResField, header); + moloch_field_string_add(tagsResField, session, lower, -1, TRUE); + g_free(lower); } moloch_plugins_cb_hp_ohv(session, parser, at, length); @@ -363,8 +358,9 @@ moloch_hp_cb_on_header_value (http_parser *parser, const char *at, size_t length if (parser->method) { if (strcasecmp("host", http->header[http->which]) == 0) { if (!http->hostString) - http->hostString = g_string_new_len("//", 2); - g_string_append_len(http->hostString, at, length); + http->hostString = g_string_new_len(at, length); + else + g_string_append_len(http->hostString, at, length); } else if (strcasecmp("cookie", http->header[http->which]) == 0) { if (!http->cookieString) http->cookieString = g_string_new_len(at, length); @@ -388,8 +384,7 @@ moloch_hp_cb_on_header_value (http_parser *parser, const char *at, size_t length return 0; } /******************************************************************************/ -int -moloch_hp_cb_on_headers_complete (http_parser *parser) +LOCAL int moloch_hp_cb_on_headers_complete (http_parser *parser) { HTTPInfo_t *http = parser->data; MolochSession_t *session = http->session; @@ -461,11 +456,11 @@ moloch_hp_cb_on_headers_complete (http_parser *parser) gboolean truncated = FALSE; if (http->urlString && http->hostString) { - char *colon = strchr(http->hostString->str+2, ':'); + char *colon = strchr(http->hostString->str, ':'); if (colon) { - moloch_field_string_add(hostField, session, http->hostString->str+2, colon - http->hostString->str-2, TRUE); + moloch_field_string_add(hostField, session, http->hostString->str, colon - http->hostString->str, TRUE); } else { - moloch_field_string_add(hostField, session, http->hostString->str+2, http->hostString->len-2, TRUE); + moloch_field_string_add(hostField, session, http->hostString->str, http->hostString->len, TRUE); } char *question = strchr(http->urlString->str, '?'); @@ -513,7 +508,7 @@ moloch_hp_cb_on_headers_complete (http_parser *parser) } if (http->urlString->str[0] != '/') { - char *result = strstr(http->urlString->str, http->hostString->str+2); + char *result = strstr(http->urlString->str, http->hostString->str); /* If the host header is in the first 8 bytes of url then just use the url */ if (result && result - http->urlString->str <= 8) { @@ -563,11 +558,11 @@ moloch_hp_cb_on_headers_complete (http_parser *parser) http->urlString = NULL; } else if (http->hostString) { - char *colon = strchr(http->hostString->str+2, ':'); + char *colon = strchr(http->hostString->str, ':'); if (colon) { - moloch_field_string_add(hostField, session, http->hostString->str+2, colon - http->hostString->str-2, TRUE); + moloch_field_string_add(hostField, session, http->hostString->str, colon - http->hostString->str, TRUE); } else { - moloch_field_string_add(hostField, session, http->hostString->str+2, http->hostString->len-2, TRUE); + moloch_field_string_add(hostField, session, http->hostString->str, http->hostString->len, TRUE); } g_string_free(http->hostString, TRUE); @@ -587,7 +582,7 @@ moloch_hp_cb_on_headers_complete (http_parser *parser) /*############################## SHARED ##############################*/ /******************************************************************************/ -int http_parse(MolochSession_t *session, void *uw, const unsigned char *data, int remaining, int which) +LOCAL int http_parse(MolochSession_t *session, void *uw, const unsigned char *data, int remaining, int which) { HTTPInfo_t *http = uw; @@ -638,7 +633,7 @@ void http_save(MolochSession_t UNUSED(*session), void *uw, int final) } /******************************************************************************/ -void http_free(MolochSession_t UNUSED(*session), void *uw) +LOCAL void http_free(MolochSession_t UNUSED(*session), void *uw) { HTTPInfo_t *http = uw; @@ -657,11 +652,15 @@ void http_free(MolochSession_t UNUSED(*session), void *uw) g_checksum_free(http->checksum[0]); g_checksum_free(http->checksum[1]); + if (config.supportSha256) { + g_checksum_free(http->checksum[2]); + g_checksum_free(http->checksum[3]); + } MOLOCH_TYPE_FREE(HTTPInfo_t, http); } /******************************************************************************/ -void http_classify(MolochSession_t *session, const unsigned char *UNUSED(data), int UNUSED(len), int UNUSED(which), void *UNUSED(uw)) +LOCAL void http_classify(MolochSession_t *session, const unsigned char *UNUSED(data), int UNUSED(len), int UNUSED(which), void *UNUSED(uw)) { if (moloch_session_has_protocol(session, "http")) return; @@ -672,6 +671,10 @@ void http_classify(MolochSession_t *session, const unsigned char *UNUSED(data), http->checksum[0] = g_checksum_new(G_CHECKSUM_MD5); http->checksum[1] = g_checksum_new(G_CHECKSUM_MD5); + if (config.supportSha256) { + http->checksum[2] = g_checksum_new(G_CHECKSUM_SHA256); + http->checksum[3] = g_checksum_new(G_CHECKSUM_SHA256); + } http_parser_init(&http->parsers[0], HTTP_BOTH); http_parser_init(&http->parsers[1], HTTP_BOTH); @@ -695,45 +698,43 @@ static const char *method_strings[] = }; hostField = moloch_field_define("http", "lotermfield", - "host.http", "Hostname", "ho", + "host.http", "Hostname", "http.host", "HTTP host header field", MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, "aliases", "[\"http.host\"]", "category", "host", NULL); - urlsField = moloch_field_define("http", "textfield", - "http.uri", "URI", "us", + urlsField = moloch_field_define("http", "termfield", + "http.uri", "URI", "http.uri", "URIs for request", - MOLOCH_FIELD_TYPE_STR_ARRAY, MOLOCH_FIELD_FLAG_CNT, - "rawField", "rawus", + MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, "category", "[\"url\",\"host\"]", NULL); xffField = moloch_field_define("http", "ip", - "ip.xff", "XFF IP", "xff", + "ip.xff", "XFF IP", "http.xffIp", "X-Forwarded-For Header", - MOLOCH_FIELD_TYPE_IP_GHASH, MOLOCH_FIELD_FLAG_SCNT | MOLOCH_FIELD_FLAG_IPPRE, + MOLOCH_FIELD_TYPE_IP_GHASH, MOLOCH_FIELD_FLAG_CNT | MOLOCH_FIELD_FLAG_IPPRE, "category", "ip", NULL); - uaField = moloch_field_define("http", "textfield", - "http.user-agent", "Useragent", "ua", + uaField = moloch_field_define("http", "termfield", + "http.user-agent", "Useragent", "http.useragent", "User-Agent Header", MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, - "rawField", "rawua", NULL); tagsReqField = moloch_field_define("http", "lotermfield", - "http.hasheader.src", "Has Src Header", "hh1", + "http.hasheader.src", "Has Src Header", "http.requestHeader", "Request has header present", - MOLOCH_FIELD_TYPE_INT_GHASH, MOLOCH_FIELD_FLAG_CNT, + MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, NULL); tagsResField = moloch_field_define("http", "lotermfield", - "http.hasheader.dst", "Has Dst Header", "hh2", + "http.hasheader.dst", "Has Dst Header", "http.responseHeader", "Response has header present", - MOLOCH_FIELD_TYPE_INT_GHASH, MOLOCH_FIELD_FLAG_CNT, + MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, NULL); moloch_field_define("http", "lotermfield", @@ -744,12 +745,21 @@ static const char *method_strings[] = NULL); md5Field = moloch_field_define("http", "lotermfield", - "http.md5", "Body MD5", "hmd5", + "http.md5", "Body MD5", "http.md5", "MD5 of http body response", MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, "category", "md5", NULL); + if (config.supportSha256) { + sha256Field = moloch_field_define("http", "lotermfield", + "http.sha256", "Body SHA256", "http.sha256", + "SHA256 of http body response", + MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, + "category", "sha256", + NULL); + } + moloch_field_define("http", "termfield", "http.version", "Version", "httpversion", "HTTP version number", @@ -758,68 +768,68 @@ static const char *method_strings[] = NULL); verReqField = moloch_field_define("http", "termfield", - "http.version.src", "Src Version", "hsver", + "http.version.src", "Src Version", "http.clientVersion", "Request HTTP version number", MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, NULL); verResField = moloch_field_define("http", "termfield", - "http.version.dst", "Dst Version", "hdver", + "http.version.dst", "Dst Version", "http.serverVersion", "Response HTTP version number", MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, NULL); pathField = moloch_field_define("http", "termfield", - "http.uri.path", "URI Path", "hpath", + "http.uri.path", "URI Path", "http.path", "Path portion of URI", MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, NULL); keyField = moloch_field_define("http", "termfield", - "http.uri.key", "QS Keys", "hkey", + "http.uri.key", "QS Keys", "http.key", "Keys from query string of URI", MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, NULL); valueField = moloch_field_define("http", "termfield", - "http.uri.value", "QS Values", "hval", + "http.uri.value", "QS Values", "http.value", "Values from query string of URI", MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, NULL); cookieKeyField = moloch_field_define("http", "termfield", - "http.cookie.key", "Cookie Keys", "hckey-term", + "http.cookie.key", "Cookie Keys", "http.cookieKey", "The keys to cookies sent up in requests", - MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_COUNT, + MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, NULL); cookieValueField = moloch_field_define("http", "termfield", - "http.cookie.value", "Cookie Values", "hcval-term", + "http.cookie.value", "Cookie Values", "http.cookieValue", "The values to cookies sent up in requests", - MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_COUNT, + MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, NULL); methodField = moloch_field_define("http", "termfield", - "http.method", "Request Method", "http.method-term", + "http.method", "Request Method", "http.method", "HTTP Request Method", - MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_COUNT, + MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, NULL); magicField = moloch_field_define("http", "termfield", - "http.bodymagic", "Body Magic", "http.bodymagic-term", + "http.bodymagic", "Body Magic", "http.bodyMagic", "The content type of body determined by libfile/magic", - MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_COUNT, + MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, NULL); userField = moloch_field_define("http", "termfield", - "http.user", "User", "huser-term", + "http.user", "User", "http.user", "HTTP Auth User", MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, "category", "user", NULL); atField = moloch_field_define("http", "lotermfield", - "http.authtype", "Auth Type", "hat-term", + "http.authtype", "Auth Type", "http.authType", "HTTP Auth Type", MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, NULL); @@ -827,11 +837,11 @@ static const char *method_strings[] = statuscodeField = moloch_field_define("http", "integer", "http.statuscode", "Status Code", "http.statuscode", "Response HTTP numeric status code", - MOLOCH_FIELD_TYPE_INT_GHASH, MOLOCH_FIELD_FLAG_COUNT, + MOLOCH_FIELD_TYPE_INT_GHASH, MOLOCH_FIELD_FLAG_CNT, NULL); reqBodyField = moloch_field_define("http", "termfield", - "http.reqbody", "Request Body", "rqbd-term", + "http.reqbody", "Request Body", "http.requestBody", "HTTP Request Body", MOLOCH_FIELD_TYPE_STR_HASH, 0, NULL); @@ -842,8 +852,8 @@ static const char *method_strings[] = moloch_config_add_header(&httpReqHeaders, "x-forwarded-for", xffField); moloch_config_add_header(&httpReqHeaders, "user-agent", uaField); moloch_config_add_header(&httpReqHeaders, "host", hostField); - moloch_config_load_header("headers-http-request", "http", "Request header ", "http.", "hdrs.hreq-", &httpReqHeaders, 0); - moloch_config_load_header("headers-http-response", "http", "Response header ", "http.", "hdrs.hres-", &httpResHeaders, 0); + moloch_config_load_header("headers-http-request", "http", "Request header ", "http.", "http.request-", &httpReqHeaders, 0); + moloch_config_load_header("headers-http-response", "http", "Response header ", "http.", "http.response-", &httpResHeaders, 0); int i; for (i = 0; method_strings[i]; i++) { diff --git a/capture/parsers/http.detail.jade b/capture/parsers/http.detail.jade index d360487794..380dc5ee94 100644 --- a/capture/parsers/http.detail.jade +++ b/capture/parsers/http.detail.jade @@ -1,23 +1,25 @@ -if (session.ho || session.ua || session.hh1 || session.hh2 || session.xff) +if (session.http) div.sessionDetailMeta.bold HTTP dl.sessionDetailMeta(suffix="http") - +arrayList(session.http, "method-term", "Method", "http.method") - +arrayList(session.http, "statuscode", "Status code", "http.statuscode") - +arrayList(session, "ho", "Hosts", "host.http") - +arrayList(session, "ua", "User Agents", "http.user-agent") - +ipArrayListPre(session, "xff", "XFFs", "xff") - +arrayList(session, "hh1", "Request Headers", "http.hasheader.src") - +arrayList(session, "hsver", "Request Versions", "http.version.src") - +arrayList(session, "hh2", "Resp Headers", "http.hasheader.dst") - +arrayList(session, "hdver", "Resp Versions", "http.version.dst") - +arrayList(session, "hmd5", "Body MD5s", "http.md5") - +arrayList(session, "hkey", "QS Keys", "http.uri.key") - +arrayList(session, "hckey-term", "Cookie Keys", "http.cookie.key") - +arrayList(session, "huser-term", "User", "http.user") - +arrayList(session, "hat-term", "Auth Type", "http.authtype") - +arrayList(session.http, "bodymagic-term", "libfile content type", "http.bodymagic") - if (session.hdrs) - each value,i in reqFields - +arrayList(session.hdrs, "hreq-" + value.name, value.name + " Header", "http." + value.name) - each value,i in resFields - +arrayList(session.hdrs, "hres-" + value.name, value.name + " Header", "http." + value.name) + +arrayList(session.http, "method", "Method", "http.method") + +arrayList(session.http, "statusCode", "Status code", "http.statuscode") + +arrayList(session.http, "host", "Hosts", "host.http") + +arrayList(session.http, "usergent", "User Agents", "http.user-agent") + +ipArrayList(session.http, "xffIp", "XFFs", "xff") + +arrayList(session.http, "requestHeader", "Request Headers", "http.hasheader.src") + +arrayList(session.http, "clientVersion", "Client Versions", "http.version.src") + +arrayList(session.http, "responseHeader", "Resp Headers", "http.hasheader.dst") + +arrayList(session.http, "serverVersion", "Server Versions", "http.version.dst") + +arrayList(session.http, "md5", "Body MD5s", "http.md5") + +arrayList(session.http, "sha256", "Body SHA256s", "http.sha256") + +arrayList(session.http, "key", "QS Keys", "http.uri.key") + +arrayList(session.http, "cookieKey", "Cookie Keys", "http.cookie.key") + +arrayList(session.http, "user", "User", "http.user") + +arrayList(session.http, "authType", "Auth Type", "http.authtype") + +arrayList(session.http, "bodyMagic", "libfile content type", "http.bodymagic") + if (session.http.requestHeader) + each value,i in session.http.requestHeader + +arrayList(session.http, "request-" + value, value + " Header", "http.request." + value) + if (session.http.responseHeader) + each value,i in session.http.responseHeader + +arrayList(session.http, "response-" + value, value + " Header", "http.response." + value) diff --git a/capture/parsers/irc.c b/capture/parsers/irc.c index b73ab4ca05..13c99010f1 100644 --- a/capture/parsers/irc.c +++ b/capture/parsers/irc.c @@ -18,11 +18,11 @@ typedef struct { int ircState; } IRCInfo_t; -static int channelsField; -static int nickField; +LOCAL int channelsField; +LOCAL int nickField; /******************************************************************************/ -int irc_parser(MolochSession_t *session, void *uw, const unsigned char *data, int remaining, int which) +LOCAL int irc_parser(MolochSession_t *session, void *uw, const unsigned char *data, int remaining, int which) { IRCInfo_t *irc = uw; @@ -75,14 +75,14 @@ int irc_parser(MolochSession_t *session, void *uw, const unsigned char *data, in return 0; } /******************************************************************************/ -void irc_free(MolochSession_t UNUSED(*session), void *uw) +LOCAL void irc_free(MolochSession_t UNUSED(*session), void *uw) { IRCInfo_t *irc = uw; MOLOCH_TYPE_FREE(IRCInfo_t, irc); } /******************************************************************************/ -void irc_classify(MolochSession_t *session, const unsigned char *data, int len, int which, void *UNUSED(uw)) +LOCAL void irc_classify(MolochSession_t *session, const unsigned char *data, int len, int which, void *UNUSED(uw)) { if (len < 8) return; @@ -109,14 +109,14 @@ void irc_classify(MolochSession_t *session, const unsigned char *data, int len, void moloch_parser_init() { nickField = moloch_field_define("irc", "termfield", - "irc.nick", "Nickname", "ircnck", + "irc.nick", "Nickname", "irc.nick", "Nicknames set", MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, "category", "user", NULL); channelsField = moloch_field_define("irc", "termfield", - "irc.channel", "Channel", "ircch", + "irc.channel", "Channel", "irc.channel", "Channels joined", MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, NULL); diff --git a/capture/parsers/irc.detail.jade b/capture/parsers/irc.detail.jade index dd9fe432ce..cdd139981f 100644 --- a/capture/parsers/irc.detail.jade +++ b/capture/parsers/irc.detail.jade @@ -1,5 +1,5 @@ -if (session.ircnck || session.ircch) +if (session.irc) div.sessionDetailMeta.bold IRC dl.sessionDetailMeta - +arrayList(session, "ircnck", "Nicks", "irc.nick") - +arrayList(session, "ircch", "Channels", "irc.channel") + +arrayList(session.irc, "nick", "Nicks", "irc.nick") + +arrayList(session.irc, "channel", "Channels", "irc.channel") diff --git a/capture/parsers/krb5.c b/capture/parsers/krb5.c index 5bc510363c..5fd5efc846 100644 --- a/capture/parsers/krb5.c +++ b/capture/parsers/krb5.c @@ -18,9 +18,9 @@ extern MolochConfig_t config; -static int realmField; -static int cnameField; -static int snameField; +LOCAL int realmField; +LOCAL int cnameField; +LOCAL int snameField; #define KRB5_MAX_SIZE 4096 typedef struct { @@ -35,7 +35,7 @@ typedef struct { -- name-string[1] SEQUENCE OF GeneralString --} */ -void krb5_parse_principal_name(MolochSession_t *session, int field, const unsigned char *data, int len) +LOCAL void krb5_parse_principal_name(MolochSession_t *session, int field, const unsigned char *data, int len) { MolochASNSeq_t seq[10]; @@ -79,7 +79,7 @@ void krb5_parse_principal_name(MolochSession_t *session, int field, const unsign -- additional-tickets[11] SEQUENCE OF Ticket OPTIONAL --} */ -void krb5_parse_req_body(MolochSession_t *session, const unsigned char *data, int len) +LOCAL void krb5_parse_req_body(MolochSession_t *session, const unsigned char *data, int len) { MolochASNSeq_t seq[12]; @@ -114,7 +114,7 @@ void krb5_parse_req_body(MolochSession_t *session, const unsigned char *data, in -- req-body[4] KDC-REQ-BODY --} */ -void krb5_parse_req(MolochSession_t *session, const unsigned char *data, int len) +LOCAL void krb5_parse_req(MolochSession_t *session, const unsigned char *data, int len) { MolochASNSeq_t seq[5]; @@ -151,7 +151,7 @@ void krb5_parse_req(MolochSession_t *session, const unsigned char *data, int len -- enc-part[6] EncryptedData --} */ -void krb5_parse_rep(MolochSession_t *UNUSED(session), const unsigned char *UNUSED(data), int UNUSED(len)) +LOCAL void krb5_parse_rep(MolochSession_t *UNUSED(session), const unsigned char *UNUSED(data), int UNUSED(len)) { } /******************************************************************************/ @@ -172,11 +172,11 @@ void krb5_parse_rep(MolochSession_t *UNUSED(session), const unsigned char *UNUSE -- e-data[12] OCTET STRING OPTIONAL --} */ -void krb5_parse_error(MolochSession_t *UNUSED(session), const unsigned char *UNUSED(data), int UNUSED(len)) +LOCAL void krb5_parse_error(MolochSession_t *UNUSED(session), const unsigned char *UNUSED(data), int UNUSED(len)) { } /******************************************************************************/ -void krb5_parse(MolochSession_t *session, const unsigned char *data, int len) +LOCAL void krb5_parse(MolochSession_t *session, const unsigned char *data, int len) { BSB obsb; uint32_t opc, msgType, olen; @@ -205,13 +205,13 @@ void krb5_parse(MolochSession_t *session, const unsigned char *data, int len) } } /******************************************************************************/ -int krb5_udp_parser(MolochSession_t *session, void *UNUSED(uw), const unsigned char *data, int len, int UNUSED(which)) +LOCAL int krb5_udp_parser(MolochSession_t *session, void *UNUSED(uw), const unsigned char *data, int len, int UNUSED(which)) { krb5_parse(session, data, len); return 0; } /******************************************************************************/ -void krb5_udp_classify(MolochSession_t *session, const unsigned char *data, int len, int UNUSED(which), void *UNUSED(uw)) +LOCAL void krb5_udp_classify(MolochSession_t *session, const unsigned char *data, int len, int UNUSED(which), void *UNUSED(uw)) { if (moloch_session_has_protocol(session, "krb5")) return; @@ -229,14 +229,14 @@ void krb5_udp_classify(MolochSession_t *session, const unsigned char *data, int } } /******************************************************************************/ -void krb5_free(MolochSession_t UNUSED(*session), void *uw) +LOCAL void krb5_free(MolochSession_t UNUSED(*session), void *uw) { KRB5Info_t *krb5 = uw; MOLOCH_TYPE_FREE(KRB5Info_t, krb5); } /******************************************************************************/ -int krb5_tcp_parser(MolochSession_t *session, void *uw, const unsigned char *data, int remaining, int which) +LOCAL int krb5_tcp_parser(MolochSession_t *session, void *uw, const unsigned char *data, int remaining, int which) { KRB5Info_t *krb5 = uw; @@ -256,7 +256,7 @@ int krb5_tcp_parser(MolochSession_t *session, void *uw, const unsigned char *dat return 0; } /******************************************************************************/ -void krb5_tcp_classify(MolochSession_t *session, const unsigned char *data, int UNUSED(len), int UNUSED(which), void *UNUSED(uw)) +LOCAL void krb5_tcp_classify(MolochSession_t *session, const unsigned char *data, int UNUSED(len), int UNUSED(which), void *UNUSED(uw)) { if (which !=0 || data[0] != 0 || data[1] != 0) return; @@ -271,19 +271,19 @@ void moloch_parser_init() { realmField = moloch_field_define("krb5", "termfield", - "krb5.realm", "Realm", "krb5.realm-term", + "krb5.realm", "Realm", "krb5.realm", "Kerberos 5 Realm", MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, NULL); cnameField = moloch_field_define("krb5", "termfield", - "krb5.cname", "cname", "krb5.cname-term", + "krb5.cname", "cname", "krb5.cname", "Kerberos 5 cname", MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, NULL); snameField = moloch_field_define("krb5", "termfield", - "krb5.sname", "sname", "krb5.sname-term", + "krb5.sname", "sname", "krb5.sname", "Kerberos 5 sname", MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, NULL); diff --git a/capture/parsers/krb5.detail.jade b/capture/parsers/krb5.detail.jade index 9cfafef1ea..d3a7a5cfeb 100644 --- a/capture/parsers/krb5.detail.jade +++ b/capture/parsers/krb5.detail.jade @@ -1,6 +1,6 @@ if (session.krb5) div.sessionDetailMeta.bold krb5 dl.sessionDetailMeta - +arrayList(session.krb5, "realm-term", "Realm", "krb5.realm") - +arrayList(session.krb5, "cname-term", "cname", "krb5.cname") - +arrayList(session.krb5, "sname-term", "sname", "krb5.sname") + +arrayList(session.krb5, "realm", "Realm", "krb5.realm") + +arrayList(session.krb5, "cname", "cname", "krb5.cname") + +arrayList(session.krb5, "sname", "sname", "krb5.sname") diff --git a/capture/parsers/ldap.c b/capture/parsers/ldap.c index 0f865c3ad8..81cd3c4891 100644 --- a/capture/parsers/ldap.c +++ b/capture/parsers/ldap.c @@ -16,15 +16,15 @@ extern MolochConfig_t config; -static int bindNameField; -static int authTypeField; +LOCAL int bindNameField; +LOCAL int authTypeField; typedef struct { unsigned char buf[2][8192]; int len[2]; } LDAPInfo_t; /******************************************************************************/ -void ldap_process(MolochSession_t *session, LDAPInfo_t *ldap, int which) +LOCAL void ldap_process(MolochSession_t *session, LDAPInfo_t *ldap, int which) { BSB obsb, ibsb; uint32_t opc, otag, olen; @@ -102,7 +102,7 @@ void ldap_process(MolochSession_t *session, LDAPInfo_t *ldap, int which) } } /******************************************************************************/ -int ldap_parser(MolochSession_t *session, void *uw, const unsigned char *data, int remaining, int which) +LOCAL int ldap_parser(MolochSession_t *session, void *uw, const unsigned char *data, int remaining, int which) { LDAPInfo_t *ldap = uw; @@ -123,7 +123,7 @@ int ldap_parser(MolochSession_t *session, void *uw, const unsigned char *data, i return 0; } /******************************************************************************/ -void ldap_save(MolochSession_t *session, void *uw, int UNUSED(final)) +LOCAL void ldap_save(MolochSession_t *session, void *uw, int UNUSED(final)) { LDAPInfo_t *ldap = uw; @@ -136,14 +136,14 @@ void ldap_save(MolochSession_t *session, void *uw, int UNUSED(final)) } } /******************************************************************************/ -void ldap_free(MolochSession_t *UNUSED(session), void *uw) +LOCAL void ldap_free(MolochSession_t *UNUSED(session), void *uw) { LDAPInfo_t *ldap = uw; MOLOCH_TYPE_FREE(LDAPInfo_t, ldap); } /******************************************************************************/ -void ldap_classify(MolochSession_t *session, const unsigned char *data, int len, int UNUSED(which), void *UNUSED(uw)) +LOCAL void ldap_classify(MolochSession_t *session, const unsigned char *data, int len, int UNUSED(which), void *UNUSED(uw)) { if (moloch_session_has_protocol(session, "ldap")) return; @@ -181,14 +181,14 @@ void moloch_parser_init() moloch_parsers_classifier_register_udp("ldap", NULL, 0, (unsigned char*)"\x30", 1, ldap_classify); authTypeField = moloch_field_define("ldap", "termfield", - "ldap.authtype", "Auth Type", "ldap.authtype-term", + "ldap.authtype", "Auth Type", "ldap.authtype", "The auth type of ldap bind", - MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_COUNT, + MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, NULL); bindNameField = moloch_field_define("ldap", "termfield", - "ldap.bindname", "Bind Name", "ldap.bindname-term", + "ldap.bindname", "Bind Name", "ldap.bindname", "The bind name of ldap bind", - MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_COUNT, + MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, NULL); } diff --git a/capture/parsers/ldap.detail.jade b/capture/parsers/ldap.detail.jade index 070fcdfd11..160e039602 100644 --- a/capture/parsers/ldap.detail.jade +++ b/capture/parsers/ldap.detail.jade @@ -1,5 +1,5 @@ if (session.ldap) div.sessionDetailMeta.bold ldap dl.sessionDetailMeta - +arrayList(session.ldap, "bindname-term", "BindNames", "ldap.bindname") - +arrayList(session.ldap, "authtype-term", "AuthTypes", "ldap.authtype") + +arrayList(session.ldap, "bindname", "BindNames", "ldap.bindname") + +arrayList(session.ldap, "authtype", "AuthTypes", "ldap.authtype") diff --git a/capture/parsers/misc.c b/capture/parsers/misc.c index ec5b87252e..52f7cdcbe3 100644 --- a/capture/parsers/misc.c +++ b/capture/parsers/misc.c @@ -16,10 +16,10 @@ extern MolochConfig_t config; -static int userField; +LOCAL int userField; /******************************************************************************/ -void rdp_classify(MolochSession_t *session, const unsigned char *data, int len, int UNUSED(which), void *UNUSED(uw)) +LOCAL void rdp_classify(MolochSession_t *session, const unsigned char *data, int len, int UNUSED(which), void *UNUSED(uw)) { if (len > 5 && data[3] <= len && data[4] == (data[3] - 5) && data[5] == 0xe0) { @@ -32,14 +32,14 @@ void rdp_classify(MolochSession_t *session, const unsigned char *data, int len, } } /******************************************************************************/ -void imap_classify(MolochSession_t *session, const unsigned char *data, int len, int UNUSED(which), void *UNUSED(uw)) +LOCAL void imap_classify(MolochSession_t *session, const unsigned char *data, int len, int UNUSED(which), void *UNUSED(uw)) { if (moloch_memstr((const char *)data+5, len-5, "IMAP", 4)) { moloch_session_add_protocol(session, "imap"); } } /******************************************************************************/ -void gh0st_classify(MolochSession_t *session, const unsigned char *data, int len, int UNUSED(which), void *UNUSED(uw)) +LOCAL void gh0st_classify(MolochSession_t *session, const unsigned char *data, int len, int UNUSED(which), void *UNUSED(uw)) { if (data[13] == 0x78 && (((data[8] == 0) && (data[7] == 0) && (((data[6]&0xff) << (uint32_t)8 | (data[5]&0xff)) == len)) || // Windows @@ -52,7 +52,7 @@ void gh0st_classify(MolochSession_t *session, const unsigned char *data, int len } } /******************************************************************************/ -void other220_classify(MolochSession_t *session, const unsigned char *data, int len, int UNUSED(which), void *UNUSED(uw)) +LOCAL void other220_classify(MolochSession_t *session, const unsigned char *data, int len, int UNUSED(which), void *UNUSED(uw)) { if (g_strstr_len((char *)data, len, "LMTP") != NULL) { moloch_session_add_protocol(session, "lmtp"); @@ -62,19 +62,19 @@ void other220_classify(MolochSession_t *session, const unsigned char *data, int } } /******************************************************************************/ -void vnc_classify(MolochSession_t *session, const unsigned char *data, int len, int UNUSED(which), void *UNUSED(uw)) +LOCAL void vnc_classify(MolochSession_t *session, const unsigned char *data, int len, int UNUSED(which), void *UNUSED(uw)) { if (len >= 12 && data[7] == '.' && data[11] == 0xa) moloch_session_add_protocol(session, "vnc"); } /******************************************************************************/ -void jabber_classify(MolochSession_t *session, const unsigned char *data, int len, int UNUSED(which), void *UNUSED(uw)) +LOCAL void jabber_classify(MolochSession_t *session, const unsigned char *data, int len, int UNUSED(which), void *UNUSED(uw)) { if (g_strstr_len((gchar*)data+5, len-5, "jabber") != NULL) moloch_session_add_protocol(session, "jabber"); } /******************************************************************************/ -void user_classify(MolochSession_t *session, const unsigned char *data, int len, int UNUSED(which), void *UNUSED(uw)) +LOCAL void user_classify(MolochSession_t *session, const unsigned char *data, int len, int UNUSED(which), void *UNUSED(uw)) { //If a USER packet must have not NICK or +iw with it so we don't pickup IRC if (len <= 5 || moloch_memstr((char *)data, len, "\nNICK ", 6) || moloch_memstr((char *)data, len, " +iw ", 5)) { @@ -89,24 +89,24 @@ void user_classify(MolochSession_t *session, const unsigned char *data, int len, moloch_field_string_add_lower(userField, session, (char*)data+5, i-5); } /******************************************************************************/ -void misc_add_protocol_classify(MolochSession_t *session, const unsigned char *UNUSED(data), int UNUSED(len), int UNUSED(which), void *uw) +LOCAL void misc_add_protocol_classify(MolochSession_t *session, const unsigned char *UNUSED(data), int UNUSED(len), int UNUSED(which), void *uw) { moloch_session_add_protocol(session, uw); } /******************************************************************************/ -void ntp_classify(MolochSession_t *session, const unsigned char *UNUSED(data), int len, int UNUSED(which), void *UNUSED(uw)) +LOCAL void ntp_classify(MolochSession_t *session, const unsigned char *data, int len, int UNUSED(which), void *UNUSED(uw)) { - if (session->port2 != 123 || // ntp port - len < 48 || // min length - data[1] > 16 // max stratum + if ((session->port1 != 123 && session->port2 != 123) || // ntp port + len < 48 || // min length + data[1] > 16 // max stratum ) { return; } moloch_session_add_protocol(session, "ntp"); } /******************************************************************************/ -void snmp_classify(MolochSession_t *session, const unsigned char *data, int len, int UNUSED(which), void *UNUSED(uw)) +LOCAL void snmp_classify(MolochSession_t *session, const unsigned char *data, int len, int UNUSED(which), void *UNUSED(uw)) { uint32_t apc, atag, alen; BSB bsb; @@ -127,7 +127,7 @@ void snmp_classify(MolochSession_t *session, const unsigned char *data, int len, moloch_session_add_protocol(session, "snmp"); } /******************************************************************************/ -void syslog_classify(MolochSession_t *session, const unsigned char *UNUSED(data), int len, int UNUSED(which), void *UNUSED(uw)) +LOCAL void syslog_classify(MolochSession_t *session, const unsigned char *UNUSED(data), int len, int UNUSED(which), void *UNUSED(uw)) { int i; for (i = 2; i < len; i++) { @@ -141,7 +141,7 @@ void syslog_classify(MolochSession_t *session, const unsigned char *UNUSED(data) } } /******************************************************************************/ -void stun_classify(MolochSession_t *session, const unsigned char *UNUSED(data), int len, int UNUSED(which), void *UNUSED(uw)) +LOCAL void stun_classify(MolochSession_t *session, const unsigned char *UNUSED(data), int len, int UNUSED(which), void *UNUSED(uw)) { if (20 + data[3] != len) return; @@ -158,13 +158,13 @@ void stun_classify(MolochSession_t *session, const unsigned char *UNUSED(data), } /******************************************************************************/ -void stun_rsp_classify(MolochSession_t *session, const unsigned char *data, int len, int UNUSED(which), void *UNUSED(uw)) +LOCAL void stun_rsp_classify(MolochSession_t *session, const unsigned char *data, int len, int UNUSED(which), void *UNUSED(uw)) { if (moloch_memstr((const char *)data+7, len-7, "STUN", 4)) moloch_session_add_protocol(session, "stun"); } /******************************************************************************/ -void flap_classify(MolochSession_t *session, const unsigned char *data, int len, int UNUSED(which), void *UNUSED(uw)) +LOCAL void flap_classify(MolochSession_t *session, const unsigned char *data, int len, int UNUSED(which), void *UNUSED(uw)) { if (len < 6) return; @@ -179,20 +179,20 @@ void flap_classify(MolochSession_t *session, const unsigned char *data, int len, moloch_session_add_protocol(session, "flap"); } /******************************************************************************/ -void tacacs_classify(MolochSession_t *session, const unsigned char *UNUSED(data), int UNUSED(len), int UNUSED(which), void *UNUSED(uw)) +LOCAL void tacacs_classify(MolochSession_t *session, const unsigned char *UNUSED(data), int UNUSED(len), int UNUSED(which), void *UNUSED(uw)) { if (session->port1 == 49 || session->port2 == 49) moloch_session_add_protocol(session, "tacacs"); } /******************************************************************************/ -void dropbox_lan_sync_classify(MolochSession_t *session, const unsigned char *data, int len, int UNUSED(which), void *UNUSED(uw)) +LOCAL void dropbox_lan_sync_classify(MolochSession_t *session, const unsigned char *data, int len, int UNUSED(which), void *UNUSED(uw)) { if (moloch_memstr((const char *)data+1, len-1, "host_int", 8)) { moloch_session_add_protocol(session, "dropbox-lan-sync"); } } /******************************************************************************/ -void kafka_classify(MolochSession_t *session, const unsigned char *data, int len, int UNUSED(which), void *UNUSED(uw)) +LOCAL void kafka_classify(MolochSession_t *session, const unsigned char *data, int len, int UNUSED(which), void *UNUSED(uw)) { if (len < 50 || data[4] != 0 || data[5] > 6|| data[7] != 0 || data[8] != 0) return; @@ -205,34 +205,20 @@ void kafka_classify(MolochSession_t *session, const unsigned char *data, int len moloch_session_add_protocol(session, "kafka"); } /******************************************************************************/ -void thrift_classify(MolochSession_t *session, const unsigned char *data, int len, int UNUSED(which), void *UNUSED(uw)) +LOCAL void thrift_classify(MolochSession_t *session, const unsigned char *data, int len, int UNUSED(which), void *UNUSED(uw)) { if (len > 20 && data[4] == 0x80 && data[5] == 0x01 && data[6] == 0) moloch_session_add_protocol(session, "thrift"); } /******************************************************************************/ -void rip_classify(MolochSession_t *session, const unsigned char *UNUSED(data), int UNUSED(len), int UNUSED(which), void *UNUSED(uw)) +LOCAL void rip_classify(MolochSession_t *session, const unsigned char *UNUSED(data), int UNUSED(len), int UNUSED(which), void *UNUSED(uw)) { if (session->port2 != 520 && session->port1 != 520) return; moloch_session_add_protocol(session, "rip"); } /******************************************************************************/ -void dhcpv6_udp_classify(MolochSession_t *session, const unsigned char *data, int UNUSED(len), int UNUSED(which), void *UNUSED(uw)) -{ - if ((data[0] != 1 && data[0] != 11) || !MOLOCH_SESSION_v6(session)) - return; - moloch_session_add_protocol(session, "dhcpv6"); -} -/******************************************************************************/ -void dhcp_udp_classify(MolochSession_t *session, const unsigned char *data, int UNUSED(len), int UNUSED(which), void *UNUSED(uw)) -{ - if (data[0] != 1 || MOLOCH_SESSION_v6(session)) - return; - moloch_session_add_protocol(session, "dhcp"); -} -/******************************************************************************/ -void isakmp_udp_classify(MolochSession_t *session, const unsigned char *data, int len, int UNUSED(which), void *UNUSED(uw)) +LOCAL void isakmp_udp_classify(MolochSession_t *session, const unsigned char *data, int len, int UNUSED(which), void *UNUSED(uw)) { if (len < 18 || (data[16] != 8 && data[16] != 33 && data[16] != 46) || @@ -242,7 +228,7 @@ void isakmp_udp_classify(MolochSession_t *session, const unsigned char *data, in moloch_session_add_protocol(session, "isakmp"); } /******************************************************************************/ -void aruba_papi_udp_classify(MolochSession_t *session, const unsigned char *data, int len, int UNUSED(which), void *UNUSED(uw)) +LOCAL void aruba_papi_udp_classify(MolochSession_t *session, const unsigned char *data, int len, int UNUSED(which), void *UNUSED(uw)) { if (len < 20 || data[0] != 0x49 || data[1] != 0x72) { return; @@ -358,7 +344,7 @@ void moloch_parser_init() moloch_parsers_classifier_register_tcp("kafka", NULL, 0, (unsigned char*)"\x00\x00", 2, kafka_classify); moloch_parsers_classifier_register_udp("steam-friends", "steam-friends", 0, (unsigned char*)"VS01", 4, misc_add_protocol_classify); - moloch_parsers_classifier_register_udp("value-a2s", "value-a2s", 0, (unsigned char*)"\xff\xff\xff\xff\x54\x53\x6f\x75", 8, misc_add_protocol_classify); + moloch_parsers_classifier_register_udp("valve-a2s", "valve-a2s", 0, (unsigned char*)"\xff\xff\xff\xff\x54\x53\x6f\x75", 8, misc_add_protocol_classify); moloch_parsers_classifier_register_tcp("stream-ihscp", "stream-ihscp", 0, (unsigned char*)"\xa4\x00\x00\x00\x56\x54\x30\x31", 8, misc_add_protocol_classify); moloch_parsers_classifier_register_tcp("honeywell-tcc", "honeywell-tcc", 0, (unsigned char*)"\x43\x42\x4b\x50\x50\x52\x05\x50", 8, misc_add_protocol_classify); @@ -375,9 +361,6 @@ void moloch_parser_init() moloch_parsers_classifier_register_tcp("nzsql", "nzsql", 0, (unsigned char*)"\x00\x00\x00\x08\x00\x01\x00\x03", 8, misc_add_protocol_classify); - moloch_parsers_classifier_register_port("dhcpv6", NULL, 547, MOLOCH_PARSERS_PORT_UDP, dhcpv6_udp_classify); - moloch_parsers_classifier_register_port("dhcp", NULL, 67, MOLOCH_PARSERS_PORT_UDP, dhcp_udp_classify); - moloch_parsers_classifier_register_tcp("splunk", "splunk", 0, (unsigned char*)"--splunk-cooked-mode-v3--", 25, misc_add_protocol_classify); moloch_parsers_classifier_register_port("isakmp", NULL, 500, MOLOCH_PARSERS_PORT_UDP, isakmp_udp_classify); diff --git a/capture/parsers/mysql.c b/capture/parsers/mysql.c index e1eb9f64ee..f87936ec3d 100644 --- a/capture/parsers/mysql.c +++ b/capture/parsers/mysql.c @@ -20,13 +20,13 @@ typedef struct { char ssl; } Info_t; -static int userField; -static int versionField; +LOCAL int userField; +LOCAL int versionField; extern MolochConfig_t config; /******************************************************************************/ -int mysql_parser(MolochSession_t *session, void *uw, const unsigned char *data, int len, int which) +LOCAL int mysql_parser(MolochSession_t *session, void *uw, const unsigned char *data, int len, int which) { Info_t *info = uw; if (which != 0) { @@ -73,7 +73,7 @@ int mysql_parser(MolochSession_t *session, void *uw, const unsigned char *data, return 0; } /******************************************************************************/ -void mysql_free(MolochSession_t UNUSED(*session), void *uw) +LOCAL void mysql_free(MolochSession_t UNUSED(*session), void *uw) { Info_t *info = uw; @@ -82,7 +82,7 @@ void mysql_free(MolochSession_t UNUSED(*session), void *uw) MOLOCH_TYPE_FREE(Info_t, info); } /******************************************************************************/ -void mysql_classify(MolochSession_t *session, const unsigned char *data, int len, int which, void *UNUSED(uw)) +LOCAL void mysql_classify(MolochSession_t *session, const unsigned char *data, int len, int which, void *UNUSED(uw)) { if (which != 1) return; @@ -117,14 +117,14 @@ void moloch_parser_init() moloch_parsers_classifier_register_tcp("mysql", NULL, 1, (unsigned char*)"\x00\x00\x00\x0a", 4, mysql_classify); userField = moloch_field_define("mysql", "lotermfield", - "mysql.user", "User", "mysql.user-term", + "mysql.user", "User", "mysql.user", "Mysql user name", MOLOCH_FIELD_TYPE_STR, MOLOCH_FIELD_FLAG_LINKED_SESSIONS, "category", "user", NULL); versionField = moloch_field_define("mysql", "termfield", - "mysql.ver", "Version", "mysql.ver-term", + "mysql.ver", "Version", "mysql.version", "Mysql server version string", MOLOCH_FIELD_TYPE_STR, MOLOCH_FIELD_FLAG_LINKED_SESSIONS, NULL); diff --git a/capture/parsers/mysql.detail.jade b/capture/parsers/mysql.detail.jade index cf2d6437d8..23c0b51a95 100644 --- a/capture/parsers/mysql.detail.jade +++ b/capture/parsers/mysql.detail.jade @@ -1,5 +1,5 @@ if (session.mysql) div.sessionDetailMeta.bold mysql dl.sessionDetailMeta - +stringList(session.mysql, "user-term", "Users", "mysql.user") - +stringList(session.mysql, "ver-term", "Versions", "mysql.ver") + +stringList(session.mysql, "user", "Users", "mysql.user") + +stringList(session.mysql, "version", "Versions", "mysql.ver") diff --git a/capture/parsers/oracle.c b/capture/parsers/oracle.c index d9ae30e36d..441dce6e3f 100644 --- a/capture/parsers/oracle.c +++ b/capture/parsers/oracle.c @@ -14,15 +14,15 @@ */ #include "moloch.h" -static int userField; -static int hostField; -static int serviceField; +LOCAL int userField; +LOCAL int hostField; +LOCAL int serviceField; extern MolochConfig_t config; // Lots of info from https://www.pythian.com/blog/repost-oracle-protocol/ /******************************************************************************/ -char *oracle_get_item(const char *data, char *needle, int needle_len, int *len) +LOCAL char *oracle_get_item(const char *data, char *needle, int needle_len, int *len) { const char *start = data + data[27]; char *item, *paren; @@ -41,7 +41,7 @@ char *oracle_get_item(const char *data, char *needle, int needle_len, int *len) return NULL; } /******************************************************************************/ -void oracle_classify(MolochSession_t *session, const unsigned char *data, int len, int which, void *UNUSED(uw)) +LOCAL void oracle_classify(MolochSession_t *session, const unsigned char *data, int len, int which, void *UNUSED(uw)) { if (which != 0 || len <= 27 || len != data[1] || (data[25] + data[27] != len)) return; @@ -49,18 +49,17 @@ void oracle_classify(MolochSession_t *session, const unsigned char *data, int le char *buf; // can't be more then 1 byte big int blen; - buf = oracle_get_item((const char *)data, "HOST=", 5, &blen); + buf = oracle_get_item((const char *)data, "HOST=", 5, &blen); // Already lowercases if (buf && !moloch_field_string_add(hostField, session, buf, blen, FALSE)) { g_free(buf); } - buf = oracle_get_item((const char *)data, "USER=", 5, &blen); - if (buf) { - moloch_field_string_add_lower(userField, session, buf, blen); + buf = oracle_get_item((const char *)data, "USER=", 5, &blen); // Already lowercases + if (buf && !moloch_field_string_add(userField, session, buf, blen, FALSE)) { g_free(buf); } - buf = oracle_get_item((const char *)data, "SERVICE_NAME=", 13, &blen); + buf = oracle_get_item((const char *)data, "SERVICE_NAME=", 13, &blen); // Already lowercases if (buf && !moloch_field_string_add(serviceField, session, buf, blen, FALSE)) { g_free(buf); } @@ -73,20 +72,20 @@ void moloch_parser_init() moloch_parsers_classifier_register_tcp("oracle", NULL, 2, (unsigned char*)"\x00\x00\x01\x00\x00\x00", 6, oracle_classify); userField = moloch_field_define("oracle", "lotermfield", - "oracle.user", "User", "oracle.user-term", + "oracle.user", "User", "oracle.user", "Oracle User", MOLOCH_FIELD_TYPE_STR, MOLOCH_FIELD_FLAG_LINKED_SESSIONS, "category", "user", NULL); hostField = moloch_field_define("oracle", "lotermfield", - "oracle.host", "Host", "oracle.host-term", + "oracle.host", "Host", "oracle.host", "Oracle Host", MOLOCH_FIELD_TYPE_STR, MOLOCH_FIELD_FLAG_LINKED_SESSIONS, NULL); serviceField = moloch_field_define("oracle", "lotermfield", - "oracle.service", "Service", "oracle.service-term", + "oracle.service", "Service", "oracle.service", "Oracle Service", MOLOCH_FIELD_TYPE_STR, MOLOCH_FIELD_FLAG_LINKED_SESSIONS, NULL); diff --git a/capture/parsers/oracle.detail.jade b/capture/parsers/oracle.detail.jade index 20352f0805..26354fc93e 100644 --- a/capture/parsers/oracle.detail.jade +++ b/capture/parsers/oracle.detail.jade @@ -1,6 +1,6 @@ if (session.oracle) div.sessionDetailMeta.bold oracle dl.sessionDetailMeta - +stringList(session.oracle, "user-term", "Users", "oracle.user") - +stringList(session.oracle, "host-term", "Hosts", "oracle.host") - +stringList(session.oracle, "service-term", "Services", "oracle.service") + +stringList(session.oracle, "user", "Users", "oracle.user") + +stringList(session.oracle, "host", "Hosts", "oracle.host") + +stringList(session.oracle, "service", "Services", "oracle.service") diff --git a/capture/parsers/postgresql.c b/capture/parsers/postgresql.c index 14a0a2872e..1c37bbbe6b 100644 --- a/capture/parsers/postgresql.c +++ b/capture/parsers/postgresql.c @@ -18,12 +18,12 @@ typedef struct { int which; } Info_t; -static int userField; -static int dbField; -static int appField; +LOCAL int userField; +LOCAL int dbField; +LOCAL int appField; /******************************************************************************/ -int postgresql_parser(MolochSession_t *session, void *uw, const unsigned char *data, int len, int which) +LOCAL int postgresql_parser(MolochSession_t *session, void *uw, const unsigned char *data, int len, int which) { Info_t *info = uw; if (which != info->which) @@ -79,14 +79,14 @@ int postgresql_parser(MolochSession_t *session, void *uw, const unsigned char *d return 0; } /******************************************************************************/ -void postgresql_free(MolochSession_t UNUSED(*session), void *uw) +LOCAL void postgresql_free(MolochSession_t UNUSED(*session), void *uw) { Info_t *info = uw; MOLOCH_TYPE_FREE(Info_t, info); } /******************************************************************************/ -void postgresql_classify(MolochSession_t *session, const unsigned char UNUSED(*data), int UNUSED(len), int which, void *UNUSED(uw)) +LOCAL void postgresql_classify(MolochSession_t *session, const unsigned char UNUSED(*data), int UNUSED(len), int which, void *UNUSED(uw)) { if (moloch_session_has_protocol(session, "postgresql")) return; @@ -105,20 +105,20 @@ void moloch_parser_init() moloch_parsers_classifier_register_tcp("postgresql", NULL, 0, (unsigned char*)"\x00\x00\x00", 3, postgresql_classify); userField = moloch_field_define("postgresql", "termfield", - "postgresql.user", "User", "postgresql.user-term", + "postgresql.user", "User", "postgresql.user", "Postgresql user name", MOLOCH_FIELD_TYPE_STR, MOLOCH_FIELD_FLAG_LINKED_SESSIONS, "category", "user", NULL); dbField = moloch_field_define("postgresql", "termfield", - "postgresql.db", "Database", "postgresql.db-term", + "postgresql.db", "Database", "postgresql.db", "Postgresql database", MOLOCH_FIELD_TYPE_STR, MOLOCH_FIELD_FLAG_LINKED_SESSIONS, NULL); appField = moloch_field_define("postgresql", "termfield", - "postgresql.app", "Application", "postgresql.app-term", + "postgresql.app", "Application", "postgresql.app", "Postgresql application", MOLOCH_FIELD_TYPE_STR, MOLOCH_FIELD_FLAG_LINKED_SESSIONS, NULL); diff --git a/capture/parsers/postgresql.detail.jade b/capture/parsers/postgresql.detail.jade index cb2204d588..a74376672b 100644 --- a/capture/parsers/postgresql.detail.jade +++ b/capture/parsers/postgresql.detail.jade @@ -1,6 +1,6 @@ if (session.postgresql) div.sessionDetailMeta.bold postgresql dl.sessionDetailMeta - +stringList(session.postgresql, "user-term", "Users", "postgresql.user") - +stringList(session.postgresql, "db-term", "DBs", "postgresql.db") - +stringList(session.postgresql, "app-term", "Applications", "postgresql.app") + +stringList(session.postgresql, "user", "Users", "postgresql.user") + +stringList(session.postgresql, "db", "DBs", "postgresql.db") + +stringList(session.postgresql, "app", "Applications", "postgresql.app") diff --git a/capture/parsers/quic.c b/capture/parsers/quic.c index 52b3ef3dac..5cf794ca7d 100644 --- a/capture/parsers/quic.c +++ b/capture/parsers/quic.c @@ -20,9 +20,9 @@ #include extern MolochConfig_t config; -static int hostField; -static int uaField; -static int versionField; +LOCAL int hostField; +LOCAL int uaField; +LOCAL int versionField; #define FBZERO_MAX_SIZE 4096 typedef struct { @@ -31,7 +31,7 @@ typedef struct { } FBZeroInfo_t; /******************************************************************************/ -int quic_chlo_parser(MolochSession_t *session, BSB dbsb) { +LOCAL int quic_chlo_parser(MolochSession_t *session, BSB dbsb) { guchar *tag = 0; int tagLen = 0; @@ -77,7 +77,7 @@ int quic_chlo_parser(MolochSession_t *session, BSB dbsb) { return 1; } /******************************************************************************/ -int quic_udp_parser(MolochSession_t *session, void *UNUSED(uw), const unsigned char *data, int len, int UNUSED(which)) +LOCAL int quic_udp_parser(MolochSession_t *session, void *UNUSED(uw), const unsigned char *data, int len, int UNUSED(which)) { int version = -1; int offset = 1; @@ -166,21 +166,21 @@ int quic_udp_parser(MolochSession_t *session, void *UNUSED(uw), const unsigned c return 0; } /******************************************************************************/ -void quic_udp_classify(MolochSession_t *session, const unsigned char *data, int len, int UNUSED(which), void *UNUSED(uw)) +LOCAL void quic_udp_classify(MolochSession_t *session, const unsigned char *data, int len, int UNUSED(which), void *UNUSED(uw)) { if (len > 100 && (data[0] & 0x83) == 0x01) { moloch_parsers_register(session, quic_udp_parser, 0, 0); } } /******************************************************************************/ -void quic_fbzero_free(MolochSession_t UNUSED(*session), void *uw) +LOCAL void quic_fbzero_free(MolochSession_t UNUSED(*session), void *uw) { FBZeroInfo_t *fbzero = uw; MOLOCH_TYPE_FREE(FBZeroInfo_t, fbzero); } /******************************************************************************/ -int quic_fb_tcp_parser(MolochSession_t *session, void *uw, const unsigned char *data, int remaining, int which) +LOCAL int quic_fb_tcp_parser(MolochSession_t *session, void *uw, const unsigned char *data, int remaining, int which) { if (which != 0) return 0; @@ -209,7 +209,7 @@ int quic_fb_tcp_parser(MolochSession_t *session, void *uw, const unsigned char * } /******************************************************************************/ -void quic_fb_tcp_classify(MolochSession_t *session, const unsigned char *UNUSED(data), int len, int which, void *UNUSED(uw)) +LOCAL void quic_fb_tcp_classify(MolochSession_t *session, const unsigned char *UNUSED(data), int len, int which, void *UNUSED(uw)) { if (which == 0 && len > 13) { FBZeroInfo_t *fbzero = MOLOCH_TYPE_ALLOC(FBZeroInfo_t); @@ -225,19 +225,19 @@ void moloch_parser_init() moloch_parsers_classifier_register_tcp("fbzero", NULL, 0, (const unsigned char *)"\x31QTV", 4, quic_fb_tcp_classify); hostField = moloch_field_define("quic", "lotermfield", - "host.quic", "Hostname", "quic.host-term", + "host.quic", "Hostname", "quic.host", "QUIC host header field", MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, "aliases", "[\"quic.host\"]", NULL); uaField = moloch_field_define("quic", "termfield", - "quic.user-agent", "User-Agent", "quic.ua-term", + "quic.user-agent", "User-Agent", "quic.useragent", "User-Agent", MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, NULL); versionField = moloch_field_define("quic", "termfield", - "quic.version", "Version", "quic.version-term", + "quic.version", "Version", "quic.version", "QUIC Version", MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, NULL); diff --git a/capture/parsers/quic.detail.jade b/capture/parsers/quic.detail.jade index ce010b0c4c..25317865a9 100644 --- a/capture/parsers/quic.detail.jade +++ b/capture/parsers/quic.detail.jade @@ -1,6 +1,6 @@ if (session.quic) div.sessionDetailMeta.bold quic dl.sessionDetailMeta - +arrayList(session.quic, "host-term", "Hosts", "host.quic") - +arrayList(session.quic, "ua-term", "User-Agents", "quic.user-agent") - +arrayList(session.quic, "version-term", "Versions", "quic.version") + +arrayList(session.quic, "host", "Hosts", "host.quic") + +arrayList(session.quic, "useragent", "User-Agents", "quic.user-agent") + +arrayList(session.quic, "version", "Versions", "quic.version") diff --git a/capture/parsers/radius.c b/capture/parsers/radius.c index a1284e2f60..9ddc737775 100644 --- a/capture/parsers/radius.c +++ b/capture/parsers/radius.c @@ -16,13 +16,13 @@ #include extern MolochConfig_t config; -static int userField; -static int macField; -static int endpointIpField; -static int framedIpField; +LOCAL int userField; +LOCAL int macField; +LOCAL int endpointIpField; +LOCAL int framedIpField; /******************************************************************************/ -int radius_udp_parser(MolochSession_t *session, void *UNUSED(uw), const unsigned char *data, int len, int UNUSED(which)) +LOCAL int radius_udp_parser(MolochSession_t *session, void *UNUSED(uw), const unsigned char *data, int len, int UNUSED(which)) { BSB bsb; @@ -53,7 +53,7 @@ int radius_udp_parser(MolochSession_t *session, void *UNUSED(uw), const unsigned break;*/ case 8: memcpy(&in.s_addr, value, 4); - moloch_field_int_add(framedIpField, session, in.s_addr); + moloch_field_ip4_add(framedIpField, session, in.s_addr); break; case 31: if (length == 12) { @@ -74,8 +74,7 @@ int radius_udp_parser(MolochSession_t *session, void *UNUSED(uw), const unsigned case 66: memcpy(str, value, length); str[length] = 0; - inet_aton(str, &in); - moloch_field_int_add(endpointIpField, session, in.s_addr); + moloch_field_ip_add_str(endpointIpField, session, str); break; /* default: @@ -85,7 +84,7 @@ int radius_udp_parser(MolochSession_t *session, void *UNUSED(uw), const unsigned return 0; } /******************************************************************************/ -void radius_udp_classify(MolochSession_t *session, const unsigned char *UNUSED(data), int len, int UNUSED(which), void *UNUSED(uw)) +LOCAL void radius_udp_classify(MolochSession_t *session, const unsigned char *UNUSED(data), int len, int UNUSED(which), void *UNUSED(uw)) { if (len != ((data[2] << 8) | data[3])) { return; @@ -103,28 +102,28 @@ void radius_udp_classify(MolochSession_t *session, const unsigned char *UNUSED(d void moloch_parser_init() { userField = moloch_field_define("radius", "termfield", - "radius.user", "User", "radius.user-term", + "radius.user", "User", "radius.user", "RADIUS user", MOLOCH_FIELD_TYPE_STR_HASH, 0, "category", "user", NULL); macField = moloch_field_define("radius", "lotermfield", - "radius.mac", "MAC", "radius.mac-term", + "radius.mac", "MAC", "radius.mac", "Radius Mac", - MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_COUNT, + MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, NULL); endpointIpField = moloch_field_define("radius", "ip", - "radius.endpoint-ip", "Endpoint IP", "radius.eip", + "radius.endpoint-ip", "Endpoint IP", "radius.endpointIp", "Radius endpoint ip addresses for session", - MOLOCH_FIELD_TYPE_IP_GHASH, MOLOCH_FIELD_FLAG_COUNT, + MOLOCH_FIELD_TYPE_IP_GHASH, MOLOCH_FIELD_FLAG_CNT, NULL); framedIpField = moloch_field_define("radius", "ip", - "radius.framed-ip", "Framed IP", "radius.fip", + "radius.framed-ip", "Framed IP", "radius.framedIp", "Radius framed ip addresses for session", - MOLOCH_FIELD_TYPE_IP_GHASH, MOLOCH_FIELD_FLAG_COUNT, + MOLOCH_FIELD_TYPE_IP_GHASH, MOLOCH_FIELD_FLAG_CNT, NULL); diff --git a/capture/parsers/radius.detail.jade b/capture/parsers/radius.detail.jade index 7928e53ec8..af3eaf2216 100644 --- a/capture/parsers/radius.detail.jade +++ b/capture/parsers/radius.detail.jade @@ -1,7 +1,7 @@ if (session.radius) div.sessionDetailMeta.bold radius dl.sessionDetailMeta - +arrayList(session.radius, "user-term", "User", "radius.user") - +arrayList(session.radius, "mac-term", "Mac", "radius.mac") - +ipArrayList(session.radius, "eip", "Endpoint IPs", "radius.endpoint-ip") - +ipArrayList(session.radius, "fip", "Framed IPs", "radius.framed-ip") + +arrayList(session.radius, "user", "User", "radius.user") + +arrayList(session.radius, "mac", "Mac", "radius.mac") + +ipArrayList(session.radius, "endpointIp", "Endpoint IPs", "radius.endpoint-ip") + +ipArrayList(session.radius, "framedIp", "Framed IPs", "radius.framed-ip") diff --git a/capture/parsers/smb.c b/capture/parsers/smb.c index afa8f85527..068dbc55f3 100644 --- a/capture/parsers/smb.c +++ b/capture/parsers/smb.c @@ -18,13 +18,13 @@ extern MolochConfig_t config; -static int domainField; -static int userField; -static int hostField; -static int osField; -static int verField; -static int fnField; -static int shareField; +LOCAL int domainField; +LOCAL int userField; +LOCAL int hostField; +LOCAL int osField; +LOCAL int verField; +LOCAL int fnField; +LOCAL int shareField; #define MAX_SMB_BUFFER 4096 typedef struct { @@ -57,7 +57,7 @@ typedef struct { #define SMB2_FLAGS_SERVER_TO_REDIR 0x00000001 /******************************************************************************/ -void smb_add_string(MolochSession_t *session, int field, char *buf, int len, int useunicode) +LOCAL void smb_add_string(MolochSession_t *session, int field, char *buf, int len, int useunicode) { GError *error = 0; gsize bread, bwritten; @@ -78,7 +78,7 @@ void smb_add_string(MolochSession_t *session, int field, char *buf, int len, int } /******************************************************************************/ // 2.2.13 AUTHENTICATE_MESSAGE from http://download.microsoft.com/download/9/5/E/95EF66AF-9026-4BB0-A41D-A4F81802D92C/[MS-NLMP].pdf -void smb_security_blob(MolochSession_t *session, unsigned char *data, int len) +LOCAL void smb_security_blob(MolochSession_t *session, unsigned char *data, int len) { BSB bsb; @@ -144,7 +144,7 @@ void smb_security_blob(MolochSession_t *session, unsigned char *data, int len) } } /******************************************************************************/ -void smb1_str_null_split(char *buf, int len, char **out, int max) +LOCAL void smb1_str_null_split(char *buf, int len, char **out, int max) { memset(out, 0, max*sizeof(char *)); out[0] = buf; @@ -157,7 +157,7 @@ void smb1_str_null_split(char *buf, int len, char **out, int max) } } /******************************************************************************/ -void smb1_parse_osverdomain(MolochSession_t *session, char *buf, int len, int useunicode) +LOCAL void smb1_parse_osverdomain(MolochSession_t *session, char *buf, int len, int useunicode) { char *out; gsize bread, bwritten; @@ -191,7 +191,7 @@ void smb1_parse_osverdomain(MolochSession_t *session, char *buf, int len, int us } } /******************************************************************************/ -void smb1_parse_userdomainosver(MolochSession_t *session, char *buf, int len, int useunicode) +LOCAL void smb1_parse_userdomainosver(MolochSession_t *session, char *buf, int len, int useunicode) { char *out; gsize bread, bwritten; @@ -227,7 +227,7 @@ void smb1_parse_userdomainosver(MolochSession_t *session, char *buf, int len, in } } /******************************************************************************/ -int smb1_parse(MolochSession_t *session, SMBInfo_t *smb, BSB *bsb, char *state, uint32_t *remlen, int which) +LOCAL int smb1_parse(MolochSession_t *session, SMBInfo_t *smb, BSB *bsb, char *state, uint32_t *remlen, int which) { unsigned char *start = BSB_WORK_PTR(*bsb); unsigned char cmd = 0; @@ -364,7 +364,7 @@ int smb1_parse(MolochSession_t *session, SMBInfo_t *smb, BSB *bsb, char *state, return 0; } /******************************************************************************/ -int smb2_parse(MolochSession_t *session, SMBInfo_t *UNUSED(smb), BSB *bsb, char *state, uint32_t *remlen, int UNUSED(which)) +LOCAL int smb2_parse(MolochSession_t *session, SMBInfo_t *UNUSED(smb), BSB *bsb, char *state, uint32_t *remlen, int UNUSED(which)) { unsigned char *start = BSB_WORK_PTR(*bsb); @@ -460,7 +460,7 @@ int smb2_parse(MolochSession_t *session, SMBInfo_t *UNUSED(smb), BSB *bsb, char return 0; } /******************************************************************************/ -int smb_parser(MolochSession_t *session, void *uw, const unsigned char *data, int remaining, int which) +LOCAL int smb_parser(MolochSession_t *session, void *uw, const unsigned char *data, int remaining, int which) { SMBInfo_t *smb = uw; char *state = &smb->state[which]; @@ -556,14 +556,14 @@ int smb_parser(MolochSession_t *session, void *uw, const unsigned char *data, in return 0; } /******************************************************************************/ -void smb_free(MolochSession_t UNUSED(*session), void *uw) +LOCAL void smb_free(MolochSession_t UNUSED(*session), void *uw) { SMBInfo_t *smb = uw; MOLOCH_TYPE_FREE(SMBInfo_t, smb); } /******************************************************************************/ -void smb_classify(MolochSession_t *session, const unsigned char *data, int UNUSED(len), int UNUSED(which), void *UNUSED(uw)) +LOCAL void smb_classify(MolochSession_t *session, const unsigned char *data, int UNUSED(len), int UNUSED(which), void *UNUSED(uw)) { if (data[4] != 0xff && data[4] != 0xfe) return; @@ -581,44 +581,44 @@ void smb_classify(MolochSession_t *session, const unsigned char *data, int UNUSE void moloch_parser_init() { shareField =moloch_field_define("smb", "termfield", - "smb.share", "Share", "smbsh", + "smb.share", "Share", "smb.share", "SMB shares connected to", MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, NULL); fnField = moloch_field_define("smb", "termfield", - "smb.fn", "Filename", "smbfn", + "smb.fn", "Filename", "smb.filename", "SMB files opened, created, deleted", MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, NULL); osField = moloch_field_define("smb", "termfield", - "smb.os", "OS", "smbos", + "smb.os", "OS", "smb.os", "SMB OS information", MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, NULL); domainField = moloch_field_define("smb", "termfield", - "smb.domain", "Domain", "smbdm", + "smb.domain", "Domain", "smb.domain", "SMB domain", MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, NULL); verField = moloch_field_define("smb", "termfield", - "smb.ver", "Version", "smbver", + "smb.ver", "Version", "smb.version", "SMB Version information", MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, NULL); userField = moloch_field_define("smb", "termfield", - "smb.user", "User", "smbuser", + "smb.user", "User", "smb.user", "SMB User", MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, "category", "user", NULL); hostField = moloch_field_define("smb", "termfield", - "host.smb", "Hostname", "smbho", + "host.smb", "Hostname", "smb.host", "SMB Host name", MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, "category", "host", diff --git a/capture/parsers/smb.detail.jade b/capture/parsers/smb.detail.jade index c3faa2cc91..ca3f8ab616 100644 --- a/capture/parsers/smb.detail.jade +++ b/capture/parsers/smb.detail.jade @@ -1,10 +1,10 @@ -if (session.smbfn || session.smbsh) +if (session.smb) div.sessionDetailMeta.bold SMB dl.sessionDetailMeta(suffix="smb") - +arrayList(session, "smbuser", "Users", "smb.user") - +arrayList(session, "smbho", "Hosts", "host.smb") - +arrayList(session, "smbdm", "Domains", "smb.domain") - +arrayList(session, "smbsh", "Shares", "smb.share") - +arrayList(session, "smbfn", "Files", "smb.fn") - +arrayList(session, "smbos", "OS", "smb.os") - +arrayList(session, "smbver", "Version", "smb.ver") + +arrayList(session.smb, "user", "Users", "smb.user") + +arrayList(session.smb, "host", "Hosts", "host.smb") + +arrayList(session.smb, "domain", "Domains", "smb.domain") + +arrayList(session.smb, "share", "Shares", "smb.share") + +arrayList(session.smb, "filename", "Files", "smb.fn") + +arrayList(session.smb, "os", "OS", "smb.os") + +arrayList(session.smb, "version", "Version", "smb.ver") diff --git a/capture/parsers/smtp.c b/capture/parsers/smtp.c index 35126ccba6..a11e9a2e19 100644 --- a/capture/parsers/smtp.c +++ b/capture/parsers/smtp.c @@ -24,24 +24,25 @@ extern unsigned char moloch_char_to_hexstr[256][3]; extern unsigned char moloch_hex_to_char[256][256]; extern uint32_t pluginsCbs; -static MolochStringHashStd_t emailHeaders; - -static int receivedField; -static int idField; -static int ipField; -static int hostField; -static int srcField; -static int dstField; +LOCAL MolochStringHashStd_t emailHeaders; + +LOCAL int receivedField; +LOCAL int idField; +LOCAL int ipField; +LOCAL int hostField; +LOCAL int srcField; +LOCAL int dstField; extern int userField; -static int hhField; -static int subField; -static int ctField; -static int md5Field; -static int fnField; -static int uaField; -static int mvField; -static int fctField; -static int magicField; +LOCAL int hhField; +LOCAL int subField; +LOCAL int ctField; +LOCAL int md5Field; +LOCAL int sha256Field; +LOCAL int fnField; +LOCAL int uaField; +LOCAL int mvField; +LOCAL int fctField; +LOCAL int magicField; typedef struct { MolochStringHead_t boundaries; @@ -51,7 +52,7 @@ typedef struct { gint state64[2]; guint save64[2]; guint bdatRemaining[2]; - GChecksum *checksum[2]; + GChecksum *checksum[4]; uint16_t base64Decode:2; uint16_t firstInContent:2; @@ -86,7 +87,7 @@ EMAIL_MIME_DATA, EMAIL_MIME_DATA_RETURN }; /******************************************************************************/ -char *smtp_remove_matching(char *str, char start, char stop) +LOCAL char *smtp_remove_matching(char *str, char start, char stop) { while (isspace(*str)) str++; @@ -103,8 +104,7 @@ char *smtp_remove_matching(char *str, char start, char stop) return startstr; } /******************************************************************************/ -void -smtp_email_add_value(MolochSession_t *session, int pos, char *s, int l) +LOCAL void smtp_email_add_value(MolochSession_t *session, int pos, char *s, int l) { while (isspace(*s)) { s++; @@ -122,24 +122,13 @@ smtp_email_add_value(MolochSession_t *session, int pos, char *s, int l) case MOLOCH_FIELD_TYPE_STR_HASH: moloch_field_string_add(pos, session, s, l, TRUE); break; - case MOLOCH_FIELD_TYPE_IP_HASH: + case MOLOCH_FIELD_TYPE_IP_GHASH: { int i; gchar **parts = g_strsplit(s, ",", 0); for (i = 0; parts[i]; i++) { - gchar *ip = parts[i]; - while (*ip == ' ') - ip++; - - in_addr_t ia = inet_addr(ip); - if (ia == 0 || ia == 0xffffffff) { - moloch_session_add_tag(session, "http:bad-xff"); - LOG("ERROR - Didn't understand ip: %s %s %d", s, ip, ia); - continue; - } - - moloch_field_int_add(pos, session, ia); + moloch_field_ip_add_str(pos, session, parts[i]); } g_strfreev(parts); @@ -148,8 +137,7 @@ smtp_email_add_value(MolochSession_t *session, int pos, char *s, int l) } /* SWITCH */ } /******************************************************************************/ -char * -smtp_quoteable_decode_inplace(char *str, gsize *olen) +LOCAL char * smtp_quoteable_decode_inplace(char *str, gsize *olen) { char *start = str; int ipos = 0; @@ -190,8 +178,7 @@ smtp_quoteable_decode_inplace(char *str, gsize *olen) } /******************************************************************************/ -void -smtp_email_add_encoded(MolochSession_t *session, int pos, char *string, int len) +LOCAL void smtp_email_add_encoded(MolochSession_t *session, int pos, char *string, int len) { /* Decode this nightmare - http://www.rfc-editor.org/rfc/rfc2047.txt */ /* =?charset?encoding?encoded-text?= */ @@ -299,7 +286,7 @@ smtp_email_add_encoded(MolochSession_t *session, int pos, char *string, int len) } } /******************************************************************************/ -void smtp_parse_email_addresses(int field, MolochSession_t *session, char *data, int len) +LOCAL void smtp_parse_email_addresses(int field, MolochSession_t *session, char *data, int len) { char *end = data+len; @@ -324,17 +311,14 @@ void smtp_parse_email_addresses(int field, MolochSession_t *session, char *data, while (data < end && *data != '>') data++; } - char *lower = g_ascii_strdown(start, data - start); - if (!moloch_field_string_add(field, session, lower, data - start, FALSE)) { - g_free(lower); - } + moloch_field_string_add_lower(field, session, start, data - start); while (data < end && *data != ',') data++; if (data < end && *data == ',') data++; } } /******************************************************************************/ -void smtp_parse_email_received(MolochSession_t *session, char *data, int len) +LOCAL void smtp_parse_email_received(MolochSession_t *session, char *data, int len) { char *start = data; char *end = data+len; @@ -351,10 +335,7 @@ void smtp_parse_email_received(MolochSession_t *session, char *data, int len) while (data < end && *data != ']') data++; *data = 0; data++; - in_addr_t ia = inet_addr(ipstart); - if (ia == 0 || ia == 0xffffffff) - continue; - moloch_field_int_add(ipField, session, ia); + moloch_field_ip_add_str(ipField, session, ipstart); continue; } @@ -364,10 +345,8 @@ void smtp_parse_email_received(MolochSession_t *session, char *data, int len) fromstart = data+1; data++; } - char *lower = g_ascii_strdown((char*)fromstart, data - fromstart); - if (!moloch_field_string_add(hostField, session, lower, data - fromstart, FALSE)) { - g_free(lower); - } + + moloch_field_string_add_lower(hostField, session, (char *)fromstart, data-fromstart); } else if (memcmp("by ", data, 3) == 0) { data += 3; while(data < end && isspace(*data)) data++; @@ -377,10 +356,7 @@ void smtp_parse_email_received(MolochSession_t *session, char *data, int len) fromstart = data+1; data++; } - char *lower = g_ascii_strdown((char*)fromstart, data - fromstart); - if (!moloch_field_string_add(hostField, session, lower, data - fromstart, FALSE)) { - g_free(lower); - } + moloch_field_string_add_lower(hostField, session, (char *)fromstart, data-fromstart); } } @@ -389,16 +365,13 @@ void smtp_parse_email_received(MolochSession_t *session, char *data, int len) char *ipstart = data; while (data < end && *data != ']') data++; *data = 0; - in_addr_t ia = inet_addr(ipstart); - if (ia == 0 || ia == 0xffffffff) - continue; - moloch_field_int_add(ipField, session, ia); + moloch_field_ip_add_str(ipField, session, ipstart); } data++; } } /******************************************************************************/ -int smtp_parser(MolochSession_t *session, void *uw, const unsigned char *data, int remaining, int which) +LOCAL int smtp_parser(MolochSession_t *session, void *uw, const unsigned char *data, int remaining, int which) { SMTPInfo_t *email = uw; GString *line = email->line[which]; @@ -432,16 +405,10 @@ int smtp_parser(MolochSession_t *session, void *uw, const unsigned char *data, i moloch_session_add_tag(session, tag); } else if (strncasecmp(line->str, "MAIL FROM:", 10) == 0) { *state = EMAIL_CMD; - char *lower = g_ascii_strdown(smtp_remove_matching(line->str+10, '<', '>'), -1); - if (!moloch_field_string_add(srcField, session, lower, -1, FALSE)) { - g_free(lower); - } + moloch_field_string_add_lower(srcField, session, smtp_remove_matching(line->str+10, '<', '>'), -1); } else if (strncasecmp(line->str, "RCPT TO:", 8) == 0) { - char *lower = g_ascii_strdown(smtp_remove_matching(line->str+8, '<', '>'), -1); - if (!moloch_field_string_add(dstField, session, lower, -1, FALSE)) { - g_free(lower); - } *state = EMAIL_CMD; + moloch_field_string_add_lower(dstField, session, smtp_remove_matching(line->str+8, '<', '>'), -1); } else if (strncasecmp(line->str, "DATA", 4) == 0) { *state = EMAIL_DATA_HEADER; email->seenHeaders |= (1 << which); @@ -612,10 +579,7 @@ int smtp_parser(MolochSession_t *session, void *uw, const unsigned char *data, i if (strcasecmp(lower, config.smtpIpHeaders[i]) == 0) { int l = strlen(config.smtpIpHeaders[i]); char *ip = smtp_remove_matching(line->str+l+1, '[', ']'); - in_addr_t ia = inet_addr(ip); - if (ia == 0 || ia == 0xffffffff) - break; - moloch_field_int_add(ipField, session, ia); + moloch_field_ip_add_str(ipField, session, ip); } } } @@ -668,12 +632,19 @@ int smtp_parser(MolochSession_t *session, void *uw, const unsigned char *data, i if (email->base64Decode & (1 << which)) { const char *md5 = g_checksum_get_string(email->checksum[which]); moloch_field_string_add(md5Field, session, (char*)md5, 32, TRUE); + if (config.supportSha256) { + const char *sha256 = g_checksum_get_string(email->checksum[which+2]); + moloch_field_string_add(sha256Field, session, (char*)sha256, 64, TRUE); + } } email->firstInContent |= (1 << which); email->base64Decode &= ~(1 << which); email->state64[which] = 0; email->save64[which] = 0; g_checksum_reset(email->checksum[which]); + if (config.supportSha256) { + g_checksum_reset(email->checksum[which+2]); + } *state = EMAIL_MIME; } else if (*state == EMAIL_MIME_DATA_RETURN) { if (email->base64Decode & (1 << which)) { @@ -683,6 +654,9 @@ int smtp_parser(MolochSession_t *session, void *uw, const unsigned char *data, i &(email->state64[which]), &(email->save64[which])); g_checksum_update(email->checksum[which], buf, b); + if (config.supportSha256) { + g_checksum_update(email->checksum[which+2], buf, b); + } if (email->firstInContent & (1 << which)) { email->firstInContent &= ~(1 << which); @@ -813,7 +787,7 @@ int smtp_parser(MolochSession_t *session, void *uw, const unsigned char *data, i return 0; } /******************************************************************************/ -void smtp_free(MolochSession_t UNUSED(*session), void *uw) +LOCAL void smtp_free(MolochSession_t UNUSED(*session), void *uw) { SMTPInfo_t *email = uw; @@ -824,6 +798,10 @@ void smtp_free(MolochSession_t UNUSED(*session), void *uw) g_checksum_free(email->checksum[0]); g_checksum_free(email->checksum[1]); + if (config.supportSha256) { + g_checksum_free(email->checksum[2]); + g_checksum_free(email->checksum[3]); + } while (DLL_POP_HEAD(s_, &email->boundaries, string)) { g_free(string->str); @@ -833,7 +811,7 @@ void smtp_free(MolochSession_t UNUSED(*session), void *uw) MOLOCH_TYPE_FREE(SMTPInfo_t, email); } /******************************************************************************/ -void smtp_classify(MolochSession_t *session, const unsigned char *data, int len, int UNUSED(which), void *UNUSED(uw)) +LOCAL void smtp_classify(MolochSession_t *session, const unsigned char *data, int len, int UNUSED(which), void *UNUSED(uw)) { if (len < 5) return; @@ -855,6 +833,10 @@ void smtp_classify(MolochSession_t *session, const unsigned char *data, int len, email->checksum[0] = g_checksum_new(G_CHECKSUM_MD5); email->checksum[1] = g_checksum_new(G_CHECKSUM_MD5); + if (config.supportSha256) { + email->checksum[2] = g_checksum_new(G_CHECKSUM_SHA256); + email->checksum[3] = g_checksum_new(G_CHECKSUM_SHA256); + } DLL_INIT(s_, &(email->boundaries)); @@ -865,7 +847,7 @@ void smtp_classify(MolochSession_t *session, const unsigned char *data, int len, void moloch_parser_init() { hostField = moloch_field_define("email", "lotermfield", - "host.email", "Hostname", "eho", + "host.email", "Hostname", "email.host", "Email hostnames", MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, "aliases", "[\"email.host\"]", @@ -873,16 +855,15 @@ void moloch_parser_init() "category", "host", NULL); - uaField = moloch_field_define("email", "lotextfield", - "email.x-mailer", "X-Mailer Header", "eua", + uaField = moloch_field_define("email", "termfield", + "email.x-mailer", "X-Mailer Header", "email.useragent", "Email X-Mailer header", MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, - "rawField", "raweua", "requiredRight", "emailSearch", NULL); srcField = moloch_field_define("email", "lotermfield", - "email.src", "Sender", "esrc", + "email.src", "Sender", "email.src", "Email from address", MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, "requiredRight", "emailSearch", @@ -890,83 +871,93 @@ void moloch_parser_init() NULL); dstField = moloch_field_define("email", "lotermfield", - "email.dst", "Receiver", "edst", + "email.dst", "Receiver", "email.dst", "Email to address", MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, "requiredRight", "emailSearch", "category", "user", NULL); - subField = moloch_field_define("email", "textfield", - "email.subject", "Subject", "esub", + subField = moloch_field_define("email", "termfield", + "email.subject", "Subject", "email.subject", "Email subject header", MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT | MOLOCH_FIELD_FLAG_FORCE_UTF8, - "rawField", "rawesub", "requiredRight", "emailSearch", NULL); idField = moloch_field_define("email", "termfield", - "email.message-id", "Id", "eid", + "email.message-id", "Id", "email.id", "Email Message-Id header", MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, "requiredRight", "emailSearch", NULL); ctField = moloch_field_define("email", "termfield", - "email.content-type", "Content-Type", "ect", + "email.content-type", "Content-Type", "email.contentType", "Email content-type header", MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, "requiredRight", "emailSearch", NULL); mvField = moloch_field_define("email", "termfield", - "email.mime-version", "Mime-Version", "emv", + "email.mime-version", "Mime-Version", "email.mimeVersion", "Email Mime-Header header", MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, "requiredRight", "emailSearch", NULL); fnField = moloch_field_define("email", "termfield", - "email.fn", "Filenames", "efn", + "email.fn", "Filenames", "email.filename", "Email attachment filenames", MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, "requiredRight", "emailSearch", NULL); md5Field = moloch_field_define("email", "termfield", - "email.md5", "Attach MD5s", "emd5", + "email.md5", "Attach MD5s", "email.md5", "Email attachment MD5s", MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, "requiredRight", "emailSearch", "category", "md5", NULL); + if (config.supportSha256) { + sha256Field = moloch_field_define("email", "termfield", + "email.sha256", "Attach SHA256s", "email.sha256", + "Email attachment SHA256s", + MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, + "requiredRight", "emailSearch", + "category", "sha256", + "disabled", "true", + NULL); + } + fctField = moloch_field_define("email", "termfield", - "email.file-content-type", "Attach Content-Type", "efct", + "email.file-content-type", "Attach Content-Type", "email.fileContentType", "Email attachment content types", MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, "requiredRight", "emailSearch", NULL); ipField = moloch_field_define("email", "ip", - "ip.email", "IP", "eip", + "ip.email", "IP", "email.ip", "Email IP address", - MOLOCH_FIELD_TYPE_IP_HASH, MOLOCH_FIELD_FLAG_CNT | MOLOCH_FIELD_FLAG_IPPRE, + MOLOCH_FIELD_TYPE_IP_GHASH, MOLOCH_FIELD_FLAG_CNT | MOLOCH_FIELD_FLAG_IPPRE, "requiredRight", "emailSearch", "category", "ip", NULL); hhField = moloch_field_define("email", "lotermfield", - "email.has-header", "Header", "ehh", + "email.has-header", "Header", "email.header", "Email has the header set", MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, "requiredRight", "emailSearch", NULL); magicField = moloch_field_define("email", "termfield", - "email.bodymagic", "Body Magic", "email.bodymagic-term", + "email.bodymagic", "Body Magic", "email.bodyMagic", "The content type of body determined by libfile/magic", - MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_COUNT, + MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, NULL); HASH_INIT(s_, emailHeaders, moloch_string_hash, moloch_string_cmp); @@ -980,7 +971,7 @@ void moloch_parser_init() moloch_config_add_header(&emailHeaders, "user-agent", uaField); moloch_config_add_header(&emailHeaders, "mime-version", mvField); moloch_config_add_header(&emailHeaders, "received", receivedField); - moloch_config_load_header("headers-email", "email", "Email header ", "email.", "hdrs.ehead-", &emailHeaders, 0); + moloch_config_load_header("headers-email", "email", "Email header ", "email.", "email.header-", &emailHeaders, 0); if (config.parseSMTP) { moloch_parsers_classifier_register_tcp("smtp", NULL, 0, (unsigned char*)"HELO", 4, smtp_classify); diff --git a/capture/parsers/socks.c b/capture/parsers/socks.c index 898da2cb63..3fe8c39479 100644 --- a/capture/parsers/socks.c +++ b/capture/parsers/socks.c @@ -27,17 +27,17 @@ typedef struct socksinfo { uint8_t state5[2]; } SocksInfo_t; -static int ipField; -static int portField; -static int userField; -static int hostField; +LOCAL int ipField; +LOCAL int portField; +LOCAL int userField; +LOCAL int hostField; //#define SOCKSDEBUG /******************************************************************************/ #define SOCKS4_STATE_REPLY 0 #define SOCKS4_STATE_DATA 1 -int socks4_parser(MolochSession_t *session, void *uw, const unsigned char *data, int remaining, int which) +LOCAL int socks4_parser(MolochSession_t *session, void *uw, const unsigned char *data, int remaining, int which) { SocksInfo_t *socks = uw; @@ -47,7 +47,7 @@ int socks4_parser(MolochSession_t *session, void *uw, const unsigned char *data, return 0; if (remaining >= 8 && data[0] == 0 && data[1] >= 0x5a && data[1] <= 0x5d) { if (socks->ip) - moloch_field_int_add(ipField, session, socks->ip); + moloch_field_ip4_add(ipField, session, socks->ip); moloch_field_int_add(portField, session, socks->port); moloch_session_add_protocol(session, "socks"); @@ -86,7 +86,7 @@ int socks4_parser(MolochSession_t *session, void *uw, const unsigned char *data, #define SOCKS5_STATE_CONN_REQUEST 5 #define SOCKS5_STATE_CONN_REPLY 6 #define SOCKS5_STATE_CONN_DATA 7 -int socks5_parser(MolochSession_t *session, void *uw, const unsigned char *data, int remaining, int which) +LOCAL int socks5_parser(MolochSession_t *session, void *uw, const unsigned char *data, int remaining, int which) { SocksInfo_t *socks = uw; int consumed; @@ -150,15 +150,13 @@ int socks5_parser(MolochSession_t *session, void *uw, const unsigned char *data, if (data[3] == 1) { // IPV4 socks->port = (data[8]&0xff) << 8 | (data[9]&0xff); memcpy(&socks->ip, data+4, 4); - moloch_field_int_add(ipField, session, socks->ip); + moloch_field_ip4_add(ipField, session, socks->ip); moloch_field_int_add(portField, session, socks->port); consumed = 4 + 4 + 2; } else if (data[3] == 3) { // Domain Name socks->port = (data[5+data[4]]&0xff) << 8 | (data[6+data[4]]&0xff); - char *lower = g_ascii_strdown((char*)data+5, data[4]); - if (!moloch_field_string_add(hostField, session, lower, data[4], FALSE)) { - g_free(lower); - } + + moloch_field_string_add_lower(hostField, session, (char *)data+5, data[4]); moloch_field_int_add(portField, session, socks->port); consumed = 4 + 1 + data[4] + 2; } else if (data[3] == 4) { // IPV6 @@ -200,7 +198,7 @@ int socks5_parser(MolochSession_t *session, void *uw, const unsigned char *data, } /******************************************************************************/ -void socks_free(MolochSession_t UNUSED(*session), void *uw) +LOCAL void socks_free(MolochSession_t UNUSED(*session), void *uw) { SocksInfo_t *socks = uw; @@ -211,7 +209,7 @@ void socks_free(MolochSession_t UNUSED(*session), void *uw) MOLOCH_TYPE_FREE(SocksInfo_t, socks); } /******************************************************************************/ -void socks4_classify(MolochSession_t *session, const unsigned char *data, int len, int which, void *UNUSED(uw)) +LOCAL void socks4_classify(MolochSession_t *session, const unsigned char *data, int len, int which, void *UNUSED(uw)) { #ifdef SOCKSDEBUG LOG("SOCKSDEBUG: enter %d %d", data[0], len); @@ -251,7 +249,7 @@ void socks4_classify(MolochSession_t *session, const unsigned char *data, int le } /******************************************************************************/ -void socks5_classify(MolochSession_t *session, const unsigned char *data, int len, int which, void *UNUSED(uw)) +LOCAL void socks5_classify(MolochSession_t *session, const unsigned char *data, int len, int which, void *UNUSED(uw)) { #ifdef SOCKSDEBUG LOG("SOCKSDEBUG: enter %d %d", data[0], len); @@ -272,16 +270,16 @@ void socks5_classify(MolochSession_t *session, const unsigned char *data, int le void moloch_parser_init() { ipField = moloch_field_define("socks", "ip", - "ip.socks", "IP", "socksip", + "ip.socks", "IP", "socks.ip", "SOCKS destination IP", MOLOCH_FIELD_TYPE_IP, MOLOCH_FIELD_FLAG_IPPRE, - "aliases", "[\"ip.socks\"]", + "aliases", "[\"socks.ip\"]", "portField", "sockspo", - "category", "user", + "portField2", "socks.port", NULL); hostField = moloch_field_define("socks", "lotermfield", - "host.socks", "Host", "socksho", + "host.socks", "Host", "socks.host", "SOCKS destination host", MOLOCH_FIELD_TYPE_STR, 0, "aliases", "[\"socks.host\"]", @@ -289,7 +287,7 @@ void moloch_parser_init() NULL); portField = moloch_field_define("socks", "integer", - "port.socks", "Port", "sockspo", + "port.socks", "Port", "socks.port", "SOCKS destination port", MOLOCH_FIELD_TYPE_INT, 0, "aliases", "[\"socks.port\"]", @@ -297,7 +295,7 @@ void moloch_parser_init() NULL); userField = moloch_field_define("socks", "termfield", - "socks.user", "User", "socksuser", + "socks.user", "User", "socks.user", "SOCKS authenticated user", MOLOCH_FIELD_TYPE_STR, 0, "aliases", "[\"socksuser\"]", diff --git a/capture/parsers/ssh.c b/capture/parsers/ssh.c index 8cee7afa9c..e7105f4956 100644 --- a/capture/parsers/ssh.c +++ b/capture/parsers/ssh.c @@ -21,14 +21,14 @@ typedef struct { uint16_t done; } SSHInfo_t; -static int verField; -static int keyField; extern MolochConfig_t config; +LOCAL int verField; +LOCAL int keyField; /******************************************************************************/ // SSH Parsing currently assumes the parts we want from a SSH Packet will be // in a single TCP packet. Kind of sucks. -int ssh_parser(MolochSession_t *session, void *uw, const unsigned char *data, int remaining, int which) +LOCAL int ssh_parser(MolochSession_t *session, void *uw, const unsigned char *data, int remaining, int which) { SSHInfo_t *ssh = uw; @@ -64,11 +64,7 @@ int ssh_parser(MolochSession_t *session, void *uw, const unsigned char *data, in if (n) { int len = (n - data); - char *str = g_ascii_strdown((char *)data, len); - - if (!moloch_field_string_add(verField, session, str, len, FALSE)) { - g_free(str); - } + moloch_field_string_add_lower(verField, session, (char *)data, len); } return 0; } @@ -127,14 +123,14 @@ int ssh_parser(MolochSession_t *session, void *uw, const unsigned char *data, in return 0; } /******************************************************************************/ -void ssh_free(MolochSession_t UNUSED(*session), void *uw) +LOCAL void ssh_free(MolochSession_t UNUSED(*session), void *uw) { SSHInfo_t *ssh = uw; MOLOCH_TYPE_FREE(SSHInfo_t, ssh); } /******************************************************************************/ -void ssh_classify(MolochSession_t *session, const unsigned char *UNUSED(data), int UNUSED(len), int UNUSED(which), void *UNUSED(uw)) +LOCAL void ssh_classify(MolochSession_t *session, const unsigned char *UNUSED(data), int UNUSED(len), int UNUSED(which), void *UNUSED(uw)) { if (moloch_session_has_protocol(session, "ssh")) return; @@ -149,13 +145,13 @@ void ssh_classify(MolochSession_t *session, const unsigned char *UNUSED(data), i void moloch_parser_init() { verField = moloch_field_define("ssh", "lotermfield", - "ssh.ver", "Version", "sshver", + "ssh.ver", "Version", "ssh.version", "SSH Software Version", MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, NULL); keyField = moloch_field_define("ssh", "termfield", - "ssh.key", "Key", "sshkey", + "ssh.key", "Key", "ssh.key", "SSH Key", MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, NULL); diff --git a/capture/parsers/ssh.detail.jade b/capture/parsers/ssh.detail.jade index 856b78130e..a07feaf036 100644 --- a/capture/parsers/ssh.detail.jade +++ b/capture/parsers/ssh.detail.jade @@ -1,5 +1,5 @@ -if (session.sshkey || session.sshver) +if (session.ssh) div.sessionDetailMeta.bold SSH dl.sessionDetailMeta - +arrayList(session, "sshkey", "Host Keys", "ssh.key") - +arrayList(session, "sshver", "Versions", "ssh.ver") + +arrayList(session.ssh, "key", "Host Keys", "ssh.key") + +arrayList(session.ssh, "version", "Versions", "ssh.ver") diff --git a/capture/parsers/tds.c b/capture/parsers/tds.c index 4f3ae13f7d..b8c67f57c4 100644 --- a/capture/parsers/tds.c +++ b/capture/parsers/tds.c @@ -20,11 +20,11 @@ typedef struct { int pos[2]; } TDSInfo_t; -static int userField; - extern MolochConfig_t config; +LOCAL int userField; + /******************************************************************************/ -int tds_parser(MolochSession_t *session, void *uw, const unsigned char *data, int remaining, int which) +LOCAL int tds_parser(MolochSession_t *session, void *uw, const unsigned char *data, int remaining, int which) { TDSInfo_t *tds = uw; @@ -53,14 +53,14 @@ int tds_parser(MolochSession_t *session, void *uw, const unsigned char *data, in return 0; } /******************************************************************************/ -void tds_free(MolochSession_t UNUSED(*session), void *uw) +LOCAL void tds_free(MolochSession_t UNUSED(*session), void *uw) { TDSInfo_t *tds = uw; MOLOCH_TYPE_FREE(TDSInfo_t, tds); } /******************************************************************************/ -void tds_classify(MolochSession_t *session, const unsigned char *UNUSED(data), int len, int which, void *UNUSED(uw)) +LOCAL void tds_classify(MolochSession_t *session, const unsigned char *UNUSED(data), int len, int which, void *UNUSED(uw)) { if (which != 0 || len < 512 || moloch_session_has_protocol(session, "tds")) return; diff --git a/capture/parsers/tls-cipher.h b/capture/parsers/tls-cipher.h index dc4d9256a9..fa5663dedf 100644 --- a/capture/parsers/tls-cipher.h +++ b/capture/parsers/tls-cipher.h @@ -951,10 +951,10 @@ NULL, "TLS_ECDHE_ECDSA_WITH_AES_256_CCM", "TLS_ECDHE_ECDSA_WITH_AES_128_CCM_8", "TLS_ECDHE_ECDSA_WITH_AES_256_CCM_8", -NULL, -NULL, -NULL, -NULL, +"TLS_ECCPWD_WITH_AES_128_GCM_SHA256", +"TLS_ECCPWD_WITH_AES_256_GCM_SHA384", +"TLS_ECCPWD_WITH_AES_128_CCM_SHA256", +"TLS_ECCPWD_WITH_AES_256_CCM_SHA384", NULL, NULL, NULL, @@ -1290,6 +1290,264 @@ NULL, NULL, NULL}; +static char *ciphers_d0[256] = { +"UNASSIGNED", +"TLS_ECDHE_PSK_WITH_AES_128_GCM_SHA256", +"TLS_ECDHE_PSK_WITH_AES_256_GCM_SHA384", +"TLS_ECDHE_PSK_WITH_AES_128_CCM_8_SHA256", +"UNASSIGNED", +"TLS_ECDHE_PSK_WITH_AES_128_CCM_SHA256", +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL, +NULL}; + static char *ciphers_e4[256] = { NULL, NULL, @@ -1759,7 +2017,7 @@ ciphers_cc, ciphers_null, ciphers_null, ciphers_null, -ciphers_null, +ciphers_d0, ciphers_null, ciphers_null, ciphers_null, diff --git a/capture/parsers/tls.c b/capture/parsers/tls.c index a45d164deb..369476e7eb 100644 --- a/capture/parsers/tls.c +++ b/capture/parsers/tls.c @@ -35,8 +35,7 @@ extern unsigned char moloch_char_to_hexstr[256][3]; LOCAL GChecksum *checksums[MOLOCH_MAX_PACKET_THREADS]; /******************************************************************************/ -void -tls_certinfo_process(MolochCertInfo_t *ci, BSB *bsb) +LOCAL void tls_certinfo_process(MolochCertInfo_t *ci, BSB *bsb) { uint32_t apc, atag, alen; char lastOid[1000]; @@ -78,8 +77,19 @@ tls_certinfo_process(MolochCertInfo_t *ci, BSB *bsb) } } /******************************************************************************/ -void -tls_alt_names(MolochCertsInfo_t *certs, BSB *bsb, char *lastOid) +LOCAL void tls_key_usage (MolochCertsInfo_t *certs, BSB *bsb) +{ + uint32_t apc, atag, alen; + + while (BSB_REMAINING(*bsb) >= 2) { + unsigned char *value = moloch_parsers_asn_get_tlv(bsb, &apc, &atag, &alen); + + if (value && atag == 4 && alen == 4) + certs->isCA = (value[3] & 0x02); + } +} +/******************************************************************************/ +LOCAL void tls_alt_names(MolochCertsInfo_t *certs, BSB *bsb, char *lastOid) { uint32_t apc, atag, alen; @@ -98,6 +108,9 @@ tls_alt_names(MolochCertsInfo_t *certs, BSB *bsb, char *lastOid) } } else if (atag == 6) { moloch_parsers_asn_decode_oid(lastOid, 100, value, alen); + if (strcmp(lastOid, "2.5.29.15") == 0) { + tls_key_usage(certs, bsb); + } if (strcmp(lastOid, "2.5.29.17") != 0) lastOid[0] = 0; } else if (lastOid[0] && atag == 4) { @@ -116,7 +129,7 @@ tls_alt_names(MolochCertsInfo_t *certs, BSB *bsb, char *lastOid) return; } /******************************************************************************/ -void tls_process_server_hello(MolochSession_t *session, const unsigned char *data, int len) +LOCAL void tls_process_server_hello(MolochSession_t *session, const unsigned char *data, int len) { BSB bsb; BSB_INIT(bsb, data, len); @@ -192,7 +205,7 @@ void tls_process_server_hello(MolochSession_t *session, const unsigned char *dat #define str4num(str) (char2num((str)[0]) * 1000 + char2num((str)[1]) * 100 + char2num((str)[2]) * 10 + char2num((str)[3])) /******************************************************************************/ -uint64_t tls_parse_time(MolochSession_t *session, int tag, unsigned char* value, int len) +LOCAL uint64_t tls_parse_time(MolochSession_t *session, int tag, unsigned char* value, int len) { int offset = 0; int pos = 0; @@ -272,7 +285,7 @@ uint64_t tls_parse_time(MolochSession_t *session, int tag, unsigned char* value, return 0; } /******************************************************************************/ -void tls_process_server_certificate(MolochSession_t *session, const unsigned char *data, int len) +LOCAL void tls_process_server_certificate(MolochSession_t *session, const unsigned char *data, int len) { BSB cbsb; @@ -383,6 +396,19 @@ void tls_process_server_certificate(MolochSession_t *session, const unsigned cha tls_alt_names(certs, &tbsb, lastOid); } + // no previous certs AND not a CA AND either no orgName or the same orgName AND the same 1 commonName + if (!session->fields[certsField] && + !certs->isCA && + ((certs->subject.orgName && certs->issuer.orgName && strcmp(certs->subject.orgName, certs->issuer.orgName) == 0) || + (certs->subject.orgName == NULL && certs->issuer.orgName == NULL)) && + certs->subject.commonName.s_count == 1 && + certs->issuer.commonName.s_count == 1 && + strcmp(certs->subject.commonName.s_next->str, certs->issuer.commonName.s_next->str) == 0) { + + moloch_session_add_tag(session, "cert:self-signed"); + } + + if (!moloch_field_certsinfo_add(certsField, session, certs, clen*2)) { moloch_field_certsinfo_free(certs); } @@ -402,8 +428,7 @@ void tls_process_server_certificate(MolochSession_t *session, const unsigned cha /* @data the data inside the record layer * @len the length of data inside record layer */ -int -tls_process_server_handshake_record(MolochSession_t *session, const unsigned char *data, int len) +LOCAL int tls_process_server_handshake_record(MolochSession_t *session, const unsigned char *data, int len) { BSB rbsb; @@ -430,8 +455,7 @@ tls_process_server_handshake_record(MolochSession_t *session, const unsigned cha } /******************************************************************************/ // https://tools.ietf.org/html/draft-davidben-tls-grease-00 -int -tls_is_grease_value(uint32_t val) +LOCAL int tls_is_grease_value(uint32_t val) { if ((val & 0x0f) != 0x0a) return 0; @@ -442,8 +466,7 @@ tls_is_grease_value(uint32_t val) return 1; } /******************************************************************************/ -void -tls_process_client(MolochSession_t *session, const unsigned char *data, int len) +LOCAL void tls_process_client(MolochSession_t *session, const unsigned char *data, int len) { BSB sslbsb; char ja3[30000]; @@ -598,7 +621,7 @@ tls_process_client(MolochSession_t *session, const unsigned char *data, int len) } /******************************************************************************/ -int tls_parser(MolochSession_t *session, void *uw, const unsigned char *data, int remaining, int which) +LOCAL int tls_parser(MolochSession_t *session, void *uw, const unsigned char *data, int remaining, int which) { TLSInfo_t *tls = uw; @@ -642,7 +665,7 @@ int tls_parser(MolochSession_t *session, void *uw, const unsigned char *data, in return 0; } /******************************************************************************/ -void tls_save(MolochSession_t *session, void *uw, int UNUSED(final)) +LOCAL void tls_save(MolochSession_t *session, void *uw, int UNUSED(final)) { TLSInfo_t *tls = uw; @@ -652,14 +675,14 @@ void tls_save(MolochSession_t *session, void *uw, int UNUSED(final)) } } /******************************************************************************/ -void tls_free(MolochSession_t *UNUSED(session), void *uw) +LOCAL void tls_free(MolochSession_t *UNUSED(session), void *uw) { TLSInfo_t *tls = uw; MOLOCH_TYPE_FREE(TLSInfo_t, tls); } /******************************************************************************/ -void tls_classify(MolochSession_t *session, const unsigned char *data, int len, int which, void *UNUSED(uw)) +LOCAL void tls_classify(MolochSession_t *session, const unsigned char *data, int len, int which, void *UNUSED(uw)) { if (len < 6 || data[2] > 0x03) return; @@ -693,111 +716,107 @@ void tls_classify(MolochSession_t *session, const unsigned char *data, int len, void moloch_parser_init() { certsField = moloch_field_define("cert", "notreal", - "cert", "tls", "tls", + "cert", "cert", "cert", "CERT Info", MOLOCH_FIELD_TYPE_CERTSINFO, MOLOCH_FIELD_FLAG_CNT | MOLOCH_FIELD_FLAG_NODB, NULL); moloch_field_define("cert", "integer", - "cert.cnt", "Cert Cnt", "tlscnt", + "cert.cnt", "Cert Cnt", "certCnt", "Count of certificates", 0, MOLOCH_FIELD_FLAG_FAKE, NULL); moloch_field_define("cert", "lotermfield", - "cert.alt", "Alt Name", "tls.alt", + "cert.alt", "Alt Name", "cert.alt", "Certificate alternative names", 0, MOLOCH_FIELD_FLAG_CNT | MOLOCH_FIELD_FLAG_FAKE, NULL); moloch_field_define("cert", "lotermfield", - "cert.serial", "Serial Number", "tls.sn", + "cert.serial", "Serial Number", "cert.serial", "Serial Number", 0, MOLOCH_FIELD_FLAG_FAKE, NULL); moloch_field_define("cert", "lotermfield", - "cert.issuer.cn", "Issuer CN", "tls.iCn", + "cert.issuer.cn", "Issuer CN", "cert.issuerCN", "Issuer's common name", 0, MOLOCH_FIELD_FLAG_FAKE, NULL); moloch_field_define("cert", "lotermfield", - "cert.subject.cn", "Subject CN", "tls.sCn", + "cert.subject.cn", "Subject CN", "cert.subjectCN", "Subject's common name", 0, MOLOCH_FIELD_FLAG_FAKE, NULL); - moloch_field_define("cert", "lotextfield", - "cert.issuer.on", "Issuer ON", "tls.iOn", + moloch_field_define("cert", "termfield", + "cert.issuer.on", "Issuer ON", "cert.issuerON", "Issuer's organization name", 0, MOLOCH_FIELD_FLAG_FAKE, - "rawField", "rawiOn", NULL); - moloch_field_define("cert", "lotextfield", - "cert.subject.on", "Subject ON", "tls.sOn", + moloch_field_define("cert", "termfield", + "cert.subject.on", "Subject ON", "cert.subjectON", "Subject's organization name", 0, MOLOCH_FIELD_FLAG_FAKE, - "rawField", "rawsOn", NULL); - moloch_field_define("cert", "lotextfield", - "cert.hash", "Hash", "tls.hash", + moloch_field_define("cert", "lotermfield", + "cert.hash", "Hash", "cert.hash", "SHA1 hash of entire certificate", 0, MOLOCH_FIELD_FLAG_FAKE, NULL); moloch_field_define("cert", "seconds", - "cert.notbefore", "Not Before", "tls.notBefore", + "cert.notbefore", "Not Before", "cert.notBefore", "Certificate is not valid before this date", 0, MOLOCH_FIELD_FLAG_FAKE, + "type2", "date", NULL); moloch_field_define("cert", "seconds", - "cert.notafter", "Not After", "tls.notAfter", + "cert.notafter", "Not After", "cert.notAfter", "Certificate is not valid after this date", 0, MOLOCH_FIELD_FLAG_FAKE, + "type2", "date", NULL); moloch_field_define("cert", "integer", - "cert.validfor", "Days Valid For", "tls.diffDays", + "cert.validfor", "Days Valid For", "cert.validDays", "Certificate is valid for this may days", 0, MOLOCH_FIELD_FLAG_FAKE, NULL); - hostField = moloch_field_define("http", "lotermfield", - "host.http", "Hostname", "ho", - "HTTP host header field", - MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, - "aliases", "[\"http.host\"]", NULL); + hostField = moloch_field_by_exp("host.http"); verField = moloch_field_define("tls", "termfield", - "tls.version", "Version", "tlsver-term", + "tls.version", "Version", "tls.version", "SSL/TLS version field", MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, NULL); cipherField = moloch_field_define("tls", "uptermfield", - "tls.cipher", "Cipher", "tlscipher-term", + "tls.cipher", "Cipher", "tls.cipher", "SSL/TLS cipher field", MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, NULL); ja3Field = moloch_field_define("tls", "lotermfield", - "tls.ja3", "JA3", "tlsja3-term", + "tls.ja3", "JA3", "tls.ja3", "SSL/TLS JA3 field", MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, NULL); dstIdField = moloch_field_define("tls", "lotermfield", - "tls.sessionid.dst", "Dst Session Id", "tlsdstid-term", + "tls.sessionid.dst", "Dst Session Id", "tls.dstSessionId", "SSL/TLS Dst Session Id", MOLOCH_FIELD_TYPE_STR_HASH, 0, NULL); srcIdField = moloch_field_define("tls", "lotermfield", - "tls.sessionid.src", "Src Session Id", "tlssrcid-term", + "tls.sessionid.src", "Src Session Id", "tls.srcSessionId", "SSL/TLS Src Session Id", MOLOCH_FIELD_TYPE_STR_HASH, 0, NULL); diff --git a/capture/parsers/tls.detail.jade b/capture/parsers/tls.detail.jade index c111c1b55a..8c76c6205e 100644 --- a/capture/parsers/tls.detail.jade +++ b/capture/parsers/tls.detail.jade @@ -1,56 +1,61 @@ -if (session.tls || session["tlsver-term"]) +if (session.tls) div.sessionDetailMeta.bold TLS dl.sessionDetailMeta - +arrayList(session, "tlsver-term", "Version", "tls.version") - +arrayList(session, "tlscipher-term", "Cipher", "tls.cipher") - +arrayList(session, "tlssrcid-term", "Src Session Id", "tls.sessionid.src") - +arrayList(session, "tlsdstid-term", "Dst Session Id", "tls.sessionid.dst") - +arrayList(session, "tlsja3-term", "JA3", "tls.ja3") - if (session.tls) - each cert in session.tls - dt Certificate - dd - - if (cert.sn) - | Serial: - a.moloch-right-click(href='#', molochexpr='cert.serial') #{cert.sn} - if (cert.hash) - | [ - a.moloch-right-click(href='#', molochexpr='cert.hash') #{cert.hash} - | ] - - if (cert.notBefore) - | Not Before: - a.formatSeconds(href='#', onclick='return addExpressionSeconds("cert.notbefore", #{cert.notBefore});') #{cert.notBefore} - - if (cert.notAfter) - | Not After: - a.formatSeconds(href='#', onclick='return addExpressionSeconds("cert.notafter", #{cert.notAfter});') #{cert.notAfter} - - if (cert.iCn && Array.isArray(cert.iCn)) - | Issuer Common: - each cn,i in cert.iCn - if (i > 0) - |, - a.moloch-right-click(href='#', molochexpr='cert.issuer.cn') #{cn} - - else if (cert.iCn) - | Issuer Common: - a.moloch-right-click(href='#', molochexpr='cert.issuer.cn') #{cert.iCn} - - if (cert.iOn) - | Issuer Org: - a.moloch-right-click(href='#', molochexpr='cert.issuer.on') #{cert.iOn} - - if (cert.sCn && Array.isArray(cert.sCn)) - | Subject Common: - each cn,i in cert.sCn - if (i > 0) - |, - a.moloch-right-click(href='#', molochexpr='cert.subject.cn') #{cn} - - else if (cert.sCn) - | Subject Common: - a.moloch-right-click(href='#', molochexpr='cert.subject.cn') #{cert.sCn} - - if (cert.sOn) - | Subject Org: - a.moloch-right-click(href='#', molochexpr='cert.subject.on') #{cert.sOn} - if (cert.alt) - | [ - each alt,i in cert.alt - if (i > 0) - |, - a.moloch-right-click(href='#', molochexpr='cert.alt') #{alt} - | ] + +arrayList(session.tls, "version", "Version", "tls.version") + +arrayList(session.tls, "cipher", "Cipher", "tls.cipher") + +arrayList(session.tls, "srcSessionId", "Src Session Id", "tls.sessionid.src") + +arrayList(session.tls, "dstSessionsId", "Dst Session Id", "tls.sessionid.dst") + +arrayList(session.tls, "ja3", "JA3", "tls.ja3") + +arrayList(session.tls, "ja3Comment", "JA3 Comment", "tls.ja3comment") + +if (session.tls) +if (session.cert) + div.sessionDetailMeta.bold Cert + dl.sessionDetailMeta + each cert in session.cert + dt Certificate + dd + - if (cert.serial) + | Serial: + a.moloch-right-click(href='#', molochexpr='cert.serial') #{cert.sn} + if (cert.hash) + | [ + a.moloch-right-click(href='#', molochexpr='cert.hash') #{cert.hash} + | ] + - if (cert.notBefore) + | Not Before: + a.formatSeconds(href='#', onclick='return addExpressionSeconds("cert.notbefore", #{cert.notBefore});') #{cert.notBefore} + - if (cert.notAfter) + | Not After: + a.formatSeconds(href='#', onclick='return addExpressionSeconds("cert.notafter", #{cert.notAfter});') #{cert.notAfter} + - if (cert.issueCN && Array.isArray(cert.issueCN)) + | Issuer Common: + each cn,i in cert.issueCN + if (i > 0) + |, + a.moloch-right-click(href='#', molochexpr='cert.issuer.cn') #{cn} + - else if (cert.issueCN) + | Issuer Common: + a.moloch-right-click(href='#', molochexpr='cert.issuer.cn') #{cert.issueCN} + - if (cert.issuerON) + | Issuer Org: + a.moloch-right-click(href='#', molochexpr='cert.issuer.on') #{cert.issuerON} + - if (cert.subjectCN && Array.isArray(cert.subjectCN)) + | Subject Common: + each cn,i in cert.subjectCN + if (i > 0) + |, + a.moloch-right-click(href='#', molochexpr='cert.subject.cn') #{cn} + - else if (cert.subjectCN) + | Subject Common: + a.moloch-right-click(href='#', molochexpr='cert.subject.cn') #{cert.subjectCN} + - if (cert.subjectON) + | Subject Org: + a.moloch-right-click(href='#', molochexpr='cert.subject.on') #{cert.subjectON} + if (cert.alt) + | [ + each alt,i in cert.alt + if (i > 0) + |, + a.moloch-right-click(href='#', molochexpr='cert.alt') #{alt} + | ] diff --git a/capture/parsers/tls.detail.pug b/capture/parsers/tls.detail.pug deleted file mode 100644 index 91d3617993..0000000000 --- a/capture/parsers/tls.detail.pug +++ /dev/null @@ -1,49 +0,0 @@ -if (session.tls || session["tlsver-term"]) - h4.sessionDetailMeta.bold TLS - dl.sessionDetailMeta - +arrayList(session, "tlsver-term", "Version", "tls.version") - +arrayList(session, "tlscipher-term", "Cipher", "tls.cipher") - +arrayList(session, "tlssrcid-term", "Src Session Id", "tls.sessionid.src") - +arrayList(session, "tlsdstid-term", "Dst Session Id", "tls.sessionid.dst") - if (session.tls) - each cert in session.tls - dt Certificate - dd - - if (cert.sn) - strong.medium Serial - +clickableValue('cert.serial', cert.sn) - if (cert.hash) - | [ - +clickableValue('cert.hash', cert.hash) - | ] - - if (cert.notBefore) - strong.medium.margined-left-lg Not Before - + clickableValue('cert.notbefore', cert.notBefore, true) - - if (cert.notAfter) - strong.medium Not After - + clickableValue('cert.notafter', cert.notAfter, true) - - if (cert.iCn && Array.isArray(cert.iCn)) - strong.medium Issuer Common - each cn,i in cert.iCn - +clickableValue('cert.issuer.cn', cn) - - else if (cert.iCn) - strong.medium Issuer Common - +clickableValue('cert.issuer.cn', cert.iCn) - - if (cert.iOn) - strong.medium Issuer Org - +clickableValue('cert.issuer.on', cert.iOn) - - if (cert.sCn && Array.isArray(cert.sCn)) - strong.medium Subject Common - each cn,i in cert.sCn - +clickableValue('cert.subject.cn', cn) - - else if (cert.sCn) - strong.medium Subject Common - +clickableValue('cert.subject.cn', cert.sCn) - - if (cert.sOn) - strong.medium Subject Org - +clickableValue('cert.subject.on', cert.sOn) - if (cert.alt) - | [ - each alt,i in cert.alt - +clickableValue('cert.alt', alt) - | ] diff --git a/capture/plugins/Makefile.in b/capture/plugins/Makefile.in index c7fed55b83..7b3ae67707 100644 --- a/capture/plugins/Makefile.in +++ b/capture/plugins/Makefile.in @@ -24,10 +24,10 @@ install: $(INSTALL) *.so *.jade *.js $(PLUGINDIR) @mkdir -p "$(WISEDIR)" $(INSTALL) wiseService/*.js wiseService/wiseService.ini.sample wiseService/package.json $(WISEDIR) - (cd $(WISEDIR) ; npm update) + (cd $(WISEDIR) ; npm install) @mkdir -p "$(PLUGINDIR)/writer-s3" $(INSTALL) writer-s3/*.js writer-s3/package.json $(PLUGINDIR)/writer-s3 - (cd $(PLUGINDIR)/writer-s3 ; npm update) + (cd $(PLUGINDIR)/writer-s3 ; npm install) distclean realclean clean: rm -f *.o *.so diff --git a/capture/plugins/daq/reader-daq.c b/capture/plugins/daq/reader-daq.c index c17ba51288..b58338e988 100644 --- a/capture/plugins/daq/reader-daq.c +++ b/capture/plugins/daq/reader-daq.c @@ -63,11 +63,11 @@ DAQ_Verdict reader_daq_packet_cb(void *batch, const DAQ_PktHdr_t *h, const uint8 return DAQ_VERDICT_PASS; } /******************************************************************************/ -static void *reader_daq_thread(gpointer handle) +LOCAL void *reader_daq_thread(gpointer handle) { + MolochPacketBatch_t batch; + moloch_packet_batch_init(&batch); while (1) { - MolochPacketBatch_t batch; - moloch_packet_batch_init(&batch); int r = daq_acquire(module, handle, 10000, reader_daq_packet_cb, &batch); moloch_packet_batch_flush(&batch); diff --git a/capture/plugins/lua/data.c b/capture/plugins/lua/data.c index e79f9b566a..1122f092b7 100644 --- a/capture/plugins/lua/data.c +++ b/capture/plugins/lua/data.c @@ -17,7 +17,7 @@ #include "molua.h" /******************************************************************************/ -static MD_t *checkMolochData (lua_State *L, int index) +LOCAL MD_t *checkMolochData (lua_State *L, int index) { MD_t *md; luaL_checktype(L, index, LUA_TUSERDATA); diff --git a/capture/plugins/lua/httpService.c b/capture/plugins/lua/httpService.c index 9d6e9427de..6579d7a620 100644 --- a/capture/plugins/lua/httpService.c +++ b/capture/plugins/lua/httpService.c @@ -30,7 +30,7 @@ typedef struct { } LuaHttp_t; /******************************************************************************/ -static void *checkMHS (lua_State *L, int index) +LOCAL void *checkMHS (lua_State *L, int index) { void **pmhs, *mhs; luaL_checktype(L, index, LUA_TUSERDATA); @@ -45,7 +45,7 @@ static void *checkMHS (lua_State *L, int index) return mhs; } /******************************************************************************/ -static void *pushMHS (lua_State *L, void *mhs) +LOCAL void *pushMHS (lua_State *L, void *mhs) { void **pmhs = (void **)lua_newuserdata(L, sizeof(void *)); *pmhs = mhs; @@ -55,7 +55,7 @@ static void *pushMHS (lua_State *L, void *mhs) } /******************************************************************************/ -static int MHS_new(lua_State *L) +LOCAL int MHS_new(lua_State *L) { if (lua_gettop(L) != 3 || !lua_isstring(L, 1) || !lua_isinteger(L, 2) || !lua_isinteger(L, 3)) { return luaL_error(L, "usage: "); @@ -66,7 +66,7 @@ static int MHS_new(lua_State *L) return 1; } /******************************************************************************/ -static void mhs_http_response_cb_process(MolochSession_t *UNUSED(session), gpointer uw1, gpointer UNUSED(uw2)) +LOCAL void mhs_http_response_cb_process(MolochSession_t *UNUSED(session), gpointer uw1, gpointer UNUSED(uw2)) { LuaHttp_t *lhttp = uw1; @@ -93,7 +93,7 @@ void mhs_http_response_cb(int code, unsigned char *data, int len, gpointer uw) moloch_session_add_cmd(&moluaFakeSessions[lhttp->thread], MOLOCH_SES_CMD_FUNC, lhttp, NULL, mhs_http_response_cb_process); } /******************************************************************************/ -static int MHS_request(lua_State *L) +LOCAL int MHS_request(lua_State *L) { if (config.debug > 2) molua_stackDump(L); @@ -140,7 +140,7 @@ static int MHS_request(lua_State *L) return 1; } /******************************************************************************/ -static int MHS_gc(lua_State *L) +LOCAL int MHS_gc(lua_State *L) { if (lua_gettop(L) != 1 || !lua_isuserdata(L, 1)) { return luaL_error(L, "usage: "); @@ -152,7 +152,7 @@ static int MHS_gc(lua_State *L) return 0; } /******************************************************************************/ -static int MHS_tostring (lua_State *L) +LOCAL int MHS_tostring (lua_State *L) { lua_pushfstring(L, "MolochHttpService: %p", lua_touserdata(L, 1)); return 1; diff --git a/capture/plugins/lua/molua.c b/capture/plugins/lua/molua.c index 94a5565c5b..3556e6c368 100644 --- a/capture/plugins/lua/molua.c +++ b/capture/plugins/lua/molua.c @@ -70,7 +70,7 @@ void lua_http_on_body_cb (MolochSession_t *session, http_parser *UNUSED(hp), con } } /******************************************************************************/ -static int M_expression_to_fieldId(lua_State *L) +LOCAL int M_expression_to_fieldId(lua_State *L) { if (lua_gettop(L) != 1 || !lua_isstring(L, 1)) { return luaL_error(L, "usage: "); diff --git a/capture/plugins/lua/session.c b/capture/plugins/lua/session.c index 409d0d0627..007211237a 100644 --- a/capture/plugins/lua/session.c +++ b/capture/plugins/lua/session.c @@ -21,7 +21,7 @@ extern lua_State *Ls[MOLOCH_MAX_PACKET_THREADS]; /******************************************************************************/ -static void *checkMolochSession (lua_State *L, int index) +LOCAL void *checkMolochSession (lua_State *L, int index) { void **pms, *ms; luaL_checktype(L, index, LUA_TUSERDATA); @@ -86,7 +86,7 @@ void molua_parsers_free_cb(MolochSession_t *session, void *uw) luaL_unref(L, LUA_REGISTRYINDEX, (long)uw); } /******************************************************************************/ -static int MS_register_tcp_classifier(lua_State *L) +LOCAL int MS_register_tcp_classifier(lua_State *L) { if (L != Ls[0]) // Only do once return 0; @@ -105,7 +105,7 @@ static int MS_register_tcp_classifier(lua_State *L) return 0; } /******************************************************************************/ -static int MS_register_udp_classifier(lua_State *L) +LOCAL int MS_register_udp_classifier(lua_State *L) { if (L != Ls[0]) // Only do once return 0; @@ -123,8 +123,8 @@ static int MS_register_udp_classifier(lua_State *L) moloch_parsers_classifier_register_udp(name, function, offset, match, match_len, molua_classify_cb); return 0; } -static char *callbackRefs[MOLUA_REF_SIZE][MOLUA_REF_MAX_CNT]; -static int callbackRefsCnt[MOLUA_REF_SIZE]; +LOCAL char *callbackRefs[MOLUA_REF_SIZE][MOLUA_REF_MAX_CNT]; +LOCAL int callbackRefsCnt[MOLUA_REF_SIZE]; /******************************************************************************/ void molua_http_on_body_cb (MolochSession_t *session, http_parser *UNUSED(hp), const char *at, size_t length) { @@ -155,7 +155,7 @@ void molua_http_on_body_cb (MolochSession_t *session, http_parser *UNUSED(hp), c } } /******************************************************************************/ -static int MS_register_body_feed(lua_State *L) +LOCAL int MS_register_body_feed(lua_State *L) { if (L != Ls[0]) // Only do once return 0; @@ -180,7 +180,7 @@ static int MS_register_body_feed(lua_State *L) return 0; } /******************************************************************************/ -static int MS_register_parser(lua_State *L) +LOCAL int MS_register_parser(lua_State *L) { if (lua_gettop(L) != 2 || !lua_isuserdata(L, 1) || !lua_isfunction(L, 2)) { return luaL_error(L, "usage: "); @@ -194,7 +194,7 @@ static int MS_register_parser(lua_State *L) return 0; } /******************************************************************************/ -static int MS_add_tag(lua_State *L) +LOCAL int MS_add_tag(lua_State *L) { if (lua_gettop(L) != 2 || !lua_isuserdata(L, 1) || !lua_isstring(L, 2)) { return luaL_error(L, "usage: "); @@ -207,7 +207,7 @@ static int MS_add_tag(lua_State *L) return 0; } /******************************************************************************/ -static int MS_incr_outstanding(lua_State *L) +LOCAL int MS_incr_outstanding(lua_State *L) { if (lua_gettop(L) != 1 || !lua_isuserdata(L, 1)) { return luaL_error(L, "usage: "); @@ -219,7 +219,7 @@ static int MS_incr_outstanding(lua_State *L) return 0; } /******************************************************************************/ -static int MS_decr_outstanding(lua_State *L) +LOCAL int MS_decr_outstanding(lua_State *L) { if (lua_gettop(L) != 1 || !lua_isuserdata(L, 1)) { return luaL_error(L, "usage: "); @@ -231,7 +231,7 @@ static int MS_decr_outstanding(lua_State *L) return 0; } /******************************************************************************/ -static int MS_add_protocol(lua_State *L) +LOCAL int MS_add_protocol(lua_State *L) { if (lua_gettop(L) != 2 || !lua_isuserdata(L, 1) || !lua_isstring(L, 2)) { return luaL_error(L, "usage: "); @@ -243,7 +243,7 @@ static int MS_add_protocol(lua_State *L) return 0; } /******************************************************************************/ -static int MS_has_protocol(lua_State *L) +LOCAL int MS_has_protocol(lua_State *L) { if (lua_gettop(L) != 2 || !lua_isuserdata(L, 1) || !lua_isstring(L, 2)) { return luaL_error(L, "usage: "); @@ -256,7 +256,7 @@ static int MS_has_protocol(lua_State *L) return 1; } /******************************************************************************/ -static int MS_add_string(lua_State *L) +LOCAL int MS_add_string(lua_State *L) { if (config.debug > 2) molua_stackDump(L); @@ -281,7 +281,7 @@ static int MS_add_string(lua_State *L) return 1; } /******************************************************************************/ -static int MS_add_int(lua_State *L) +LOCAL int MS_add_int(lua_State *L) { if (config.debug > 2) molua_stackDump(L); @@ -305,7 +305,7 @@ static int MS_add_int(lua_State *L) return 1; } /******************************************************************************/ -static int MS_tostring(lua_State *L) +LOCAL int MS_tostring(lua_State *L) { MolochSession_t *session = checkMolochSession(L, 1); lua_pushfstring(L, "MolochSession: %p", session); @@ -313,7 +313,7 @@ static int MS_tostring(lua_State *L) } /******************************************************************************/ -static int MS_table(lua_State *L) +LOCAL int MS_table(lua_State *L) { MolochSession_t *session = checkMolochSession(L, 1); MoluaPlugin_t *mp = session->pluginData[molua_pluginIndex]; diff --git a/capture/plugins/netflow.c b/capture/plugins/netflow.c index 46b80bc818..c820e36581 100644 --- a/capture/plugins/netflow.c +++ b/capture/plugins/netflow.c @@ -58,7 +58,7 @@ LOCAL int bufCount[MOLOCH_MAX_PACKET_THREADS]; LOCAL uint32_t totalFlows[MOLOCH_MAX_PACKET_THREADS]; /******************************************************************************/ -void netflow_send(const int thread) +LOCAL void netflow_send(const int thread) { BSB hbsb; @@ -103,7 +103,7 @@ void netflow_send(const int thread) /* * Called by moloch when a session is about to be saved */ -void netflow_plugin_save(MolochSession_t *session, int UNUSED(final)) +LOCAL void netflow_plugin_save(MolochSession_t *session, int UNUSED(final)) { static char zero[8] = {0, 0, 0, 0, 0, 0, 0, 0}; const int thread = session->thread; @@ -228,7 +228,7 @@ void netflow_plugin_save(MolochSession_t *session, int UNUSED(final)) /* * Called by moloch when moloch is quiting */ -void netflow_plugin_exit() +LOCAL void netflow_plugin_exit() { int thread; for (thread = 0; thread < config.packetThreads; thread++) { diff --git a/capture/plugins/pfring/reader-pfring.c b/capture/plugins/pfring/reader-pfring.c index 94339fe57b..54a5c90189 100644 --- a/capture/plugins/pfring/reader-pfring.c +++ b/capture/plugins/pfring/reader-pfring.c @@ -62,7 +62,7 @@ void reader_pfring_packet_cb(const struct pfring_pkthdr *h, const u_char *p, con moloch_packet_batch_flush(batch); } /******************************************************************************/ -static void *reader_pfring_thread(void *ringv) +LOCAL void *reader_pfring_thread(void *ringv) { pfring *ring = ringv; diff --git a/capture/plugins/scrubspi.c b/capture/plugins/scrubspi.c index 4049fb049c..8997ad01ff 100644 --- a/capture/plugins/scrubspi.c +++ b/capture/plugins/scrubspi.c @@ -31,7 +31,7 @@ LOCAL SS_t ss[MAX_SS]; /******************************************************************************/ -void scrubspi_plugin_save(MolochSession_t *session, int UNUSED(final)) +LOCAL void scrubspi_plugin_save(MolochSession_t *session, int UNUSED(final)) { int s; guint i; @@ -72,12 +72,15 @@ void scrubspi_plugin_save(MolochSession_t *session, int UNUSED(final)) } ); + break; + case MOLOCH_FIELD_TYPE_STR_GHASH: + LOGEXIT("MOLOCH_FIELD_TYPE_STR_GHASH not supported yet, open an issue"); break; } } } /******************************************************************************/ -void scrubspi_add_entry(char *key, char *value) +LOCAL void scrubspi_add_entry(char *key, char *value) { char spliton[2] = {0, 0}; spliton[0] = value[0]; @@ -103,7 +106,8 @@ void scrubspi_add_entry(char *key, char *value) MolochFieldInfo_t *field = config.fields[pos]; if (field->type != MOLOCH_FIELD_TYPE_STR && field->type != MOLOCH_FIELD_TYPE_STR_ARRAY && - field->type != MOLOCH_FIELD_TYPE_STR_HASH) { + field->type != MOLOCH_FIELD_TYPE_STR_HASH && + field->type != MOLOCH_FIELD_TYPE_STR_GHASH) { LOGEXIT("Field %s in [scrubspi] is not of type string", keys[j]); } ss[ssLen].pos = pos; diff --git a/capture/plugins/snf/reader-snf.c b/capture/plugins/snf/reader-snf.c index d251f393cc..dbd9cdb70c 100644 --- a/capture/plugins/snf/reader-snf.c +++ b/capture/plugins/snf/reader-snf.c @@ -141,6 +141,9 @@ void reader_snf_init(char *UNUSED(name)) for (r = 0; r < snfNumRings; r++) { err = snf_ring_open(handles[i], &rings[i][r]); + if (err != 0) { + LOGEXIT("Mryicom: Couldn't open ring %d for interface '%s' %d", r, config.interface[i], err); + } } } @@ -153,6 +156,5 @@ void reader_snf_init(char *UNUSED(name)) /******************************************************************************/ void moloch_plugin_init() { - LOG("ALW START"); moloch_readers_add("snf", reader_snf_init); } diff --git a/capture/plugins/tagger.c b/capture/plugins/tagger.c index 4cae68d9f9..9dc6bae4ef 100644 --- a/capture/plugins/tagger.c +++ b/capture/plugins/tagger.c @@ -37,15 +37,15 @@ extern MolochConfig_t config; extern void *esServer; -static int tagsField; -static int httpHostField; -static int httpXffField; -static int httpMd5Field; -static int httpPathField; -static int emailMd5Field; -static int emailSrcField; -static int emailDstField; -static int dnsHostField; +LOCAL int tagsField; +LOCAL int httpHostField; +LOCAL int httpXffField; +LOCAL int httpMd5Field; +LOCAL int httpPathField; +LOCAL int emailMd5Field; +LOCAL int emailSrcField; +LOCAL int emailDstField; +LOCAL int dnsHostField; /******************************************************************************/ @@ -103,10 +103,10 @@ TaggerStringHash_t allURIs; HASH_VAR(s_, allFiles, TaggerFileHead_t, 101); -static patricia_tree_t *allIps; +LOCAL patricia_tree_t *allIps; /******************************************************************************/ -void tagger_process_match(MolochSession_t *session, GPtrArray *infos) +LOCAL void tagger_process_match(MolochSession_t *session, GPtrArray *infos) { uint32_t f, t; for (f = 0; f < infos->len; f++) { @@ -122,7 +122,7 @@ void tagger_process_match(MolochSession_t *session, GPtrArray *infos) /* * Called by moloch when a session is about to be saved */ -void tagger_plugin_save(MolochSession_t *session, int UNUSED(final)) +LOCAL void tagger_plugin_save(MolochSession_t *session, int UNUSED(final)) { TaggerString_t *tstring; @@ -162,34 +162,27 @@ void tagger_plugin_save(MolochSession_t *session, int UNUSED(final)) tagger_process_match(session, ((TaggerIP_t *)(nodes[i]->data))->infos); } - // ALW - Fix when we support ipv6 for other ips - prefix.family = AF_INET; - prefix.bitlen = 32; if (httpXffField != -1 && session->fields[httpXffField]) { - if (config.fields[httpXffField]->type == MOLOCH_FIELD_TYPE_IP_HASH) { - MolochIntHashStd_t *ihash = session->fields[httpXffField]->ihash; - MolochInt_t *xff; - - HASH_FORALL(i_, *ihash, xff, - prefix.add.sin.s_addr = xff->i_hash; - cnt = patricia_search_all(allIps, &prefix, 1, nodes); - for (i = 0; i < cnt; i++) { - tagger_process_match(session, ((TaggerIP_t *)(nodes[i]->data))->infos); - } - ); - } else { - GHashTable *ghash; - GHashTableIter iter; - gpointer ikey; - - ghash = session->fields[httpXffField]->ghash; - g_hash_table_iter_init (&iter, ghash); - while (g_hash_table_iter_next (&iter, &ikey, NULL)) { - prefix.add.sin.s_addr = (int)(long)ikey; - cnt = patricia_search_all(allIps, &prefix, 1, nodes); - for (i = 0; i < cnt; i++) { - tagger_process_match(session, ((TaggerIP_t *)(nodes[i]->data))->infos); - } + GHashTable *ghash; + GHashTableIter iter; + gpointer ikey; + + ghash = session->fields[httpXffField]->ghash; + g_hash_table_iter_init (&iter, ghash); + while (g_hash_table_iter_next (&iter, &ikey, NULL)) { + if (IN6_IS_ADDR_V4MAPPED((struct in6_addr*)ikey)) { + prefix.family = AF_INET; + prefix.bitlen = 32; + prefix.add.sin.s_addr = MOLOCH_V6_TO_V4(*(struct in6_addr*)ikey); + } else { + prefix.family = AF_INET6; + prefix.bitlen = 128; + memcpy(&prefix.add.sin6.s6_addr, ikey, 16); + } + + cnt = patricia_search_all(allIps, &prefix, 1, nodes); + for (i = 0; i < cnt; i++) { + tagger_process_match(session, ((TaggerIP_t *)(nodes[i]->data))->infos); } } } @@ -272,7 +265,7 @@ void tagger_plugin_save(MolochSession_t *session, int UNUSED(final)) } /******************************************************************************/ -void tagger_free_ip (TaggerIP_t *tip) +LOCAL void tagger_free_ip (TaggerIP_t *tip) { g_ptr_array_free(tip->infos, TRUE); MOLOCH_TYPE_FREE(TaggerIP_t, tip); @@ -281,7 +274,7 @@ void tagger_free_ip (TaggerIP_t *tip) /* * Called by moloch when moloch is quiting */ -void tagger_plugin_exit() +LOCAL void tagger_plugin_exit() { TaggerString_t *tstring; HASH_FORALL_POP_HEAD(s_, allDomains, tstring, @@ -321,7 +314,8 @@ void tagger_plugin_exit() Destroy_Patricia(allIps, tagger_free_ip); } -void tagger_remove_file(GPtrArray *infos, TaggerFile_t *file) +/******************************************************************************/ +LOCAL void tagger_remove_file(GPtrArray *infos, TaggerFile_t *file) { int f; for (f = 0; f < (int)infos->len; f++) { @@ -335,7 +329,7 @@ void tagger_remove_file(GPtrArray *infos, TaggerFile_t *file) /* * Free most of the memory used by a file */ -void tagger_unload_file(TaggerFile_t *file) { +LOCAL void tagger_unload_file(TaggerFile_t *file) { int i; if (file->type[0] == 'i') { prefix_t prefix; @@ -394,7 +388,7 @@ void tagger_unload_file(TaggerFile_t *file) { file->md5 = NULL; } /******************************************************************************/ -void tagger_info_free(gpointer data) +LOCAL void tagger_info_free(gpointer data) { TaggerInfo_t *info = data; @@ -405,7 +399,7 @@ void tagger_info_free(gpointer data) /* * File data from ES */ -void tagger_load_file_cb(int UNUSED(code), unsigned char *data, int data_len, gpointer uw) +LOCAL void tagger_load_file_cb(int UNUSED(code), unsigned char *data, int data_len, gpointer uw) { TaggerFile_t *file = uw; uint32_t out[4*100]; @@ -459,11 +453,6 @@ void tagger_load_file_cb(int UNUSED(code), unsigned char *data, int data_len, gp } } - int tag = 0; - for (tag = 0; file->tags[tag]; tag++) { - moloch_db_get_tag(NULL, tagsField, file->tags[tag], NULL); - } - patricia_node_t *node; TaggerIP_t *tip; @@ -558,7 +547,7 @@ void tagger_load_file_cb(int UNUSED(code), unsigned char *data, int data_len, gp /* * Start loading a file from database */ -void tagger_load_file(TaggerFile_t *file) +LOCAL void tagger_load_file(TaggerFile_t *file) { char key[500]; int key_len; @@ -571,7 +560,7 @@ void tagger_load_file(TaggerFile_t *file) /* * Process the list of files from ES */ -void tagger_fetch_files_cb(int UNUSED(code), unsigned char *data, int data_len, gpointer UNUSED(uw)) +LOCAL void tagger_fetch_files_cb(int UNUSED(code), unsigned char *data, int data_len, gpointer UNUSED(uw)) { uint32_t hits_len; unsigned char *hits = moloch_js0n_get(data, data_len, "hits", &hits_len); @@ -628,7 +617,7 @@ void tagger_fetch_files_cb(int UNUSED(code), unsigned char *data, int data_len, /* * Get the list of files from ES, when called at start up it will be a sync call */ -gboolean tagger_fetch_files (gpointer sync) +LOCAL gboolean tagger_fetch_files (gpointer sync) { char key[500]; int key_len; @@ -678,16 +667,15 @@ void moloch_plugin_init() NULL ); - tagsField = moloch_field_by_db("ta"); - httpHostField = moloch_field_by_db("ho"); - httpXffField = moloch_field_by_db("xff"); - httpMd5Field = moloch_field_by_db("hmd5"); - httpPathField = moloch_field_by_db("hpath"); - emailMd5Field = moloch_field_by_db("emd5"); - emailSrcField = moloch_field_by_db("esrc"); - emailDstField = moloch_field_by_db("edst"); - dnsHostField = moloch_field_by_db("dnsho"); - + tagsField = moloch_field_by_db("tags"); + httpHostField = moloch_field_by_db("http.host"); + httpXffField = moloch_field_by_db("http.xffIp"); + httpMd5Field = moloch_field_by_db("http.md5"); + httpPathField = moloch_field_by_db("http.path"); + emailMd5Field = moloch_field_by_db("email.md5"); + emailSrcField = moloch_field_by_db("email.src"); + emailDstField = moloch_field_by_db("email.dst"); + dnsHostField = moloch_field_by_db("dns.host"); /* Call right away sync, and schedule every 60 seconds async */ tagger_fetch_files((gpointer)1); diff --git a/capture/plugins/taggerUpload.pl b/capture/plugins/taggerUpload.pl index 474159d5d6..3500815f40 100755 --- a/capture/plugins/taggerUpload.pl +++ b/capture/plugins/taggerUpload.pl @@ -55,7 +55,6 @@ ($) my $md5hex = md5_hex($elements); my $content = '{' . $fields . '"tags": "' . join(',', @ARGV[3 .. $#ARGV]) . '", "md5":"' . $md5hex .'", "type":"' . $ARGV[1] . '", "data":"' . $elements . '"}'. "\n"; -#print $content,"\n"; $response = $userAgent->post("$host/tagger/file/$ARGV[2]", "Content-Type" => "application/json;charset=UTF-8", Content => $content); print $response->content, "\n"; diff --git a/capture/plugins/wise.c b/capture/plugins/wise.c index fe279e5dfd..eef2a2b5cd 100644 --- a/capture/plugins/wise.c +++ b/capture/plugins/wise.c @@ -37,7 +37,9 @@ LOCAL char udpTuple; LOCAL int httpHostField; LOCAL int httpXffField; LOCAL int httpMd5Field; +LOCAL int httpSha256Field; LOCAL int emailMd5Field; +LOCAL int emailSha256Field; LOCAL int emailSrcField; LOCAL int emailDstField; LOCAL int dnsHostField; @@ -66,9 +68,10 @@ LOCAL const int validDNS[256] = { #define INTEL_TYPE_URL 4 #define INTEL_TYPE_TUPLE 5 #define INTEL_TYPE_JA3 6 -#define INTEL_TYPE_SIZE 7 +#define INTEL_TYPE_SHA256 7 +#define INTEL_TYPE_SIZE 8 -LOCAL char *wiseStrings[] = {"ip", "domain", "md5", "email", "url", "tuple", "ja3"}; +LOCAL char *wiseStrings[] = {"ip", "domain", "md5", "email", "url", "tuple", "ja3", "sha256"}; #define INTEL_STAT_LOOKUP 0 #define INTEL_STAT_CACHE 1 @@ -122,7 +125,7 @@ LOCAL MOLOCH_LOCK_DEFINE(iRequest); LOCAL char *iBuf = 0; /******************************************************************************/ -int wise_item_cmp(const void *keyv, const void *elementv) +LOCAL int wise_item_cmp(const void *keyv, const void *elementv) { char *key = (char*)keyv; WiseItem_t *element = (WiseItem_t *)elementv; @@ -130,7 +133,7 @@ int wise_item_cmp(const void *keyv, const void *elementv) return strcmp(key, element->key) == 0; } /******************************************************************************/ -void wise_print_stats() +LOCAL void wise_print_stats() { int i; for (i = 0; i < INTEL_TYPE_SIZE; i++) { @@ -146,7 +149,7 @@ void wise_print_stats() } } /******************************************************************************/ -void wise_load_fields() +LOCAL void wise_load_fields() { char key[500]; int key_len; @@ -183,7 +186,7 @@ void wise_load_fields() free(data); } /******************************************************************************/ -void wise_session_cmd_cb(MolochSession_t *session, gpointer uw1, gpointer UNUSED(uw2)) +LOCAL void wise_session_cmd_cb(MolochSession_t *session, gpointer uw1, gpointer UNUSED(uw2)) { WiseItem_t *wi = uw1; @@ -193,7 +196,7 @@ void wise_session_cmd_cb(MolochSession_t *session, gpointer uw1, gpointer UNUSED moloch_session_decr_outstanding(session); } /******************************************************************************/ -void wise_free_item_unlocked(WiseItem_t *wi) +LOCAL void wise_free_item_unlocked(WiseItem_t *wi) { int i; HASH_REMOVE(wih_, itemHash[(int)wi->type], wi); @@ -209,7 +212,7 @@ void wise_free_item_unlocked(WiseItem_t *wi) MOLOCH_TYPE_FREE(WiseItem_t, wi); } /******************************************************************************/ -void wise_cb(int UNUSED(code), unsigned char *data, int data_len, gpointer uw) +LOCAL void wise_cb(int UNUSED(code), unsigned char *data, int data_len, gpointer uw) { BSB bsb; @@ -290,7 +293,7 @@ void wise_cb(int UNUSED(code), unsigned char *data, int data_len, gpointer uw) MOLOCH_TYPE_FREE(WiseRequest_t, request); } /******************************************************************************/ -void wise_lookup(MolochSession_t *session, WiseRequest_t *request, char *value, int type) +LOCAL void wise_lookup(MolochSession_t *session, WiseRequest_t *request, char *value, int type) { if (*value == 0) @@ -368,7 +371,7 @@ void wise_lookup(MolochSession_t *session, WiseRequest_t *request, char *value, MOLOCH_UNLOCK(item); } /******************************************************************************/ -void wise_lookup_domain(MolochSession_t *session, WiseRequest_t *request, char *domain) +LOCAL void wise_lookup_domain(MolochSession_t *session, WiseRequest_t *request, char *domain) { unsigned char *end = (unsigned char*)domain; unsigned char *colon = 0; @@ -498,7 +501,6 @@ LOCAL gboolean wise_flush(gpointer UNUSED(user_data)) void wise_plugin_pre_save(MolochSession_t *session, int UNUSED(final)) { MolochString_t *hstring; - uint32_t i; MOLOCH_LOCK(iRequest); if (!iRequest) { @@ -564,6 +566,29 @@ void wise_plugin_pre_save(MolochSession_t *session, int UNUSED(final)) ); } + //SHA256s + if (config.supportSha256) { + if (session->fields[httpSha256Field]) { + MolochStringHashStd_t *shash = session->fields[httpSha256Field]->shash; + HASH_FORALL(s_, *shash, hstring, + if (hstring->uw) { + char str[1000]; + snprintf(str, sizeof(str), "%s;%s", hstring->str, (char*)hstring->uw); + wise_lookup(session, iRequest, str, INTEL_TYPE_SHA256); + } else { + wise_lookup(session, iRequest, hstring->str, INTEL_TYPE_SHA256); + } + ); + } + + if (session->fields[emailSha256Field]) { + MolochStringHashStd_t *shash = session->fields[emailSha256Field]->shash; + HASH_FORALL(s_, *shash, hstring, + wise_lookup(session, iRequest, hstring->str, INTEL_TYPE_SHA256); + ); + } + } + //Email if (session->fields[emailSrcField]) { MolochStringHashStd_t *shash = session->fields[emailSrcField]->shash; @@ -581,6 +606,16 @@ void wise_plugin_pre_save(MolochSession_t *session, int UNUSED(final)) //URLs if (session->fields[httpUrlField]) { + MolochStringHashStd_t *shash = session->fields[httpUrlField]->shash; + HASH_FORALL(s_, *shash, hstring, + if (hstring->str[0] == 'h' && memcmp("http://", hstring->str, 7) == 0) { + wise_lookup_url(session, iRequest, hstring->str+7); + } else { + wise_lookup_url(session, iRequest, hstring->str); + } + ); + + /* GPtrArray *sarray = session->fields[httpUrlField]->sarray; for(i = 0; i < sarray->len; i++) { @@ -592,7 +627,7 @@ void wise_plugin_pre_save(MolochSession_t *session, int UNUSED(final)) wise_lookup_url(session, iRequest, str+7); } else wise_lookup_url(session, iRequest, str); - } + }*/ } // Tuples @@ -615,7 +650,7 @@ void wise_plugin_pre_save(MolochSession_t *session, int UNUSED(final)) MOLOCH_UNLOCK(iRequest); } /******************************************************************************/ -void wise_plugin_exit() +LOCAL void wise_plugin_exit() { MOLOCH_LOCK(item); int h; @@ -630,7 +665,7 @@ void wise_plugin_exit() MOLOCH_UNLOCK(item); } /******************************************************************************/ -uint32_t wise_plugin_outstanding() +LOCAL uint32_t wise_plugin_outstanding() { int count; MOLOCH_LOCK(iRequest); @@ -658,17 +693,22 @@ void moloch_plugin_init() int port = moloch_config_int(NULL, "wisePort", 8081, 1, 0xffff); char *host = moloch_config_str(NULL, "wiseHost", "127.0.0.1"); - httpHostField = moloch_field_by_db("ho"); - httpXffField = moloch_field_by_db("xff"); - httpMd5Field = moloch_field_by_db("hmd5"); - emailMd5Field = moloch_field_by_db("emd5"); - emailSrcField = moloch_field_by_db("esrc"); - emailDstField = moloch_field_by_db("edst"); - dnsHostField = moloch_field_by_db("dnsho"); - tagsField = moloch_field_by_db("ta"); - httpUrlField = moloch_field_by_db("us"); - protocolField = moloch_field_by_db("prot-term"); - ja3Field = moloch_field_by_db("tlsja3-term"); + httpHostField = moloch_field_by_db("http.host"); + httpXffField = moloch_field_by_db("http.xffIp"); + httpMd5Field = moloch_field_by_db("http.md5"); + emailMd5Field = moloch_field_by_db("email.md5"); + emailSrcField = moloch_field_by_db("email.src"); + emailDstField = moloch_field_by_db("email.dst"); + dnsHostField = moloch_field_by_db("dns.host"); + tagsField = moloch_field_by_db("tags"); + httpUrlField = moloch_field_by_db("http.uri"); + protocolField = moloch_field_by_db("protocol"); + ja3Field = moloch_field_by_db("tls.ja3"); + + if (config.supportSha256) { + httpSha256Field = moloch_field_by_db("http.sha256"); + emailSha256Field = moloch_field_by_db("email.sha256"); + } char hoststr[200]; snprintf(hoststr, sizeof(hoststr), "http://%s:%d", host, port); diff --git a/capture/plugins/wiseService/package.json b/capture/plugins/wiseService/package.json index 1ca5e51ede..93481d5139 100644 --- a/capture/plugins/wiseService/package.json +++ b/capture/plugins/wiseService/package.json @@ -4,23 +4,9 @@ "private": true, "license": "Apache-2.0", "dependencies": { - "async": "^2.5.0", - "bson": "^0.5.5", - "connect-timeout": "^1.7.0", - "console-stamp": "^0.2.2", - "csv": "^1.1.0", - "elasticsearch": "^13.3.0", - "express": "^4.16.1", - "glob": "^7.1.0", "hashtable": "^2.0.2", - "iniparser": "http://github.com/awick/node-iniparser/tarball/master", "iptrie": "http://github.com/postwait/node-iptrie/tarball/master", "lru-cache": "^4.0.1", - "morgan": "^1.9.0", - "native-dns": "^0.7.0", - "redis": "^2.6.2", - "request": "^2.75.0", - "sqlite3": "^3.1.4", - "unzip": "^0.1.11" + "native-dns": "^0.7.0" } } diff --git a/capture/plugins/wiseService/simpleSource.js b/capture/plugins/wiseService/simpleSource.js index 8f6f6c88af..86795940d4 100644 --- a/capture/plugins/wiseService/simpleSource.js +++ b/capture/plugins/wiseService/simpleSource.js @@ -105,6 +105,9 @@ SimpleSource.prototype.initSimple = function() { case "ja3": this.getJa3 = this.sendResult; break; + case "sha256": + this.getSha256 = this.sendResult; + break; default: console.log(this.section, "- ERROR not loading since unknown type specified in config file", this.type); return false; diff --git a/capture/plugins/wiseService/source.alienvault.js b/capture/plugins/wiseService/source.alienvault.js index 2479fbaf0c..c0be952004 100644 --- a/capture/plugins/wiseService/source.alienvault.js +++ b/capture/plugins/wiseService/source.alienvault.js @@ -37,15 +37,15 @@ function AlienVaultSource (api, section) { this.api.addSource("alienvault", this); this.idField = this.api.addField("field:alienvault.id;db:alienvault.id;kind:integer;friendly:Id;help:Alien Vault ID;count:true"); - this.reliabilityField = this.api.addField("field:alienvault.reliability;db:alienvault.reliability;kind:integer;friendly:Reliability;help:Alient Vault Reliability;count:true"); - this.threatlevelField = this.api.addField("field:alienvault.threat-level;db:alienvault.threatlevel;kind:integer;friendly:Threat Level;help:Alient Vault Threat Level;count:true"); - this.activityField = this.api.addField("field:alienvault.activity;db:alienvault.activity-term;kind:termfield;friendly:Activity;help:Alient Vault Activity;count:true"); + this.reliabilityField = this.api.addField("field:alienvault.reliability;db:alienvault.reliability;kind:integer;friendly:Reliability;help:Alien Vault Reliability;count:true"); + this.threatlevelField = this.api.addField("field:alienvault.threat-level;db:alienvault.threatlevel;kind:integer;friendly:Threat Level;help:Alien Vault Threat Level;count:true"); + this.activityField = this.api.addField("field:alienvault.activity;db:alienvault.activity;kind:termfield;friendly:Activity;help:Alien Vault Activity;count:true"); this.api.addView("alienvault", "if (session.alienvault)\n" + " div.sessionDetailMeta.bold AlienVault\n" + " dl.sessionDetailMeta\n" + - " +arrayList(session.alienvault, 'activity-term', 'Activity', 'alienvault.activity')\n" + + " +arrayList(session.alienvault, 'activity', 'Activity', 'alienvault.activity')\n" + " +arrayList(session.alienvault, 'threatlevel', 'Threat Level', 'alienvault.threat-level')\n" + " +arrayList(session.alienvault, 'reliability', 'Reliability', 'alienvault.reliability')\n" + " +arrayList(session.alienvault, 'id', 'Id', 'alienvault.id')\n" diff --git a/capture/plugins/wiseService/source.emergingthreats.js b/capture/plugins/wiseService/source.emergingthreats.js index cb62ab29ce..a3d75a514b 100644 --- a/capture/plugins/wiseService/source.emergingthreats.js +++ b/capture/plugins/wiseService/source.emergingthreats.js @@ -42,13 +42,13 @@ function EmergingThreatsSource (api, section) { this.api.addSource("emergingthreats", this); this.scoreField = this.api.addField("field:emergingthreats.score;db:et.score;kind:integer;friendly:Score;help:Emerging Threats Score;count:true"); - this.categoryField = this.api.addField("field:emergingthreats.category;db:et.category-term;kind:termfield;friendly:Category;help:Emerging Threats Category;count:true"); + this.categoryField = this.api.addField("field:emergingthreats.category;db:et.category;kind:termfield;friendly:Category;help:Emerging Threats Category;count:true"); this.api.addView("emergingthreats", "if (session.et)\n" + " div.sessionDetailMeta.bold Emerging Threats\n" + " dl.sessionDetailMeta\n" + - " +arrayList(session.et, 'category-term', 'Category', 'emergingthreats.category')\n" + + " +arrayList(session.et, 'category', 'Category', 'emergingthreats.category')\n" + " +arrayList(session.et, 'score', 'Score', 'emergingthreats.score')\n" ); diff --git a/capture/plugins/wiseService/source.opendns.js b/capture/plugins/wiseService/source.opendns.js index 00bc9d333a..8d762c6e89 100644 --- a/capture/plugins/wiseService/source.opendns.js +++ b/capture/plugins/wiseService/source.opendns.js @@ -40,17 +40,17 @@ function OpenDNSSource (api, section) { setInterval(this.getCategories.bind(this), 10*60*1000); setInterval(this.performQuery.bind(this), 500); - this.statusField = this.api.addField("field:opendns.domain.status;db:opendns.dmstatus-term;kind:lotermfield;friendly:Status;help:OpenDNS domain security status;count:true"); - this.scField = this.api.addField("field:opendns.domain.security;db:opendns.dmscat-term;kind:termfield;friendly:Security;help:OpenDNS domain security category;count:true"); - this.ccField = this.api.addField("field:opendns.domain.content;db:opendns.dmccat-term;kind:termfield;friendly:Security;help:OpenDNS domain content category;count:true"); + this.statusField = this.api.addField("field:opendns.domain.status;db:opendns.statusdmstatus;kind:lotermfield;friendly:Status;help:OpenDNS domain security status;count:true"); + this.scField = this.api.addField("field:opendns.domain.security;db:opendns.securityCategory;kind:termfield;friendly:Security;help:OpenDNS domain security category;count:true"); + this.ccField = this.api.addField("field:opendns.domain.content;db:opendns.contentCategory;kind:termfield;friendly:Security;help:OpenDNS domain content category;count:true"); this.api.addView("opendns", "if (session.opendns)\n" + " div.sessionDetailMeta.bold OpenDNS\n" + " dl.sessionDetailMeta\n" + - " +arrayList(session.opendns, 'dmstatus-term', 'Status', 'opendns.domain.status')\n" + - " +arrayList(session.opendns, 'dmscat-term', 'Security Cat', 'opendns.domain.security')\n" + - " +arrayList(session.opendns, 'dmccat-term', 'Content Cat', 'opendns.domain.content')\n" + " +arrayList(session.opendns, 'status', 'Status', 'opendns.domain.status')\n" + + " +arrayList(session.opendns, 'securityCategory', 'Security Cat', 'opendns.domain.security')\n" + + " +arrayList(session.opendns, 'contentCategory', 'Content Cat', 'opendns.domain.content')\n" ); this.api.addRightClick("opendnsip", {name:"OpenDNS", url:"https://sgraph.opendns.com/ip-view/%TEXT%", category:"ip"}); diff --git a/capture/plugins/wiseService/source.passivetotal.js b/capture/plugins/wiseService/source.passivetotal.js index 678e40d6cd..76e22acff8 100644 --- a/capture/plugins/wiseService/source.passivetotal.js +++ b/capture/plugins/wiseService/source.passivetotal.js @@ -48,9 +48,9 @@ function PassiveTotalSource (api, section) { "if (session.passivetotal)\n" + " div.sessionDetailMeta.bold PassiveTotal\n" + " dl.sessionDetailMeta\n" + - " +arrayList(session.passivetotal, 'tags-term', 'Tags', 'passivetotal.tags')\n"; + " +arrayList(session.passivetotal, 'tags', 'Tags', 'passivetotal.tags')\n"; - this.tagsField = this.api.addField("field:passivetotal.tags;db:passivetotal.tags-term;kind:termfield;friendly:Tags;help:PassiveTotal Tags;count:true"); + this.tagsField = this.api.addField("field:passivetotal.tags;db:passivetotal.tags;kind:termfield;friendly:Tags;help:PassiveTotal Tags;count:true"); this.api.addView("passivetotal", str); } diff --git a/capture/plugins/wiseService/source.threatq.js b/capture/plugins/wiseService/source.threatq.js index 92db798400..9354888859 100644 --- a/capture/plugins/wiseService/source.threatq.js +++ b/capture/plugins/wiseService/source.threatq.js @@ -48,18 +48,18 @@ function ThreatQSource (api, section) { this.idField = this.api.addField("field:threatq.id;db:threatq.id;kind:integer;friendly:Id;help:ThreatQ Reference ID;shortcut:0;count:true"); - this.typeField = this.api.addField("field:threatq.type;db:threatq.type-term;kind:lotermfield;friendly:Type;help:Indicator Type;shortcut:1;count:true"); - this.sourceField = this.api.addField("field:threatq.source;db:threatq.source-term;kind:lotermfield;friendly:Source;help:Indicator Release Source;shortcut:2;count:true"); - this.campaignField = this.api.addField("field:threatq.campaign;db:threatq.campaign-term;kind:lotermfield;friendly:Campaign;help:Campaign Attribution;shortcut:3;count:true"); + this.typeField = this.api.addField("field:threatq.type;db:threatq.type;kind:lotermfield;friendly:Type;help:Indicator Type;shortcut:1;count:true"); + this.sourceField = this.api.addField("field:threatq.source;db:threatq.source;kind:lotermfield;friendly:Source;help:Indicator Release Source;shortcut:2;count:true"); + this.campaignField = this.api.addField("field:threatq.campaign;db:threatq.campaign;kind:lotermfield;friendly:Campaign;help:Campaign Attribution;shortcut:3;count:true"); this.api.addView("threatq", "if (session.threatq)\n" + " div.sessionDetailMeta.bold ThreatQ\n" + " dl.sessionDetailMeta\n" + " +arrayList(session.threatq, 'id', 'Id', 'threatq.id')\n" + - " +arrayList(session.threatq, 'type-term', 'Type', 'threatq.type')\n" + - " +arrayList(session.threatq, 'source-term', 'Source', 'threatq.source')\n" + - " +arrayList(session.threatq, 'campaign-term', 'Campaign', 'threatq.campaign')\n" + " +arrayList(session.threatq, 'type', 'Type', 'threatq.type')\n" + + " +arrayList(session.threatq, 'source', 'Source', 'threatq.source')\n" + + " +arrayList(session.threatq, 'campaign', 'Campaign', 'threatq.campaign')\n" ); this.api.addRightClick("threatqip", {name:"ThreatQ", url:`https://${this.host}/search.php?search=%TEXT%`, category:"ip"}); diff --git a/capture/plugins/wiseService/source.threatstream.js b/capture/plugins/wiseService/source.threatstream.js index c5acaef13c..336b55fd66 100644 --- a/capture/plugins/wiseService/source.threatstream.js +++ b/capture/plugins/wiseService/source.threatstream.js @@ -102,25 +102,25 @@ function ThreatStreamSource (api, section) { process.exit(0); } - this.severityField = this.api.addField("field:threatstream.severity;db:threatstream.severity-term;kind:lotermfield;friendly:Severity;help:Threatstream Severity;count:true"); + this.severityField = this.api.addField("field:threatstream.severity;db:threatstream.severity;kind:lotermfield;friendly:Severity;help:Threatstream Severity;count:true"); this.confidenceField = this.api.addField("field:threatstream.confidence;db:threatstream.confidence;kind:integer;friendly:Confidence;help:Threatstream Confidence;count:true"); this.idField = this.api.addField("field:threatstream.id;db:threatstream.id;kind:integer;friendly:Id;help:Threatstream Id;count:true"); - this.typeField = this.api.addField("field:threatstream.type;db:threatstream.type-term;kind:lotermfield;friendly:Type;help:Threatstream Type;count:true"); - this.maltypeField = this.api.addField("field:threatstream.maltype;db:threatstream.maltype-term;kind:lotermfield;friendly:Malware Type;help:Threatstream Malware Type;count:true"); - this.sourceField = this.api.addField("field:threatstream.source;db:threatstream.source-term;kind:termfield;friendly:Source;help:Threatstream Source;count:true"); + this.typeField = this.api.addField("field:threatstream.type;db:threatstream.type;kind:lotermfield;friendly:Type;help:Threatstream Type;count:true"); + this.maltypeField = this.api.addField("field:threatstream.maltype;db:threatstream.maltype;kind:lotermfield;friendly:Malware Type;help:Threatstream Malware Type;count:true"); + this.sourceField = this.api.addField("field:threatstream.source;db:threatstream.source;kind:termfield;friendly:Source;help:Threatstream Source;count:true"); this.importIdField = this.api.addField("field:threatstream.importId;db:threatstream.importId;kind:integer;friendly:Import Id;help:Threatstream Import Id;count:true"); this.api.addView("threatstream", "if (session.threatstream)\n" + " div.sessionDetailMeta.bold Threatstream\n" + " dl.sessionDetailMeta\n" + - " +arrayList(session.threatstream, 'severity-term', 'Severity', 'threatstream.severity')\n" + + " +arrayList(session.threatstream, 'severity', 'Severity', 'threatstream.severity')\n" + " +arrayList(session.threatstream, 'confidence', 'Confidence', 'threatstream.confidence')\n" + " +arrayList(session.threatstream, 'id', 'Id', 'threatstream.id')\n" + " +arrayList(session.threatstream, 'importId', 'Import Id', 'threatstream.importId')\n" + - " +arrayList(session.threatstream, 'type-term', 'Type', 'threatstream.type')\n" + - " +arrayList(session.threatstream, 'maltype-term', 'Malware Type', 'threatstream.maltype')\n" + - " +arrayList(session.threatstream, 'source-term', 'Source', 'threatstream.source')\n" + " +arrayList(session.threatstream, 'type', 'Type', 'threatstream.type')\n" + + " +arrayList(session.threatstream, 'maltype', 'Malware Type', 'threatstream.maltype')\n" + + " +arrayList(session.threatstream, 'source', 'Source', 'threatstream.source')\n" ); this.api.addRightClick("threatstreamip", {name:"Threatstream", url:"https://ui.threatstream.com/detail/ip/%TEXT%", category:"ip"}); diff --git a/capture/plugins/wiseService/source.virustotal.js b/capture/plugins/wiseService/source.virustotal.js index 15cfce107b..0d68d7137a 100644 --- a/capture/plugins/wiseService/source.virustotal.js +++ b/capture/plugins/wiseService/source.virustotal.js @@ -56,17 +56,17 @@ function VirusTotalSource (api, section) { " div.sessionDetailMeta.bold VirusTotal\n" + " dl.sessionDetailMeta\n" + " +arrayList(session.virustotal, 'hits', 'Hits', 'virustotal.hits')\n" + - " +arrayList(session.virustotal, 'links-term', 'Links', 'virustotal.links')\n"; + " +arrayList(session.virustotal, 'links', 'Links', 'virustotal.links')\n"; for(var i = 0; i < this.dataSources.length; i++) { var uc = this.dataSources[i]; var lc = this.dataSourcesLC[i]; - this.dataFields[i] = this.api.addField(`field:virustotal.${lc};db:virustotal.${lc}-term;kind:lotermfield;friendly:${uc};help:VirusTotal ${uc} Status;count:true`); - str += " +arrayList(session.virustotal, '" + lc + "-term', '" + uc + "', 'virustotal." + lc + "')\n"; + this.dataFields[i] = this.api.addField(`field:virustotal.${lc};db:virustotal.${lc};kind:lotermfield;friendly:${uc};help:VirusTotal ${uc} Status;count:true`); + str += " +arrayList(session.virustotal, '" + lc + "', '" + uc + "', 'virustotal." + lc + "')\n"; } this.hitsField = this.api.addField("field:virustotal.hits;db:virustotal.hits;kind:integer;friendly:Hits;help:VirusTotal Hits;count:true"); - this.linksField = this.api.addField("field:virustotal.links;db:virustotal.links-term;kind:termfield;friendly:Link;help:VirusTotal Link;count:true"); + this.linksField = this.api.addField("field:virustotal.links;db:virustotal.links;kind:termfield;friendly:Link;help:VirusTotal Link;count:true"); this.api.addRightClick("virustotallinks", {name:"Open", url:"%TEXT%", fields:"virustotal.links"}); diff --git a/capture/plugins/wiseService/wiseCache.js b/capture/plugins/wiseService/wiseCache.js index 13dfcb994f..cd04cd02b7 100644 --- a/capture/plugins/wiseService/wiseCache.js +++ b/capture/plugins/wiseService/wiseCache.js @@ -30,7 +30,7 @@ var LRU = require('lru-cache') function WISEMemoryCache (options) { var cacheSize = +options.cacheSize || 100000; - this.cache = [LRU({max: cacheSize}), LRU({max: cacheSize}), LRU({max: cacheSize}), LRU({max: cacheSize}), LRU({max: cacheSize}), LRU({max: cacheSize}), LRU({max: cacheSize})]; + this.cache = [LRU({max: cacheSize}), LRU({max: cacheSize}), LRU({max: cacheSize}), LRU({max: cacheSize}), LRU({max: cacheSize}), LRU({max: cacheSize}), LRU({max: cacheSize}), LRU({max: cacheSize})]; } ////////////////////////////////////////////////////////////////////////////////// @@ -53,7 +53,7 @@ exports.WISEMemoryCache = WISEMemoryCache; function WISERedisCache (options) { options = options || {}; var cacheSize = +options.cacheSize || 10000; - this.cache = [LRU({max: cacheSize}), LRU({max: cacheSize}), LRU({max: cacheSize}), LRU({max: cacheSize}), LRU({max: cacheSize}), LRU({max: cacheSize}), LRU({max: cacheSize})]; + this.cache = [LRU({max: cacheSize}), LRU({max: cacheSize}), LRU({max: cacheSize}), LRU({max: cacheSize}), LRU({max: cacheSize}), LRU({max: cacheSize}), LRU({max: cacheSize}), LRU({max: cacheSize})]; options.return_buffers = true; // force buffers on for the bson decoding to work this.client = redis.createClient(options); diff --git a/capture/plugins/wiseService/wiseService.js b/capture/plugins/wiseService/wiseService.js index f9ecbfe53b..3dedc81efd 100644 --- a/capture/plugins/wiseService/wiseService.js +++ b/capture/plugins/wiseService/wiseService.js @@ -62,6 +62,9 @@ var internals = { ja3: { sources: [] }, + sha256: { + sources: [] + }, sources: [], requestStats: [0,0,0,0,0,0,0], foundStats: [0,0,0,0,0,0,0], @@ -186,7 +189,7 @@ internals.sourceApi = { }, debug: internals.debug, addSource: function(section, src) { - src.srcInProgress = {ip: {}, domain: {}, email: {}, md5: {}, url: {}, tuple: {}, ja3: {}}; + src.srcInProgress = {ip: {}, domain: {}, email: {}, md5: {}, url: {}, tuple: {}, ja3: {}, sha256: {}}; internals.sources[section] = src; if (src.getIp) { internals.ip.sources.push(src); @@ -209,6 +212,9 @@ internals.sourceApi = { if (src.getJa3) { internals.ja3.sources.push(src); } + if (src.getSha256) { + internals.sha256.sources.push(src); + } }, app: app }; @@ -236,9 +242,9 @@ app.get("/rightClicks", function(req, res) { res.send(internals.rightClicks); }); ////////////////////////////////////////////////////////////////////////////////// -internals.type2Func = ["getIp", "getDomain", "getMd5", "getEmail", "getURL", "getTuple", "getJa3"]; -internals.type2Name = ["ip", "domain", "md5", "email", "url", "tuple", "ja3"]; -internals.name2Type = {ip:0, 0:0, domain:1, 1:1, md5:2, 2:2, email:3, 3:3, url:4, 4:4, tuple:5, 5:5, ja3:6, 6:6}; +internals.type2Func = ["getIp", "getDomain", "getMd5", "getEmail", "getURL", "getTuple", "getJa3", "getSha256"]; +internals.type2Name = ["ip", "domain", "md5", "email", "url", "tuple", "ja3", "sha256"]; +internals.name2Type = {ip:0, 0:0, domain:1, 1:1, md5:2, 2:2, email:3, 3:3, url:4, 4:4, tuple:5, 5:5, ja3:6, 6:6, sha256:7, 7:7}; ////////////////////////////////////////////////////////////////////////////////// function processQuery(req, query, cb) { @@ -246,7 +252,7 @@ function processQuery(req, query, cb) { var funcName = internals.type2Func[query.type]; var typeInfo = internals[typeName]; - if (query.type === 2) { + if (query.type === 2 || query.type == 7) { var parts = query.value.split(";"); query.value = parts[0]; query.contentType = parts[1]; @@ -512,16 +518,16 @@ if (getConfig("wiseService", "regressionTests")) { ////////////////////////////////////////////////////////////////////////////////// function printStats() { - console.log(sprintf("REQUESTS: domain: %7d ip: %7d email: %7d md5: %7d url: %7d tuple: %7d ja3: %7d", - internals.requestStats[1], internals.requestStats[0], internals.requestStats[3], internals.requestStats[2], internals.requestStats[4], internals.requestStats[5], internals.requestStats[6])); - console.log(sprintf("FOUND: domain: %7d ip: %7d email: %7d md5: %7d url: %7d tuple: %7d ja3: %7d", - internals.foundStats[1], internals.foundStats[0], internals.foundStats[3], internals.foundStats[2], internals.foundStats[4], internals.foundStats[5], internals.foundStats[6])); - console.log(sprintf("CACHE HIT: domain: %7d ip: %7d email: %7d md5: %7d url: %7d tuple: %7d ja3: %7d", - internals.cacheHitStats[1], internals.cacheHitStats[0], internals.cacheHitStats[3], internals.cacheHitStats[2], internals.cacheHitStats[4], internals.cacheHitStats[5], internals.cacheHitStats[6])); - console.log(sprintf("CACHE SRC HIT: domain: %7d ip: %7d email: %7d md5: %7d url: %7d tuple: %7d ja3: %7d", - internals.cacheSrcHitStats[1], internals.cacheSrcHitStats[0], internals.cacheSrcHitStats[3], internals.cacheSrcHitStats[2], internals.cacheSrcHitStats[4], internals.cacheSrcHitStats[5], internals.cacheSrcHitStats[6])); - console.log(sprintf("CACHE SRC REFRESH: domain: %7d ip: %7d email: %7d md5: %7d url: %7d tuple: %7d ja3: %7d", - internals.cacheSrcRefreshStats[1], internals.cacheSrcRefreshStats[0], internals.cacheSrcRefreshStats[3], internals.cacheSrcRefreshStats[2], internals.cacheSrcRefreshStats[4], internals.cacheSrcRefreshStats[5], internals.cacheSrcRefreshStats[6])); + console.log(sprintf("REQUESTS: domain: %7d ip: %7d email: %7d md5: %7d url: %7d tuple: %7d ja3: %7d sha256: %7d", + internals.requestStats[1], internals.requestStats[0], internals.requestStats[3], internals.requestStats[2], internals.requestStats[4], internals.requestStats[5], internals.requestStats[6], internals.requestStats[7])); + console.log(sprintf("FOUND: domain: %7d ip: %7d email: %7d md5: %7d url: %7d tuple: %7d ja3: %7d sha256: %7d", + internals.foundStats[1], internals.foundStats[0], internals.foundStats[3], internals.foundStats[2], internals.foundStats[4], internals.foundStats[5], internals.foundStats[6], internals.foundStats[7])); + console.log(sprintf("CACHE HIT: domain: %7d ip: %7d email: %7d md5: %7d url: %7d tuple: %7d ja3: %7d sha256: %7d", + internals.cacheHitStats[1], internals.cacheHitStats[0], internals.cacheHitStats[3], internals.cacheHitStats[2], internals.cacheHitStats[4], internals.cacheHitStats[5], internals.cacheHitStats[6], internals.cacheHitStats[7])); + console.log(sprintf("CACHE SRC HIT: domain: %7d ip: %7d email: %7d md5: %7d url: %7d tuple: %7d ja3: %7d sha256: %7d", + internals.cacheSrcHitStats[1], internals.cacheSrcHitStats[0], internals.cacheSrcHitStats[3], internals.cacheSrcHitStats[2], internals.cacheSrcHitStats[4], internals.cacheSrcHitStats[5], internals.cacheSrcHitStats[6], internals.cacheSrcHitStats[7])); + console.log(sprintf("CACHE SRC REFRESH: domain: %7d ip: %7d email: %7d md5: %7d url: %7d tuple: %7d ja3: %7d sha256: %7d", + internals.cacheSrcRefreshStats[1], internals.cacheSrcRefreshStats[0], internals.cacheSrcRefreshStats[3], internals.cacheSrcRefreshStats[2], internals.cacheSrcRefreshStats[4], internals.cacheSrcRefreshStats[5], internals.cacheSrcRefreshStats[6], internals.cacheSrcRefreshStats[7])); for (var section in internals.sources) { let src = internals.sources[section]; @@ -585,6 +591,7 @@ internals.tuple.global_allowed = function(value) { return true; }; internals.ja3.global_allowed = function(value) {return true;}; +internals.sha256.global_allowed = function(value) {return true;}; internals.ip.source_allowed = function(src, value) { if (src.excludeIPs.find(value)) { @@ -644,6 +651,7 @@ internals.tuple.source_allowed = function(src, value) { return true; }; internals.ja3.source_allowed = function(src, value) {return true;}; +internals.sha256.source_allowed = function(src, value) {return true;}; ////////////////////////////////////////////////////////////////////////////////// function loadExcludes() { ["excludeDomains", "excludeEmails", "excludeURLs", "excludeTuples"].forEach((type) => { diff --git a/capture/plugins/wiseService/wiseSource.js b/capture/plugins/wiseService/wiseSource.js index 47dbed413c..f95b0b0dbc 100644 --- a/capture/plugins/wiseService/wiseSource.js +++ b/capture/plugins/wiseService/wiseSource.js @@ -341,7 +341,7 @@ WISESource.prototype.formatSetting = function () { return true; } ////////////////////////////////////////////////////////////////////////////////// -var typeName2Func = {ip: "getIp", domain: "getDomain", md5: "getMd5", email: "getEmail", url: "getURL", tuple: "getTuple", ja3: "getJa3"}; +var typeName2Func = {ip: "getIp", domain: "getDomain", md5: "getMd5", email: "getEmail", url: "getURL", tuple: "getTuple", ja3: "getJa3", sha256: "getSha256"}; WISESource.prototype.typeSetting = function () { this.type = this.api.getConfig(this.section, "type"); diff --git a/capture/plugins/writer-s3.c b/capture/plugins/writer-s3.c index 209d56a1d7..fea95e9a1c 100644 --- a/capture/plugins/writer-s3.c +++ b/capture/plugins/writer-s3.c @@ -48,30 +48,30 @@ typedef struct writer_s3_file { char *partNumbers[2001]; } SavepcapS3File_t; -static char *outputBuffer; -static uint32_t outputPos; -static uint32_t outputId; -static uint64_t outputFilePos = 0; +LOCAL char *outputBuffer; +LOCAL uint32_t outputPos; +LOCAL uint32_t outputId; +LOCAL uint64_t outputFilePos = 0; SavepcapS3File_t *currentFile; -static SavepcapS3File_t fileQ; +LOCAL SavepcapS3File_t fileQ; -static void * s3Server = 0; -static char *s3Region; -static char s3Host[100]; -static char *s3Bucket; -static char *s3AccessKeyId; -static char *s3SecretAccessKey; -static char s3Compress; -static uint32_t s3MaxConns; -static uint32_t s3MaxRequests; +LOCAL void * s3Server = 0; +LOCAL char *s3Region; +LOCAL char s3Host[100]; +LOCAL char *s3Bucket; +LOCAL char *s3AccessKeyId; +LOCAL char *s3SecretAccessKey; +LOCAL char s3Compress; +LOCAL uint32_t s3MaxConns; +LOCAL uint32_t s3MaxRequests; -static int inprogress; +LOCAL int inprogress; void writer_s3_request(char *method, char *path, char *qs, unsigned char *data, int len, gboolean reduce, MolochHttpResponse_cb cb, gpointer uw); -static MOLOCH_LOCK_DEFINE(output); +LOCAL MOLOCH_LOCK_DEFINE(output); /******************************************************************************/ uint32_t writer_s3_queue_length() { diff --git a/capture/plugins/writer-s3/package.json b/capture/plugins/writer-s3/package.json index d375a2e493..c2c212d431 100644 --- a/capture/plugins/writer-s3/package.json +++ b/capture/plugins/writer-s3/package.json @@ -4,7 +4,6 @@ "description": "", "main": "index.js", "dependencies": { - "async": "", - "aws-sdk": "" + "aws-sdk": "^2.188.0" } } diff --git a/capture/reader-libpcap-file.c b/capture/reader-libpcap-file.c index 720cf42c12..d4b41231e3 100644 --- a/capture/reader-libpcap-file.c +++ b/capture/reader-libpcap-file.c @@ -36,7 +36,7 @@ LOCAL char offlinePcapFilename[PATH_MAX+1]; LOCAL char *offlinePcapName; LOCAL int pktsToRead; -void reader_libpcapfile_opened(); +LOCAL void reader_libpcapfile_opened(); LOCAL MolochPacketBatch_t batch; @@ -45,9 +45,9 @@ LOCAL MolochPacketBatch_t batch; LOCAL int monitorFd; LOCAL GHashTable *wdHashTable; -void reader_libpcapfile_monitor_dir(char *dirname); +LOCAL void reader_libpcapfile_monitor_dir(char *dirname); -void reader_libpcapfile_monitor_do(struct inotify_event *event) +LOCAL void reader_libpcapfile_monitor_do(struct inotify_event *event) { gchar *dirname = g_hash_table_lookup(wdHashTable, (void *)(long)event->wd); gchar *fullfilename = g_build_filename (dirname, event->name, NULL); @@ -80,7 +80,7 @@ void reader_libpcapfile_monitor_do(struct inotify_event *event) return; } /******************************************************************************/ -gboolean reader_libpcapfile_monitor_read() +LOCAL gboolean reader_libpcapfile_monitor_read() { char buf[20 * (sizeof(struct inotify_event) + NAME_MAX + 1)] __attribute__ ((aligned(8))); struct inotify_event *event; @@ -101,7 +101,7 @@ gboolean reader_libpcapfile_monitor_read() return TRUE; } /******************************************************************************/ -void reader_libpcapfile_monitor_dir(char *dirname) +LOCAL void reader_libpcapfile_monitor_dir(char *dirname) { if (config.debug) LOG("Monitoring %s", dirname); @@ -145,7 +145,7 @@ void reader_libpcapfile_monitor_dir(char *dirname) g_dir_close(dir); } /******************************************************************************/ -void reader_libpcapfile_init_monitor() +LOCAL void reader_libpcapfile_init_monitor() { int dir; monitorFd = inotify_init1(IN_NONBLOCK); @@ -161,13 +161,13 @@ void reader_libpcapfile_init_monitor() } } #else -void reader_libpcapfile_init_monitor() +LOCAL void reader_libpcapfile_init_monitor() { LOGEXIT("Monitoring not supporting on this OS"); } #endif /******************************************************************************/ -int reader_libpcapfile_next() +LOCAL int reader_libpcapfile_next() { char errbuf[1024]; gchar *fullfilename; @@ -331,7 +331,7 @@ int reader_libpcapfile_next() return 0; } /******************************************************************************/ -gboolean reader_libpcapfile_monitor_gfunc (gpointer UNUSED(user_data)) +LOCAL gboolean reader_libpcapfile_monitor_gfunc (gpointer UNUSED(user_data)) { if (DLL_COUNT(s_, &monitorQ) == 0) return TRUE; @@ -343,7 +343,7 @@ gboolean reader_libpcapfile_monitor_gfunc (gpointer UNUSED(user_data)) return TRUE; } /******************************************************************************/ -int reader_libpcapfile_stats(MolochReaderStats_t *stats) +LOCAL int reader_libpcapfile_stats(MolochReaderStats_t *stats) { struct pcap_stat ps; if (!pcap) { @@ -360,7 +360,7 @@ int reader_libpcapfile_stats(MolochReaderStats_t *stats) return 0; } /******************************************************************************/ -void reader_libpcapfile_pcap_cb(u_char *UNUSED(user), const struct pcap_pkthdr *h, const u_char *bytes) +LOCAL void reader_libpcapfile_pcap_cb(u_char *UNUSED(user), const struct pcap_pkthdr *h, const u_char *bytes) { MolochPacket_t *packet = MOLOCH_TYPE_ALLOC0(MolochPacket_t); @@ -382,7 +382,7 @@ void reader_libpcapfile_pcap_cb(u_char *UNUSED(user), const struct pcap_pkthdr * moloch_packet_batch(&batch, packet); } /******************************************************************************/ -gboolean reader_libpcapfile_read() +LOCAL gboolean reader_libpcapfile_read() { // pause reading if too many waiting disk operations if (moloch_writer_queue_length() > 10) { @@ -400,7 +400,6 @@ gboolean reader_libpcapfile_read() } int r; - moloch_packet_batch_init(&batch); if (pktsToRead > 0) { r = pcap_dispatch(pcap, MIN(pktsToRead, 5000), reader_libpcapfile_pcap_cb, NULL); @@ -438,7 +437,7 @@ gboolean reader_libpcapfile_read() return TRUE; } /******************************************************************************/ -void reader_libpcapfile_opened() +LOCAL void reader_libpcapfile_opened() { int dlt_to_linktype(int dlt); @@ -472,7 +471,7 @@ void reader_libpcapfile_opened() } /******************************************************************************/ -void reader_libpcapfile_start() { +LOCAL void reader_libpcapfile_start() { reader_libpcapfile_next(); if (!pcap) { if (config.pcapMonitor) { @@ -492,4 +491,5 @@ void reader_libpcapfile_init(char *UNUSED(name)) reader_libpcapfile_init_monitor(); DLL_INIT(s_, &monitorQ); + moloch_packet_batch_init(&batch); } diff --git a/capture/reader-libpcap.c b/capture/reader-libpcap.c index 221f212053..db95729f07 100644 --- a/capture/reader-libpcap.c +++ b/capture/reader-libpcap.c @@ -24,7 +24,7 @@ extern MolochConfig_t config; -static pcap_t *pcaps[MAX_INTERFACES]; +LOCAL pcap_t *pcaps[MAX_INTERFACES]; /******************************************************************************/ int reader_libpcap_stats(MolochReaderStats_t *stats) @@ -63,10 +63,11 @@ void reader_libpcap_pcap_cb(u_char *batch, const struct pcap_pkthdr *h, const u_ moloch_packet_batch((MolochPacketBatch_t *)batch, packet); } /******************************************************************************/ -static void *reader_libpcap_thread(gpointer pcapv) +LOCAL void *reader_libpcap_thread(gpointer pcapv) { pcap_t *pcap = pcapv; - LOG("THREAD %p", (gpointer)pthread_self()); + if (config.debug) + LOG("THREAD %p", (gpointer)pthread_self()); MolochPacketBatch_t batch; moloch_packet_batch_init(&batch); @@ -112,7 +113,7 @@ void reader_libpcap_start() { } } /******************************************************************************/ -void reader_libpcap_stop() +void reader_libpcap_stop() { int i; for (i = 0; i < MAX_INTERFACES && config.interface[i]; i++) { diff --git a/capture/reader-tpacketv3.c b/capture/reader-tpacketv3.c index 4b6032590d..09e5bd2da9 100644 --- a/capture/reader-tpacketv3.c +++ b/capture/reader-tpacketv3.c @@ -90,7 +90,7 @@ int reader_tpacketv3_stats(MolochReaderStats_t *stats) return 0; } /******************************************************************************/ -static void *reader_tpacketv3_thread(gpointer infov) +LOCAL void *reader_tpacketv3_thread(gpointer infov) { long info = (long)infov; struct pollfd pfd; @@ -171,7 +171,7 @@ void reader_tpacketv3_start() { char name[100]; for (i = 0; i < MAX_INTERFACES && config.interface[i]; i++) { for (t = 0; t < numThreads; t++) { - snprintf(name, sizeof(name), "moloch-pcap%d-%d", i, t); + snprintf(name, sizeof(name), "moloch-af3%d-%d", i, t); g_thread_new(name, &reader_tpacketv3_thread, (gpointer)(long)i); } } diff --git a/capture/readers.c b/capture/readers.c index b8a32e46f7..6e7fda12cb 100644 --- a/capture/readers.c +++ b/capture/readers.c @@ -20,7 +20,7 @@ extern MolochConfig_t config; -static MolochStringHashStd_t readersHash; +LOCAL MolochStringHashStd_t readersHash; void reader_libpcapfile_init(char*); void reader_libpcap_init(char*); diff --git a/capture/rules.c b/capture/rules.c index 8897fc567e..103e89a68e 100644 --- a/capture/rules.c +++ b/capture/rules.c @@ -52,11 +52,13 @@ typedef struct { } YamlNode_t; typedef struct { - char *fields; + uint8_t *fields; char *filename; char *bpf; struct bpf_program bpfp; GHashTable *hash[MOLOCH_FIELDS_MAX]; + patricia_tree_t *tree4[MOLOCH_FIELDS_MAX]; + patricia_tree_t *tree6[MOLOCH_FIELDS_MAX]; MolochFieldOps_t ops; int fieldsLen; int saveFlags; @@ -66,9 +68,11 @@ typedef struct { // Has all possible values to array of rules LOCAL GHashTable *fieldsHash[MOLOCH_FIELDS_MAX]; +LOCAL patricia_tree_t *fieldsTree4[MOLOCH_FIELDS_MAX]; +LOCAL patricia_tree_t *fieldsTree6[MOLOCH_FIELDS_MAX]; LOCAL int rulesLen[MOLOCH_RULE_TYPE_NUM]; -LOCAL MolochRule_t *rules[MOLOCH_RULE_TYPE_NUM][MOLOCH_RULES_MAX]; +LOCAL MolochRule_t *rules[MOLOCH_RULE_TYPE_NUM][MOLOCH_RULES_MAX+1]; LOCAL pcap_t *deadPcap; extern MolochPcapFileHdr_t pcapFileHeader; @@ -84,7 +88,7 @@ void moloch_rules_free_node(YamlNode_t *node) MOLOCH_TYPE_FREE(YamlNode_t, node); } /******************************************************************************/ -YamlNode_t *moloch_rules_add_node(YamlNode_t *parent, char *key, char *value) +YamlNode_t *moloch_rules_add_node(YamlNode_t *parent, char *key, char *value) { YamlNode_t *node = MOLOCH_TYPE_ALLOC(YamlNode_t); node->key = key; @@ -182,7 +186,7 @@ void moloch_rules_parse_print(YamlNode_t *node, int level) /******************************************************************************/ YamlNode_t *moloch_rules_get(YamlNode_t *node, char *path) { - + while (1) { char *colon = strchr(path, ':'); int len; @@ -228,10 +232,10 @@ GPtrArray *moloch_rules_get_values(YamlNode_t *parent, char *path) /******************************************************************************/ void moloch_rules_process_add_field(MolochRule_t *rule, int pos, char *key) { - struct in_addr in; - uint32_t n; - char *key2; - GPtrArray *rules; + uint32_t n; + char *key2; + GPtrArray *rules; + patricia_node_t *node; config.fields[pos]->ruleEnabled = 1; @@ -239,6 +243,7 @@ void moloch_rules_process_add_field(MolochRule_t *rule, int pos, char *key) case MOLOCH_FIELD_TYPE_INT: case MOLOCH_FIELD_TYPE_INT_ARRAY: case MOLOCH_FIELD_TYPE_INT_HASH: + case MOLOCH_FIELD_TYPE_INT_GHASH: if (!fieldsHash[pos]) fieldsHash[pos] = g_hash_table_new(NULL, NULL); @@ -252,27 +257,34 @@ void moloch_rules_process_add_field(MolochRule_t *rule, int pos, char *key) } g_ptr_array_add(rules, rule); break; + case MOLOCH_FIELD_TYPE_IP: case MOLOCH_FIELD_TYPE_IP_GHASH: - case MOLOCH_FIELD_TYPE_INT_GHASH: - case MOLOCH_FIELD_TYPE_IP_HASH: - if (!fieldsHash[pos]) - fieldsHash[pos] = g_hash_table_new(NULL, NULL); - - inet_aton(key, &in); - g_hash_table_add(rule->hash[pos], (void *)(long)in.s_addr); + if (!fieldsTree4[pos]) { + fieldsTree4[pos] = New_Patricia(32); + fieldsTree6[pos] = New_Patricia(128); + } - rules = g_hash_table_lookup(fieldsHash[pos], (void *)(long)in.s_addr); - if (!rules) { - rules = g_ptr_array_new(); - g_hash_table_insert(fieldsHash[pos], (void *)(long)in.s_addr, rules); + if (strchr(key, '.') != 0) { + make_and_lookup(rule->tree4[pos], key); + node = make_and_lookup(fieldsTree4[pos], key); + } else { + make_and_lookup(rule->tree6[pos], key); + node = make_and_lookup(fieldsTree6[pos], key); + } + if (node->data) { + rules = node->data; + } else { + node->data = rules = g_ptr_array_new(); } g_ptr_array_add(rules, rule); break; + case MOLOCH_FIELD_TYPE_STR: case MOLOCH_FIELD_TYPE_STR_ARRAY: case MOLOCH_FIELD_TYPE_STR_HASH: + case MOLOCH_FIELD_TYPE_STR_GHASH: if (!fieldsHash[pos]) fieldsHash[pos] = g_hash_table_new(g_str_hash, g_str_equal); @@ -378,19 +390,23 @@ void moloch_rules_process_rule(char *filename, YamlNode_t *parent) case MOLOCH_FIELD_TYPE_INT: case MOLOCH_FIELD_TYPE_INT_ARRAY: case MOLOCH_FIELD_TYPE_INT_HASH: - case MOLOCH_FIELD_TYPE_IP: - case MOLOCH_FIELD_TYPE_IP_GHASH: case MOLOCH_FIELD_TYPE_INT_GHASH: - case MOLOCH_FIELD_TYPE_IP_HASH: rule->hash[pos] = g_hash_table_new(NULL, NULL); break; + case MOLOCH_FIELD_TYPE_IP: + case MOLOCH_FIELD_TYPE_IP_GHASH: + rule->tree4[pos] = New_Patricia(32); + rule->tree6[pos] = New_Patricia(128); + break; + case MOLOCH_FIELD_TYPE_STR: case MOLOCH_FIELD_TYPE_STR_ARRAY: case MOLOCH_FIELD_TYPE_STR_HASH: + case MOLOCH_FIELD_TYPE_STR_GHASH: rule->hash[pos] = g_hash_table_new(g_str_hash, g_str_equal); - break; + case MOLOCH_FIELD_TYPE_CERTSINFO: LOGEXIT("%s: Currently don't support any certs fields", filename); } @@ -439,32 +455,43 @@ void moloch_rules_process(char *filename, YamlNode_t *parent) } } /******************************************************************************/ +/* Called at the start on main thread or each time a new file is open on single thread */ void moloch_rules_recompile() { - int t, i; + int t, r; if (deadPcap) pcap_close(deadPcap); deadPcap = pcap_open_dead(pcapFileHeader.linktype, pcapFileHeader.snaplen); + MolochRule_t *rule; for (t = 0; t < MOLOCH_RULE_TYPE_NUM; t++) { - for (i = 0; i < rulesLen[t]; i++) { - if (!rules[t][i]->bpf) + for (r = 0; (rule = rules[t][r]); r++) { + if (!rule->bpf) continue; - pcap_freecode(&rules[t][i]->bpfp); + pcap_freecode(&rule->bpfp); if (pcapFileHeader.linktype != 239) { - if (pcap_compile(deadPcap, &rules[t][i]->bpfp, rules[t][i]->bpf, 1, PCAP_NETMASK_UNKNOWN) == -1) { - LOGEXIT("ERROR - Couldn't compile filter %s: '%s' with %s", rules[t][i]->filename, rules[t][i]->bpf, pcap_geterr(deadPcap)); + if (pcap_compile(deadPcap, &rule->bpfp, rule->bpf, 1, PCAP_NETMASK_UNKNOWN) == -1) { + LOGEXIT("ERROR - Couldn't compile filter %s: '%s' with %s", rule->filename, rule->bpf, pcap_geterr(deadPcap)); } } else { - rules[t][i]->bpfp.bf_len = 0; + rule->bpfp.bf_len = 0; } } } } /******************************************************************************/ -void moloch_rules_check_rule_fields(MolochSession_t *session, MolochRule_t *rule, int skipPos) +LOCAL gboolean moloch_rules_check_ip(const MolochRule_t *rule, const int p, const struct in6_addr *ip) +{ + if (IN6_IS_ADDR_V4MAPPED(ip)) { + return patricia_search_best3 (rule->tree4[p], ((u_char *)ip->s6_addr) + 12, 32) != NULL; + } else { + return patricia_search_best3 (rule->tree6[p], (u_char *)ip->s6_addr, 128) != NULL; + } +} +/******************************************************************************/ +LOCAL void moloch_rules_check_rule_fields(MolochSession_t *session, MolochRule_t *rule, int skipPos) { MolochString_t *hstring; MolochInt_t *hint; @@ -478,10 +505,29 @@ void moloch_rules_check_rule_fields(MolochSession_t *session, MolochRule_t *rule int good = 1; for (f = 0; good && f < rule->fieldsLen; f++) { - if (rule->fields[f] == skipPos) - continue; int p = rule->fields[f]; + if (p == skipPos) + continue; + + if (p >= session->maxFields) { + switch (p) { + case MOLOCH_FIELD_EXSPECIAL_SRC_IP: + good = moloch_rules_check_ip(rule, p, &session->addr1); + break; + case MOLOCH_FIELD_EXSPECIAL_SRC_PORT: + good = g_hash_table_contains(rule->hash[p], (gpointer)(long)session->port1); + break; + case MOLOCH_FIELD_EXSPECIAL_DST_IP: + good = moloch_rules_check_ip(rule, p, &session->addr2); + break; + case MOLOCH_FIELD_EXSPECIAL_DST_PORT: + good = g_hash_table_contains(rule->hash[p], (gpointer)(long)session->port2); + break; + } + continue; + } + if (!session->fields[p]) { good = 0; break; @@ -489,6 +535,9 @@ void moloch_rules_check_rule_fields(MolochSession_t *session, MolochRule_t *rule switch (config.fields[p]->type) { case MOLOCH_FIELD_TYPE_IP: + good = moloch_rules_check_ip(rule, p, session->fields[p]->ip); + break; + case MOLOCH_FIELD_TYPE_INT: good = g_hash_table_contains(rule->hash[p], (gpointer)(long)session->fields[p]->i); break; @@ -503,7 +552,6 @@ void moloch_rules_check_rule_fields(MolochSession_t *session, MolochRule_t *rule } break; case MOLOCH_FIELD_TYPE_INT_HASH: - case MOLOCH_FIELD_TYPE_IP_HASH: ihash = session->fields[p]->ihash; good = 0; HASH_FORALL(i_, *ihash, hint, @@ -514,6 +562,17 @@ void moloch_rules_check_rule_fields(MolochSession_t *session, MolochRule_t *rule ); break; case MOLOCH_FIELD_TYPE_IP_GHASH: + ghash = session->fields[p]->ghash; + g_hash_table_iter_init (&iter, ghash); + good = 0; + while (g_hash_table_iter_next (&iter, &ikey, NULL)) { + if (moloch_rules_check_ip(rule, p, ikey)) { + good = 1; + break; + } + } + break; + case MOLOCH_FIELD_TYPE_STR_GHASH: case MOLOCH_FIELD_TYPE_INT_GHASH: ghash = session->fields[p]->ghash; g_hash_table_iter_init (&iter, ghash); @@ -558,30 +617,64 @@ void moloch_rules_run_field_set(MolochSession_t *session, int pos, const gpointe { int r; - GPtrArray *rules = g_hash_table_lookup(fieldsHash[pos], value); - if (!rules) - return; + if (config.fields[pos]->type == MOLOCH_FIELD_TYPE_IP || + config.fields[pos]->type == MOLOCH_FIELD_TYPE_IP_GHASH) { - for (r = 0; r < (int)rules->len; r++) { - MolochRule_t *rule = g_ptr_array_index(rules, r); - if (rule->fieldsLen == 1) { - moloch_field_ops_run(session, &rule->ops); + patricia_node_t *nodes[MOLOCH_RULES_MAX]; + + int cnt; + if (IN6_IS_ADDR_V4MAPPED((struct in6_addr *)value)) { + cnt = patricia_search_all2(fieldsTree4[pos], ((u_char *)value) + 12, 32, nodes, MOLOCH_RULES_MAX); + } else { + cnt = patricia_search_all2(fieldsTree6[pos], (u_char *)value, 128, nodes, MOLOCH_RULES_MAX); + } + if (cnt == 0) return; + + // These are all the possible rules that match + int i; + for (i = 0; i < cnt; i++) { + GPtrArray *rules = nodes[i]->data; + + for (r = 0; r < (int)rules->len; r++) { + MolochRule_t *rule = g_ptr_array_index(rules, r); + + // If there is only 1 field we are checking for then the ops can be run since it matched above + if (rule->fieldsLen == 1) { + moloch_field_ops_run(session, &rule->ops); + continue; + } + + // Need to check other fields in rule + moloch_rules_check_rule_fields(session, rule, pos); + } + } + } else { + // See if this value is in the hash table of values we are watching for + GPtrArray *rules = g_hash_table_lookup(fieldsHash[pos], value); + if (!rules) + return; + + for (r = 0; r < (int)rules->len; r++) { + MolochRule_t *rule = g_ptr_array_index(rules, r); + + // If there is only 1 field we are checking for then the ops can be run since it matched above + if (rule->fieldsLen == 1) { + moloch_field_ops_run(session, &rule->ops); + return; + } + + // Need to check other fields in rule + moloch_rules_check_rule_fields(session, rule, pos); } - moloch_rules_check_rule_fields(session, rule, pos); } } /******************************************************************************/ -int moloch_rules_run_every_packet(MolochPacket_t *UNUSED(packet)) -{ - return 1; -} -/******************************************************************************/ void moloch_rules_run_session_setup(MolochSession_t *session, MolochPacket_t *packet) { int r; - for (r = 0; r < rulesLen[MOLOCH_RULE_TYPE_SESSION_SETUP]; r++) { - MolochRule_t *rule = rules[MOLOCH_RULE_TYPE_SESSION_SETUP][r]; + MolochRule_t *rule; + for (r = 0; (rule = rules[MOLOCH_RULE_TYPE_SESSION_SETUP][r]); r++) { if (rule->fieldsLen) { moloch_rules_check_rule_fields(session, rule, -1); } else if (rule->bpfp.bf_len && bpf_filter(rule->bpfp.bf_insns, packet->pkt, packet->pktlen, packet->pktlen)) { @@ -592,9 +685,9 @@ void moloch_rules_run_session_setup(MolochSession_t *session, MolochPacket_t *pa /******************************************************************************/ void moloch_rules_run_after_classify(MolochSession_t *session) { - int r; - for (r = 0; r < rulesLen[MOLOCH_RULE_TYPE_AFTER_CLASSIFY]; r++) { - MolochRule_t *rule = rules[MOLOCH_RULE_TYPE_AFTER_CLASSIFY][r]; + int r; + MolochRule_t *rule; + for (r = 0; (rule = rules[MOLOCH_RULE_TYPE_AFTER_CLASSIFY][r]); r++) { if (rule->fieldsLen) { moloch_rules_check_rule_fields(session, rule, -1); } @@ -603,10 +696,10 @@ void moloch_rules_run_after_classify(MolochSession_t *session) /******************************************************************************/ void moloch_rules_run_before_save(MolochSession_t *session, int final) { - int r; - final = 1 >> final; - for (r = 0; r < rulesLen[MOLOCH_RULE_TYPE_BEFORE_SAVE]; r++) { - MolochRule_t *rule = rules[MOLOCH_RULE_TYPE_BEFORE_SAVE][r]; + int r; + final = 1 >> final; + MolochRule_t *rule; + for (r = 0; (rule = rules[MOLOCH_RULE_TYPE_BEFORE_SAVE][r]); r++) { if ((rule->saveFlags & final) == 0) { continue; } @@ -617,6 +710,24 @@ void moloch_rules_run_before_save(MolochSession_t *session, int final) } } /******************************************************************************/ +void moloch_rules_session_create(MolochSession_t *session) +{ + switch (session->protocol) { + case IPPROTO_TCP: + case IPPROTO_UDP: + if (config.fields[MOLOCH_FIELD_EXSPECIAL_SRC_IP]->ruleEnabled) + moloch_rules_run_field_set(session, MOLOCH_FIELD_EXSPECIAL_SRC_IP, &session->addr1); + if (config.fields[MOLOCH_FIELD_EXSPECIAL_DST_IP]->ruleEnabled) + moloch_rules_run_field_set(session, MOLOCH_FIELD_EXSPECIAL_DST_IP, &session->addr2); + case IPPROTO_ICMP: + if (config.fields[MOLOCH_FIELD_EXSPECIAL_SRC_PORT]->ruleEnabled) + moloch_rules_run_field_set(session, MOLOCH_FIELD_EXSPECIAL_SRC_PORT, (gpointer)(long)session->port1); + if (config.fields[MOLOCH_FIELD_EXSPECIAL_DST_PORT]->ruleEnabled) + moloch_rules_run_field_set(session, MOLOCH_FIELD_EXSPECIAL_DST_PORT, (gpointer)(long)session->port2); + break; + } +} +/******************************************************************************/ void moloch_rules_init() { char **rulesFiles = moloch_config_str_list(NULL, "rulesFiles", NULL); @@ -650,14 +761,13 @@ void moloch_rules_init() char **bpfs; GRegex *regex = g_regex_new(":\\s*(\\d+)\\s*$", 0, 0, 0); - int type = MOLOCH_RULE_TYPE_SESSION_SETUP; bpfs = moloch_config_str_list(NULL, "dontSaveBPFs", NULL); int pos = moloch_field_by_exp("_maxPacketsToSave"); gint start_pos; if (bpfs) { for (i = 0; bpfs[i]; i++) { - int n = rulesLen[type]++; - MolochRule_t *rule = rules[type][n] = MOLOCH_TYPE_ALLOC0(MolochRule_t); + int n = rulesLen[MOLOCH_RULE_TYPE_SESSION_SETUP]++; + MolochRule_t *rule = rules[MOLOCH_RULE_TYPE_SESSION_SETUP][n] = MOLOCH_TYPE_ALLOC0(MolochRule_t); rule->filename = "dontSaveBPFs"; moloch_field_ops_init(&rule->ops, 1, MOLOCH_FIELD_OPS_FLAGS_COPY); @@ -679,8 +789,8 @@ void moloch_rules_init() pos = moloch_field_by_exp("_minPacketsBeforeSavingSPI"); if (bpfs) { for (i = 0; bpfs[i]; i++) { - int n = rulesLen[type]++; - MolochRule_t *rule = rules[type][n] = MOLOCH_TYPE_ALLOC0(MolochRule_t); + int n = rulesLen[MOLOCH_RULE_TYPE_SESSION_SETUP]++; + MolochRule_t *rule = rules[MOLOCH_RULE_TYPE_SESSION_SETUP][n] = MOLOCH_TYPE_ALLOC0(MolochRule_t); rule->filename = "minPacketsSaveBPFs"; moloch_field_ops_init(&rule->ops, 1, MOLOCH_FIELD_OPS_FLAGS_COPY); diff --git a/capture/session.c b/capture/session.c index 726f173908..0d874e20fb 100644 --- a/capture/session.c +++ b/capture/session.c @@ -142,7 +142,6 @@ uint32_t moloch_session_hash(const void *key) uint32_t *p = (uint32_t *)key; const uint32_t *end = (uint32_t *)((unsigned char *)key + ((unsigned char *)key)[0] - 4); uint32_t h = ((uint8_t *)key)[((uint8_t *)key)[0]-1]; // There is one extra byte at the end - while (p < end) { h ^= *p; @@ -175,35 +174,22 @@ void moloch_session_add_cmd(MolochSession_t *session, MolochSesCmd icmd, gpointe MOLOCH_UNLOCK(sessionCmds[session->thread].lock); } /******************************************************************************/ -void moloch_session_get_tag_cb(void *sessionV, int tagType, const char *tagName, uint32_t tag, gboolean async) +void moloch_session_add_cmd_thread(int thread, gpointer uw1, gpointer uw2, MolochCmd_func func) { - MolochSession_t *session = sessionV; - - if (tag == 0) { - LOG("ERROR - Not adding tag %s type %d couldn't get tag num", tagName, tagType); - moloch_session_decr_outstanding(session); - } else if (async) { - moloch_session_add_cmd(session, MOLOCH_SES_CMD_ADD_TAG, (gpointer)(long)tagType, (gpointer)(long)tag, NULL); - } else { - moloch_field_int_add(tagType, session, tag); - moloch_session_decr_outstanding(session); - } - -} -/******************************************************************************/ -gboolean moloch_session_has_tag(MolochSession_t *session, const char *tagName) -{ - uint32_t tagValue; - - if (!session->fields[config.tagsField]) - return FALSE; + static MolochSession_t fakeSessions[MOLOCH_MAX_PACKET_THREADS]; - if ((tagValue = moloch_db_peek_tag(tagName)) == 0) - return FALSE; + fakeSessions[thread].thread = thread; - MolochInt_t *hint; - HASH_FIND_INT(i_, *(session->fields[config.tagsField]->ihash), tagValue, hint); - return hint != 0; + MolochSesCmd_t *cmd = MOLOCH_TYPE_ALLOC(MolochSesCmd_t); + cmd->cmd = MOLOCH_SES_CMD_FUNC; + cmd->session = &fakeSessions[thread]; + cmd->uw1 = uw1; + cmd->uw2 = uw2; + cmd->func = func; + MOLOCH_LOCK(sessionCmds[thread].lock); + DLL_PUSH_TAIL(cmd_, &sessionCmds[thread], cmd); + moloch_packet_thread_wake(thread); + MOLOCH_UNLOCK(sessionCmds[thread].lock); } /******************************************************************************/ void moloch_session_add_protocol(MolochSession_t *session, const char *protocol) @@ -222,33 +208,7 @@ gboolean moloch_session_has_protocol(MolochSession_t *session, const char *proto } /******************************************************************************/ void moloch_session_add_tag(MolochSession_t *session, const char *tag) { - moloch_session_incr_outstanding(session); - moloch_db_get_tag(session, config.tagsField, tag, moloch_session_get_tag_cb); moloch_field_string_add(config.tagsStringField, session, tag, -1, TRUE); - - if (session->stopSaving == 0 && HASH_COUNT(s_, config.dontSaveTags)) { - MolochString_t *tstring; - - HASH_FIND(s_, config.dontSaveTags, tag, tstring); - if (tstring) { - session->stopSaving = (int)(long)tstring->uw; - } - } -} - -/******************************************************************************/ -void moloch_session_add_tag_type(MolochSession_t *session, int tagtype, const char *tag) { - moloch_session_incr_outstanding(session); - moloch_db_get_tag(session, tagtype, tag, moloch_session_get_tag_cb); - - if (session->stopSaving == 0 && HASH_COUNT(s_, config.dontSaveTags)) { - MolochString_t *tstring; - - HASH_FIND(s_, config.dontSaveTags, tag, tstring); - if (tstring) { - session->stopSaving = (long)tstring->uw; - } - } } /******************************************************************************/ void moloch_session_mark_for_close (MolochSession_t *session, int ses) @@ -349,13 +309,6 @@ void moloch_session_mid_save(MolochSession_t *session, uint32_t tv_sec) moloch_rules_run_before_save(session, 0); -#ifdef FIXLATER - /* If we are parsing pcap its ok to pause and make sure all tags are loaded */ - while (session->outstandingQueries > 0 && config.pcapReadOffline) { - g_main_context_iteration (g_main_context_default(), TRUE); - } -#endif - if (!session->rootId) { session->rootId = "ROOT"; } @@ -519,10 +472,6 @@ void moloch_session_process_commands(int thread) break; switch (cmd->cmd) { - case MOLOCH_SES_CMD_ADD_TAG: - moloch_field_int_add((long)cmd->uw1, cmd->session, (long)cmd->uw2); - moloch_session_decr_outstanding(cmd->session); - break; case MOLOCH_SES_CMD_FUNC: cmd->func(cmd->session, cmd->uw1, cmd->uw2); break; @@ -615,9 +564,9 @@ void moloch_session_init() if (p == 12) p = 11; protocolField = moloch_field_define("general", "termfield", - "protocols", "Protocols", "prot-term", + "protocols", "Protocols", "protocol", "Protocols set for session", - MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_COUNT | MOLOCH_FIELD_FLAG_LINKED_SESSIONS, + MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT | MOLOCH_FIELD_FLAG_LINKED_SESSIONS, NULL); if (config.debug) @@ -642,7 +591,7 @@ void moloch_session_init() moloch_add_can_quit(moloch_session_need_save_outstanding, "session save outstanding"); } /******************************************************************************/ -static void moloch_session_flush_close(MolochSession_t *session, gpointer UNUSED(uw1), gpointer UNUSED(uw2)) +LOCAL void moloch_session_flush_close(MolochSession_t *session, gpointer UNUSED(uw1), gpointer UNUSED(uw2)) { int thread = session->thread; int i; @@ -661,12 +610,9 @@ void moloch_session_flush() { moloch_packet_flush(); - static MolochSession_t fakeSessions[MOLOCH_MAX_PACKET_THREADS]; - int thread; for (thread = 0; thread < config.packetThreads; thread++) { - fakeSessions[thread].thread = thread; - moloch_session_add_cmd(&fakeSessions[thread], MOLOCH_SES_CMD_FUNC, NULL, NULL, moloch_session_flush_close); + moloch_session_add_cmd_thread(thread, NULL, NULL, moloch_session_flush_close); } } /******************************************************************************/ diff --git a/capture/thirdparty/patricia.c b/capture/thirdparty/patricia.c index 6e2b94a235..0b2d12f97a 100644 --- a/capture/thirdparty/patricia.c +++ b/capture/thirdparty/patricia.c @@ -12,20 +12,8 @@ #include #include "patricia.h" -/* - * prefix_tochar convert prefix information to bytes - */ -u_char * -prefix_tochar(prefix_t * prefix) -{ - if (prefix == NULL) - return (NULL); - - return ((u_char *) & prefix->add.sin); -} - static inline int -comp_with_mask(void *addr, void *dest, u_int mask) +comp_with_mask(void *addr, const void *dest, u_int mask) { if ( /* mask/8 == 0 || */ memcmp(addr, dest, mask / 8) == 0) { @@ -505,6 +493,52 @@ patricia_search_best2(patricia_tree_t * patricia, prefix_t * prefix, return (NULL); } +patricia_node_t * +patricia_search_best3(patricia_tree_t * patricia, const u_char *addr, int bitlen) +{ + patricia_node_t *node; + patricia_node_t *stack[PATRICIA_MAXBITS + 1]; + int cnt = 0; + + if (!patricia || !addr) + return NULL; + + if (patricia->head == NULL) + return (NULL); + + node = patricia->head; + + while (node->bit < bitlen) { + + if (node->prefix) { + stack[cnt++] = node; + } + + if (BIT_TEST(addr[node->bit >> 3], 0x80 >> (node->bit & 0x07))) { + node = node->r; + } else { + node = node->l; + } + + if (node == NULL) + break; + } + + if (node && node->prefix) + stack[cnt++] = node; + + if (cnt <= 0) + return (NULL); + + while (--cnt >= 0) { + node = stack[cnt]; + if (comp_with_mask(prefix_touchar(node->prefix), addr, node->prefix->bitlen)) { + return (node); + } + } + return (NULL); +} + /* * if inclusive != 0, "best" may be the given prefix itself */ @@ -550,6 +584,43 @@ patricia_search_all(patricia_tree_t * patricia, prefix_t * prefix, int inclusive return cnt; } +int +patricia_search_all2(patricia_tree_t * patricia, u_char *addr, int bitlen, patricia_node_t **results, int resultsize) +{ + patricia_node_t *node; + int cnt = 0; + + node = patricia->head; + + if (node == NULL) + return 0; + + while (node->bit < bitlen && cnt < resultsize) { + + if (node->prefix && node->data && + comp_with_mask(prefix_tochar(node->prefix), + addr, node->prefix->bitlen)) { + results[cnt++] = node; + } + + if (BIT_TEST(addr[node->bit >> 3], 0x80 >> (node->bit & 0x07))) { + node = node->r; + } else { + node = node->l; + } + + if (!node) + return cnt; + } + + if (node->prefix && node->data && + comp_with_mask(prefix_touchar(node->prefix), addr, node->prefix->bitlen)) { + results[cnt++] = node; + } + + return cnt; +} + patricia_node_t * patricia_search_best(patricia_tree_t * patricia, diff --git a/capture/thirdparty/patricia.h b/capture/thirdparty/patricia.h index 4acb83fbdb..1c366db879 100644 --- a/capture/thirdparty/patricia.h +++ b/capture/thirdparty/patricia.h @@ -57,6 +57,7 @@ typedef void (*void_fn_t)(); /* { from defs.h */ #define prefix_touchar(prefix) ((u_char *)&(prefix)->add.sin) +#define prefix_tochar(prefix) (prefix?((u_char *)&(prefix)->add.sin):NULL) #define MAXLINE 1024 #define BIT_TEST(f, b) ((f) & (b)) /* } */ @@ -113,15 +114,17 @@ typedef struct _patricia_tree_t { patricia_node_t *patricia_search_exact (patricia_tree_t *patricia, prefix_t *prefix); patricia_node_t *patricia_search_best (patricia_tree_t *patricia, prefix_t *prefix); -patricia_node_t * patricia_search_best2 (patricia_tree_t *patricia, prefix_t *prefix, - int inclusive); +patricia_node_t *patricia_search_best2 (patricia_tree_t *patricia, prefix_t *prefix, int inclusive); +patricia_node_t *patricia_search_best3(patricia_tree_t * patricia, const u_char *addr, int bitlen); int patricia_search_all(patricia_tree_t * patricia, prefix_t * prefix, int inclusive, patricia_node_t **results); +int patricia_search_all2(patricia_tree_t * patricia, u_char *addr, int bitlen, patricia_node_t **results, int resultsize); patricia_node_t *patricia_lookup (patricia_tree_t *patricia, prefix_t *prefix); void patricia_remove (patricia_tree_t *patricia, patricia_node_t *node); patricia_tree_t *New_Patricia (int maxbits); void Clear_Patricia (patricia_tree_t *patricia, void_fn_t func); void Destroy_Patricia (patricia_tree_t *patricia, void_fn_t func); void patricia_process (patricia_tree_t *patricia, void_fn_t func); +prefix_t *New_Prefix2(int family, void *dest, int bitlen, prefix_t * prefix); void Deref_Prefix (prefix_t * prefix); diff --git a/capture/writer-disk.c b/capture/writer-disk.c index f22d72c9d8..37bb205e06 100644 --- a/capture/writer-disk.c +++ b/capture/writer-disk.c @@ -30,6 +30,7 @@ #endif extern MolochConfig_t config; +extern MolochPcapFileHdr_t pcapFileHeader; typedef struct moloch_output { @@ -45,42 +46,42 @@ typedef struct moloch_output { } MolochDiskOutput_t; -static MolochDiskOutput_t *output; -static MOLOCH_LOCK_DEFINE(output); +LOCAL MolochDiskOutput_t *output; +LOCAL MOLOCH_LOCK_DEFINE(output); -static MolochDiskOutput_t outputQ; -static MOLOCH_LOCK_DEFINE(outputQ); -static MOLOCH_COND_DEFINE(outputQ); +LOCAL MolochDiskOutput_t outputQ; +LOCAL MOLOCH_LOCK_DEFINE(outputQ); +LOCAL MOLOCH_COND_DEFINE(outputQ); -static MolochIntHead_t freeOutputBufs; -static MOLOCH_LOCK_DEFINE(freeOutputBufs); +LOCAL MolochIntHead_t freeOutputBufs; +LOCAL MOLOCH_LOCK_DEFINE(freeOutputBufs); -static uint32_t outputId; -static char *outputFileName; -static uint64_t outputFilePos = 0; -static struct timeval outputFileTime; +LOCAL uint32_t outputId; +LOCAL char *outputFileName; +LOCAL uint64_t outputFilePos = 0; +LOCAL struct timeval outputFileTime; #define MOLOCH_WRITE_NORMAL 0x00 -#define MOLOCH_WRITE_DIRECT 0x01 +#define MOLOCH_WRITE_DIRECT 0x01 #define MOLOCH_WRITE_MMAP 0x02 #define MOLOCH_WRITE_THREAD 0x04 -static int writeMethod; -static int pageSize; +LOCAL int writeMethod; +LOCAL int pageSize; /******************************************************************************/ -uint32_t writer_disk_queue_length_thread() +LOCAL uint32_t writer_disk_queue_length_thread() { int count = DLL_COUNT(mo_, &outputQ); return count; } /******************************************************************************/ -uint32_t writer_disk_queue_length_nothread() +LOCAL uint32_t writer_disk_queue_length_nothread() { return DLL_COUNT(mo_, &outputQ); } /******************************************************************************/ -void writer_disk_alloc_buf(MolochDiskOutput_t *out) +LOCAL void writer_disk_alloc_buf(MolochDiskOutput_t *out) { if (writeMethod & MOLOCH_WRITE_THREAD) MOLOCH_LOCK(freeOutputBufs); @@ -100,7 +101,7 @@ void writer_disk_alloc_buf(MolochDiskOutput_t *out) MOLOCH_UNLOCK(freeOutputBufs); } /******************************************************************************/ -void writer_disk_free_buf(MolochDiskOutput_t *out) +LOCAL void writer_disk_free_buf(MolochDiskOutput_t *out) { if (writeMethod & MOLOCH_WRITE_THREAD) MOLOCH_LOCK(freeOutputBufs); @@ -119,7 +120,7 @@ void writer_disk_free_buf(MolochDiskOutput_t *out) /******************************************************************************/ /* Only used in non thread mode to write out data */ -gboolean writer_disk_output_cb(gint fd, GIOCondition UNUSED(cond), gpointer UNUSED(data)) +LOCAL gboolean writer_disk_output_cb(gint fd, GIOCondition UNUSED(cond), gpointer UNUSED(data)) { if (fd && config.quitting) return FALSE; @@ -200,9 +201,10 @@ gboolean writer_disk_output_cb(gint fd, GIOCondition UNUSED(cond), gpointer UNUS return DLL_COUNT(mo_, &outputQ) > 0; } /******************************************************************************/ -void *writer_disk_output_thread(void *UNUSED(arg)) +LOCAL void *writer_disk_output_thread(void *UNUSED(arg)) { - LOG("THREAD %p", (gpointer)pthread_self()); + if (config.debug) + LOG("THREAD %p", (gpointer)pthread_self()); MolochDiskOutput_t *out; int outputFd = 0; @@ -260,7 +262,7 @@ void *writer_disk_output_thread(void *UNUSED(arg)) } } /******************************************************************************/ -void writer_disk_flush(gboolean all) +LOCAL void writer_disk_flush(gboolean all) { if (unlikely(config.dryRun || !output)) { return; @@ -308,7 +310,7 @@ void writer_disk_flush(gboolean all) output = noutput; } /******************************************************************************/ -void writer_disk_exit() +LOCAL void writer_disk_exit() { writer_disk_flush(TRUE); outputFileName = 0; @@ -324,8 +326,7 @@ void writer_disk_exit() } } /******************************************************************************/ -extern MolochPcapFileHdr_t pcapFileHeader; -void writer_disk_create(MolochPacket_t * const packet) +LOCAL void writer_disk_create(MolochPacket_t * const packet) { outputFileName = moloch_db_create_file(packet->ts.tv_sec, NULL, 0, 0, &outputId); outputFilePos = 24; @@ -349,8 +350,7 @@ struct pcap_sf_pkthdr { uint32_t caplen; /* length of portion present */ uint32_t pktlen; /* length this packet (off wire) */ }; -void -writer_disk_write(const MolochSession_t * const UNUSED(session), MolochPacket_t * const packet) +LOCAL void writer_disk_write(const MolochSession_t * const UNUSED(session), MolochPacket_t * const packet) { struct pcap_sf_pkthdr hdr; @@ -384,8 +384,7 @@ writer_disk_write(const MolochSession_t * const UNUSED(session), MolochPacket_t MOLOCH_UNLOCK(output); } /******************************************************************************/ -gboolean -writer_disk_file_time_gfunc (gpointer UNUSED(user_data)) +LOCAL gboolean writer_disk_file_time_gfunc (gpointer UNUSED(user_data)) { static struct timeval tv; gettimeofday(&tv, 0); diff --git a/capture/writer-inplace.c b/capture/writer-inplace.c index d70eb9faee..f3c3f7172c 100644 --- a/capture/writer-inplace.c +++ b/capture/writer-inplace.c @@ -29,23 +29,19 @@ extern MolochConfig_t config; LOCAL GHashTable *filePtr2Id; -MOLOCH_LOCK_DEFINE(filePtr2Id); +LOCAL MOLOCH_LOCK_DEFINE(filePtr2Id); /******************************************************************************/ -uint32_t writer_inplace_queue_length() +LOCAL uint32_t writer_inplace_queue_length() { return 0; } /******************************************************************************/ -void writer_inplace_flush(gboolean UNUSED(all)) +LOCAL void writer_inplace_exit() { } /******************************************************************************/ -void writer_inplace_exit() -{ -} -/******************************************************************************/ -long writer_inplace_create(MolochPacket_t * const packet) +LOCAL long writer_inplace_create(MolochPacket_t * const packet) { struct stat st; @@ -59,7 +55,7 @@ long writer_inplace_create(MolochPacket_t * const packet) } /******************************************************************************/ -void writer_inplace_write(const MolochSession_t * const UNUSED(session), MolochPacket_t * const packet) +LOCAL void writer_inplace_write(const MolochSession_t * const UNUSED(session), MolochPacket_t * const packet) { MOLOCH_LOCK(filePtr2Id); long outputId = (long)g_hash_table_lookup(filePtr2Id, packet->readerName); @@ -71,7 +67,7 @@ void writer_inplace_write(const MolochSession_t * const UNUSED(session), MolochP packet->writerFilePos = packet->readerFilePos; } /******************************************************************************/ -void writer_inplace_write_dryrun(const MolochSession_t * const UNUSED(session), MolochPacket_t * const packet) +LOCAL void writer_inplace_write_dryrun(const MolochSession_t * const UNUSED(session), MolochPacket_t * const packet) { packet->writerFilePos = packet->readerFilePos; } diff --git a/capture/writer-null.c b/capture/writer-null.c index 75b542316c..8a5df38e5f 100644 --- a/capture/writer-null.c +++ b/capture/writer-null.c @@ -26,23 +26,19 @@ extern MolochConfig_t config; -static uint32_t outputFilePos = 24; +LOCAL uint32_t outputFilePos = 24; /******************************************************************************/ -uint32_t writer_null_queue_length() +LOCAL uint32_t writer_null_queue_length() { return 0; } /******************************************************************************/ -void writer_null_flush(gboolean UNUSED(all)) +LOCAL void writer_null_exit() { } /******************************************************************************/ -void writer_null_exit() -{ -} -/******************************************************************************/ -void writer_null_write(const MolochSession_t * const UNUSED(session), MolochPacket_t * const packet) +LOCAL void writer_null_write(const MolochSession_t * const UNUSED(session), MolochPacket_t * const packet) { packet->writerFileNum = 0; packet->writerFilePos = outputFilePos; diff --git a/capture/writer-simple.c b/capture/writer-simple.c index 1bd0511f81..c6aaf4a8aa 100644 --- a/capture/writer-simple.c +++ b/capture/writer-simple.c @@ -35,6 +35,7 @@ extern MolochConfig_t config; extern MolochPcapFileHdr_t pcapFileHeader; + typedef struct { EVP_CIPHER_CTX *cipher_ctx; uint64_t pos; @@ -58,15 +59,15 @@ typedef struct { MOLOCH_LOCK_EXTERN(lock); } MolochSimpleHead_t; -static MolochSimpleHead_t simpleQ; -static MOLOCH_LOCK_DEFINE(simpleQ); -static MOLOCH_COND_DEFINE(simpleQ); +LOCAL MolochSimpleHead_t simpleQ; +LOCAL MOLOCH_LOCK_DEFINE(simpleQ); +LOCAL MOLOCH_COND_DEFINE(simpleQ); enum MolochSimpleMode { MOLOCH_SIMPLE_NORMAL, MOLOCH_SIMPLE_XOR2048, MOLOCH_SIMPLE_AES256CTR}; LOCAL MolochSimple_t *currentInfo[MOLOCH_MAX_PACKET_THREADS]; LOCAL MolochSimpleHead_t freeList[MOLOCH_MAX_PACKET_THREADS]; -LOCAL int pageSize; +LOCAL uint32_t pageSize; LOCAL enum MolochSimpleMode simpleMode; LOCAL char *simpleKEKId; LOCAL uint8_t simpleKEK[EVP_MAX_KEY_LENGTH]; @@ -74,14 +75,15 @@ LOCAL int simpleKEKLen; LOCAL uint8_t simpleIV[EVP_MAX_IV_LENGTH]; LOCAL const EVP_CIPHER *cipher; LOCAL int openOptions; +LOCAL struct timeval lastSave[MOLOCH_MAX_PACKET_THREADS]; /******************************************************************************/ -uint32_t writer_simple_queue_length() +LOCAL uint32_t writer_simple_queue_length() { return DLL_COUNT(simple_, &simpleQ); } /******************************************************************************/ -MolochSimple_t *writer_simple_alloc(int thread, MolochSimple_t *previous) +LOCAL MolochSimple_t *writer_simple_alloc(int thread, MolochSimple_t *previous) { MolochSimple_t *info; @@ -118,7 +120,7 @@ MolochSimple_t *writer_simple_alloc(int thread, MolochSimple_t *previous) return info; } /******************************************************************************/ -void writer_simple_free(MolochSimple_t *info) +LOCAL void writer_simple_free(MolochSimple_t *info) { int thread = info->thread; @@ -147,19 +149,29 @@ void writer_simple_free(MolochSimple_t *info) } /******************************************************************************/ -void writer_simple_process_buf(int thread, int closing) +LOCAL void writer_simple_process_buf(int thread, int closing) { MolochSimple_t *info = currentInfo[thread]; info->closing = closing; if (!closing) { + // Round down to nearest pagesize + int writeSize = (info->bufpos/pageSize) * pageSize; + + // Create next buffer currentInfo[thread] = writer_simple_alloc(thread, info); - memcpy(currentInfo[thread]->buf, info->buf + config.pcapWriteSize, info->bufpos - config.pcapWriteSize); - currentInfo[thread]->bufpos = info->bufpos - config.pcapWriteSize; + + // Copy what we aren't going to write to next buffer + memcpy(currentInfo[thread]->buf, info->buf + writeSize, info->bufpos - writeSize); + currentInfo[thread]->bufpos = info->bufpos - writeSize; + + // Set what we are going to write + info->bufpos = writeSize; } else { currentInfo[thread] = NULL; } MOLOCH_LOCK(simpleQ); + gettimeofday(&lastSave[thread], NULL); DLL_PUSH_TAIL(simple_, &simpleQ, info); if ((DLL_COUNT(simple_, &simpleQ) % 100) == 0) { LOG("WARNING - Disk Q of %d is too large, check the Moloch FAQ about testing disk speed", DLL_COUNT(simple_, &simpleQ)); @@ -168,7 +180,7 @@ void writer_simple_process_buf(int thread, int closing) MOLOCH_UNLOCK(simpleQ); } /******************************************************************************/ -void writer_simple_encrypt_key(uint8_t *inkey, int inkeylen, char *outkeyhex) +LOCAL void writer_simple_encrypt_key(uint8_t *inkey, int inkeylen, char *outkeyhex) { uint8_t ciphertext[1024]; @@ -196,7 +208,7 @@ struct pcap_sf_pkthdr { uint32_t pktlen; /* length this packet (off wire) */ }; /******************************************************************************/ -void writer_simple_write(const MolochSession_t * const session, MolochPacket_t * const packet) +LOCAL void writer_simple_write(const MolochSession_t * const session, MolochPacket_t * const packet) { char dekhex[1024]; int thread = session->thread; @@ -273,12 +285,13 @@ void writer_simple_write(const MolochSession_t * const session, MolochPacket_t * } } /******************************************************************************/ -void *writer_simple_thread(void *UNUSED(arg)) +LOCAL void *writer_simple_thread(void *UNUSED(arg)) { MolochSimple_t *info; if (config.debug) LOG("THREAD %p", (gpointer)pthread_self()); + while (1) { MOLOCH_LOCK(simpleQ); while (DLL_COUNT(simple_, &simpleQ) == 0) { @@ -288,14 +301,11 @@ void *writer_simple_thread(void *UNUSED(arg)) MOLOCH_UNLOCK(simpleQ); uint32_t pos = 0; - uint32_t total; + uint32_t total = info->bufpos; if (info->closing) { - total = info->bufpos; - if (total % pageSize != 0) { - total = (total - (total % pageSize) + pageSize); - } - } else { - total = config.pcapWriteSize; + // Round up to next page size + if (total % pageSize != 0) + total = ((total/pageSize)+1)*pageSize; } switch(simpleMode) { @@ -336,7 +346,7 @@ void *writer_simple_thread(void *UNUSED(arg)) return NULL; } /******************************************************************************/ -void writer_simple_exit() +LOCAL void writer_simple_exit() { int thread; @@ -352,6 +362,46 @@ void writer_simple_exit() } } /******************************************************************************/ +// Called inside each packet thread +LOCAL void writer_simple_check(MolochSession_t *session, void *UNUSED(uw1), void *UNUSED(uw2)) +{ + struct timeval now; + gettimeofday(&now, NULL); + + // No data or not enough bytes, reset the time + if (!currentInfo[session->thread] || currentInfo[session->thread]->bufpos < (uint32_t)pageSize) { + lastSave[session->thread] = now; + return; + } + + // Last add must be 10 seconds ago and have more then pageSize bytes + if (now.tv_sec - lastSave[session->thread].tv_sec < 10) + return; + + writer_simple_process_buf(session->thread, 0); +} +/******************************************************************************/ +/* Called in the main thread. Check all the timestamps, and if out of date + * schedule something in each writer thread to do the partial write since there + * is no locks around buffering. + */ +LOCAL gboolean writer_simple_check_gfunc (gpointer UNUSED(user_data)) +{ + struct timeval now; + gettimeofday(&now, NULL); + + MOLOCH_LOCK(simpleQ); + int thread; + for (thread = 0; thread < config.packetThreads; thread++) { + if (now.tv_sec - lastSave[thread].tv_sec >= 10) { + moloch_session_add_cmd_thread(thread, NULL, NULL, writer_simple_check); + } + } + MOLOCH_UNLOCK(simpleQ); + + return TRUE; +} +/******************************************************************************/ void writer_simple_init(char *name) { moloch_writer_queue_length = writer_simple_queue_length; @@ -386,6 +436,7 @@ void writer_simple_init(char *name) } pageSize = getpagesize(); + if (config.pcapWriteSize % pageSize != 0) { config.pcapWriteSize = ((config.pcapWriteSize + pageSize - 1) / pageSize) * pageSize; LOG ("INFO: Reseting pcapWriteSize to %u since it must be a multiple of %u", config.pcapWriteSize, pageSize); @@ -404,11 +455,17 @@ void writer_simple_init(char *name) DLL_INIT(simple_, &simpleQ); + struct timeval now; + gettimeofday(&now, NULL); + int thread; for (thread = 0; thread < config.packetThreads; thread++) { + lastSave[thread] = now; DLL_INIT(simple_, &freeList[thread]); MOLOCH_LOCK_INIT(freeList[thread].lock); } g_thread_new("moloch-simple", &writer_simple_thread, NULL); + + g_timeout_add_seconds(1, writer_simple_check_gfunc, 0); } diff --git a/capture/writers.c b/capture/writers.c index 4c0ee5a5ca..1d72e86bab 100644 --- a/capture/writers.c +++ b/capture/writers.c @@ -26,7 +26,7 @@ MolochWriterExit moloch_writer_exit; /******************************************************************************/ extern MolochConfig_t config; -static MolochStringHashStd_t writersHash; +LOCAL MolochStringHashStd_t writersHash; /******************************************************************************/ void moloch_writers_start(char *name) { diff --git a/capture/yara.c b/capture/yara.c index 8f60ba0c04..a811d8baa0 100644 --- a/capture/yara.c +++ b/capture/yara.c @@ -35,8 +35,6 @@ char *moloch_yara_version() { #else /* YR_MAJOR_VERSION */ #ifdef STRING_IS_HEX snprintf(buf, sizeof(buf), "2.x"); - #else /* STRING_IS_HEX */ - snprintf(buf, sizeof(buf), "1.x"); #endif /* STRING_IS_HEX */ #endif /* YR_MAJOR_VERSION */ return buf; @@ -45,10 +43,10 @@ char *moloch_yara_version() { #if YR_MAJOR_VERSION == 3 && YR_MINOR_VERSION >= 4 // Yara 3 -static YR_COMPILER *yCompiler = 0; -static YR_COMPILER *yEmailCompiler = 0; -static YR_RULES *yRules = 0; -static YR_RULES *yEmailRules = 0; +LOCAL YR_COMPILER *yCompiler = 0; +LOCAL YR_COMPILER *yEmailCompiler = 0; +LOCAL YR_RULES *yRules = 0; +LOCAL YR_RULES *yEmailRules = 0; @@ -84,12 +82,61 @@ void moloch_yara_open(char *filename, YR_COMPILER **compiler, YR_RULES **rules) } } /******************************************************************************/ +void moloch_yara_load(char *name) +{ + static YR_COMPILER *yCompilerOld; + static YR_RULES *yRulesOld; + + if (!name) { + yr_rules_destroy(yRulesOld); + yr_compiler_destroy(yCompilerOld); + yRulesOld = NULL; + yCompilerOld = NULL; + return; + } + + YR_COMPILER *compiler; + YR_RULES *rules; + moloch_yara_open(name, &compiler, &rules); + + yCompilerOld = yCompiler; + yRulesOld = yRules; + yCompiler = compiler; + yRules = rules; +} +/******************************************************************************/ +void moloch_yara_load_email(char *name) +{ + static YR_COMPILER *yCompilerOld; + static YR_RULES *yRulesOld; + + if (!name) { + yr_rules_destroy(yRulesOld); + yr_compiler_destroy(yCompilerOld); + yRulesOld = NULL; + yCompilerOld = NULL; + return; + } + + YR_COMPILER *compiler; + YR_RULES *rules; + moloch_yara_open(name, &compiler, &rules); + + yCompilerOld = yEmailCompiler; + yRulesOld = yEmailRules; + yEmailCompiler = compiler; + yEmailRules = rules; +} +/******************************************************************************/ void moloch_yara_init() { yr_initialize(); - moloch_yara_open(config.yara, &yCompiler, &yRules); - moloch_yara_open(config.emailYara, &yEmailCompiler, &yEmailRules); + if (config.yara) + moloch_config_monitor_file("yara file", config.yara, moloch_yara_load); + + if (config.emailYara) + moloch_config_monitor_file("yara email file", config.emailYara, moloch_yara_load_email); } /******************************************************************************/ @@ -139,10 +186,10 @@ void moloch_yara_exit() } #elif defined(YR_COMPILER_H) // Yara 3 -static YR_COMPILER *yCompiler = 0; -static YR_COMPILER *yEmailCompiler = 0; -static YR_RULES *yRules = 0; -static YR_RULES *yEmailRules = 0; +LOCAL YR_COMPILER *yCompiler = 0; +LOCAL YR_COMPILER *yEmailCompiler = 0; +LOCAL YR_RULES *yRules = 0; +LOCAL YR_RULES *yEmailRules = 0; /******************************************************************************/ @@ -232,10 +279,10 @@ void moloch_yara_exit() } #elif defined(STRING_IS_HEX) // Yara 2.x -static YR_COMPILER *yCompiler = 0; -static YR_COMPILER *yEmailCompiler = 0; -static YR_RULES *yRules = 0; -static YR_RULES *yEmailRules = 0; +LOCAL YR_COMPILER *yCompiler = 0; +LOCAL YR_COMPILER *yEmailCompiler = 0; +LOCAL YR_RULES *yRules = 0; +LOCAL YR_RULES *yEmailRules = 0; /******************************************************************************/ @@ -324,120 +371,5 @@ void moloch_yara_exit() yr_finalize(); } #else -// Yara 1.x - -static YARA_CONTEXT *yContext[MOLOCH_MAX_PACKET_THREADS]; -static YARA_CONTEXT *yEmailContext[MOLOCH_MAX_PACKET_THREADS]; - - -/******************************************************************************/ -void moloch_yara_report_error(const char* file_name, int line_number, const char* error_message) -{ - LOG("%s:%d: %s\n", file_name, line_number, error_message); -} -/******************************************************************************/ -YARA_CONTEXT *moloch_yara_open(char *filename) -{ - YARA_CONTEXT *context; - - context = yr_create_context(); - context->error_report_function = moloch_yara_report_error; - - if (filename) { - FILE *rule_file; - - rule_file = fopen(filename, "r"); - - if (rule_file != NULL) { - yr_push_file_name(context, filename); - - int errors = yr_compile_file(rule_file, context); - - fclose(rule_file); - - if (errors) { - exit (0); - } - } else { - printf("yara could not open file: %s\n", filename); - exit(1); - } - } - return context; -} -/******************************************************************************/ -void moloch_yara_init() -{ - yr_init(); - - int t; - - for (t = 0; t < config.packetThreads; t++) { - yContext[t] = moloch_yara_open(config.yara); - yEmailContext[t] = moloch_yara_open(config.emailYara); - } -} - -/******************************************************************************/ -int moloch_yara_callback(RULE* rule, MolochSession_t* session) -{ - char tagname[256]; - TAG* tag; - - if (rule->flags & RULE_FLAGS_MATCH) { - snprintf(tagname, sizeof(tagname), "yara:%s", rule->identifier); - moloch_session_add_tag(session, tagname); - tag = rule->tag_list_head; - while(tag != NULL) { - if (tag->identifier) { - snprintf(tagname, sizeof(tagname), "yara:%s", tag->identifier); - moloch_session_add_tag(session, tagname); - } - tag = tag->next; - } - } - - return CALLBACK_CONTINUE; -} -/******************************************************************************/ -int yr_scan_mem_blocks(MEMORY_BLOCK* block, YARA_CONTEXT* context, YARACALLBACK callback, void* user_data); - -void moloch_yara_execute(MolochSession_t *session, const uint8_t *data, int len, int UNUSED(first)) -{ - MEMORY_BLOCK block; - - block.data = (uint8_t *)data; - block.size = len; - block.base = 0; - block.next = NULL; - - yr_scan_mem_blocks(&block, yContext[session->thread], (YARACALLBACK)moloch_yara_callback, session); - return; -} -/******************************************************************************/ -void moloch_yara_email_execute(MolochSession_t *session, const uint8_t *data, int len, int UNUSED(first)) -{ - MEMORY_BLOCK block; - - if (!config.emailYara) - return; - - block.data = (uint8_t *)data; - block.size = len; - block.base = 0; - block.next = NULL; - - yr_scan_mem_blocks(&block, yEmailContext[session->thread], (YARACALLBACK)moloch_yara_callback, session); - return; -} -/******************************************************************************/ -void moloch_yara_exit() -{ - int t; - - for (t = 0; t < config.packetThreads; t++) { - yr_destroy_context(yContext[t]); - yr_destroy_context(yEmailContext[t]); - } -} +#error "Yara 1.x not supported" #endif diff --git a/configure b/configure index b222f94586..27224728a5 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.63 for moloch 0.50.1. +# Generated by GNU Autoconf 2.63 for moloch 1.0.0-rc2. # # Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, # 2002, 2003, 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc. @@ -594,8 +594,8 @@ SHELL=${CONFIG_SHELL-/bin/sh} # Identity of this package. PACKAGE_NAME='moloch' PACKAGE_TARNAME='moloch' -PACKAGE_VERSION='0.50.1' -PACKAGE_STRING='moloch 0.50.1' +PACKAGE_VERSION='1.0.0-rc2' +PACKAGE_STRING='moloch 1.0.0-rc2' PACKAGE_BUGREPORT='' ac_default_prefix="/data/moloch" @@ -647,8 +647,8 @@ CURL_LIBS CURL_CFLAGS GLIB2_LIBS GLIB2_CFLAGS -GEOIP_LIBS -GEOIP_CFLAGS +MAXMINDDB_LIBS +MAXMINDDB_CFLAGS YARA_LIBS YARA_CFLAGS PCAP_LIBS @@ -759,7 +759,7 @@ with_pfring with_libpcap with_libnl with_yara -with_GeoIP +with_maxminddb with_glib2 with_curl with_lua @@ -1329,7 +1329,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures moloch 0.50.1 to adapt to many kinds of systems. +\`configure' configures moloch 1.0.0-rc2 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1399,7 +1399,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of moloch 0.50.1:";; + short | recursive ) echo "Configuration of moloch 1.0.0-rc2:";; esac cat <<\_ACEOF @@ -1418,7 +1418,7 @@ Optional Packages: --without-libnl disable libnl support [default=yes, on Linux, if present] --with-yara=DIR use yara build directory - --with-GeoIP=DIR use GeoIP build directory + --with-maxminddb=DIR use maxminddb build directory --with-glib2=DIR use glib2 build directory --with-curl=DIR use curl build directory --with-lua=DIR use lua build directory @@ -1501,7 +1501,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -moloch configure 0.50.1 +moloch configure 1.0.0-rc2 generated by GNU Autoconf 2.63 Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, @@ -1515,7 +1515,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by moloch $as_me 0.50.1, which was +It was created by moloch $as_me 1.0.0-rc2, which was generated by GNU Autoconf 2.63. Invocation command line was $ $0 $@ @@ -2364,7 +2364,7 @@ fi # Define the identity of the package. PACKAGE='moloch' - VERSION='0.50.1' + VERSION='1.0.0-rc2' cat >>confdefs.h <<_ACEOF @@ -5396,7 +5396,7 @@ darwin*) ;; *) SHARED_FLAGS="--shared" - UNDEFINED_FLAGS="-u g_checksum_update -u g_hmac_update" + UNDEFINED_FLAGS="-u g_checksum_update -u g_hmac_update -u g_uri_unescape_segment" esac @@ -5963,12 +5963,12 @@ fi -{ $as_echo "$as_me:$LINENO: checking for GeoIP" >&5 -$as_echo_n "checking for GeoIP... " >&6; } +{ $as_echo "$as_me:$LINENO: checking for maxminddb" >&5 +$as_echo_n "checking for maxminddb... " >&6; } -# Check whether --with-GeoIP was given. -if test "${with_GeoIP+set}" = set; then - withval=$with_GeoIP; case "$withval" in +# Check whether --with-maxminddb was given. +if test "${with_maxminddb+set}" = set; then + withval=$with_maxminddb; case "$withval" in yes|no) { $as_echo "$as_me:$LINENO: result: no" >&5 $as_echo "no" >&6; } @@ -5976,51 +5976,51 @@ $as_echo "no" >&6; } *) { $as_echo "$as_me:$LINENO: result: $withval" >&5 $as_echo "$withval" >&6; } - if test -f $withval/libGeoIP/GeoIP.h -a -f $withval/libGeoIP/.libs/libGeoIP.a; then + if test -f $withval/include/maxminddb.h -a -f $withval/src/.libs/libmaxminddb.a; then owd=`pwd` if cd $withval; then withval=`pwd`; cd $owd; fi - GEOIP_CFLAGS="-I$withval/libGeoIP" - GEOIP_LIBS="$withval/libGeoIP/.libs/libGeoIP.a" - elif test -f $withval/include/GeoIP.h -a -f $withval/lib/libGeoIP.a; then + MAXMINDDB_CFLAGS="-I$withval/include" + MAXMINDDB_LIBS="$withval/src/.libs/libmaxminddb.a" + elif test -f $withval/include/maxminddb.h -a -f $withval/lib/libmaxminddb.a; then owd=`pwd` if cd $withval; then withval=`pwd`; cd $owd; fi - GEOIP_CFLAGS="-I$withval/include" - GEOIP_LIBS="$withval/lib/libGeoIP.a" + MAXMINDDB_CFLAGS="-I$withval/include" + MAXMINDDB_LIBS="$withval/lib/libmaxminddb.a" else - { { $as_echo "$as_me:$LINENO: error: GeoIP.h or GeoIP.a not found in $withval" >&5 -$as_echo "$as_me: error: GeoIP.h or GeoIP.a not found in $withval" >&2;} + { { $as_echo "$as_me:$LINENO: error: maxminddb.h or libmaxminddb.a not found in $withval" >&5 +$as_echo "$as_me: error: maxminddb.h or libmaxminddb.a not found in $withval" >&2;} { (exit 1); exit 1; }; } fi ;; esac else - if test -f ${prefix}/include/GeoIP/GeoIP.h; then - GEOIP_CFLAGS="-I${prefix}/include/GeoIP" - GEOIP_LIBS="-L${exec_prefix}/lib -lGeoIP" - elif test -f /usr/include/GeoIP/GeoIP.h; then - GEOIP_CFLAGS="" - GEOIP_LIBS="-lGeoIP" + if test -f ${prefix}/include/maxminddb.h; then + MAXMINDDB_CFLAGS="-I${prefix}/include" + MAXMINDDB_LIBS="-L${exec_prefix}/lib -lmaxminddb" + elif test -f /usr/include/maxminddb.h; then + MAXMINDDB_CFLAGS="" + MAXMINDDB_LIBS="-lmaxminddb" else TMP=$LIBS - LIBS="-lGeoIP $LIBS" + LIBS="-lmaxminddb $LIBS" cat >conftest.$ac_ext <<_ACEOF /* confdefs.h. */ _ACEOF cat confdefs.h >>conftest.$ac_ext cat >>conftest.$ac_ext <<_ACEOF /* end confdefs.h. */ -#include +#include int main () { -GeoIP_open(NULL, GEOIP_MEMORY_CACHE) +MMDB_open(NULL, MMDB_MODE_MMAP, NULL) ; return 0; } @@ -6046,23 +6046,23 @@ $as_echo "$ac_try_echo") >&5 test "$cross_compiling" = yes || $as_test_x conftest$ac_exeext }; then - LIBGEOIP_FOUND=1 + LIBMAXMINDDB_FOUND=1 else $as_echo "$as_me: failed program was:" >&5 sed 's/^/| /' conftest.$ac_ext >&5 - LIBGEOIP_FOUND=0 + LIBMAXMINDDB_FOUND=0 fi rm -rf conftest.dSYM rm -f core conftest.err conftest.$ac_objext conftest_ipa8_conftest.oo \ conftest$ac_exeext conftest.$ac_ext LIBS=$TMP - if test $LIBGEOIP_FOUND = 1 ; then - GEOIP_LIBS="-lGeoIP" + if test $LIBMAXMINDDB_FOUND = 1 ; then + MAXMINDDB_LIBS="-lmaxminddb" else - { { $as_echo "$as_me:$LINENO: error: GeoIP not found" >&5 -$as_echo "$as_me: error: GeoIP not found" >&2;} + { { $as_echo "$as_me:$LINENO: error: maxminddb not found" >&5 +$as_echo "$as_me: error: maxminddb not found" >&2;} { (exit 1); exit 1; }; } fi fi @@ -6970,7 +6970,7 @@ exec 6>&1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by moloch $as_me 0.50.1, which was +This file was extended by moloch $as_me 1.0.0-rc2, which was generated by GNU Autoconf 2.63. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -7033,7 +7033,7 @@ Report bugs to ." _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_version="\\ -moloch config.status 0.50.1 +moloch config.status 1.0.0-rc2 configured by $0, generated by GNU Autoconf 2.63, with options \\"`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`\\" diff --git a/configure.ac b/configure.ac index 466983cf6b..6feb0726e2 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_INIT([moloch], [0.50.1]) +AC_INIT([moloch], [1.0.0-rc2]) AM_INIT_AUTOMAKE([-Wall -Werror foreign]) AC_PROG_CC AC_PROG_CXX @@ -42,7 +42,7 @@ darwin*) ;; *) SHARED_FLAGS="--shared" - UNDEFINED_FLAGS="-u g_checksum_update -u g_hmac_update" + UNDEFINED_FLAGS="-u g_checksum_update -u g_hmac_update -u g_uri_unescape_segment" esac AC_SUBST(SHARED_FLAGS) AC_SUBST(UNDEFINED_FLAGS) @@ -277,57 +277,57 @@ AC_SUBST(YARA_CFLAGS) AC_SUBST(YARA_LIBS) -dnl Checks for GeoIP -AC_MSG_CHECKING(for GeoIP) -AC_ARG_WITH(GeoIP, -[ --with-GeoIP=DIR use GeoIP build directory], +dnl Checks for maxminddb +AC_MSG_CHECKING(for maxminddb) +AC_ARG_WITH(maxminddb, +[ --with-maxminddb=DIR use maxminddb build directory], [ case "$withval" in yes|no) AC_MSG_RESULT(no) ;; *) AC_MSG_RESULT($withval) - if test -f $withval/libGeoIP/GeoIP.h -a -f $withval/libGeoIP/.libs/libGeoIP.a; then + if test -f $withval/include/maxminddb.h -a -f $withval/src/.libs/libmaxminddb.a; then owd=`pwd` if cd $withval; then withval=`pwd`; cd $owd; fi - GEOIP_CFLAGS="-I$withval/libGeoIP" - GEOIP_LIBS="$withval/libGeoIP/.libs/libGeoIP.a" - elif test -f $withval/include/GeoIP.h -a -f $withval/lib/libGeoIP.a; then + MAXMINDDB_CFLAGS="-I$withval/include" + MAXMINDDB_LIBS="$withval/src/.libs/libmaxminddb.a" + elif test -f $withval/include/maxminddb.h -a -f $withval/lib/libmaxminddb.a; then owd=`pwd` if cd $withval; then withval=`pwd`; cd $owd; fi - GEOIP_CFLAGS="-I$withval/include" - GEOIP_LIBS="$withval/lib/libGeoIP.a" + MAXMINDDB_CFLAGS="-I$withval/include" + MAXMINDDB_LIBS="$withval/lib/libmaxminddb.a" else - AC_ERROR(GeoIP.h or GeoIP.a not found in $withval) + AC_ERROR(maxminddb.h or libmaxminddb.a not found in $withval) fi ;; esac ], [ - if test -f ${prefix}/include/GeoIP/GeoIP.h; then - GEOIP_CFLAGS="-I${prefix}/include/GeoIP" - GEOIP_LIBS="-L${exec_prefix}/lib -lGeoIP" - elif test -f /usr/include/GeoIP/GeoIP.h; then - GEOIP_CFLAGS="" - GEOIP_LIBS="-lGeoIP" + if test -f ${prefix}/include/maxminddb.h; then + MAXMINDDB_CFLAGS="-I${prefix}/include" + MAXMINDDB_LIBS="-L${exec_prefix}/lib -lmaxminddb" + elif test -f /usr/include/maxminddb.h; then + MAXMINDDB_CFLAGS="" + MAXMINDDB_LIBS="-lmaxminddb" else TMP=$LIBS - LIBS="-lGeoIP $LIBS" - AC_TRY_LINK([#include ], GeoIP_open(NULL, GEOIP_MEMORY_CACHE), LIBGEOIP_FOUND=1,LIBGEOIP_FOUND=0) + LIBS="-lmaxminddb $LIBS" + AC_TRY_LINK([#include ], MMDB_open(NULL, MMDB_MODE_MMAP, NULL), LIBMAXMINDDB_FOUND=1,LIBMAXMINDDB_FOUND=0) LIBS=$TMP - if test $LIBGEOIP_FOUND = 1 ; then - GEOIP_LIBS="-lGeoIP" + if test $LIBMAXMINDDB_FOUND = 1 ; then + MAXMINDDB_LIBS="-lmaxminddb" else - AC_ERROR(GeoIP not found) + AC_ERROR(maxminddb not found) fi fi AC_MSG_RESULT(yes) ]) -AC_SUBST(GEOIP_CFLAGS) -AC_SUBST(GEOIP_LIBS) +AC_SUBST(MAXMINDDB_CFLAGS) +AC_SUBST(MAXMINDDB_LIBS) dnl Checks for glib2, these are wrong diff --git a/db/db.pl b/db/db.pl index 187b5941e4..383900c829 100755 --- a/db/db.pl +++ b/db/db.pl @@ -52,7 +52,7 @@ use POSIX; use strict; -my $VERSION = 37; +my $VERSION = 50; my $verbose = 0; my $PREFIX = ""; my $NOCHANGES = 0; @@ -267,42 +267,6 @@ sub esAlias esPost("/_aliases", '{ "actions": [ { "' . $cmd . '": { "index": "' . $PREFIX . $index . '", "alias" : "'. $PREFIX . $alias .'" } } ] }', 1); } -################################################################################ -sub tagsCreate -{ - my $settings = ' -{ - "settings": { - "number_of_shards": 1, - "number_of_replicas": 0, - "auto_expand_replicas": "0-3" - } -}'; - - print "Creating tags_v3 index\n" if ($verbose > 0); - esPut("/${PREFIX}tags_v3", $settings); - esAlias("add", "tags_v3", "tags"); - tagsUpdate(); -} - -################################################################################ -sub tagsUpdate -{ - my $mapping = ' -{ - "tag": { - "dynamic": "strict", - "properties": { - "n": { - "type": "integer" - } - } - } -}'; - - print "Setting tags_v3 mapping\n" if ($verbose > 0); - esPut("/${PREFIX}tags_v3/tag/_mapping", $mapping); -} ################################################################################ sub sequenceCreate { @@ -390,15 +354,13 @@ sub filesUpdate "type": "long" }, "node": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "first": { "type": "long" }, "name": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "filesize": { "type": "long" @@ -455,12 +417,10 @@ sub statsUpdate ], "properties": { "hostname": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "nodeName": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "currentTime": { "type": "date", @@ -521,8 +481,7 @@ sub dstatsUpdate ], "properties": { "nodeName": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "interval": { "type": "short" @@ -568,7 +527,7 @@ sub fieldsUpdate "string_template": { "match_mapping_type": "string", "mapping": { - "index": "not_analyzed" + "type": "keyword" } } } @@ -585,6 +544,7 @@ sub fieldsUpdate "help": "Search all ip fields", "type": "ip", "dbField": "ipall", + "dbField2": "ipall", "portField": "portall", "noFacet": "true" }'); @@ -594,6 +554,7 @@ sub fieldsUpdate "help": "Search all port fields", "type": "integer", "dbField": "portall", + "dbField2": "portall", "regex": "(^port\\\\.(?:(?!\\\\.cnt$).)*$|\\\\.port$)" }'); esPost("/${PREFIX}fields_v1/field/rir", '{ @@ -601,7 +562,8 @@ sub fieldsUpdate "group": "general", "help": "Search all rir fields", "type": "uptermfield", - "dbField": "all", + "dbField": "rirall", + "dbField2": "rirall", "regex": "(^rir\\\\.(?:(?!\\\\.cnt$).)*$|\\\\.rir$)" }'); esPost("/${PREFIX}fields_v1/field/country", '{ @@ -609,23 +571,26 @@ sub fieldsUpdate "group": "general", "help": "Search all country fields", "type": "uptermfield", - "dbField": "all", + "dbField": "geoall", + "dbField2": "geoall", "regex": "(^country\\\\.(?:(?!\\\\.cnt$).)*$|\\\\.country$)" }'); esPost("/${PREFIX}fields_v1/field/asn", '{ "friendlyName": "All ASN fields", "group": "general", "help": "Search all ASN fields", - "type": "textfield", - "dbField": "all", + "type": "termfield", + "dbField": "asnall", + "dbField2": "asnall", "regex": "(^asn\\\\.(?:(?!\\\\.cnt$).)*$|\\\\.asn$)" }'); esPost("/${PREFIX}fields_v1/field/host", '{ "friendlyName": "All Host fields", "group": "general", "help": "Search all Host fields", - "type": "lotextfield", - "dbField": "all", + "type": "lotermfield", + "dbField": "hostall", + "dbField2": "hostall", "regex": "(^host\\\\.(?:(?!\\\\.cnt$).)*$|\\\\.host$)" }'); esPost("/${PREFIX}fields_v1/field/ip.src", '{ @@ -634,7 +599,9 @@ sub fieldsUpdate "help": "Source IP", "type": "ip", "dbField": "a1", + "dbField2": "srcIp", "portField": "p1", + "portField2": "srcPort", "category": "ip" }'); esPost("/${PREFIX}fields_v1/field/port.src", '{ @@ -643,14 +610,16 @@ sub fieldsUpdate "help": "Source Port", "type": "integer", "dbField": "p1", + "dbField2": "srcPort", "category": "port" }'); esPost("/${PREFIX}fields_v1/field/asn.src", '{ "friendlyName": "Src ASN", "group": "general", "help": "GeoIP ASN string calculated from the source IP", - "type": "textfield", + "type": "termfield", "dbField": "as1", + "dbField2": "srcASN", "rawField": "rawas1", "category": "asn" }'); @@ -660,6 +629,7 @@ sub fieldsUpdate "help": "Source Country", "type": "uptermfield", "dbField": "g1", + "dbField2": "srcGEO", "category": "country" }'); esPost("/${PREFIX}fields_v1/field/rir.src", '{ @@ -668,6 +638,7 @@ sub fieldsUpdate "help": "Source RIR", "type": "uptermfield", "dbField": "rir1", + "dbField2": "srcRIR", "category": "rir" }'); esPost("/${PREFIX}fields_v1/field/ip.dst", '{ @@ -676,7 +647,9 @@ sub fieldsUpdate "help": "Destination IP", "type": "ip", "dbField": "a2", + "dbField2": "dstIp", "portField": "p2", + "portField2": "dstPort", "category": "ip" }'); esPost("/${PREFIX}fields_v1/field/port.dst", '{ @@ -685,14 +658,16 @@ sub fieldsUpdate "help": "Source Port", "type": "integer", "dbField": "p2", + "dbField2": "dstPort", "category": "port" }'); esPost("/${PREFIX}fields_v1/field/asn.dst", '{ "friendlyName": "Dst ASN", "group": "general", "help": "GeoIP ASN string calculated from the destination IP", - "type": "textfield", + "type": "termfield", "dbField": "as2", + "dbField2": "dstASN", "rawField": "rawas2", "category": "asn" }'); @@ -702,6 +677,7 @@ sub fieldsUpdate "help": "Destination Country", "type": "uptermfield", "dbField": "g2", + "dbField2": "dstGEO", "category": "country" }'); esPost("/${PREFIX}fields_v1/field/rir.dst", '{ @@ -710,6 +686,7 @@ sub fieldsUpdate "help": "Destination RIR", "type": "uptermfield", "dbField": "rir2", + "dbField2": "dstRIR", "category": "rir" }'); esPost("/${PREFIX}fields_v1/field/bytes", '{ @@ -717,63 +694,72 @@ sub fieldsUpdate "group": "general", "help": "Total number of raw bytes sent AND received in a session", "type": "integer", - "dbField": "by" + "dbField": "by", + "dbField2": "totBytes" }'); esPost("/${PREFIX}fields_v1/field/bytes.src", '{ "friendlyName": "Src Bytes", "group": "general", "help": "Total number of raw bytes sent by source in a session", "type": "integer", - "dbField": "by1" + "dbField": "by1", + "dbField2": "srcBytes" }'); esPost("/${PREFIX}fields_v1/field/bytes.dst", '{ "friendlyName": "Dst Bytes", "group": "general", "help": "Total number of raw bytes sent by destination in a session", "type": "integer", - "dbField": "by2" + "dbField": "by2", + "dbField2": "dstBytes" }'); esPost("/${PREFIX}fields_v1/field/databytes", '{ "friendlyName": "Data bytes", "group": "general", "help": "Total number of data bytes sent AND received in a session", "type": "integer", - "dbField": "db" + "dbField": "db", + "dbField2": "totDataBytes" }'); esPost("/${PREFIX}fields_v1/field/databytes.src", '{ "friendlyName": "Src data bytes", "group": "general", "help": "Total number of data bytes sent by source in a session", "type": "integer", - "dbField": "db1" + "dbField": "db1", + "dbField2": "srcDataBytes" }'); esPost("/${PREFIX}fields_v1/field/databytes.dst", '{ "friendlyName": "Dst data bytes", "group": "general", "help": "Total number of data bytes sent by destination in a session", "type": "integer", - "dbField": "db2" + "dbField": "db2", + "dbField2": "dstDataBytes" }'); esPost("/${PREFIX}fields_v1/field/packets", '{ "friendlyName": "Packets", "group": "general", "help": "Total number of packets sent AND received in a session", "type": "integer", - "dbField": "pa" + "dbField": "pa", + "dbField2": "totPackets" }'); esPost("/${PREFIX}fields_v1/field/packets.src", '{ "friendlyName": "Src Packets", "group": "general", "help": "Total number of packets sent by source in a session", "type": "integer", - "dbField": "pa1" + "dbField": "pa1", + "dbField2": "srcPackets" }'); esPost("/${PREFIX}fields_v1/field/packets.dst", '{ "friendlyName": "Dst Packets", "group": "general", "help": "Total number of packets sent by destination in a session", "type": "integer", - "dbField": "pa2" + "dbField": "pa2", + "dbField2": "dstPackets" }'); esPost("/${PREFIX}fields_v1/field/ip.protocol", '{ "friendlyName": "IP Protocol", @@ -781,6 +767,7 @@ sub fieldsUpdate "help": "IP protocol number or friendly name", "type": "lotermfield", "dbField": "pr", + "dbField2": "ipProtocol", "transform": "ipProtocolLookup" }'); esPost("/${PREFIX}fields_v1/field/id", '{ @@ -789,6 +776,7 @@ sub fieldsUpdate "help": "Moloch ID for the session", "type": "termfield", "dbField": "_id", + "dbField2": "_id", "noFacet": "true" }'); @@ -797,21 +785,24 @@ sub fieldsUpdate "group": "general", "help": "Moloch ID of the first session in a multi session stream", "type": "termfield", - "dbField": "ro" + "dbField": "ro", + "dbField2": "rootId" }'); esPost("/${PREFIX}fields_v1/field/node", '{ "friendlyName": "Moloch Node", "group": "general", "help": "Moloch node name the session was recorded on", "type": "termfield", - "dbField": "no" + "dbField": "no", + "dbField2": "node" }'); esPost("/${PREFIX}fields_v1/field/file", '{ "friendlyName": "Filename", "group": "general", "help": "Moloch offline pcap filename", "type": "fileand", - "dbField": "fileand" + "dbField": "fileand", + "dbField2": "fileand" }'); esPost("/${PREFIX}fields_v1/field/payload8.src.hex", '{ "friendlyName": "Payload Src Hex", @@ -819,6 +810,7 @@ sub fieldsUpdate "help": "First 8 bytes of source payload in hex", "type": "lotermfield", "dbField": "fb1", + "dbField2": "srcPayload8", "aliases": ["payload.src"] }'); esPost("/${PREFIX}fields_v1/field/payload8.src.utf8", '{ @@ -827,6 +819,7 @@ sub fieldsUpdate "help": "First 8 bytes of source payload in utf8", "type": "termfield", "dbField": "fb1", + "dbField2": "srcPayload8", "transform": "utf8ToHex", "noFacet": "true" }'); @@ -836,6 +829,7 @@ sub fieldsUpdate "help": "First 8 bytes of destination payload in hex", "type": "lotermfield", "dbField": "fb2", + "dbField2": "dstPayload8", "aliases": ["payload.dst"] }'); esPost("/${PREFIX}fields_v1/field/payload8.dst.utf8", '{ @@ -844,6 +838,7 @@ sub fieldsUpdate "help": "First 8 bytes of destination payload in utf8", "type": "termfield", "dbField": "fb2", + "dbField2": "dstPayload8", "transform": "utf8ToHex", "noFacet": "true" }'); @@ -853,6 +848,7 @@ sub fieldsUpdate "help": "First 8 bytes of payload in hex", "type": "lotermfield", "dbField": "fballhex", + "dbField2": "fballhex", "regex": "^payload8.(src|dst).hex$" }'); esPost("/${PREFIX}fields_v1/field/payload8.utf8", '{ @@ -861,6 +857,7 @@ sub fieldsUpdate "help": "First 8 bytes of payload in hex", "type": "lotermfield", "dbField": "fballutf8", + "dbField2": "fballutf8", "regex": "^payload8.(src|dst).utf8$" }'); esPost("/${PREFIX}fields_v1/field/scrubbed.by", '{ @@ -868,7 +865,8 @@ sub fieldsUpdate "group": "general", "help": "SPI data was scrubbed by", "type": "lotermfield", - "dbField": "scrubby" + "dbField": "scrubby", + "dbField2": "scrubby" }'); esPost("/${PREFIX}fields_v1/field/view", '{ "friendlyName": "View Name", @@ -876,6 +874,7 @@ sub fieldsUpdate "help": "Moloch view name", "type": "viewand", "dbField": "viewand", + "dbField2": "viewand", "noFacet": "true" }'); esPost("/${PREFIX}fields_v1/field/starttime", '{ @@ -883,14 +882,18 @@ sub fieldsUpdate "group": "general", "help": "Session Start Time", "type": "seconds", - "dbField": "fp" + "type2": "date", + "dbField": "fp", + "dbField2": "firstPacket" }'); esPost("/${PREFIX}fields_v1/field/stoptime", '{ "friendlyName": "Stop Time", "group": "general", "help": "Session Stop Time", "type": "seconds", - "dbField": "lp" + "type2": "date", + "dbField": "lp", + "dbField2": "lastPacket" }'); } @@ -921,8 +924,7 @@ sub queriesUpdate "dynamic": "strict", "properties": { "name": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "enabled": { "type": "boolean" @@ -937,20 +939,16 @@ sub queriesUpdate "type": "long" }, "query": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "action": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "creator": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "tags": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" } } } @@ -962,47 +960,38 @@ sub queriesUpdate } ################################################################################ -sub sessionsUpdate +sub sessions2Update { my $mapping = ' { "session": { + "_meta": { + "molochDbVersion": ' . $VERSION . ' + }, "_all": {"enabled": "false"}, "dynamic": "true", "dynamic_templates": [ { - "template_hdrs": { - "path_match": "hdrs.*", - "match_mapping_type": "string", + "template_ip_end": { + "match": "*Ip", "mapping": { - "type": "string", - "index": "no", - "fields": { - "snow": {"type": "string", "analyzer" : "snowball"}, - "raw": {"type": "string", "index" : "not_analyzed"} - } + "type": "ip" } } - }, { - "template_georir": { - "match_pattern": "regex", - "path_match": ".*-(geo|rir|term)$", - "match_mapping_type": "string", + }, + { + "template_ip_alone": { + "match": "ip", "mapping": { - "type": "string", - "index": "not_analyzed" + "type": "ip" } } - }, { + }, + { "template_string": { "match_mapping_type": "string", "mapping": { - "type": "string", - "index": "no", - "fields": { - "snow" : {"type": "string", "analyzer" : "snowball"}, - "raw" : {"type": "string", "index" : "not_analyzed"} - } + "type": "keyword" } } } @@ -1017,597 +1006,24 @@ sub sessionsUpdate "lastPacket": { "type": "date" }, - "ipSrc": { - "type": "ip" - }, - "portSrc": { - "type": "integer" - }, - "ipDst": { - "type": "ip" - }, - "portDst": { - "type": "integer" - }, - "us": { - "type": "string", - "analyzer": "url_analyzer", - "copy_to": "rawus", - "norms": {"enabled": "false"} - }, - "rawus": { - "type": "string", - "index": "not_analyzed" - }, - "uscnt": { - "type": "integer" - }, - "ua": { - "type": "string", - "analyzer": "snowball", - "copy_to": "rawua", - "norms": {"enabled": "false"} - }, - "rawua": { - "type": "string", - "index": "not_analyzed" - }, - "uacnt": { - "type": "integer" - }, - "ps": { - "type": "long", - "index": "no" - }, - "psl": { - "type": "integer", - "index": "no" - }, - "fs": { - "type": "long" - }, - "lp": { - "type": "long", - "doc_values": "true" - }, - "lpd": { - "type": "date", - "doc_values": "true" - }, - "fp": { - "type": "long", - "doc_values": "true" - }, - "fpd": { - "type": "date", - "doc_values": "true" - }, - "a1": { - "type": "long", - "doc_values": "true" - }, - "g1": { - "type": "string", - "index": "not_analyzed" - }, - "as1": { - "type": "string", - "analyzer": "snowball", - "copy_to": "rawas1", - "norms": {"enabled": "false"} - }, - "rawas1": { - "type": "string", - "index": "not_analyzed" - }, - "rir1": { - "type": "string", - "index": "not_analyzed" - }, - "p1": { - "type": "integer", - "doc_values": "true" - }, - "fb1": { - "type": "string", - "index": "not_analyzed" - }, - "a2": { + "packetPosArray": { "type": "long", - "doc_values": "true" - }, - "g2": { - "type": "string", - "index": "not_analyzed" - }, - "as2": { - "type": "string", - "analyzer": "snowball", - "copy_to": "rawas2", - "norms": {"enabled": "false"} - }, - "rawas2": { - "type": "string", - "index": "not_analyzed" + "index": false }, - "rir2": { - "type": "string", - "index": "not_analyzed" - }, - "p2": { + "packetLenArray": { "type": "integer", - "doc_values": "true" - }, - "fb2": { - "type": "string", - "index": "not_analyzed" - }, - "xff": { - "type": "long" - }, - "xffcnt": { - "type": "integer" - }, - "xffscnt": { - "type": "integer" - }, - "gxff": { - "type": "string", - "index": "not_analyzed" - }, - "asxff": { - "type": "string", - "analyzer": "snowball", - "copy_to": "rawasxff", - "norms": {"enabled": "false"} - }, - "rawasxff": { - "type": "string", - "index": "not_analyzed" - }, - "rirxff": { - "type": "string", - "index": "not_analyzed" - }, - "hmd5cnt": { - "type": "short" - }, - "hmd5": { - "type": "string", - "index": "not_analyzed" - }, - "dnshocnt": { - "type": "integer" - }, - "dnsho": { - "type": "string", - "index": "not_analyzed" - }, - "dnsip": { - "type": "long" - }, - "dnsipcnt": { - "type": "integer" - }, - "gdnsip": { - "type": "string", - "index": "not_analyzed" - }, - "asdnsip": { - "type": "string", - "analyzer": "snowball", - "copy_to": "rawasdnsip", - "norms": {"enabled": "false"} - }, - "rawasdnsip": { - "type": "string", - "index": "not_analyzed" - }, - "rirdnsip": { - "type": "string", - "index": "not_analyzed" - }, - "pr": { - "type": "short" - }, - "pa": { - "type": "integer" - }, - "pa1": { - "type": "integer" - }, - "pa2": { - "type": "integer" - }, - "by": { - "type": "long" - }, - "by1": { - "type": "long" - }, - "by2": { - "type": "long" - }, - "db": { - "type": "long" - }, - "db1": { - "type": "long" - }, - "db2": { - "type": "long" - }, - "ro": { - "type": "string", - "index": "not_analyzed" - }, - "no": { - "type": "string", - "index": "not_analyzed" - }, - "ho": { - "type": "string", - "index": "not_analyzed" - }, - "hocnt": { - "type": "integer" - }, - "ta": { - "type": "integer" - }, - "tacnt": { - "type": "integer" - }, - "hh": { - "type": "integer" - }, - "hh1": { - "type": "integer" - }, - "hh2": { - "type": "integer" - }, - "hh1cnt": { - "type": "integer" - }, - "hh2cnt": { - "type": "integer" - }, - "hsver": { - "type": "string", - "index": "not_analyzed" - }, - "hsvercnt": { - "type": "integer" - }, - "hdver": { - "type": "string", - "index": "not_analyzed" + "index": false }, - "hdvercnt": { - "type": "integer" - }, - "hpath": { - "type": "string", - "index": "not_analyzed" - }, - "hpathcnt": { - "type": "integer" - }, - "hkey": { - "type": "string", - "index": "not_analyzed" - }, - "hkeycnt": { - "type": "integer" - }, - "hval": { - "type": "string", - "index": "not_analyzed" - }, - "hvalcnt": { - "type": "integer" - }, - "user": { - "type": "string", - "index": "not_analyzed" - }, - "usercnt": { - "type": "integer" - }, - "tls": { + "cert": { "type": "object", - "dynamic": "strict", "properties": { - "iCn": { - "type": "string", - "index": "not_analyzed" - }, - "iOn": { - "type": "string", - "analyzer": "snowball", - "norms": {"enabled": "false"}, - "fields": { - "rawiOn": {"type": "string", "index": "not_analyzed"} - } - }, - "sCn": { - "type": "string", - "index": "not_analyzed" - }, - "sOn": { - "type": "string", - "analyzer": "snowball", - "norms": {"enabled": "false"}, - "fields": { - "rawsOn": {"type": "string", "index": "not_analyzed"} - } - }, - "sn": { - "type": "string", - "index": "not_analyzed" - }, - "alt": { - "type": "string", - "index": "not_analyzed" - }, - "altcnt": { - "type": "integer" - }, "notBefore": { - "type": "long" + "type": "date" }, "notAfter": { - "type": "long" - }, - "diffDays": { - "type": "integer" - }, - "hash": { - "type": "string", - "index": "not_analyzed" + "type": "date" } } - }, - "tlscnt": { - "type": "integer" - }, - "sshkey": { - "type": "string", - "index": "not_analyzed" - }, - "sshkeycnt": { - "type": "short" - }, - "sshver": { - "type": "string", - "index": "not_analyzed" - }, - "sshvercnt": { - "type": "short" - }, - "euacnt": { - "type": "short" - }, - "eua": { - "type": "string", - "analyzer": "snowball", - "copy_to": "raweua", - "norms": {"enabled": "false"} - }, - "raweua": { - "type": "string", - "index": "not_analyzed" - }, - "esubcnt": { - "type": "short" - }, - "esub": { - "type": "string", - "analyzer": "snowball", - "copy_to": "rawesub", - "norms": {"enabled": "false"} - }, - "rawesub": { - "type": "string", - "index": "not_analyzed" - }, - "eidcnt": { - "type": "short" - }, - "eid": { - "type": "string", - "index": "not_analyzed" - }, - "ectcnt": { - "type": "short" - }, - "ect": { - "type": "string", - "index": "not_analyzed" - }, - "emvcnt": { - "type": "short" - }, - "emv": { - "type": "string", - "index": "not_analyzed" - }, - "efncnt": { - "type": "short" - }, - "efn": { - "type": "string", - "index": "not_analyzed" - }, - "emd5cnt": { - "type": "short" - }, - "emd5": { - "type": "string", - "index": "not_analyzed" - }, - "esrccnt": { - "type": "short" - }, - "esrc": { - "type": "string", - "index": "not_analyzed" - }, - "edstcnt": { - "type": "short" - }, - "edst": { - "type": "string", - "index": "not_analyzed" - }, - "eho": { - "type": "string", - "index": "not_analyzed" - }, - "ehocnt": { - "type": "integer" - }, - "eip": { - "type": "long" - }, - "eipcnt": { - "type": "integer" - }, - "ehh": { - "type": "string", - "index": "not_analyzed" - }, - "ehhcnt": { - "type": "integer" - }, - "geip": { - "type": "string", - "index": "not_analyzed" - }, - "aseip": { - "type": "string", - "analyzer": "snowball", - "copy_to": "rawaseip", - "norms": {"enabled": "false"} - }, - "rawaseip": { - "type": "string", - "index": "not_analyzed" - }, - "rireip": { - "type": "string", - "index": "not_analyzed" - }, - "ircnck": { - "type": "string", - "index": "not_analyzed" - }, - "ircnckcnt": { - "type": "integer" - }, - "ircch": { - "type": "string", - "index": "not_analyzed" - }, - "ircchcnt": { - "type": "integer" - }, - "hdrs": { - "type": "object", - "dynamic": "true" - }, - "plugin": { - "type": "object", - "dynamic": "true" - }, - "scrubat": { - "type": "date" - }, - "scrubby": { - "type": "string", - "index": "not_analyzed" - }, - "smbdmcnt": { - "type": "short" - }, - "smbdm": { - "type": "string", - "index": "not_analyzed" - }, - "smbfncnt": { - "type": "short" - }, - "smbfn": { - "type": "string", - "index": "not_analyzed" - }, - "smbhocnt": { - "type": "short" - }, - "smbho": { - "type": "string", - "index": "not_analyzed" - }, - "smboscnt": { - "type": "short" - }, - "smbos": { - "type": "string", - "index": "not_analyzed" - }, - "smbshcnt": { - "type": "short" - }, - "smbsh": { - "type": "string", - "index": "not_analyzed" - }, - "smbusercnt": { - "type": "short" - }, - "smbuser": { - "type": "string", - "index": "not_analyzed" - }, - "smbvercnt": { - "type": "short" - }, - "smbver": { - "type": "string", - "index": "not_analyzed" - }, - "socksip": { - "type": "long" - }, - "gsocksip": { - "type": "string", - "index": "not_analyzed" - }, - "assocksip": { - "type": "string", - "analyzer": "snowball", - "copy_to": "rawassocksip", - "norms": {"enabled": "false"} - }, - "rawassocksip": { - "type": "string", - "index": "not_analyzed" - }, - "rirsocksip": { - "type": "string", - "index": "not_analyzed" - }, - "sockspo": { - "type": "integer" - }, - "socksuser": { - "type": "string", - "index": "not_analyzed" - }, - "socksho": { - "type": "string", - "index": "not_analyzed" } } } @@ -1619,34 +1035,24 @@ sub sessionsUpdate my $template = ' { - "template": "' . $PREFIX . 'sessions-*", + "template": "' . $PREFIX . 'sessions2-*", "settings": { "index": { "routing.allocation.total_shards_per_node": ' . $shardsPerNode . ', "refresh_interval": "60s", "number_of_shards": ' . $SHARDS . ', - "number_of_replicas": ' . $REPLICAS . ', - "analysis": { - "analyzer": { - "url_analyzer": { - "type": "custom", - "tokenizer": "pattern", - "filter": ["lowercase"] - } - } - } + "number_of_replicas": ' . $REPLICAS . ' } }, "mappings":' . $mapping . ' }'; print "Creating sessions template\n" if ($verbose > 0); - #print "$template\n"; - esPut("/_template/${PREFIX}sessions_template", $template); + esPut("/_template/${PREFIX}sessions2_template", $template); - my $indices = esGet("/${PREFIX}sessions-*/_alias", 1); + my $indices = esGet("/${PREFIX}sessions2-*/_alias", 1); - print "Updating sessions mapping for ", scalar(keys %{$indices}), " indices\n" if (scalar(keys %{$indices}) != 0); + print "Updating sessions2 mapping for ", scalar(keys %{$indices}), " indices\n" if (scalar(keys %{$indices}) != 0); foreach my $i (keys %{$indices}) { progress("$i "); esPut("/$i/session/_mapping", $mapping, 1); @@ -1666,24 +1072,19 @@ sub historyUpdate "dynamic": "strict", "properties": { "uiPage": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "userId": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "method": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "api": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "expression": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "view": { "type": "object", @@ -1696,8 +1097,7 @@ sub historyUpdate "type": "integer" }, "query": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "queryTime": { "type": "integer" @@ -1773,12 +1173,10 @@ sub usersUpdate "dynamic": "strict", "properties": { "userId": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "userName": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "enabled": { "type": "boolean" @@ -1799,12 +1197,10 @@ sub usersUpdate "type": "boolean" }, "passStore": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "expression": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "settings": { "type": "object", @@ -1903,6 +1299,19 @@ sub dbESVersion { ################################################################################ sub dbVersion { my ($loud) = @_; + my $version; + + $version = esGet("/_template/${PREFIX}sessions2_template?filter_path=**._meta", 1); + + if (defined $version && + exists $version->{"${PREFIX}sessions2_template"} && + exists $version->{"${PREFIX}sessions2_template"}->{mappings}->{session} && + exists $version->{"${PREFIX}sessions2_template"}->{mappings}->{session}->{_meta} && + exists $version->{"${PREFIX}sessions2_template"}->{mappings}->{session}->{_meta}->{molochDbVersion} + ) { + $main::versionNumber = $version->{"${PREFIX}sessions2_template"}->{mappings}->{session}->{_meta}->{molochDbVersion}; + return; + } my $version = esGet("/${PREFIX}dstats/version/version", 1); @@ -1957,12 +1366,12 @@ sub dbCheck { $main::esVersion = int($parts[0]*100*100) + int($parts[1]*100) + int($parts[2]); if ($main::esVersion < 50500 || - $main::esVersion >= 60000) + $main::esVersion >= 70000) { print("Currently using Elasticsearch version ", $esversion->{version}->{number}, " which isn't supported\n", "* < 5.5.0 are not supported\n", "* 5.6.x is recommended\n", - "* >= 6.x are not supported\n", + "* >= 6.x is supported but not well tested\n", "\n", "Instructions: https://github.com/aol/moloch/wiki/FAQ#How_do_I_upgrade_elasticsearch\n", "Make sure to restart any viewer or capture after upgrading!\n" @@ -2039,7 +1448,7 @@ sub progress { ################################################################################ sub optimizeOther { print "Optimizing Admin Indices\n"; - foreach my $i ("${PREFIX}stats_v2", "${PREFIX}dstats_v2", "${PREFIX}files_v4", "${PREFIX}sequence_v1", "${PREFIX}tags_v3", "${PREFIX}users_v4") { + foreach my $i ("${PREFIX}stats_v2", "${PREFIX}dstats_v2", "${PREFIX}files_v4", "${PREFIX}sequence_v1", "${PREFIX}users_v4") { progress("$i "); esPost("/$i/_forcemerge?max_num_segments=1", "", 1); esPost("/$i/_upgrade", "", 1); @@ -2121,10 +1530,10 @@ sub parseArgs { showHelp("Invalid expire ") if ($ARGV[2] !~ /^(hourly|daily|weekly|monthly)$/); # First handle sessions expire - my $indices = esGet("/${PREFIX}sessions-*/_alias", 1); + my $indices = esGet("/${PREFIX}sessions2-*/_alias", 1); my $endTime = time(); - my $endTimeIndex = time2index($ARGV[2], "sessions-", $endTime); + my $endTimeIndex = time2index($ARGV[2], "sessions2-", $endTime); delete $indices->{$endTimeIndex}; my @startTime = gmtime; @@ -2143,7 +1552,7 @@ sub parseArgs { my $optimizecnt = 0; my $startTime = mktime(@startTime); while ($startTime <= $endTime) { - my $iname = time2index($ARGV[2], "sessions-", $startTime); + my $iname = time2index($ARGV[2], "sessions2-", $startTime); if (exists $indices->{$iname} && $indices->{$iname}->{OPTIMIZEIT} != 1) { $indices->{$iname}->{OPTIMIZEIT} = 1; $optimizecnt++; @@ -2175,7 +1584,6 @@ sub parseArgs { esDelete("/$i", 1); } } - # Now figure out history expire my $hindices = esGet("/${PREFIX}history_v1-*/_alias", 1); @@ -2205,9 +1613,10 @@ sub parseArgs { esDelete("/$i", 1); } } + esPost("/_flush/synced", "", 1); exit 0; } elsif ($ARGV[1] eq "optimize") { - my $indices = esGet("/${PREFIX}sessions-*/_alias", 1); + my $indices = esGet("/${PREFIX}sessions2-*/_alias", 1); dbESVersion(); $main::userAgent->timeout(3600); @@ -2218,6 +1627,7 @@ sub parseArgs { esPost("/$i/_forcemerge?max_num_segments=4", "", 1); esPost("/$i/_upgrade", "", 1); } + esPost("/_flush/synced", "", 1); print "\n"; exit 0; } elsif ($ARGV[1] eq "info") { @@ -2228,9 +1638,9 @@ sub parseArgs { my $sessions = 0; my $sessionsBytes = 0; - my @sessions = grep /^${PREFIX}sessions-/, keys %{$status->{indices}}; + my @sessions = grep /^${PREFIX}sessions2-/, keys %{$status->{indices}}; foreach my $index (@sessions) { - next if ($index !~ /^${PREFIX}sessions-/); + next if ($index !~ /^${PREFIX}sessions2-/); $sessions += $status->{indices}->{$index}->{primaries}->{docs}->{count}; $sessionsBytes += $status->{indices}->{$index}->{primaries}->{store}->{size_in_bytes}; } @@ -2255,7 +1665,7 @@ sub parseArgs { printf "DB Version: %10s\n", $main::versionNumber; printf "ES Nodes: %10s/%s\n", commify(dataNodes($nodes->{nodes})), commify(scalar(keys %{$nodes->{nodes}})); printf "Session Indices: %10s\n", commify(scalar(@sessions)); - printf "Sessions: %10s (%s bytes)\n", commify($sessions), commify($sessionsBytes); + printf "Sessions2: %10s (%s bytes)\n", commify($sessions), commify($sessionsBytes); if (scalar(@sessions) > 0) { printf "Session Density: %10s (%s bytes)\n", commify(int($sessions/(scalar(keys %{$nodes->{nodes}})*scalar(@sessions)))), commify(int($sessionsBytes/(scalar(keys %{$nodes->{nodes}})*scalar(@sessions)))); @@ -2268,8 +1678,6 @@ sub parseArgs { } printIndex($status, "files_v4"); printIndex($status, "files_v3"); - printIndex($status, "tags_v3"); - printIndex($status, "tags_v2"); printIndex($status, "users_v4"); printIndex($status, "users_v3"); exit 0; @@ -2507,12 +1915,11 @@ sub dataNodes sleep(1); print "Creating\n"; - tagsCreate(); sequenceCreate(); filesCreate(); statsCreate(); dstatsCreate(); - sessionsUpdate(); + sessions2Update(); fieldsCreate(); historyUpdate(); if ($ARGV[1] =~ "init") { @@ -2525,8 +1932,8 @@ sub dataNodes # Remaing is upgrade or upgradenoprompt # For really old versions don't support upgradenoprompt - if ($main::versionNumber < 19) { - print "No longer supported. Please upgrade to Moloch 0.17.0 first. (Db version $main::VersionNumber)\n\n"; + if ($main::versionNumber < 37) { + print "No longer supported. Please upgrade to Moloch 0.20.x first. (Db version $main::VersionNumber)\n\n"; exit 1; } @@ -2540,58 +1947,21 @@ sub dataNodes print "Starting Upgrade\n"; - if ($main::versionNumber <= 31) { - dbCheckForActivity(); - esGet("/_flush", 0); - esGet("/_refresh", 0); - - sequenceUpgrade(); - - if ($main::versionNumber < 20) { - queriesCreate(); - } else { - createAliasedFromNonAliased("queries", "queries_v1", \&queriesCreate); - } - - esDelete("/${PREFIX}tags_v1", 1); - createAliasedFromNonAliased("fields", "fields_v1", \&fieldsCreate); - createNewAliasesFromOld("tags", "tags_v3", "tags_v2", \&tagsCreate); - - esDelete("/${PREFIX}users_v1", 1); - esDelete("/${PREFIX}users_v2", 1); - createNewAliasesFromOld("users", "users_v4", "users_v3", \&usersCreate); - - esDelete("/${PREFIX}files_v1", 1); - esDelete("/${PREFIX}files_v2", 1); - createNewAliasesFromOld("files", "files_v4", "files_v3", \&filesCreate); - - if ($main::versionNumber <= 30) { - createNewAliasesFromOld("dstats", "dstats_v2", "dstats_v1", \&dstatsCreate); - createAliasedFromNonAliased("stats", "stats_v2", \&statsCreate); - } - - esDelete("/_template/${PREFIX}template_1", 1); - historyUpdate(); - sessionsUpdate(); - checkForOldIndices(); - } elsif ($main::versionNumber <= 33) { - createNewAliasesFromOld("stats", "stats_v2", "stats_v1", \&statsCreate); + if ($main::versionNumber <= 38) { + esDelete("/_template/${PREFIX}sessions_template", 1); usersUpdate(); historyUpdate(); - sessionsUpdate(); - checkForOldIndices(); - } elsif ($main::versionNumber <= 37) { - usersUpdate(); - historyUpdate(); - sessionsUpdate(); + sessions2Update(); checkForOldIndices(); + fieldsUpdate(); + } elsif ($main::versionNumber <= 50) { + sessions2Update(); } else { print "db.pl is hosed\n"; } - } print "Finished\n"; sleep 1; -esPost("/${PREFIX}dstats/version/version", "{\"version\": $VERSION}"); +#esPost("/${PREFIX}dstats/version/version", "{\"version\": $VERSION}"); diff --git a/easybutton-build.sh b/easybutton-build.sh index a3f97738d7..82990eb6b7 100755 --- a/easybutton-build.sh +++ b/easybutton-build.sh @@ -13,12 +13,12 @@ GLIB=2.54.3 YARA=3.7.1 -GEOIP=1.6.11 +MAXMIND=1.3.2 PCAP=1.8.1 CURL=7.58.0 LUA=5.3.4 DAQ=2.0.6 -NODE=6.12.3 +NODE=8.9.4 TDIR="/data/moloch" DOPFRING=0 @@ -72,7 +72,7 @@ MAKE=make # Installing dependencies echo "MOLOCH: Installing Dependencies" if [ -f "/etc/redhat-release" ]; then - sudo yum -y install wget curl pcre pcre-devel pkgconfig flex bison gcc-c++ zlib-devel e2fsprogs-devel openssl-devel file-devel make gettext libuuid-devel perl-JSON bzip2-libs bzip2-devel perl-libwww-perl libpng-devel xz libffi-devel readline-devel libtool libyaml-devel + sudo yum -y install wget curl pcre pcre-devel pkgconfig flex bison gcc-c++ zlib-devel e2fsprogs-devel openssl-devel file-devel make gettext libuuid-devel perl-JSON bzip2-libs bzip2-devel perl-libwww-perl libpng-devel xz libffi-devel readline-devel libtool libyaml-devel perl-Socket6 if [ $? -ne 0 ]; then echo "MOLOCH: yum failed" exit 1 @@ -80,7 +80,7 @@ if [ -f "/etc/redhat-release" ]; then fi if [ -f "/etc/debian_version" ]; then - sudo apt-get -y install wget curl libpcre3-dev uuid-dev libmagic-dev pkg-config g++ flex bison zlib1g-dev libffi-dev gettext libgeoip-dev make libjson-perl libbz2-dev libwww-perl libpng-dev xz-utils libffi-dev libssl-dev libreadline-dev libtool libyaml-dev dh-autoreconf + sudo apt-get -y install wget curl libpcre3-dev uuid-dev libmagic-dev pkg-config g++ flex bison zlib1g-dev libffi-dev gettext libgeoip-dev make libjson-perl libbz2-dev libwww-perl libpng-dev xz-utils libffi-dev libssl-dev libreadline-dev libtool libyaml-dev dh-autoreconf libsocket6-perl if [ $? -ne 0 ]; then echo "MOLOCH: apt-get failed" exit 1 @@ -143,35 +143,29 @@ else echo "MOLOCH: Not rebuilding yara" fi -# GeoIP -if [ ! -f "GeoIP-$GEOIP.tar.gz" ]; then - wget https://github.com/maxmind/geoip-api-c/releases/download/v$GEOIP/GeoIP-$GEOIP.tar.gz +# Maxmind +if [ ! -f "libmaxminddb-$MAXMIND.tar.gz" ]; then + wget https://github.com/maxmind/libmaxminddb/releases/download/$MAXMIND/libmaxminddb-$MAXMIND.tar.gz fi -if [ ! -f "GeoIP-$GEOIP/libGeoIP/.libs/libGeoIP.a" ]; then -tar zxf GeoIP-$GEOIP.tar.gz +if [ ! -f "libmaxminddb-$MAXMIND/src/.libs/libmaxminddb.a" ]; then + tar zxf libmaxminddb-$MAXMIND.tar.gz -# Crossing fingers, this is no longer needed -# Not sure why this is required on some platforms -# if [ -f "/usr/bin/libtoolize" ]; then -# (cd GeoIP-$GEOIP ; libtoolize -f) -# fi - - (cd GeoIP-$GEOIP ; ./configure --enable-static; $MAKE) + (cd libmaxminddb-$MAXMIND ; ./configure --enable-static; $MAKE) if [ $? -ne 0 ]; then echo "MOLOCH: $MAKE failed" exit 1 fi else - echo "MOLOCH: Not rebuilding libGeoIP" + echo "MOLOCH: Not rebuilding libmaxmind" fi # libpcap if [ ! -f "libpcap-$PCAP.tar.gz" ]; then wget http://www.tcpdump.org/release/libpcap-$PCAP.tar.gz fi -tar zxf libpcap-$PCAP.tar.gz if [ ! -f "libpcap-$PCAP/libpcap.a" ]; then + tar zxf libpcap-$PCAP.tar.gz echo "MOLOCH: Building libpcap"; (cd libpcap-$PCAP; ./configure --disable-dbus --disable-usb --disable-canusb --disable-bluetooth --with-snf=no; $MAKE) if [ $? -ne 0 ]; then @@ -238,8 +232,8 @@ fi # Now build moloch echo "MOLOCH: Building capture" cd .. -echo "./configure --prefix=$TDIR $PCAPBUILD --with-yara=thirdparty/yara/yara-$YARA --with-GeoIP=thirdparty/GeoIP-$GEOIP $WITHGLIB --with-curl=thirdparty/curl-$CURL --with-lua=thirdparty/lua-$LUA" -./configure --prefix=$TDIR $PCAPBUILD --with-yara=thirdparty/yara/yara-$YARA --with-GeoIP=thirdparty/GeoIP-$GEOIP $WITHGLIB --with-curl=thirdparty/curl-$CURL --with-lua=thirdparty/lua-$LUA +echo "./configure --prefix=$TDIR $PCAPBUILD --with-yara=thirdparty/yara/yara-$YARA --with-maxminddb=thirdparty/libmaxminddb-$MAXMIND $WITHGLIB --with-curl=thirdparty/curl-$CURL --with-lua=thirdparty/lua-$LUA" +./configure --prefix=$TDIR $PCAPBUILD --with-yara=thirdparty/yara/yara-$YARA --with-maxminddb=thirdparty/libmaxminddb-$MAXMIND $WITHGLIB --with-curl=thirdparty/curl-$CURL --with-lua=thirdparty/lua-$LUA if [ $DOCLEAN -eq 1 ]; then $MAKE clean diff --git a/package.json b/package.json new file mode 100644 index 0000000000..2df5add85e --- /dev/null +++ b/package.json @@ -0,0 +1,41 @@ +{ + "name": "moloch", + "version": "1.0.0", + "description": "", + "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "https://github.com/aol/moloch.git" + }, + "dependencies": { + "async": "^2.5.0", + "body-parser": "^1.18.2", + "bson": "^0.5.5", + "connect-timeout": "^1.7.0", + "console-stamp": "^0.2.2", + "csv": "^1.1.0", + "elasticsearch": "^13.3.0", + "express": "^4.16.1", + "font-awesome": "^4.7.0", + "glob": "^7.1.2", + "iniparser": "http://github.com/awick/node-iniparser/tarball/master", + "moment": "^2.19.1", + "morgan": "^1.9.0", + "pug": "^2.0.0", + "redis": "^2.6.2", + "request": "^2.75.0", + "request-promise": "^4.2.2", + "stylus": "^0.54.5", + "sqlite3": "^3.1.4", + "vue": "^2.5.13", + "vue-server-renderer": "^2.5.13", + "vue-router": "^3.0.1", + "unzip": "^0.1.11" + }, + "scripts": { + }, + "devDependencies": { + }, + "optionalDependencies": { + } +} diff --git a/parliament/.angular-cli.json b/parliament/.angular-cli.json index 6401267a4d..e4e3163d84 100644 --- a/parliament/.angular-cli.json +++ b/parliament/.angular-cli.json @@ -21,7 +21,7 @@ "prefix": "app", "styles": [ "../node_modules/bootstrap/dist/css/bootstrap.min.css", - "../node_modules/font-awesome/css/font-awesome.min.css", + "../../node_modules/font-awesome/css/font-awesome.min.css", "styles.css" ], "scripts": [], diff --git a/parliament/package.json b/parliament/package.json index 536a6d06cb..b935fa0d79 100644 --- a/parliament/package.json +++ b/parliament/package.json @@ -15,14 +15,8 @@ }, "dependencies": { "bcrypt": "^1.0.3", - "body-parser": "^1.18.2", - "express": "^4.16.2", - "font-awesome": "^4.7.0", - "glob": "^7.1.2", "jsonwebtoken": "^8.1.0", - "morgan": "^1.9.0", "notifme-sdk": "^1.6.0", - "request-promise": "^4.2.2", "serve-favicon": "^2.4.5" }, "devDependencies": { diff --git a/parliament/parliament.js b/parliament/parliament.js index 5a259bf7ba..725d3eb12a 100644 --- a/parliament/parliament.js +++ b/parliament/parliament.js @@ -85,12 +85,21 @@ const issueTypes = { i++; break; + case '--regressionTests': + app.set('regressionTests', 1); + break; + + case '--debug': + // Someday support debug :) + break; + case '-h': case '--help': help(); break; default: + console.log(`Unknown option ${appArgs[i]}`); help(); break; } @@ -105,6 +114,13 @@ const issueTypes = { app.set('file', file || './parliament.json'); }()); +if (!!app.get("regressionTests")) { + app.post('/shutdown', function(req, res) { + process.exit(0); + throw new Error("Exiting"); + }); +}; + // get the parliament file or create it if it doesn't exist let parliament; try { @@ -797,7 +813,8 @@ router.put('/settings', verifyToken, (req, res, next) => { // Get parliament with stats router.get('/parliament', (req, res, next) => { let parliamentClone = JSON.parse(JSON.stringify(parliament)); - if (parliamentClone.password) { parliamentClone.password = undefined; } + delete parliamentClone.settings + delete parliamentClone.password return res.json(parliamentClone); }); @@ -1206,6 +1223,13 @@ router.post('/testAlert', (req, res, next) => { }); +/* SIGNALS! ----------------------------------------------------------------- */ +// Explicit sigint handler for running under docker +// See https://github.com/nodejs/node/issues/4182 +process.on('SIGINT', function() { + process.exit(); +}); + /* LISTEN! ----------------------------------------------------------------- */ let server; if (app.get('keyFile') && app.get('certFile')) { diff --git a/parliament/src/styles.css b/parliament/src/styles.css index 53660f52db..19ab6b6630 100644 --- a/parliament/src/styles.css +++ b/parliament/src/styles.css @@ -1,4 +1,4 @@ -@import "~font-awesome/css/font-awesome.css"; +@import "../../node_modules/font-awesome/css/font-awesome.css"; /* general styles -------------------------------------- */ diff --git a/release/Configure b/release/Configure index 3bb5417e0e..6aea1fc040 100755 --- a/release/Configure +++ b/release/Configure @@ -139,7 +139,7 @@ chmod 0700 $MOLOCH_INSTALL_DIR/raw ################################################################################ if [ "$MOLOCH_LOCALELASTICSEARCH" == "yes" ]; then echo "Moloch - Downloading and installing demo Elasticsearch" - ES_VERSION=5.6.0 + ES_VERSION=5.6.7 mkdir $MOLOCH_INSTALL_DIR/data chown nobody $MOLOCH_INSTALL_DIR/data if [ -f "/etc/redhat-release" ]; then diff --git a/release/Vagrantfile b/release/Vagrantfile index 1e30896d12..de511b41bc 100644 --- a/release/Vagrantfile +++ b/release/Vagrantfile @@ -6,7 +6,7 @@ Vagrant.configure("2") do |config| config.vm.provider "virtualbox" do |vb| - vb.customize ["modifyvm", :id, "--memory", "2500"] + vb.customize ["modifyvm", :id, "--memory", "4000"] end config.vm.define "ubuntu-14.04" do |ubuntu1404| @@ -25,9 +25,9 @@ Vagrant.configure("2") do |config| centos7.vm.box = "bento/centos-7.2" end - config.vm.define "debian8" do |debian8| - debian8.vm.box = "debian/jessie64" - end +# config.vm.define "debian8" do |debian8| +# debian8.vm.box = "debian/jessie64" +# end config.vm.provision "ansible" do |ansible| diff --git a/release/build.yml b/release/build.yml index 7d260e8935..f406823663 100644 --- a/release/build.yml +++ b/release/build.yml @@ -44,7 +44,11 @@ - centos-release-scl - name: install devtoolset centos 6 - yum: state=present name=devtoolset-3-toolchain + yum: state=present name={{item}} + with_items: + - devtoolset-3-toolchain + - python27-python-devel + - python27-python-pip # TPACKET_V3 block end when: ansible_distribution_major_version == "6" @@ -125,19 +129,24 @@ # Actually build - block: - name: git clone - git: repo=https://github.com/aol/moloch dest={{moloch_name}} update=yes force=yes + git: repo=https://github.com/aol/moloch dest={{moloch_name}} update=yes force=yes version=v1.0.0-rc1 - - name: build + - name: build centos 6 + shell: cd {{moloch_name}}; make clean; scl enable devtoolset-3 python27 "./easybutton-build.sh -d /data/{{moloch_name}} --daq" + when: ansible_distribution_major_version == "6" + + - name: build not centos 6 shell: cd {{moloch_name}}; make clean; ./easybutton-build.sh -d /data/{{moloch_name}} --daq + when: ansible_distribution_major_version != "6" - name: install centos 6 - shell: cd {{moloch_name}}; scl enable devtoolset-3 "make install" + shell: npm -g config set user root; cd {{moloch_name}}; scl enable devtoolset-3 "make install" environment: PATH: "/data/{{moloch_name}}/bin:{{ansible_env.PATH}}" when: ansible_distribution_major_version == "6" - name: install not centos 6 - shell: cd {{moloch_name}}; make install + shell: npm -g config set user root; cd {{moloch_name}}; make install environment: PATH: "/data/{{moloch_name}}/bin:{{ansible_env.PATH}}" when: ansible_distribution_major_version != "6" diff --git a/release/config.ini.sample b/release/config.ini.sample index 38889a32f3..88db161738 100644 --- a/release/config.ini.sample +++ b/release/config.ini.sample @@ -1,25 +1,25 @@ # Latest settings documentation: https://github.com/aol/moloch/wiki/Settings # # Moloch uses a tiered system for configuration variables. This allows Moloch -# to share one config file for many machines. The ordering of sections in this -# file doesn't matter. +# to share one config file for many machines. The ordering of sections in this +# file doesn't matter. # # Order of config variables: # 1st) [optional] The section titled with the node name is used first. # Moloch will always tag sessions with node: # 2nd) [optional] If a node has a nodeClass variable, the section titled with -# the nodeClass name is used next. Sessions will be tagged with -# node: which is useful if watching different +# the nodeClass name is used next. Sessions will be tagged with +# node: which is useful if watching different # network classes. # 3rd) The section titled "default" is used last. [default] -# Comma seperated list of elasticsearch host:port combinations. If not using a -# elasticsearch VIP, a different elasticsearch node in the cluster can be specified +# Comma seperated list of elasticsearch host:port combinations. If not using a +# elasticsearch VIP, a different elasticsearch node in the cluster can be specified # for each Moloch node to help spread load on high volume clusters elasticsearch=MOLOCH_ELASTICSEARCH -# How often to create a new elasticsearch index. hourly,daily,weekly,monthly +# How often to create a new elasticsearch index. hourly,hourly6,daily,weekly,monthly # Changing the value will cause previous sessions to be unreachable rotateIndex=daily @@ -33,13 +33,18 @@ rotateIndex=daily # Private key file to use, comment out to use http instead # keyFile=MOLOCH_INSTALL_DIR/etc/moloch.key -# S2S and Password Hash secret - Must be in default section. Since elasticsearch +# Password Hash and S2S secret - Must be in default section. Since elasticsearch # is wide open by default, we encrypt the stored password hashes with this # so a malicous person can't insert a working new account. It is also used # for secure S2S communication. Comment out for no user authentication. # Changing the value will make all previously stored passwords no longer work. +# Make this RANDOM, you never need to type in passwordSecret = MOLOCH_PASSWORD +# Use a different password for S2S communication then passwordSecret. +# Must be in default section. Make this RANDOM, you never need to type in +#serverSecret= + # HTTP Digest Realm - Must be in default section. Changing the value # will make all previously stored passwords no longer work httpRealm = Moloch @@ -51,36 +56,22 @@ httpRealm = Moloch # Semicolon ';' seperated list of interfaces to listen on for traffic interface=MOLOCH_INTERFACE -# The bpf filter +# The bpf filter of traffic to ignore #bpf=not port 9200 # The yara file name #yara= -## Start wiseService configuration # Host to connect to for wiseService #wiseHost=127.0.0.1 -# Number of seconds to cache results before asking wiseService again -#wiseCacheSecs=600 - -# Max number of items to store in the wise cache that is local to each moloch-capture node -#wiseMaxCache=100000 - -# Number of connections to wiseService, this is also the number of concurrent wise queries. -#wiseMaxConns=10 - -# Number of oustanding requests to the wiseService -#wiseMaxRequests=100 -## End wiseService configuration - -# Uncomment to log access requests to a different log file +# Log viewer access requests to a different log file #accessLogFile = MOLOCH_INSTALL_DIR/logs/access.log # The directory to save raw pcap files to pcapDir = MOLOCH_INSTALL_DIR/raw -# The max raw pcap file size in gigabytes, with a max value of 36G. +# The max raw pcap file size in gigabytes, with a max value of 36G. # The disk should have room for at least 10*maxFileSizeG maxFileSizeG = 12 @@ -88,32 +79,32 @@ maxFileSizeG = 12 # only rotate based on current file size and the maxFileSizeG variable #maxFileTimeM = 60 -# TCP timeout value. Moloch writes a session record after this many seconds +# TCP timeout value. Moloch writes a session record after this many seconds # of inactivity. tcpTimeout = 600 -# Moloch writes a session record after this many seconds, no matter if +# Moloch writes a session record after this many seconds, no matter if # active or inactive tcpSaveTimeout = 720 -# UDP timeout value. Moloch assumes the UDP session is ended after this +# UDP timeout value. Moloch assumes the UDP session is ended after this # many seconds of inactivity. udpTimeout = 30 -# ICMP timeout value. Moloch assumes the ICMP session is ended after this +# ICMP timeout value. Moloch assumes the ICMP session is ended after this # many seconds of inactivity. icmpTimeout = 10 -# An aproximiate maximum number of active sessions Moloch/libnids will try +# An aproximiate maximum number of active sessions Moloch/libnids will try # and monitor maxStreams = 1000000 # Moloch writes a session record after this many packets maxPackets = 10000 -# Delete pcap files when free space is lower then this in gigabytes OR it can be -# expressed as a percentage (ex: 5%). This does NOT delete the session records in -# the database. It is recommended this value is between 5% and 10% of the disk. +# Delete pcap files when free space is lower then this in gigabytes OR it can be +# expressed as a percentage (ex: 5%). This does NOT delete the session records in +# the database. It is recommended this value is between 5% and 10% of the disk. # Database deletes are done by the db.pl expire script freeSpaceG = 5% @@ -127,17 +118,21 @@ viewPort = 8005 #viewUrl = https://HOSTNAME:8005 # Path of the maxmind geoip country file. Download free version from: -# http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz -geoipFile = MOLOCH_INSTALL_DIR/etc/GeoIP.dat +# https://updates.maxmind.com/app/update_secure?edition_id=GeoLite2-Country +geoLite2Country = MOLOCH_INSTALL_DIR/etc/GeoLite2-Country.mmdb # Path of the maxmind geoip ASN file. Download free version from: -# http://www.maxmind.com/download/geoip/database/asnum/GeoIPASNum.dat.gz -geoipASNFile = MOLOCH_INSTALL_DIR/etc/GeoIPASNum.dat +# https://updates.maxmind.com/app/update_secure?edition_id=GeoLite2-ASN +geoLite2ASN = MOLOCH_INSTALL_DIR/etc/GeoLite2-ASN.mmdb # Path of the rir assignments file # https://www.iana.org/assignments/ipv4-address-space/ipv4-address-space.csv rirFile = MOLOCH_INSTALL_DIR/etc/ipv4-address-space.csv +# Path of the OUI file from whareshark +# https://raw.githubusercontent.com/wireshark/wireshark/master/manuf +ouiFile = MOLOCH_INSTALL_DIR/etc/oui.txt + # User to drop privileges to. The pcapDir must be writable by this user or group below dropUser=nobody @@ -145,13 +140,13 @@ dropUser=nobody dropGroup=daemon # Semicolon ';' seperated list of tags which once capture sets for a session causes the -# remaining pcap from being saved for the session. It is likely that the initial packets +# remaining pcap from being saved for the session. It is likely that the initial packets # WILL be saved for the session since tags usually aren't set until after several packets # Each tag can optionally be followed by a : which specifies how many total packets to save #dontSaveTags= # Header to use for determining the username to check in the database for instead of -# using http digest. Use this if apache or something else is doing the auth. +# using http digest. Use this if apache or something else is doing the auth. # Set viewHost to localhost or use iptables # Might need something like this in the httpd.conf # RewriteRule .* - [E=ENV_RU:%{REMOTE_USER}] @@ -167,8 +162,12 @@ parseSMB=true # Should we parse HTTP QS Values parseQSValue=false +# Should we calculate sha256 for bodies +supportSha256=false + # Only index HTTP request bodies less than this number of bytes */ -maxReqBody=0 +maxReqBody=64 + # Only store request bodies that Utf-8? config.reqBodyOnlyUtf8 = true @@ -202,7 +201,7 @@ pluginsDir=MOLOCH_INSTALL_DIR/plugins # Specify the max number of indices we calculate spidata for. # ES will blow up if we allow the spiData to search too many indices. -spiDataMaxIndices=3 +spiDataMaxIndices=4 # Uncomment the following to allow direct uploads. This is experimental #uploadCommand=MOLOCH_INSTALL_DIR/bin/moloch-capture --copy -n {NODE} -r {TMPFILE} -c {CONFIG} {TAGS} @@ -226,23 +225,18 @@ packetThreads=2 #includes= # ADVANCED - How is pcap written to disk -# simple = use O_DIRECT if available, writes in pcapWriteSize chunks, -# a file per packet thread. +# simple = use O_DIRECT if available, writes in pcapWriteSize chunks, +# a file per packet thread. +# simple-nodirect = don't use O_DIRECT. Required for zfs and others pcapWriteMethod=simple -# ADVANCED - Buffer size when writing pcap files. Should be a multiple of the raid 5 or xfs +# ADVANCED - Buffer size when writing pcap files. Should be a multiple of the raid 5 or xfs # stripe size. Defaults to 256k pcapWriteSize = 262143 -# ADVANCED - value for pcap_set_buffer_size, may not be used depending on kernel etc -pcapBufferSize = 30000000 - # ADVANCED - Number of bytes to bulk index at a time dbBulkSize = 300000 -# ADVANCED - Number of seconds before we force a flush to ES -dbFlushTimeout = 5 - # ADVANCED - Compress requests to ES, reduces ES bandwidth by ~80% at the cost # of increased CPU. MUST have "http.compression: true" in elasticsearch.yml file compressES = false @@ -258,7 +252,7 @@ maxESRequests = 500 # Decreasing may cause more dropped packets packetsPerPoll = 50000 -# ADVANCED - Moloch will try to compensate for SYN packet drops by swapping +# ADVANCED - Moloch will try to compensate for SYN packet drops by swapping # the source and destination addresses when a SYN-acK packet was captured first. # Probably useful to set it false, when running Moloch in wild due to SYN floods. antiSynDrop = true @@ -277,6 +271,21 @@ logESRequests = true logFileCreation = true +### High Performance settings +# https://github.com/aol/moloch/wiki/Settings#High_Performance_Settings +# magicMode=basic +# pcapReadMethod=tpacketv3 +# tpacketv3NumThreads=2 +# pcapWriteMethod=simple +# pcapWriteSize = 2560000 +# packetThreads=5 +# maxPacketsInQueue = 200000 + +### Low Bandwidth settings +# packetThreads=1 +# pcapWriteSize = 65536 + + ############################################################################## # Classes of nodes # Can override most default values, and create a tag call node: @@ -293,7 +302,7 @@ nodeClass = class1 # Might use a different elasticsearch node elasticsearch=elasticsearchhost1 -# Uncomment if this node should process the cron queries, only ONE node should process cron queries +# Uncomment if this node should process the cron queries, only ONE node should process cron queries # cronQueries = true [node2] @@ -307,17 +316,17 @@ interface = eth4 # override-ips is a special section that overrides the MaxMind databases for # the fields set, but fields not set will still use MaxMind (example if you set # tags but not country it will use MaxMind for the country) -# Spaces and capitalization is very important. +# Spaces and capitalization is very important. # IP Can be a single IP or a CIDR # Up to 10 tags can be added -# +# # ip=tag:TAGNAME1;tag:TAGNAME2;country:3LetterUpperCaseCountry;asn:ASN STRING #[override-ips] #10.1.0.0/16=tag:ny-office;country:USA;asn:AS0000 This is an ASN ############################################################################## # It is now possible to define in the config file extra http/email headers -# to index. They are accessed using the expression http. and +# to index. They are accessed using the expression http. and # email. with optional .cnt expressions # # Possible config atributes for all headers diff --git a/release/doit.sh b/release/doit.sh index 4c5c522b76..3ba4d31e51 100755 --- a/release/doit.sh +++ b/release/doit.sh @@ -1,10 +1,14 @@ #!/bin/sh +vagrant destroy -f +sleep 2 vagrant up --no-provision -sleep 1 +sleep 2 + +echo "UP" for i in ubuntu-14.04 ubuntu-16.04 centos-6 centos-7; do echo $i - sleep 1 sh -c "vagrant provision $i > $i.out 2>&1 || echo \"Error Occurred: $i\"" & + sleep 3 done wait diff --git a/release/moloch_update_geo.sh b/release/moloch_update_geo.sh index 6e718f07d0..5158ff6f51 100755 --- a/release/moloch_update_geo.sh +++ b/release/moloch_update_geo.sh @@ -2,7 +2,13 @@ cd BUILD_MOLOCH_INSTALL_DIR/etc wget -N -nv --no-check-certificate https://www.iana.org/assignments/ipv4-address-space/ipv4-address-space.csv -wget -N -nv http://www.maxmind.com/download/geoip/database/asnum/GeoIPASNum.dat.gz; gunzip -f GeoIPASNum.dat.gz -wget -N -nv http://download.maxmind.com/download/geoip/database/asnum/GeoIPASNumv6.dat.gz; gunzip -f GeoIPASNumv6.dat.gz -wget -N -nv http://www.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz; gunzip -f GeoIP.dat.gz -wget -N -nv http://geolite.maxmind.com/download/geoip/database/GeoIPv6.dat.gz; gunzip -f GeoIPv6.dat.gz + +wget -N -nv -O GeoLite2-Country.mmdb.gz 'https://updates.maxmind.com/app/update_secure?edition_id=GeoLite2-Country' +/bin/rm -f GeoLite2-Country.mmdb +zcat GeoLite2-Country.mmdb.gz > GeoLite2-Country.mmdb + +wget -N -nv -O GeoLite2-ASN.mmdb.gz 'https://updates.maxmind.com/app/update_secure?edition_id=GeoLite2-ASN' +/bin/rm -f GeoLite2-ASN.mmdb +zcat GeoLite2-ASN.mmdb.gz > GeoLite2-ASN.mmdb + +wget -nv -O oui.txt https://raw.githubusercontent.com/wireshark/wireshark/master/manuf diff --git a/screwdriver.yaml b/screwdriver.yaml index 277bd9af20..63387562ee 100644 --- a/screwdriver.yaml +++ b/screwdriver.yaml @@ -4,10 +4,15 @@ shared: channels: - moloch statuses: - - SUCCESS - FAILURE - ABORTED - - RUNNING +# annotations: +# beta.screwdriver.cd/cpu: HIGH +# beta.screwdriver.cd/ram: HIGH + + environment: + MOLOCH_COPY_BRANCH: "origin/1.0.0" + MOLOCH_FILE_NAME: 1 jobs: centos-6: @@ -22,10 +27,10 @@ jobs: - test-capture: (cd tests ; ./tests.pl) - build-package: | export MOLOCH_VERSION=`sed 's/.*"\(.*\)\".*$/\1/' /data/moloch/viewer/version.js | tr "-" "_"` - if [ "$GIT_BRANCH" = "origin/master" ]; then + if [ "$GIT_BRANCH" = "$MOLOCH_COPY_BRANCH" ]; then scl enable ruby193 "/opt/rh/ruby193/root/usr/local/bin/fpm -s dir -t rpm -n moloch -v $MOLOCH_VERSION --iteration $SD_BUILD_ID --template-scripts --after-install 'release/afterinstall.sh' --url "http://molo.ch" --description 'Moloch Full Packet System' -d perl-libwww-perl -d perl-JSON -d ethtool -d libyaml-devel /data/moloch" - scl enable python27 "aws s3 cp --quiet moloch*.x86_64.rpm s3://files.molo.ch/moloch-master.centos6.x86_64.rpm --acl public-read" - scl enable python27 "aws s3api put-object-acl --bucket files.molo.ch --key moloch-master.centos6.x86_64.rpm --acl public-read" + scl enable python27 "aws s3 cp --quiet moloch*.x86_64.rpm s3://files.molo.ch/moloch-${MOLOCH_FILE_NAME}.centos6.x86_64.rpm --acl public-read" + scl enable python27 "aws s3api put-object-acl --bucket files.molo.ch --key moloch-${MOLOCH_FILE_NAME}.centos6.x86_64.rpm --acl public-read" fi secrets: - AWS_ACCESS_KEY_ID @@ -46,10 +51,10 @@ jobs: - test-ui: (cd viewer; npm install ; npm test) - build-package: | export MOLOCH_VERSION=`sed 's/.*"\(.*\)\".*$/\1/' /data/moloch/viewer/version.js | tr "-" "_"` - if [ "$GIT_BRANCH" = "origin/master" ]; then + if [ "$GIT_BRANCH" = "$MOLOCH_COPY_BRANCH" ]; then fpm -s dir -t rpm -n moloch -v $MOLOCH_VERSION --iteration $SD_BUILD_ID --template-scripts --after-install "release/afterinstall.sh" --url "http://molo.ch" --description "Moloch Full Packet System" -d perl-libwww-perl -d perl-JSON -d ethtool -d libyaml-devel /data/moloch - aws s3 cp --quiet moloch*.x86_64.rpm s3://files.molo.ch/moloch-master.centos7.x86_64.rpm --acl public-read - aws s3api put-object-acl --bucket files.molo.ch --key moloch-master.centos7.x86_64.rpm --acl public-read + aws s3 cp --quiet moloch*.x86_64.rpm s3://files.molo.ch/moloch-${MOLOCH_FILE_NAME}.centos7.x86_64.rpm --acl public-read + aws s3api put-object-acl --bucket files.molo.ch --key moloch-${MOLOCH_FILE_NAME}.centos7.x86_64.rpm --acl public-read fi secrets: - AWS_ACCESS_KEY_ID @@ -67,10 +72,10 @@ jobs: - test-capture: (cd tests ; ./tests.pl) - build-package: | export MOLOCH_VERSION=`sed 's/.*"\(.*\)\".*$/\1/' /data/moloch/viewer/version.js | tr "-" "_"` - if [ "$GIT_BRANCH" = "origin/master" ]; then + if [ "$GIT_BRANCH" = "$MOLOCH_COPY_BRANCH" ]; then fpm -s dir -t deb -n moloch -v $MOLOCH_VERSION --iteration $SD_BUILD_ID --template-scripts --after-install "release/afterinstall.sh" --url "http://molo.ch" --description "Moloch Full Packet System" -d libwww-perl -d libjson-perl -d ethtool -d libyaml-dev /data/moloch - aws s3 cp --quiet moloch*amd64.deb s3://files.molo.ch/moloch_master_ubuntu14_amd64.deb --acl public-read - aws s3api put-object-acl --bucket files.molo.ch --key moloch_master_ubuntu14_amd64.deb --acl public-read + aws s3 cp --quiet moloch*amd64.deb s3://files.molo.ch/moloch_${MOLOCH_FILE_NAME}_ubuntu14_amd64.deb --acl public-read + aws s3api put-object-acl --bucket files.molo.ch --key moloch_${MOLOCH_FILE_NAME}_ubuntu14_amd64.deb --acl public-read fi secrets: - AWS_ACCESS_KEY_ID @@ -81,55 +86,64 @@ jobs: image: andywick/moloch-build-16:0.50.1 steps: - ln -s /thirdparty . + - apt-get update +# - docker: | +# apt-get install -y docker.io +# service docker start +# sleep 1 +# docker run -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" docker.elastic.co/elasticsearch/elasticsearch:5.5.3 & +# sleep 1 - build: ./easybutton-build.sh - export PATH=/data/moloch/bin:$PATH - installing: make install - cp -r capture/plugins/lua/samples /data/moloch/lua - test-capture: (cd tests ; ./tests.pl) - export TZ=US/Eastern +# - test-viewer: (cd tests ; ./tests.pl --viewer) - build-package: | export MOLOCH_VERSION=`sed 's/.*"\(.*\)\".*$/\1/' /data/moloch/viewer/version.js | tr "-" "_"` - if [ "$GIT_BRANCH" = "origin/master" ]; then + if [ "$GIT_BRANCH" = "$MOLOCH_COPY_BRANCH" ]; then fpm -s dir -t deb -n moloch -v $MOLOCH_VERSION --iteration $SD_BUILD_ID --template-scripts --after-install "release/afterinstall.sh" --url "http://molo.ch" --description "Moloch Full Packet System" -d libwww-perl -d libjson-perl -d ethtool -d libyaml-dev /data/moloch - aws s3 cp --quiet moloch*amd64.deb s3://files.molo.ch/moloch-master_ubuntu16_amd64.deb --acl public-read - aws s3api put-object-acl --bucket files.molo.ch --key moloch-master_ubuntu16_amd64.deb --acl public-read + aws s3 cp --quiet moloch*amd64.deb s3://files.molo.ch/moloch-${MOLOCH_FILE_NAME}_ubuntu16_amd64.deb --acl public-read + aws s3api put-object-acl --bucket files.molo.ch --key moloch-${MOLOCH_FILE_NAME}_ubuntu16_amd64.deb --acl public-read fi secrets: - AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY - ubuntu-18: - requires: [~pr, ~commit] - image: andywick/moloch-build-18:0.50.1 - steps: - - ln -s /thirdparty . - - build: ./easybutton-build.sh - - export PATH=/data/moloch/bin:$PATH - - installing: make install - - cp -r capture/plugins/lua/samples /data/moloch/lua -# Doesn't seem to work on ubuntu-18 currently -# - test-capture: (cd tests ; ./tests.pl) - - build-package: | - export MOLOCH_VERSION=`sed 's/.*"\(.*\)\".*$/\1/' /data/moloch/viewer/version.js | tr "-" "_"` - if [ "$GIT_BRANCH" = "origin/master" ]; then - fpm -s dir -t deb -n moloch -v $MOLOCH_VERSION --iteration $SD_BUILD_ID --template-scripts --after-install "release/afterinstall.sh" --url "http://molo.ch" --description "Moloch Full Packet System" -d libwww-perl -d libjson-perl -d ethtool -d libyaml-dev /data/moloch - aws s3 cp --quiet moloch*amd64.deb s3://files.molo.ch/moloch-master_ubuntu18_amd64.deb --acl public-read - aws s3api put-object-acl --bucket files.molo.ch --key moloch-master_ubuntu18_amd64.deb --acl public-read - fi - secrets: - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY +# ubuntu-18: +# requires: [~pr, ~commit] +# image: andywick/moloch-build-18:0.50.1 +# steps: +# - apt-get update +# - ln -s /thirdparty . +# - build: ./easybutton-build.sh +# - export PATH=/data/moloch/bin:$PATH +# - installing: make install +# - cp -r capture/plugins/lua/samples /data/moloch/lua +## Doesn't seem to work on ubuntu-18 currently +## - test-capture: (cd tests ; ./tests.pl) +# - build-package: | +# export MOLOCH_VERSION=`sed 's/.*"\(.*\)\".*$/\1/' /data/moloch/viewer/version.js | tr "-" "_"` +# if [ "$GIT_BRANCH" = "$MOLOCH_COPY_BRANCH" ]; then +# fpm -s dir -t deb -n moloch -v $MOLOCH_VERSION --iteration $SD_BUILD_ID --template-scripts --after-install "release/afterinstall.sh" --url "http://molo.ch" --description "Moloch Full Packet System" -d libwww-perl -d libjson-perl -d ethtool -d libyaml-dev /data/moloch +# aws s3 cp --quiet moloch*amd64.deb s3://files.molo.ch/moloch-${MOLOCH_FILE_NAME}_ubuntu18_amd64.deb --acl public-read +# aws s3api put-object-acl --bucket files.molo.ch --key moloch-${MOLOCH_FILE_NAME}_ubuntu18_amd64.deb --acl public-read +# fi +# secrets: +# - AWS_ACCESS_KEY_ID +# - AWS_SECRET_ACCESS_KEY slack: - requires: [centos-6, centos-7, ubuntu-14, ubuntu-16, ubuntu-18] + requires: [centos-6, centos-7, ubuntu-14, ubuntu-16] image: andywick/moloch-build-7:6 steps: - slack-success: | - export MOLOCH_VERSION=`grep PACKAGE_VERSION= configure | sed "s/.*'\(.*\)\'.*$/\1/"` - if [ "$GIT_BRANCH" = "origin/master" ]; then - BUILD_VERSION=`git describe --tags` + export MOLOCH_VERSION=`grep PACKAGE_VERSION= configure | sed "s/.*'\(.*\)'.*$/\1/"` + echo MOLOCH_VERSION: $MOLOCH_VERSION + if [ "$GIT_BRANCH" = "$MOLOCH_COPY_BRANCH" ]; then MSG=`git log -1 --format=%s` BODY="{\"icon_emoji\": \":sushi:\", \"username\": \"MolochBuild\", \"text\":\"It worked: $GIT_BRANCH - moloch-$MOLOCH_VERSION-$SD_BUILD_ID - $MSG\"}" curl -XPOST -H "Content-type: application/json" --data "$BODY" $SLACK diff --git a/tests/MolochTest.pm b/tests/MolochTest.pm index 5068660728..fc4677c4e1 100644 --- a/tests/MolochTest.pm +++ b/tests/MolochTest.pm @@ -3,7 +3,7 @@ use Exporter; use strict; use Test::More; @MolochTest::ISA = qw(Exporter); -@MolochTest::EXPORT = qw (esGet esPost esDelete esCopy viewerGet viewerGetToken viewerGet2 viewerDelete viewerPost viewerPost2 viewerPostToken viewerPostToken2 countTest countTest2 errTest bin2hex mesGet mesPost multiGet getTokenCookie getTokenCookie2); +@MolochTest::EXPORT = qw (esGet esPost esDelete esCopy viewerGet viewerGetToken viewerGet2 viewerDelete viewerPost viewerPost2 viewerPostToken viewerPostToken2 countTest countTest2 errTest bin2hex mesGet mesPost multiGet getTokenCookie getTokenCookie2 parliamentGet parliamentGetToken parliamentPost parliamentPut parliamentDelete parliamentDeleteToken); use LWP::UserAgent; use HTTP::Request::Common; @@ -236,5 +236,52 @@ sub getTokenCookie2 { return $1; } ################################################################################ +sub parliamentGet { +my ($url, $debug) = @_; + my $response = $MolochTest::userAgent->get("http://$MolochTest::host:8008$url"); + diag $url, " response:", $response->content if ($debug); + my $json = from_json($response->content); + return ($json); +} +################################################################################ +sub parliamentGetToken { +my ($url, $token, $debug) = @_; + my $response = $MolochTest::userAgent->get("http://$MolochTest::host:8008$url", "x-access-token" => $token); + diag $url, " response:", $response->content if ($debug); + my $json = from_json($response->content); + return ($json); +} +################################################################################ +sub parliamentPost { +my ($url, $content, $debug) = @_; + my $response = $MolochTest::userAgent->post("http://$MolochTest::host:8008$url", Content => $content, "Content-Type" => "application/json;charset=UTF-8"); + diag $url, " response:", $response->content if ($debug); + my $json = from_json($response->content); + return ($json); +} +################################################################################ +sub parliamentPut { +my ($url, $content, $debug) = @_; + my $response = $MolochTest::userAgent->request(HTTP::Request::Common::PUT("http://$MolochTest::host:8008$url", Content => $content, "Content-Type" => "application/json;charset=UTF-8")); + diag $url, " response:", $response->content if ($debug); + my $json = from_json($response->content); + return ($json); +} +################################################################################ +sub parliamentDelete { +my ($url, $debug) = @_; + my $response = $MolochTest::userAgent->request(HTTP::Request::Common::DELETE("http://$MolochTest::host:8008$url")); + diag $url, " response:", $response->content if ($debug); + my $json = from_json($response->content); + return ($json); +} +################################################################################ +sub parliamentDeleteToken { +my ($url, $token, $debug) = @_; + my $response = $MolochTest::userAgent->request(HTTP::Request::Common::DELETE("http://$MolochTest::host:8008$url", "x-access-token" => $token)); + diag $url, " response:", $response->content if ($debug); + my $json = from_json($response->content); + return ($json); +} return 1; diff --git a/tests/README b/tests/README index 793c963227..069c69f80b 100644 --- a/tests/README +++ b/tests/README @@ -26,7 +26,8 @@ pppoe.pcap - Subset of http://www.pcapr.net/view/tyson.key/2009/8/0/14/AOLTraffi fbzero-android.pcap - Subset of https://github.com/ntop/nDPI/issues/300#issuecomment-261893575 wireshark-bdat.pcap - https://www.wireshark.org/lists/wireshark-bugs/201610/msg00410.html mpls-basic.pcap - https://wiki.wireshark.org/SampleCaptures?action=AttachFile&do=get&target=mpls-basic.cap - +wireshark-dhcp.pcap - https://wiki.wireshark.org/SampleCaptures?action=AttachFile&do=get&target=dhcp.pcap +CVE-2018-6794.pcap - https://redmine.openinfosecfoundation.org/issues/2427 2) Viewer diff --git a/tests/api-connections.t b/tests/api-connections.t index 0ed6598c66..b2cb2355c7 100644 --- a/tests/api-connections.t +++ b/tests/api-connections.t @@ -7,39 +7,57 @@ use Test::Differences; use Data::Dumper; use strict; -my $pwd = getcwd() . "/pcap"; +my $pwd = "*"; my $files = "(file=$pwd/socks-http-example.pcap||file=$pwd/socks-http-pass.pcap||file=$pwd/socks-https-example.pcap||file=$pwd/socks5-http-302.pcap||file=$pwd/socks5-rdp.pcap||file=$pwd/socks5-reverse.pcap||file=$pwd/socks5-smtp-503.pcap)"; -my $json; -# a1 to a2 +my ($json, $mjson); +# srcIp to dstIp $json = viewerGet("/connections.json?date=-1&expression=" . uri_escape("$files")); delete $json->{health}; - eq_or_diff($json, from_json('{ "nodes": [ { "id": "10.0.0.1", "db": 26034, "by": 30979, "pa": 86, "cnt": 2, "sessions": 3, "type": 3, "pos": 0 }, { "id": "10.0.0.2", "db": 26119, "by": 31647, "pa": 96, "cnt": 3, "sessions": 4, "type": 3, "pos": 1 }, { "id": "10.180.156.185", "db": 33866, "by": 46190, "pa": 184, "cnt": 1, "sessions": 9, "type": 1, "pos": 2 }, { "id": "10.180.156.249", "db": 33866, "by": 46190, "pa": 184, "cnt": 1, "sessions": 9, "type": 2, "pos": 3 }, { "id": "10.0.0.3", "db": 85, "by": 668, "pa": 10, "cnt": 1, "sessions": 1, "type": 1, "pos": 4 } ], "links": [ { "value": 2, "source": 0, "target": 1, "by": 29487, "db": 25707, "pa": 66, "no": { "test": 1 } }, { "value": 1, "source": 1, "target": 0, "by": 1492, "db": 327, "pa": 20, "no": { "test": 1 } }, { "value": 9, "source": 2, "target": 3, "by": 46190, "db": 33866, "pa": 184, "no": { "test": 1 } }, { "value": 1, "source": 4, "target": 1, "by": 668, "db": 85, "pa": 10, "no": { "test": 1 } } ], "recordsFiltered": 13 }', {relaxed => 1}), "a1 to a2", { context => 3 }); + eq_or_diff($json, from_json('{ "nodes": [ + { "id": "10.0.0.1", "db": 26034, "by": 30979, "pa": 86, "cnt": 2, "sessions": 3, "type": 3, "pos": 0 }, + { "id": "10.0.0.2", "db": 26119, "by": 31647, "pa": 96, "cnt": 3, "sessions": 4, "type": 3, "pos": 1 }, + { "id": "10.0.0.3", "db": 85, "by": 668, "pa": 10, "cnt": 1, "sessions": 1, "type": 1, "pos": 2 }, + { "id": "10.180.156.185", "db": 33866, "by": 46190, "pa": 184, "cnt": 1, "sessions": 9, "type": 1, "pos": 3 }, + { "id": "10.180.156.249", "db": 33866, "by": 46190, "pa": 184, "cnt": 1, "sessions": 9, "type": 2, "pos": 4 } ], "links": [ { "value": 2, "source": 0, "target": 1, "by": 29487, "db": 25707, "pa": 66, "node": { "test": 1 } }, { "value": 1, "source": 1, "target": 0, "by": 1492, "db": 327, "pa": 20, "node": { "test": 1 } }, { "value": 9, "source": 3, "target": 4, "by": 46190, "db": 33866, "pa": 184, "node": { "test": 1 } }, { "value": 1, "source": 2, "target": 1, "by": 668, "db": 85, "pa": 10, "node": { "test": 1 } } ], "recordsFiltered": 13 }', {relaxed => 1}), "srcIp to dstIp", { context => 3 }); - $json = multiGet("/connections.json?date=-1&expression=" . uri_escape("$files")); - delete $json->{health}; - eq_or_diff($json, from_json('{ "nodes": [ { "id": "10.0.0.1", "db": 26034, "by": 30979, "pa": 86, "cnt": 2, "sessions": 3, "type": 3, "pos": 0 }, { "id": "10.0.0.2", "db": 26119, "by": 31647, "pa": 96, "cnt": 3, "sessions": 4, "type": 3, "pos": 1 }, { "id": "10.180.156.185", "db": 33866, "by": 46190, "pa": 184, "cnt": 1, "sessions": 9, "type": 1, "pos": 2 }, { "id": "10.180.156.249", "db": 33866, "by": 46190, "pa": 184, "cnt": 1, "sessions": 9, "type": 2, "pos": 3 }, { "id": "10.0.0.3", "db": 85, "by": 668, "pa": 10, "cnt": 1, "sessions": 1, "type": 1, "pos": 4 } ], "links": [ { "value": 2, "source": 0, "target": 1, "by": 29487, "db": 25707, "pa": 66, "no": { "test": 1 } }, { "value": 1, "source": 1, "target": 0, "by": 1492, "db": 327, "pa": 20, "no": { "test": 1 } }, { "value": 9, "source": 2, "target": 3, "by": 46190, "db": 33866, "pa": 184, "no": { "test": 1 } }, { "value": 1, "source": 4, "target": 1, "by": 668, "db": 85, "pa": 10, "no": { "test": 1 } } ], "recordsFiltered": 13 }', {relaxed => 1}), "multi a1 to a2", { context => 3 }); + $mjson = multiGet("/connections.json?date=-1&expression=" . uri_escape("$files")); + delete $mjson->{health}; + eq_or_diff($mjson, $json, "multi: srcIp to dstIp", { context => 3 }); -# a1 to ip.dst +# srcIp to ip.dst $json = viewerGet("/connections.json?date=-1&dstField=ip.dst:port&expression=" . uri_escape("$files")); delete $json->{health}; - eq_or_diff($json, from_json('{ "nodes": [ { "id": "10.0.0.1", "db": 25707, "by": 29487, "pa": 66, "cnt": 2, "sessions": 2, "type": 1, "pos": 0 }, { "id": "10.0.0.2:21477", "db": 1361, "by": 2176, "pa": 14, "cnt": 1, "sessions": 1, "type": 2, "pos": 1 }, { "id": "10.0.0.2", "db": 327, "by": 1492, "pa": 20, "cnt": 1, "sessions": 1, "type": 1, "pos": 2 }, { "id": "10.0.0.1:1080", "db": 327, "by": 1492, "pa": 20, "cnt": 1, "sessions": 1, "type": 2, "pos": 3 }, { "id": "10.180.156.185", "db": 33866, "by": 46190, "pa": 184, "cnt": 1, "sessions": 9, "type": 1, "pos": 4 }, { "id": "10.180.156.249:1080", "db": 33866, "by": 46190, "pa": 184, "cnt": 1, "sessions": 9, "type": 2, "pos": 5 }, { "id": "10.0.0.3", "db": 85, "by": 668, "pa": 10, "cnt": 1, "sessions": 1, "type": 1, "pos": 6 }, { "id": "10.0.0.2:42356", "db": 85, "by": 668, "pa": 10, "cnt": 1, "sessions": 1, "type": 2, "pos": 7 }, { "id": "10.0.0.2:8855", "db": 24346, "by": 27311, "pa": 52, "cnt": 1, "sessions": 1, "type": 2, "pos": 8 } ], "links": [ { "value": 1, "source": 0, "target": 1, "by": 2176, "db": 1361, "pa": 14, "no": { "test": 1 } }, { "value": 1, "source": 2, "target": 3, "by": 1492, "db": 327, "pa": 20, "no": { "test": 1 } }, { "value": 9, "source": 4, "target": 5, "by": 46190, "db": 33866, "pa": 184, "no": { "test": 1 } }, { "value": 1, "source": 6, "target": 7, "by": 668, "db": 85, "pa": 10, "no": { "test": 1 } }, { "value": 1, "source": 0, "target": 8, "by": 27311, "db": 24346, "pa": 52, "no": { "test": 1 } } ], "recordsFiltered": 13 }', {relaxed => 1}), "a1 to ip.dst", { context => 3 }); - - $json = multiGet("/connections.json?date=-1&dstField=ip.dst:port&expression=" . uri_escape("$files")); - delete $json->{health}; - eq_or_diff($json, from_json('{ "nodes": [ { "id": "10.0.0.1", "db": 25707, "by": 29487, "pa": 66, "cnt": 2, "sessions": 2, "type": 1, "pos": 0 }, { "id": "10.0.0.2:21477", "db": 1361, "by": 2176, "pa": 14, "cnt": 1, "sessions": 1, "type": 2, "pos": 1 }, { "id": "10.0.0.2", "db": 327, "by": 1492, "pa": 20, "cnt": 1, "sessions": 1, "type": 1, "pos": 2 }, { "id": "10.0.0.1:1080", "db": 327, "by": 1492, "pa": 20, "cnt": 1, "sessions": 1, "type": 2, "pos": 3 }, { "id": "10.180.156.185", "db": 33866, "by": 46190, "pa": 184, "cnt": 1, "sessions": 9, "type": 1, "pos": 4 }, { "id": "10.180.156.249:1080", "db": 33866, "by": 46190, "pa": 184, "cnt": 1, "sessions": 9, "type": 2, "pos": 5 }, { "id": "10.0.0.3", "db": 85, "by": 668, "pa": 10, "cnt": 1, "sessions": 1, "type": 1, "pos": 6 }, { "id": "10.0.0.2:42356", "db": 85, "by": 668, "pa": 10, "cnt": 1, "sessions": 1, "type": 2, "pos": 7 }, { "id": "10.0.0.2:8855", "db": 24346, "by": 27311, "pa": 52, "cnt": 1, "sessions": 1, "type": 2, "pos": 8 } ], "links": [ { "value": 1, "source": 0, "target": 1, "by": 2176, "db": 1361, "pa": 14, "no": { "test": 1 } }, { "value": 1, "source": 2, "target": 3, "by": 1492, "db": 327, "pa": 20, "no": { "test": 1 } }, { "value": 9, "source": 4, "target": 5, "by": 46190, "db": 33866, "pa": 184, "no": { "test": 1 } }, { "value": 1, "source": 6, "target": 7, "by": 668, "db": 85, "pa": 10, "no": { "test": 1 } }, { "value": 1, "source": 0, "target": 8, "by": 27311, "db": 24346, "pa": 52, "no": { "test": 1 } } ], "recordsFiltered": 13 }', {relaxed => 1}), "multi a1 to ip.dst", { context => 3 }); + eq_or_diff($json, from_json('{ "nodes": [ + { "id": "10.0.0.1", "db": 25707, "by": 29487, "pa": 66, "cnt": 2, "sessions": 2, "type": 1, "pos": 0 }, + { "id": "10.0.0.1:1080", "db": 327, "by": 1492, "pa": 20, "cnt": 1, "sessions": 1, "type": 2, "pos": 1 }, + { "id": "10.0.0.2", "db": 327, "by": 1492, "pa": 20, "cnt": 1, "sessions": 1, "type": 1, "pos": 2 }, + { "id": "10.0.0.2:21477", "db": 1361, "by": 2176, "pa": 14, "cnt": 1, "sessions": 1, "type": 2, "pos": 3 }, + { "id": "10.0.0.2:42356", "db": 85, "by": 668, "pa": 10, "cnt": 1, "sessions": 1, "type": 2, "pos": 4 }, + { "id": "10.0.0.2:8855", "db": 24346, "by": 27311, "pa": 52, "cnt": 1, "sessions": 1, "type": 2, "pos": 5 }, + { "id": "10.0.0.3", "db": 85, "by": 668, "pa": 10, "cnt": 1, "sessions": 1, "type": 1, "pos": 6 }, + { "id": "10.180.156.185", "db": 33866, "by": 46190, "pa": 184, "cnt": 1, "sessions": 9, "type": 1, "pos": 7 }, + { "id": "10.180.156.249:1080", "db": 33866, "by": 46190, "pa": 184, "cnt": 1, "sessions": 9, "type": 2, "pos": 8 } + ], "links": [ { "value": 1, "source": 0, "target": 3, "by": 2176, "db": 1361, "pa": 14, "node": { "test": 1 } }, { "value": 1, "source": 2, "target": 1, "by": 1492, "db": 327, "pa": 20, "node": { "test": 1 } }, { "value": 9, "source": 7, "target": 8, "by": 46190, "db": 33866, "pa": 184, "node": { "test": 1 } }, { "value": 1, "source": 6, "target": 4, "by": 668, "db": 85, "pa": 10, "node": { "test": 1 } }, { "value": 1, "source": 0, "target": 5, "by": 27311, "db": 24346, "pa": 52, "node": { "test": 1 } } ], "recordsFiltered": 13 }', {relaxed => 1}), "srcIp to ip.dst", { context => 3 }); + $mjson = multiGet("/connections.json?date=-1&dstField=ip.dst:port&expression=" . uri_escape("$files")); + delete $mjson->{health}; + eq_or_diff($mjson, $json, "multi: srcIp to ip.dst:port", { context => 3 }); -# a1 to tls.notAfter - $json = viewerGet("/connections.json?date=-1&dstField=tls.notAfter&expression=" . uri_escape("$files")); +# srcIp to cert.notAfter + $json = viewerGet("/connections.json?date=-1&dstField=cert.notAfter&expression=" . uri_escape("$files")); delete $json->{health}; - eq_or_diff($json, from_json('{ "nodes": [ { "id": "1418212800", "db": 26760, "by": 32958, "pa": 93, "cnt": 1, "sessions": 3, "type": 2, "pos": 0 }, { "id": "1648944000", "db": 26760, "by": 32958, "pa": 93, "cnt": 1, "sessions": 3, "type": 2, "pos": 1 }, { "id": "10.180.156.185", "db": 53520, "by": 65916, "pa": 186, "cnt": 2, "sessions": 6, "type": 1, "pos": 2 } ], "links": [ { "value": 3, "source": 2, "target": 0, "by": 32958, "db": 26760, "pa": 93, "no": { "test": 1 } }, { "value": 3, "source": 2, "target": 1, "by": 32958, "db": 26760, "pa": 93, "no": { "test": 1 } } ], "recordsFiltered": 3 }', {relaxed => 1}), "a1 to tls.notAfter", { context => 3 }); + eq_or_diff($json, from_json('{ "nodes": [ + { "id": "10.180.156.185", "db": 53520, "by": 65916, "pa": 186, "cnt": 2, "sessions": 6, "type": 1, "pos": 0 }, + { "id": "1418212800000", "db": 26760, "by": 32958, "pa": 93, "cnt": 1, "sessions": 3, "type": 2, "pos": 1 }, + { "id": "1648944000000", "db": 26760, "by": 32958, "pa": 93, "cnt": 1, "sessions": 3, "type": 2, "pos": 2 } + ], "links": [ { "value": 3, "source": 0, "target": 1, "by": 32958, "db": 26760, "pa": 93, "node": { "test": 1 } }, { "value": 3, "source": 0, "target": 2, "by": 32958, "db": 26760, "pa": 93, "node": { "test": 1 } } ], "recordsFiltered": 3 }', {relaxed => 1}), "srcIp to cert.notAfter", { context => 3 }); - $json = multiGet("/connections.json?date=-1&dstField=tls.notAfter&expression=" . uri_escape("$files")); - delete $json->{health}; - eq_or_diff($json, from_json('{ "nodes": [ { "id": "1418212800", "db": 26760, "by": 32958, "pa": 93, "cnt": 1, "sessions": 3, "type": 2, "pos": 0 }, { "id": "1648944000", "db": 26760, "by": 32958, "pa": 93, "cnt": 1, "sessions": 3, "type": 2, "pos": 1 }, { "id": "10.180.156.185", "db": 53520, "by": 65916, "pa": 186, "cnt": 2, "sessions": 6, "type": 1, "pos": 2 } ], "links": [ { "value": 3, "source": 2, "target": 0, "by": 32958, "db": 26760, "pa": 93, "no": { "test": 1 } }, { "value": 3, "source": 2, "target": 1, "by": 32958, "db": 26760, "pa": 93, "no": { "test": 1 } } ], "recordsFiltered": 3 }', {relaxed => 1}), "multi a1 to tls.notAfter", { context => 3 }); + $mjson = multiGet("/connections.json?date=-1&dstField=cert.notAfter&expression=" . uri_escape("$files")); + delete $mjson->{health}; + eq_or_diff($mjson, $json, "multi: srcIp to cert.notAfter", { context => 3 }); # ip.protocol unknown $json = viewerGet("/connections.json?date=-1&expression=" . uri_escape("$files&&ip.protocol==blah")); diff --git a/tests/api-files.t b/tests/api-files.t new file mode 100644 index 0000000000..f9e644780f --- /dev/null +++ b/tests/api-files.t @@ -0,0 +1,71 @@ +use Test::More tests => 19; +use Cwd; +use URI::Escape; +use MolochTest; +use JSON; +use Test::Differences; +use Data::Dumper; +use strict; + +my $json; +my $mjson; + +# Basic list +$json = viewerGet("/file/list"); +$mjson = multiGet("/file/list"); +eq_or_diff($mjson, $json, "single doesn't match multi", { context => 3 }); + +cmp_ok ($json->{recordsTotal}, ">=", 108); +cmp_ok ($json->{recordsFiltered}, ">=", 108); +delete $json->{data}->[0]->{first}; +cmp_ok ($json->{data}->[0]->{num}, "<", $json->{data}->[1]->{num}); + +# name sort +$json = viewerGet("/file/list?sortField=name"); +$mjson = multiGet("/file/list?sortField=name"); +eq_or_diff($mjson, $json, "single doesn't match multi", { context => 3 }); + +cmp_ok ($json->{data}->[0]->{name}, "lt", $json->{data}->[1]->{name}); + +# reverse name sort +$json = viewerGet("/file/list?sortField=name&desc=true"); +$mjson = multiGet("/file/list?sortField=name&desc=true"); +eq_or_diff($mjson, $json, "single doesn't match multi", { context => 3 }); + +cmp_ok ($json->{data}->[0]->{name}, "gt", $json->{data}->[1]->{name}); + +# filter +$json = viewerGet("/file/list?sortField=name&desc=true&filter=v6-http"); +$mjson = multiGet("/file/list?sortField=name&desc=true&filter=v6-http"); +eq_or_diff($mjson, $json, "single doesn't match multi", { context => 3 }); + +cmp_ok ($json->{recordsTotal}, ">=", 108); +cmp_ok ($json->{recordsFiltered}, "==", 1); +delete $json->{data}->[0]->{id}; +delete $json->{data}->[0]->{num}; +delete $json->{data}->[0]->{first}; +eq_or_diff($json->{data}->[0], from_json('{"locked":1,"filesize":9159,"node":"test","name":"' . getcwd() . '/pcap/v6-http.pcap"}')); + +# filter 2 +$json = viewerGet("/file/list?sortField=name&desc=true&filter=v6"); +$mjson = multiGet("/file/list?sortField=name&desc=true&filter=v6"); +eq_or_diff($mjson, $json, "single doesn't match multi", { context => 3 }); + +cmp_ok ($json->{recordsTotal}, ">=", 108); +cmp_ok ($json->{recordsFiltered}, "==", 2); +delete $json->{data}->[0]->{id}; +delete $json->{data}->[0]->{num}; +delete $json->{data}->[0]->{first}; +delete $json->{data}->[1]->{id}; +delete $json->{data}->[1]->{num}; +delete $json->{data}->[1]->{first}; +eq_or_diff($json->{data}, from_json('[{"locked":1,"filesize":28251,"node":"test","name":"' . getcwd() . '/pcap/v6.pcap"},' . + '{"locked":1,"filesize":9159,"node":"test","name":"' . getcwd() . '/pcap/v6-http.pcap"}]')); + +# filter emptry +$json = viewerGet("/file/list?sortField=name&desc=true&filter=sillyname"); +$mjson = multiGet("/file/list?sortField=name&desc=true&filter=sillyname"); +eq_or_diff($mjson, $json, "single doesn't match multi", { context => 3 }); + +cmp_ok ($json->{recordsTotal}, ">=", 108); +cmp_ok ($json->{recordsFiltered}, "==", 0); diff --git a/tests/api-fresh.t b/tests/api-fresh.t index bccb3ff64c..672b6b20af 100644 --- a/tests/api-fresh.t +++ b/tests/api-fresh.t @@ -81,5 +81,5 @@ my $json; is (!exists $json->{graph}, 1, "Shouldn't have connections.json graph"); is (!exists $json->{map}, 1, "Shouldn't have connections.json map"); - $json = viewerGet2("/uniqueValue.json?type=tags"); - is (scalar @{$json}, 0, "Empty uniqueValue"); + my $txt = $MolochTest::userAgent->get("http://$MolochTest::host:8124/unique.txt?field=tags")->content; + is ($txt, "", "Empty unique.txt"); diff --git a/tests/api-history.t b/tests/api-history.t index ea7a8c8895..72144fb633 100644 --- a/tests/api-history.t +++ b/tests/api-history.t @@ -1,4 +1,4 @@ -use Test::More tests => 36; +use Test::More tests => 37; use Cwd; use URI::Escape; use MolochTest; @@ -7,7 +7,7 @@ use Test::Differences; use Data::Dumper; use strict; -my $pwd = getcwd() . "/pcap"; +my $pwd = "*/pcap"; my $token = getTokenCookie(); @@ -80,6 +80,8 @@ my $pwd = getcwd() . "/pcap"; is ($item->{api}, "/sessions.json", "Test4: api"); $json = viewerGet("/history/list?userId=historytest1&api=somethingsilly"); is ($json->{recordsFiltered}, 0, "Test4: recordsFiltered"); + $json = viewerGet("/history/list?userId=somethingsilly&api=sessions"); + is ($json->{recordsFiltered}, 0, "Test4: recordsFiltered"); # Should be able to filter by time range my $current = time; diff --git a/tests/api-multies.t b/tests/api-multies.t index fbcff31a70..052271ebe3 100644 --- a/tests/api-multies.t +++ b/tests/api-multies.t @@ -1,4 +1,4 @@ -use Test::More tests => 30; +use Test::More tests => 27; use Cwd; use URI::Escape; use MolochTest; @@ -12,33 +12,28 @@ my $json; is ($json->{tagline}, "You Know, for Search", "ES tagline"); ok (!exists $json->{status} || $json->{status} == 200, "ES no status or 200 status"); - $json = mesGet("/MULTIPREFIX_dstats/version/version"); - cmp_ok($json->{_source}->{version}, '>=', 20, "dstats version is at least 20"); - is ($json->{_index}, "MULTIPREFIX_dstats_v2", "Correct dstats index name"); + $json = mesGet("/_template/MULTIPREFIX_sessions2_template?filter_path=**._meta"); + cmp_ok($json->{MULTIPREFIX_sessions2_template}->{mappings}->{session}->{_meta}->{molochDbVersion}, '>=', 50, "dstats version is at least 50"); #_stats $json = mesGet("/MULTIPREFIX_stats/_stats"); is ($json->{_node}, "127.0.0.1:9200,prefix:tests", "Correct _node status"); is (exists $json->{indices}->{MULTIPREFIX_stats_v2}, 1, "Correct stats/_stats index"); - $json = mesGet("/MULTIPREFIX_tags/_stats"); - is (exists $json->{indices}->{MULTIPREFIX_tags_v3}, 1, "Correct tags/_stats version"); - cmp_ok($json->{indices}->{MULTIPREFIX_tags_v3}->{total}->{docs}->{count}, '>=', 60, "tags count is at least 60"); - $json = mesGet("/MULTIPREFIX_files/_stats"); cmp_ok($json->{indices}->{MULTIPREFIX_files_v4}->{total}->{docs}->{count}, '>=', 60, "files count is at least 60"); $json = mesGet("/MULTIPREFIX_sequence/_stats"); - cmp_ok($json->{indices}->{MULTIPREFIX_sequence_v1}->{total}->{docs}->{count}, '>=', 2, "sequence count is at least 2"); + cmp_ok($json->{indices}->{MULTIPREFIX_sequence_v1}->{total}->{docs}->{count}, '>=', 1, "sequence count is at least 1"); $json = mesGet("/MULTIPREFIX_dstats/_stats"); - cmp_ok($json->{indices}->{MULTIPREFIX_dstats_v2}->{total}->{docs}->{count}, '>=', 2, "dstats count is at least 60"); + cmp_ok($json->{indices}->{MULTIPREFIX_dstats_v2}->{total}->{docs}->{count}, '>=', 1, "dstats count is at least 1"); # _count $json = mesPost("/MULTIPREFIX_users/_count?ignore_unavailable=true", ""); is ($json->{count}, 0, "Correct count number of users"); - $json = mesPost("/MULTIPREFIX_sessions-*/_count?ignore_unavailable=true", ""); + $json = mesPost("/MULTIPREFIX_sessions2-*/_count?ignore_unavailable=true", ""); cmp_ok ($json->{count}, '>=', 80, "Correct count number of sessions"); $json = mesPost("/MULTIPREFIX_stats*/_count?ignore_unavailable=true", ""); @@ -61,9 +56,9 @@ my $json; is ($json->{_node}, "127.0.0.1:9200,prefix:tests", "Correct _node status"); # aliases - $json = mesGet("/MULTIPREFIX_sessions-*/_aliases"); - is (exists $json->{"MULTIPREFIX_sessions-050330"}, 1, "Correct session alias"); - is (exists $json->{"MULTIPREFIX_sessions-140113"}, 1, "Correct session alias"); + $json = mesGet("/MULTIPREFIX_sessions2-*/_alias"); + is (exists $json->{"MULTIPREFIX_sessions2-050330"}, 1, "Correct session alias"); + is (exists $json->{"MULTIPREFIX_sessions2-140113"}, 1, "Correct session alias"); # _search @@ -80,8 +75,8 @@ my $json; cmp_ok($json->{hits}->{total}, '<', 300, "fields count is less then 300"); is ($json->{hits}->{hits}->[0]->{_index}, "MULTIPREFIX_fields_v1", "Correct fields index name"); - $json = mesGet("/MULTIPREFIX_sessions-141015/session/_search?preference=_primary_first&ignore_unavailable=true"); - is ($json->{hits}->{hits}->[0]->{_index}, "MULTIPREFIX_sessions-141015", "Correct sessions index name"); + $json = mesGet("/MULTIPREFIX_sessions2-141015/session/_search?preference=_primary_first&ignore_unavailable=true"); + is ($json->{hits}->{hits}->[0]->{_index}, "MULTIPREFIX_sessions2-141015", "Correct sessions index name"); cmp_ok($json->{hits}->{total}, '>=', 6, "sessions count is at least 6"); #print Dumper($json); diff --git a/tests/api-scrub.t b/tests/api-scrub.t index 52f259e118..ab8ee6fffb 100644 --- a/tests/api-scrub.t +++ b/tests/api-scrub.t @@ -2,10 +2,10 @@ use Test::More tests => 34; use Cwd; use URI::Escape; use MolochTest; +use Data::Dumper; use JSON; use strict; -my $pwd = getcwd() . "/pcap"; my $copytest = getcwd() . "/copytest.pcap"; countTest(0, "date=-1&expression=" . uri_escape("file=$copytest")); diff --git a/tests/api-sessionDetail.t b/tests/api-sessionDetail.t index 63964c7866..27303552c4 100644 --- a/tests/api-sessionDetail.t +++ b/tests/api-sessionDetail.t @@ -7,7 +7,7 @@ use JSON; use Data::Dumper; use strict; -my $pwd = getcwd() . "/pcap"; +my $pwd = "*/pcap"; # new /detail api my $sdId = viewerGet("/sessions.json?date=-1&expression=" . uri_escape("file=$pwd/http-content-gzip.pcap")); diff --git a/tests/api-sessions.t b/tests/api-sessions.t index 9953fac270..96228b42b8 100644 --- a/tests/api-sessions.t +++ b/tests/api-sessions.t @@ -7,7 +7,7 @@ use Test::Differences; use Data::Dumper; use strict; -my $pwd = getcwd() . "/pcap"; +my $pwd = "*/pcap"; # bigendian pcap file tests my $json = viewerGet("/sessions.json?date=-1&expression=" . uri_escape("file=$pwd/bigendian.pcap")); @@ -24,7 +24,7 @@ my $pwd = getcwd() . "/pcap"; # Check facets short $json = viewerGet("/sessions.json?startTime=1386004308&stopTime=1386004400&facets=1&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); - eq_or_diff($json->{map}, from_json('{"dst":{"USA": 3}, "src":{"USA": 3}}'), "map short"); + eq_or_diff($json->{map}, from_json('{"dst":{"US": 3}, "src":{"US": 3}}'), "map short"); eq_or_diff($json->{graph}->{lpHisto}, from_json('[["1386004309000", 1], ["1386004312000", 1], [1386004317000, 1]]'), "lpHisto short"); eq_or_diff($json->{graph}->{pa1Histo}, from_json('[["1386004309000", 8], ["1386004312000", 8], [1386004317000, 10]]'), "pa1Histo short"); eq_or_diff($json->{graph}->{pa2Histo}, from_json('[["1386004309000", 6], ["1386004312000", 7], [1386004317000, 7]]'), "pa2Histo short"); @@ -38,7 +38,7 @@ my $pwd = getcwd() . "/pcap"; # multi Check facets short $json = multiGet("/sessions.json?startTime=1386004308&stopTime=1386004400&facets=1&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); - eq_or_diff($json->{map}, from_json('{"dst":{"USA": 3}, "src":{"USA": 3}}'), "multi map short"); + eq_or_diff($json->{map}, from_json('{"dst":{"US": 3}, "src":{"US": 3}}'), "multi map short"); eq_or_diff($json->{graph}->{lpHisto}, from_json('[["1386004309000", 1], ["1386004312000", 1], [1386004317000, 1]]'), "multi lpHisto short"); eq_or_diff($json->{graph}->{pa1Histo}, from_json('[["1386004309000", 8], ["1386004312000", 8], [1386004317000, 10]]'), "multi pa1Histo short"); eq_or_diff($json->{graph}->{pa2Histo}, from_json('[["1386004309000", 6], ["1386004312000", 7], [1386004317000, 7]]'), "multi pa2Histo short"); @@ -52,7 +52,7 @@ my $pwd = getcwd() . "/pcap"; # Check facets medium $json = viewerGet("/sessions.json?startTime=1386004308&stopTime=1386349908&facets=1&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); - eq_or_diff($json->{map}, from_json('{"dst":{"USA": 3}, "src":{"USA": 3}}'), "map medium"); + eq_or_diff($json->{map}, from_json('{"dst":{"US": 3}, "src":{"US": 3}}'), "map medium"); eq_or_diff($json->{graph}->{lpHisto}, from_json('[["1386004260000", 3]]'), "lpHisto medium"); eq_or_diff($json->{graph}->{pa1Histo}, from_json('[["1386004260000", 26]]'), "pa1Histo medium"); eq_or_diff($json->{graph}->{pa2Histo}, from_json('[["1386004260000", 20]]'), "pa2Histo medium"); @@ -66,7 +66,7 @@ my $pwd = getcwd() . "/pcap"; # mutli Check facets medium $json = multiGet("/sessions.json?startTime=1386004308&stopTime=1386349908&facets=1&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); - eq_or_diff($json->{map}, from_json('{"dst":{"USA": 3}, "src":{"USA": 3}}'), "multi map medium"); + eq_or_diff($json->{map}, from_json('{"dst":{"US": 3}, "src":{"US": 3}}'), "multi map medium"); eq_or_diff($json->{graph}->{lpHisto}, from_json('[["1386004260000", 3]]'), "multi lpHisto medium"); eq_or_diff($json->{graph}->{pa1Histo}, from_json('[["1386004260000", 26]]'), "multi pa1Histo medium"); eq_or_diff($json->{graph}->{pa2Histo}, from_json('[["1386004260000", 20]]'), "multi pa2Histo medium"); @@ -80,7 +80,7 @@ my $pwd = getcwd() . "/pcap"; # Check facets ALL $json = viewerGet("/sessions.json?date=-1&facets=1&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); - eq_or_diff($json->{map}, from_json('{"dst":{"USA": 3, "CAN": 1}, "src":{"USA": 3, "RUS":1}}'), "map ALL"); + eq_or_diff($json->{map}, from_json('{"dst":{"US": 3, "CA": 1}, "src":{"US": 3, "RU":1}}'), "map ALL"); eq_or_diff($json->{graph}->{lpHisto}, from_json('[["1335956400000", 1], ["1386003600000", 3], [1387742400000, 1], [1482552000000,1]]'), "lpHisto ALL"); eq_or_diff($json->{graph}->{pa1Histo}, from_json('[["1335956400000", 2], ["1386003600000", 26], [1387742400000, 3], [1482552000000,3]]'), "pa1Histo ALL"); eq_or_diff($json->{graph}->{pa2Histo}, from_json('[["1335956400000", 0], ["1386003600000", 20], [1387742400000, 1], [1482552000000,1]]'), "pa2Histo ALL"); @@ -92,7 +92,7 @@ my $pwd = getcwd() . "/pcap"; # multi Check facets ALL $json = multiGet("/sessions.json?date=-1&facets=1&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); - eq_or_diff($json->{map}, from_json('{"dst":{"USA": 3, "CAN": 1}, "src":{"USA": 3, "RUS":1}}'), "multi map ALL"); + eq_or_diff($json->{map}, from_json('{"dst":{"US": 3, "CA": 1}, "src":{"US": 3, "RU":1}}'), "multi map ALL"); eq_or_diff($json->{graph}->{lpHisto}, from_json('[["1335956400000", 1], ["1386003600000", 3], [1387742400000, 1], [1482552000000,1]]'), "multi lpHisto ALL"); eq_or_diff($json->{graph}->{pa1Histo}, from_json('[["1335956400000", 2], ["1386003600000", 26], [1387742400000, 3], [1482552000000,3]]'), "multi pa1Histo ALL"); eq_or_diff($json->{graph}->{pa2Histo}, from_json('[["1335956400000", 0], ["1386003600000", 20], [1387742400000, 1], [1482552000000,1]]'), "multi pa2Histo ALL"); @@ -102,20 +102,20 @@ my $pwd = getcwd() . "/pcap"; is ($json->{graph}->{interval}, 3600, "multi correct interval ALL"); # Check ip.protocol=blah - my $json = viewerGet("/sessions.json?date=-1&&spi=a1&expression=" . uri_escape("file=$pwd/bigendian.pcap&&ip.protocol==blah")); + my $json = viewerGet("/sessions.json?date=-1&&spi=ipsrc&expression=" . uri_escape("file=$pwd/bigendian.pcap&&ip.protocol==blah")); is($json->{bsqErr}, "Unknown protocol string blah", "ip.protocol==blah"); # multi Check ip.protocol=blah - my $json = multiGet("/sessions.json?date=-1&&spi=a1&expression=" . uri_escape("file=$pwd/bigendian.pcap&&ip.protocol==blah")); + my $json = multiGet("/sessions.json?date=-1&&spi=ipsrc&expression=" . uri_escape("file=$pwd/bigendian.pcap&&ip.protocol==blah")); is($json->{bsqErr}, "Unknown protocol string blah", "multi ip.protocol==blah"); # csv my $csv = $MolochTest::userAgent->get("http://$MolochTest::host:8123/sessions.csv?date=-1&expression=" . uri_escape("file=$pwd/socks-http-example.pcap"))->content; $csv =~ s/\r//g; eq_or_diff ($csv, 'IP Protocol, Start Time, Stop Time, Src IP, Src Port, Src Country, Dst IP, Dst Port, Dst Country, Bytes, Data bytes, Packets, Moloch Node -tcp,1386004309,1386004309,10.180.156.185,53533,USA,10.180.156.249,1080,USA,2698,1754,14,test -tcp,1386004312,1386004312,10.180.156.185,53534,USA,10.180.156.249,1080,USA,2780,1770,15,test -tcp,1386004317,1386004317,10.180.156.185,53535,USA,10.180.156.249,1080,USA,2905,1763,17,test +tcp,1386004309468,1386004309478,10.180.156.185,53533,US,10.180.156.249,1080,US,2698,1754,14,test +tcp,1386004312331,1386004312384,10.180.156.185,53534,US,10.180.156.249,1080,US,2780,1770,15,test +tcp,1386004317979,1386004317989,10.180.156.185,53535,US,10.180.156.249,1080,US,2905,1763,17,test ', "CSV Expression"); my $idQuery = viewerGet("/sessions.json?date=-1&expression=" . uri_escape("file=$pwd/socks-http-example.pcap")); @@ -123,22 +123,22 @@ tcp,1386004317,1386004317,10.180.156.185,53535,USA,10.180.156.249,1080,USA,2905, $csv =~ s/\r//g; eq_or_diff ($csv, 'IP Protocol, Start Time, Stop Time, Src IP, Src Port, Src Country, Dst IP, Dst Port, Dst Country, Bytes, Data bytes, Packets, Moloch Node -tcp,1386004309,1386004309,10.180.156.185,53533,USA,10.180.156.249,1080,USA,2698,1754,14,test +tcp,1386004309468,1386004309478,10.180.156.185,53533,US,10.180.156.249,1080,US,2698,1754,14,test ', "CSV Ids"); - my $csv = $MolochTest::userAgent->get("http://$MolochTest::host:8123/sessions.csv?fields=fp,lp,a1,g1,a2,g2,pa,no,tcpflags.rst,tcpflags.psh&date=-1&expression=" . uri_escape("file=$pwd/socks-http-example.pcap"))->content; + my $csv = $MolochTest::userAgent->get("http://$MolochTest::host:8123/sessions.csv?fields=firstPacket,lastPacket,srcIp,srcGEO,dstIp,dstGEO,totPackets,node,tcpflags.rst,tcpflags.psh&date=-1&expression=" . uri_escape("file=$pwd/socks-http-example.pcap"))->content; $csv =~ s/\r//g; eq_or_diff ($csv, 'Start Time, Stop Time, Src IP, Src Country, Dst IP, Dst Country, Packets, Moloch Node, TCP Flag RST, TCP Flag PSH -1386004309,1386004309,10.180.156.185,USA,10.180.156.249,USA,14,test,0,4 -1386004312,1386004312,10.180.156.185,USA,10.180.156.249,USA,15,test,0,4 -1386004317,1386004317,10.180.156.185,USA,10.180.156.249,USA,17,test,0,6 +1386004309468,1386004309478,10.180.156.185,US,10.180.156.249,US,14,test,0,4 +1386004312331,1386004312384,10.180.156.185,US,10.180.156.249,US,15,test,0,4 +1386004317979,1386004317989,10.180.156.185,US,10.180.156.249,US,17,test,0,6 ', "CSV Expression"); # bigendian pcap fs tests - my $json = viewerGet("/sessions.json?date=-1&fields=fs&expression=" . uri_escape("file=$pwd/bigendian.pcap")); - ok ($json->{data}->[0]->{fs}->[0] =~ /bigendian.pcap/, "correct fs"); + my $json = viewerGet("/sessions.json?date=-1&fields=fileId&expression=" . uri_escape("file=$pwd/bigendian.pcap")); + ok ($json->{data}->[0]->{fileId}->[0] =~ /bigendian.pcap/, "correct fs"); # bigendian pcap fs tests 2 fields - my $json = viewerGet("/sessions.json?date=-1&fields=tls&fields=fs&expression=" . uri_escape("file=$pwd/bigendian.pcap")); - ok ($json->{data}->[0]->{fs}->[0] =~ /bigendian.pcap/, "correct fs"); + my $json = viewerGet("/sessions.json?date=-1&fields=tls&fields=fileId&expression=" . uri_escape("file=$pwd/bigendian.pcap")); + ok ($json->{data}->[0]->{fileId}->[0] =~ /bigendian.pcap/, "correct fs"); diff --git a/tests/api-spigraph.t b/tests/api-spigraph.t index 0e2754d7f7..1988506c95 100644 --- a/tests/api-spigraph.t +++ b/tests/api-spigraph.t @@ -1,4 +1,4 @@ -use Test::More tests => 63; +use Test::More tests => 77; use Cwd; use URI::Escape; use MolochTest; @@ -7,114 +7,133 @@ use Test::Differences; use Data::Dumper; use strict; -my $pwd = getcwd() . "/pcap"; +my $pwd = "*/pcap"; -my $json; +my ($json, $mjson); #node - $json = viewerGet("/spigraph.json?date=-1&field=no&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); - eq_or_diff($json->{map}, from_json('{"dst":{"USA": 3, "CAN": 1}, "src":{"USA": 3, "RUS":1}}'), "map field: no"); - eq_or_diff($json->{graph}->{lpHisto}, from_json('[["1335956400000", 1], ["1386003600000", 3], [1387742400000, 1], [1482552000000, 1]]'), "lpHisto field: no"); - eq_or_diff($json->{graph}->{pa1Histo}, from_json('[["1335956400000", 2], ["1386003600000", 26], [1387742400000, 3], [1482552000000, 3]]'), "pa1Histo field: no"); - eq_or_diff($json->{graph}->{pa2Histo}, from_json('[["1335956400000", 0], ["1386003600000", 20], [1387742400000, 1], [1482552000000, 1]]'), "pa2Histo field: no"); - eq_or_diff($json->{graph}->{db1Histo}, from_json('[["1335956400000", 0], ["1386003600000", 486], [1387742400000, 68], [1482552000000, 68]]'), "db1Histo field: no"); - eq_or_diff($json->{graph}->{db2Histo}, from_json('[["1335956400000", 0], ["1386003600000", 4801], [1387742400000, 0], [1482552000000, 0]]'), "db2Histo field: no"); - eq_or_diff($json->{items}, from_json('[{"graph":{"lpHisto":[[1335956400000,1],[1386003600000,3],[1387742400000,1],[1482552000000,1]],"xmin":1335956400000,"db2Histo":[[1335956400000,0],[1386003600000,4801],[1387742400000,0],[1482552000000,0]],"interval":3600,"xmax":1482552000000,"db1Histo":[[1335956400000,0],[1386003600000,486],[1387742400000,68],[1482552000000,68]],"pa2Histo":[[1335956400000,0],[1386003600000,20],[1387742400000,1],[1482552000000,1]],"pa1Histo":[[1335956400000,2],[1386003600000,26],[1387742400000,3],[1482552000000,3]]},"dbHisto":5423,"map":{"dst":{"CAN":1,"USA":3},"src":{"USA":3,"RUS":1}},"count":6,"name":"test","paHisto":44,"lpHisto":6}]'), "items field: no"); + $json = viewerGet("/spigraph.json?date=-1&field=node&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); + eq_or_diff($json->{map}, from_json('{"dst":{"US": 3, "CA": 1}, "src":{"US": 3, "RU":1}}'), "map field: no"); + eq_or_diff($json->{graph}->{lpHisto}, from_json('[["1335956400000", 1], ["1386003600000", 3], [1387742400000, 1], [1482552000000, 1]]'), "lpHisto field: node"); + eq_or_diff($json->{graph}->{pa1Histo}, from_json('[["1335956400000", 2], ["1386003600000", 26], [1387742400000, 3], [1482552000000, 3]]'), "pa1Histo field: node"); + eq_or_diff($json->{graph}->{pa2Histo}, from_json('[["1335956400000", 0], ["1386003600000", 20], [1387742400000, 1], [1482552000000, 1]]'), "pa2Histo field: node"); + eq_or_diff($json->{graph}->{db1Histo}, from_json('[["1335956400000", 0], ["1386003600000", 486], [1387742400000, 68], [1482552000000, 68]]'), "db1Histo field: node"); + eq_or_diff($json->{graph}->{db2Histo}, from_json('[["1335956400000", 0], ["1386003600000", 4801], [1387742400000, 0], [1482552000000, 0]]'), "db2Histo field: node"); + eq_or_diff($json->{items}, from_json('[{"graph":{"lpHisto":[[1335956400000,1],[1386003600000,3],[1387742400000,1],[1482552000000,1]],"xmin":1335956400000,"db2Histo":[[1335956400000,0],[1386003600000,4801],[1387742400000,0],[1482552000000,0]],"interval":3600,"xmax":1482552000000,"db1Histo":[[1335956400000,0],[1386003600000,486],[1387742400000,68],[1482552000000,68]],"pa2Histo":[[1335956400000,0],[1386003600000,20],[1387742400000,1],[1482552000000,1]],"pa1Histo":[[1335956400000,2],[1386003600000,26],[1387742400000,3],[1482552000000,3]]},"dbHisto":5423,"map":{"dst":{"CA":1,"US":3},"src":{"US":3,"RU":1}},"count":6,"name":"test","paHisto":56,"lpHisto":6}]'), "items field: node", { context => 3 }); + cmp_ok ($json->{recordsTotal}, '>=', 194); + cmp_ok ($json->{recordsFiltered}, '==', 6); #node multi - $json = multiGet("/spigraph.json?date=-1&field=no&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); - eq_or_diff($json->{map}, from_json('{"dst":{"USA": 3, "CAN": 1}, "src":{"USA": 3, "RUS":1}}'), "multi map field: no"); - eq_or_diff($json->{graph}->{lpHisto}, from_json('[["1335956400000", 1], ["1386003600000", 3], [1387742400000, 1], [1482552000000, 1]]'), "multi lpHisto field: no"); - eq_or_diff($json->{graph}->{pa1Histo}, from_json('[["1335956400000", 2], ["1386003600000", 26], [1387742400000, 3], [1482552000000, 3]]'), "multi pa1Histo field: no"); - eq_or_diff($json->{graph}->{pa2Histo}, from_json('[["1335956400000", 0], ["1386003600000", 20], [1387742400000, 1], [1482552000000, 1]]'), "multi pa2Histo field: no"); - eq_or_diff($json->{graph}->{db1Histo}, from_json('[["1335956400000", 0], ["1386003600000", 486], [1387742400000, 68], [1482552000000, 68]]'), "multi db1Histo field: no"); - eq_or_diff($json->{graph}->{db2Histo}, from_json('[["1335956400000", 0], ["1386003600000", 4801], [1387742400000, 0], [1482552000000, 0]]'), "multi db2Histo field: no"); - eq_or_diff($json->{items}, from_json('[{"graph":{"lpHisto":[[1335956400000,1],[1386003600000,3],[1387742400000,1],[1482552000000,1]],"xmin":1335956400000,"db2Histo":[[1335956400000,0],[1386003600000,4801],[1387742400000,0],[1482552000000,0]],"interval":3600,"xmax":1482552000000,"db1Histo":[[1335956400000,0],[1386003600000,486],[1387742400000,68],[1482552000000,68]],"pa2Histo":[[1335956400000,0],[1386003600000,20],[1387742400000,1],[1482552000000,1]],"pa1Histo":[[1335956400000,2],[1386003600000,26],[1387742400000,3],[1482552000000,3]]},"dbHisto":5423,"map":{"dst":{"CAN":1,"USA":3},"src":{"USA":3,"RUS":1}},"count":6,"name":"test","paHisto":44,"lpHisto":6}]'), "multi items field: no"); - - - -#ta - $json = viewerGet("/spigraph.json?date=-1&field=ta&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); - eq_or_diff($json->{map}, from_json('{"dst":{"USA": 3, "CAN": 1}, "src":{"USA": 3, "RUS":1}}'), "map field: ta"); - eq_or_diff($json->{graph}->{lpHisto}, from_json('[["1335956400000", 1], ["1386003600000", 3], [1387742400000, 1], [1482552000000, 1]]'), "lpHisto field: ta"); - eq_or_diff($json->{graph}->{pa1Histo}, from_json('[["1335956400000", 2], ["1386003600000", 26], [1387742400000, 3], [1482552000000, 3]]'), "pa1Histo field: ta"); - eq_or_diff($json->{graph}->{pa2Histo}, from_json('[["1335956400000", 0], ["1386003600000", 20], [1387742400000, 1], [1482552000000, 1]]'), "pa2Histo field: ta"); - eq_or_diff($json->{graph}->{db1Histo}, from_json('[["1335956400000", 0], ["1386003600000", 486], [1387742400000, 68], [1482552000000, 68]]'), "db1Histo field: ta"); - eq_or_diff($json->{graph}->{db2Histo}, from_json('[["1335956400000", 0], ["1386003600000", 4801], [1387742400000, 0], [1482552000000, 0]]'), "db2Histo field: ta"); + $mjson = multiGet("/spigraph.json?date=-1&field=node&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); + eq_or_diff($mjson->{map}, $json->{map}, "multi map field: node"); + eq_or_diff($mjson->{graph}->{lpHisto}, $json->{graph}->{lpHisto}, "multi lpHisto field: node"); + eq_or_diff($mjson->{graph}->{pa1Histo}, $json->{graph}->{pa1Histo}, "multi pa1Histo field: node"); + eq_or_diff($mjson->{graph}->{pa2Histo}, $json->{graph}->{pa2Histo}, "multi pa2Histo field: node"); + eq_or_diff($mjson->{graph}->{db1Histo}, $json->{graph}->{db1Histo}, "multi db1Histo field: node"); + eq_or_diff($mjson->{graph}->{db2Histo}, $json->{graph}->{db2Histo}, "multi db2Histo field: node"); + eq_or_diff($mjson->{items}, $json->{items}, "multi items field: node"); + cmp_ok ($mjson->{recordsTotal}, '>=', 194, "recordsTotal"); + cmp_ok ($mjson->{recordsFiltered}, '==', 6); + + + +#tags + $json = viewerGet("/spigraph.json?date=-1&field=tags&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); + eq_or_diff($json->{map}, from_json('{"dst":{"US": 3, "CA": 1}, "src":{"US": 3, "RU":1}}'), "map field: tags"); + eq_or_diff($json->{graph}->{lpHisto}, from_json('[["1335956400000", 1], ["1386003600000", 3], [1387742400000, 1], [1482552000000, 1]]'), "lpHisto field: tags"); + eq_or_diff($json->{graph}->{pa1Histo}, from_json('[["1335956400000", 2], ["1386003600000", 26], [1387742400000, 3], [1482552000000, 3]]'), "pa1Histo field: tags"); + eq_or_diff($json->{graph}->{pa2Histo}, from_json('[["1335956400000", 0], ["1386003600000", 20], [1387742400000, 1], [1482552000000, 1]]'), "pa2Histo field: tags"); + eq_or_diff($json->{graph}->{db1Histo}, from_json('[["1335956400000", 0], ["1386003600000", 486], [1387742400000, 68], [1482552000000, 68]]'), "db1Histo field: tags"); + eq_or_diff($json->{graph}->{db2Histo}, from_json('[["1335956400000", 0], ["1386003600000", 4801], [1387742400000, 0], [1482552000000, 0]]'), "db2Histo field: tags"); + cmp_ok ($json->{recordsTotal}, '>=', 194); + cmp_ok ($json->{recordsFiltered}, '==', 6); my @items = sort({$a->{name} cmp $b->{name}} @{$json->{items}}); - eq_or_diff(\@items, from_json('[{"count":3,"map":{"dst":{"USA":3},"src":{"USA":3}},"lpHisto":3,"paHisto":40,"name":"byhost2","dbHisto":5287,"graph":{"db2Histo":[[1386003600000,4801]],"xmin":1335956400000,"pa2Histo":[[1386003600000,20]],"lpHisto":[[1386003600000,3]],"xmax":1482552000000,"db1Histo":[[1386003600000,486]],"interval":3600,"pa1Histo":[[1386003600000,26]]}},{"paHisto":0,"dbHisto":0,"name":"byip2","graph":{"pa1Histo":[[1335956400000,2]],"interval":3600,"db1Histo":[[1335956400000,0]],"xmax":1482552000000,"lpHisto":[[1335956400000,1]],"pa2Histo":[[1335956400000,0]],"db2Histo":[[1335956400000,0]],"xmin":1335956400000},"count":1,"map":{"src":{},"dst":{}},"lpHisto":1},{"name":"domainwise","paHisto":40,"dbHisto":5287,"graph":{"xmin":1335956400000,"db2Histo":[[1386003600000,4801]],"pa2Histo":[[1386003600000,20]],"lpHisto":[[1386003600000,3]],"xmax":1482552000000,"db1Histo":[[1386003600000,486]],"interval":3600,"pa1Histo":[[1386003600000,26]]},"count":3,"map":{"src":{"USA":3},"dst":{"USA":3}},"lpHisto":3},{"dbHisto":68,"paHisto":2,"name":"dstip","graph":{"pa2Histo":[[1387742400000,1]],"xmin":1335956400000,"db2Histo":[[1387742400000,0]],"lpHisto":[[1387742400000,1]],"xmax":1482552000000,"pa1Histo":[[1387742400000,3]],"interval":3600,"db1Histo":[[1387742400000,68]]},"lpHisto":1,"map":{"src":{"RUS":1},"dst":{"CAN":1}},"count":1},{"dbHisto":5287,"paHisto":40,"name":"hosttaggertest1","graph":{"interval":3600,"pa1Histo":[[1386003600000,26]],"db1Histo":[[1386003600000,486]],"xmax":1482552000000,"lpHisto":[[1386003600000,3]],"pa2Histo":[[1386003600000,20]],"db2Histo":[[1386003600000,4801]],"xmin":1335956400000},"lpHisto":3,"map":{"dst":{"USA":3},"src":{"USA":3}},"count":3},{"lpHisto":3,"count":3,"map":{"dst":{"USA":3},"src":{"USA":3}},"paHisto":40,"dbHisto":5287,"graph":{"xmax":1482552000000,"pa1Histo":[[1386003600000,26]],"interval":3600,"db1Histo":[[1386003600000,486]],"pa2Histo":[[1386003600000,20]],"xmin":1335956400000,"db2Histo":[[1386003600000,4801]],"lpHisto":[[1386003600000,3]]},"name":"hosttaggertest2"},{"paHisto":0,"dbHisto":0,"name":"iptaggertest1","graph":{"pa2Histo":[[1335956400000,0]],"db2Histo":[[1335956400000,0]],"xmin":1335956400000,"lpHisto":[[1335956400000,1]],"xmax":1482552000000,"interval":3600,"pa1Histo":[[1335956400000,2]],"db1Histo":[[1335956400000,0]]},"lpHisto":1,"count":1,"map":{"src":{},"dst":{}}},{"lpHisto":1,"map":{"src":{},"dst":{}},"count":1,"paHisto":0,"graph":{"db1Histo":[[1335956400000,0]],"interval":3600,"pa1Histo":[[1335956400000,2]],"xmax":1482552000000,"lpHisto":[[1335956400000,1]],"db2Histo":[[1335956400000,0]],"xmin":1335956400000,"pa2Histo":[[1335956400000,0]]},"dbHisto":0,"name":"iptaggertest2"},{"map":{"dst":{},"src":{}},"count":1,"lpHisto":1,"paHisto":0,"graph":{"interval":3600,"pa1Histo":[[1335956400000,2]],"db1Histo":[[1335956400000,0]],"xmax":1482552000000,"lpHisto":[[1335956400000,1]],"pa2Histo":[[1335956400000,0]],"xmin":1335956400000,"db2Histo":[[1335956400000,0]]},"dbHisto":0,"name":"ipwise"},{"dbHisto":68,"paHisto":2,"name":"ipwisecsv","graph":{"lpHisto":[[1387742400000,1]],"pa2Histo":[[1387742400000,1]],"db2Histo":[[1387742400000,0]],"xmin":1335956400000,"interval":3600,"pa1Histo":[[1387742400000,3]],"db1Histo":[[1387742400000,68]],"xmax":1482552000000},"count":1,"map":{"src":{"RUS":1},"dst":{"CAN":1}},"lpHisto":1},{"count":1,"map":{"src":{"RUS":1},"dst":{"CAN":1}},"lpHisto":1,"paHisto":2,"name":"srcip","dbHisto":68,"graph":{"lpHisto":[[1387742400000,1]],"pa2Histo":[[1387742400000,1]],"xmin":1335956400000,"db2Histo":[[1387742400000,0]],"interval":3600,"pa1Histo":[[1387742400000,3]],"db1Histo":[[1387742400000,68]],"xmax":1482552000000}},{"lpHisto":3,"count":3,"map":{"src":{"USA":3},"dst":{"USA":3}},"paHisto":40,"dbHisto":5287,"graph":{"lpHisto":[[1386003600000,3]],"xmin":1335956400000,"db2Histo":[[1386003600000,4801]],"pa2Histo":[[1386003600000,20]],"db1Histo":[[1386003600000,486]],"interval":3600,"pa1Histo":[[1386003600000,26]],"xmax":1482552000000},"name":"wisebyhost2"},{"paHisto":0,"name":"wisebyip2","dbHisto":0,"graph":{"lpHisto":[[1335956400000,1]],"pa2Histo":[[1335956400000,0]],"xmin":1335956400000,"db2Histo":[[1335956400000,0]],"pa1Histo":[[1335956400000,2]],"interval":3600,"db1Histo":[[1335956400000,0]],"xmax":1482552000000},"count":1,"map":{"dst":{},"src":{}},"lpHisto":1}]'), "items field: ta"); - -#ta multi - $json = multiGet("/spigraph.json?date=-1&field=ta&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); - eq_or_diff($json->{map}, from_json('{"dst":{"USA": 3, "CAN": 1}, "src":{"USA": 3, "RUS":1}}'), "multi map field: ta"); - eq_or_diff($json->{graph}->{lpHisto}, from_json('[["1335956400000", 1], ["1386003600000", 3], [1387742400000, 1], [1482552000000, 1]]'), "multi lpHisto field: ta"); - eq_or_diff($json->{graph}->{pa1Histo}, from_json('[["1335956400000", 2], ["1386003600000", 26], [1387742400000, 3], [1482552000000, 3]]'), "multi pa1Histo field: ta"); - eq_or_diff($json->{graph}->{pa2Histo}, from_json('[["1335956400000", 0], ["1386003600000", 20], [1387742400000, 1], [1482552000000, 1]]'), "multi pa2Histo field: ta"); - eq_or_diff($json->{graph}->{db1Histo}, from_json('[["1335956400000", 0], ["1386003600000", 486], [1387742400000, 68], [1482552000000, 68]]'), "multi db1Histo field: ta"); - eq_or_diff($json->{graph}->{db2Histo}, from_json('[["1335956400000", 0], ["1386003600000", 4801], [1387742400000, 0], [1482552000000, 0]]'), "multi db2Histo field: ta"); - - - - my @items = sort({$a->{name} cmp $b->{name}} @{$json->{items}}); - eq_or_diff(\@items, from_json('[{"count":3,"map":{"dst":{"USA":3},"src":{"USA":3}},"lpHisto":3,"paHisto":40,"name":"byhost2","dbHisto":5287,"graph":{"db2Histo":[[1386003600000,4801]],"xmin":1335956400000,"pa2Histo":[[1386003600000,20]],"lpHisto":[[1386003600000,3]],"xmax":1482552000000,"db1Histo":[[1386003600000,486]],"interval":3600,"pa1Histo":[[1386003600000,26]]}},{"paHisto":0,"dbHisto":0,"name":"byip2","graph":{"pa1Histo":[[1335956400000,2]],"interval":3600,"db1Histo":[[1335956400000,0]],"xmax":1482552000000,"lpHisto":[[1335956400000,1]],"pa2Histo":[[1335956400000,0]],"db2Histo":[[1335956400000,0]],"xmin":1335956400000},"count":1,"map":{"src":{},"dst":{}},"lpHisto":1},{"name":"domainwise","paHisto":40,"dbHisto":5287,"graph":{"xmin":1335956400000,"db2Histo":[[1386003600000,4801]],"pa2Histo":[[1386003600000,20]],"lpHisto":[[1386003600000,3]],"xmax":1482552000000,"db1Histo":[[1386003600000,486]],"interval":3600,"pa1Histo":[[1386003600000,26]]},"count":3,"map":{"src":{"USA":3},"dst":{"USA":3}},"lpHisto":3},{"dbHisto":68,"paHisto":2,"name":"dstip","graph":{"pa2Histo":[[1387742400000,1]],"xmin":1335956400000,"db2Histo":[[1387742400000,0]],"lpHisto":[[1387742400000,1]],"xmax":1482552000000,"pa1Histo":[[1387742400000,3]],"interval":3600,"db1Histo":[[1387742400000,68]]},"lpHisto":1,"map":{"src":{"RUS":1},"dst":{"CAN":1}},"count":1},{"dbHisto":5287,"paHisto":40,"name":"hosttaggertest1","graph":{"interval":3600,"pa1Histo":[[1386003600000,26]],"db1Histo":[[1386003600000,486]],"xmax":1482552000000,"lpHisto":[[1386003600000,3]],"pa2Histo":[[1386003600000,20]],"db2Histo":[[1386003600000,4801]],"xmin":1335956400000},"lpHisto":3,"map":{"dst":{"USA":3},"src":{"USA":3}},"count":3},{"lpHisto":3,"count":3,"map":{"dst":{"USA":3},"src":{"USA":3}},"paHisto":40,"dbHisto":5287,"graph":{"xmax":1482552000000,"pa1Histo":[[1386003600000,26]],"interval":3600,"db1Histo":[[1386003600000,486]],"pa2Histo":[[1386003600000,20]],"xmin":1335956400000,"db2Histo":[[1386003600000,4801]],"lpHisto":[[1386003600000,3]]},"name":"hosttaggertest2"},{"paHisto":0,"dbHisto":0,"name":"iptaggertest1","graph":{"pa2Histo":[[1335956400000,0]],"db2Histo":[[1335956400000,0]],"xmin":1335956400000,"lpHisto":[[1335956400000,1]],"xmax":1482552000000,"interval":3600,"pa1Histo":[[1335956400000,2]],"db1Histo":[[1335956400000,0]]},"lpHisto":1,"count":1,"map":{"src":{},"dst":{}}},{"lpHisto":1,"map":{"src":{},"dst":{}},"count":1,"paHisto":0,"graph":{"db1Histo":[[1335956400000,0]],"interval":3600,"pa1Histo":[[1335956400000,2]],"xmax":1482552000000,"lpHisto":[[1335956400000,1]],"db2Histo":[[1335956400000,0]],"xmin":1335956400000,"pa2Histo":[[1335956400000,0]]},"dbHisto":0,"name":"iptaggertest2"},{"map":{"dst":{},"src":{}},"count":1,"lpHisto":1,"paHisto":0,"graph":{"interval":3600,"pa1Histo":[[1335956400000,2]],"db1Histo":[[1335956400000,0]],"xmax":1482552000000,"lpHisto":[[1335956400000,1]],"pa2Histo":[[1335956400000,0]],"xmin":1335956400000,"db2Histo":[[1335956400000,0]]},"dbHisto":0,"name":"ipwise"},{"dbHisto":68,"paHisto":2,"name":"ipwisecsv","graph":{"lpHisto":[[1387742400000,1]],"pa2Histo":[[1387742400000,1]],"db2Histo":[[1387742400000,0]],"xmin":1335956400000,"interval":3600,"pa1Histo":[[1387742400000,3]],"db1Histo":[[1387742400000,68]],"xmax":1482552000000},"count":1,"map":{"src":{"RUS":1},"dst":{"CAN":1}},"lpHisto":1},{"count":1,"map":{"src":{"RUS":1},"dst":{"CAN":1}},"lpHisto":1,"paHisto":2,"name":"srcip","dbHisto":68,"graph":{"lpHisto":[[1387742400000,1]],"pa2Histo":[[1387742400000,1]],"xmin":1335956400000,"db2Histo":[[1387742400000,0]],"interval":3600,"pa1Histo":[[1387742400000,3]],"db1Histo":[[1387742400000,68]],"xmax":1482552000000}},{"lpHisto":3,"count":3,"map":{"src":{"USA":3},"dst":{"USA":3}},"paHisto":40,"dbHisto":5287,"graph":{"lpHisto":[[1386003600000,3]],"xmin":1335956400000,"db2Histo":[[1386003600000,4801]],"pa2Histo":[[1386003600000,20]],"db1Histo":[[1386003600000,486]],"interval":3600,"pa1Histo":[[1386003600000,26]],"xmax":1482552000000},"name":"wisebyhost2"},{"paHisto":0,"name":"wisebyip2","dbHisto":0,"graph":{"lpHisto":[[1335956400000,1]],"pa2Histo":[[1335956400000,0]],"xmin":1335956400000,"db2Histo":[[1335956400000,0]],"pa1Histo":[[1335956400000,2]],"interval":3600,"db1Histo":[[1335956400000,0]],"xmax":1482552000000},"count":1,"map":{"dst":{},"src":{}},"lpHisto":1}]'), "items field: ta"); - - -#a1 - $json = viewerGet("/spigraph.json?date=-1&field=a1&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); - eq_or_diff($json->{map}, from_json('{"dst":{"USA": 3, "CAN": 1}, "src":{"USA": 3, "RUS":1}}'), "map field: a1"); - eq_or_diff($json->{graph}->{lpHisto}, from_json('[["1335956400000", 1], ["1386003600000", 3], [1387742400000, 1], [1482552000000, 1]]'), "lpHisto field: a1"); - eq_or_diff($json->{graph}->{pa1Histo}, from_json('[["1335956400000", 2], ["1386003600000", 26], [1387742400000, 3], [1482552000000, 3]]'), "pa1Histo field: a1"); - eq_or_diff($json->{graph}->{pa2Histo}, from_json('[["1335956400000", 0], ["1386003600000", 20], [1387742400000, 1], [1482552000000, 1]]'), "pa2Histo field: a1"); - eq_or_diff($json->{graph}->{db1Histo}, from_json('[["1335956400000", 0], ["1386003600000", 486], [1387742400000, 68], [1482552000000, 68]]'), "db1Histo field: a1"); - eq_or_diff($json->{graph}->{db2Histo}, from_json('[["1335956400000", 0], ["1386003600000", 4801], [1387742400000, 0], [1482552000000, 0]]'), "db2Histo field: a1"); - my @items = sort({$a->{name} cmp $b->{name}} @{$json->{items}}); - eq_or_diff(\@items, from_json('[{"paHisto":2,"name":"10.0.0.1","count":1,"lpHisto":1,"map":{"src":{"RUS":1},"dst":{"CAN":1}},"dbHisto":68,"graph":{"lpHisto":[[1387742400000,1]],"xmin":1335956400000,"db2Histo":[[1387742400000,0]],"db1Histo":[[1387742400000,68]],"pa2Histo":[[1387742400000,1]],"xmax":1482552000000,"pa1Histo":[[1387742400000,3]],"interval":3600}},{"name":"10.10.10.10","paHisto":2,"count":1,"lpHisto":1,"map":{"dst":{},"src":{}},"graph":{"interval":3600,"pa1Histo":[[1482552000000,3]],"xmax":1482552000000,"pa2Histo":[[1482552000000,1]],"db2Histo":[[1482552000000,0]],"db1Histo":[[1482552000000,68]],"xmin":1335956400000,"lpHisto":[[1482552000000,1]]},"dbHisto":68},{"graph":{"db2Histo":[[1386003600000,4801]],"db1Histo":[[1386003600000,486]],"lpHisto":[[1386003600000,3]],"xmin":1335956400000,"pa1Histo":[[1386003600000,26]],"interval":3600,"xmax":1482552000000,"pa2Histo":[[1386003600000,20]]},"dbHisto":5287,"lpHisto":3,"count":3,"map":{"dst":{"USA":3},"src":{"USA":3}},"name":"10.180.156.185","paHisto":40},{"graph":{"xmax":1482552000000,"pa2Histo":[[1335956400000,0]],"pa1Histo":[[1335956400000,2]],"interval":3600,"lpHisto":[[1335956400000,1]],"xmin":1335956400000,"db1Histo":[[1335956400000,0]],"db2Histo":[[1335956400000,0]]},"dbHisto":0,"map":{"dst":{},"src":{}},"lpHisto":1,"count":1,"name":"192.168.177.160","paHisto":0}]'), "items field: a1"); - -#a1 multi - $json = multiGet("/spigraph.json?date=-1&field=a1&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); - eq_or_diff($json->{map}, from_json('{"dst":{"USA": 3, "CAN": 1}, "src":{"USA": 3, "RUS":1}}'), "multi map field: a1"); - eq_or_diff($json->{graph}->{lpHisto}, from_json('[["1335956400000", 1], ["1386003600000", 3], [1387742400000, 1], [1482552000000, 1]]'), "multi lpHisto field: a1"); - eq_or_diff($json->{graph}->{pa1Histo}, from_json('[["1335956400000", 2], ["1386003600000", 26], [1387742400000, 3], [1482552000000, 3]]'), "multi pa1Histo field: a1"); - eq_or_diff($json->{graph}->{pa2Histo}, from_json('[["1335956400000", 0], ["1386003600000", 20], [1387742400000, 1], [1482552000000, 1]]'), "multi pa2Histo field: a1"); - eq_or_diff($json->{graph}->{db1Histo}, from_json('[["1335956400000", 0], ["1386003600000", 486], [1387742400000, 68], [1482552000000, 68]]'), "multi db1Histo field: a1"); - eq_or_diff($json->{graph}->{db2Histo}, from_json('[["1335956400000", 0], ["1386003600000", 4801], [1387742400000, 0], [1482552000000, 0]]'), "multi db2Histo field: a1"); - my @items = sort({$a->{name} cmp $b->{name}} @{$json->{items}}); - eq_or_diff(\@items, from_json('[{"paHisto":2,"name":"10.0.0.1","count":1,"lpHisto":1,"map":{"src":{"RUS":1},"dst":{"CAN":1}},"dbHisto":68,"graph":{"lpHisto":[[1387742400000,1]],"xmin":1335956400000,"db2Histo":[[1387742400000,0]],"db1Histo":[[1387742400000,68]],"pa2Histo":[[1387742400000,1]],"xmax":1482552000000,"pa1Histo":[[1387742400000,3]],"interval":3600}},{"name":"10.10.10.10","paHisto":2,"count":1,"lpHisto":1,"map":{"dst":{},"src":{}},"graph":{"interval":3600,"pa1Histo":[[1482552000000,3]],"xmax":1482552000000,"pa2Histo":[[1482552000000,1]],"db2Histo":[[1482552000000,0]],"db1Histo":[[1482552000000,68]],"xmin":1335956400000,"lpHisto":[[1482552000000,1]]},"dbHisto":68},{"graph":{"db2Histo":[[1386003600000,4801]],"db1Histo":[[1386003600000,486]],"lpHisto":[[1386003600000,3]],"xmin":1335956400000,"pa1Histo":[[1386003600000,26]],"interval":3600,"xmax":1482552000000,"pa2Histo":[[1386003600000,20]]},"dbHisto":5287,"lpHisto":3,"count":3,"map":{"dst":{"USA":3},"src":{"USA":3}},"name":"10.180.156.185","paHisto":40},{"graph":{"xmax":1482552000000,"pa2Histo":[[1335956400000,0]],"pa1Histo":[[1335956400000,2]],"interval":3600,"lpHisto":[[1335956400000,1]],"xmin":1335956400000,"db1Histo":[[1335956400000,0]],"db2Histo":[[1335956400000,0]]},"dbHisto":0,"map":{"dst":{},"src":{}},"lpHisto":1,"count":1,"name":"192.168.177.160","paHisto":0}]'), "multi items field: a1"); - -#hh1 - $json = viewerGet("/spigraph.json?date=-1&field=hh1&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); - eq_or_diff($json->{map}, from_json('{"dst":{"USA": 3, "CAN": 1}, "src":{"USA": 3, "RUS":1}}'), "map field: hh1"); - eq_or_diff($json->{graph}->{lpHisto}, from_json('[["1335956400000", 1], ["1386003600000", 3], [1387742400000, 1], [1482552000000, 1]]'), "lpHisto field: h1"); - eq_or_diff($json->{graph}->{pa1Histo}, from_json('[["1335956400000", 2], ["1386003600000", 26], [1387742400000, 3], [1482552000000, 3]]'), "pa1Histo field: h1"); - eq_or_diff($json->{graph}->{pa2Histo}, from_json('[["1335956400000", 0], ["1386003600000", 20], [1387742400000, 1], [1482552000000, 1]]'), "pa2Histo field: h1"); - eq_or_diff($json->{graph}->{db1Histo}, from_json('[["1335956400000", 0], ["1386003600000", 486], [1387742400000, 68], [1482552000000, 68]]'), "db1Histo field: h1"); - eq_or_diff($json->{graph}->{db2Histo}, from_json('[["1335956400000", 0], ["1386003600000", 4801], [1387742400000, 0], [1482552000000, 0]]'), "db2Histo field: h1"); - my @items = sort({$a->{name} cmp $b->{name}} @{$json->{items}}); - eq_or_diff(\@items, from_json('[{"name":"accept","count":3,"dbHisto":5287,"map":{"dst":{"USA":3},"src":{"USA":3}},"graph":{"pa2Histo":[[1386003600000,20]],"pa1Histo":[[1386003600000,26]],"interval":3600,"lpHisto":[[1386003600000,3]],"xmax":1482552000000,"db1Histo":[[1386003600000,486]],"xmin":1335956400000,"db2Histo":[[1386003600000,4801]]},"paHisto":40,"lpHisto":3},{"lpHisto":3,"name":"host","count":3,"map":{"dst":{"USA":3},"src":{"USA":3}},"dbHisto":5287,"graph":{"db2Histo":[[1386003600000,4801]],"lpHisto":[[1386003600000,3]],"xmax":1482552000000,"db1Histo":[[1386003600000,486]],"xmin":1335956400000,"pa2Histo":[[1386003600000,20]],"pa1Histo":[[1386003600000,26]],"interval":3600},"paHisto":40},{"lpHisto":3,"name":"user-agent","count":3,"dbHisto":5287,"graph":{"interval":3600,"pa2Histo":[[1386003600000,20]],"pa1Histo":[[1386003600000,26]],"db1Histo":[[1386003600000,486]],"xmin":1335956400000,"lpHisto":[[1386003600000,3]],"xmax":1482552000000,"db2Histo":[[1386003600000,4801]]},"map":{"src":{"USA":3},"dst":{"USA":3}},"paHisto":40}]'), "items field: hh1"); - -#hh1 multi - $json = multiGet("/spigraph.json?date=-1&field=hh1&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); - eq_or_diff($json->{map}, from_json('{"dst":{"USA": 3, "CAN": 1}, "src":{"USA": 3, "RUS":1}}'), "multi map field: hh1"); - eq_or_diff($json->{graph}->{lpHisto}, from_json('[["1335956400000", 1], ["1386003600000", 3], [1387742400000, 1], [1482552000000, 1]]'), "multi lpHisto field: h1"); - eq_or_diff($json->{graph}->{pa1Histo}, from_json('[["1335956400000", 2], ["1386003600000", 26], [1387742400000, 3], [1482552000000, 3]]'), "multi pa1Histo field: h1"); - eq_or_diff($json->{graph}->{pa2Histo}, from_json('[["1335956400000", 0], ["1386003600000", 20], [1387742400000, 1], [1482552000000, 1]]'), "multi pa2Histo field: h1"); - eq_or_diff($json->{graph}->{db1Histo}, from_json('[["1335956400000", 0], ["1386003600000", 486], [1387742400000, 68], [1482552000000, 68]]'), "multi db1Histo field: h1"); - eq_or_diff($json->{graph}->{db2Histo}, from_json('[["1335956400000", 0], ["1386003600000", 4801], [1387742400000, 0], [1482552000000, 0]]'), "multi db2Histo field: h1"); - my @items = sort({$a->{name} cmp $b->{name}} @{$json->{items}}); - eq_or_diff(\@items, from_json('[{"name":"accept","count":3,"dbHisto":5287,"map":{"dst":{"USA":3},"src":{"USA":3}},"graph":{"pa2Histo":[[1386003600000,20]],"pa1Histo":[[1386003600000,26]],"interval":3600,"lpHisto":[[1386003600000,3]],"xmax":1482552000000,"db1Histo":[[1386003600000,486]],"xmin":1335956400000,"db2Histo":[[1386003600000,4801]]},"paHisto":40,"lpHisto":3},{"lpHisto":3,"name":"host","count":3,"map":{"dst":{"USA":3},"src":{"USA":3}},"dbHisto":5287,"graph":{"db2Histo":[[1386003600000,4801]],"lpHisto":[[1386003600000,3]],"xmax":1482552000000,"db1Histo":[[1386003600000,486]],"xmin":1335956400000,"pa2Histo":[[1386003600000,20]],"pa1Histo":[[1386003600000,26]],"interval":3600},"paHisto":40},{"lpHisto":3,"name":"user-agent","count":3,"dbHisto":5287,"graph":{"interval":3600,"pa2Histo":[[1386003600000,20]],"pa1Histo":[[1386003600000,26]],"db1Histo":[[1386003600000,486]],"xmin":1335956400000,"lpHisto":[[1386003600000,3]],"xmax":1482552000000,"db2Histo":[[1386003600000,4801]]},"map":{"src":{"USA":3},"dst":{"USA":3}},"paHisto":40}]'), "multi items field: hh1"); - -#rawua - $json = viewerGet("/spigraph.json?date=-1&field=rawua&expression=" . uri_escape("file=$pwd/socks5-reverse.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); + eq_or_diff(\@items, from_json('[{"count":3,"map":{"dst":{"US":3},"src":{"US":3}},"lpHisto":3,"paHisto":46,"name":"byhost2","dbHisto":5287,"graph":{"db2Histo":[[1386003600000,4801]],"xmin":1335956400000,"pa2Histo":[[1386003600000,20]],"lpHisto":[[1386003600000,3]],"xmax":1482552000000,"db1Histo":[[1386003600000,486]],"interval":3600,"pa1Histo":[[1386003600000,26]]}},{"paHisto":2,"dbHisto":0,"name":"byip2","graph":{"pa1Histo":[[1335956400000,2]],"interval":3600,"db1Histo":[[1335956400000,0]],"xmax":1482552000000,"lpHisto":[[1335956400000,1]],"pa2Histo":[[1335956400000,0]],"db2Histo":[[1335956400000,0]],"xmin":1335956400000},"count":1,"map":{"src":{},"dst":{}},"lpHisto":1},{"name":"domainwise","paHisto":46,"dbHisto":5287,"graph":{"xmin":1335956400000,"db2Histo":[[1386003600000,4801]],"pa2Histo":[[1386003600000,20]],"lpHisto":[[1386003600000,3]],"xmax":1482552000000,"db1Histo":[[1386003600000,486]],"interval":3600,"pa1Histo":[[1386003600000,26]]},"count":3,"map":{"src":{"US":3},"dst":{"US":3}},"lpHisto":3},{"dbHisto":68,"paHisto":4,"name":"dstip","graph":{"pa2Histo":[[1387742400000,1]],"xmin":1335956400000,"db2Histo":[[1387742400000,0]],"lpHisto":[[1387742400000,1]],"xmax":1482552000000,"pa1Histo":[[1387742400000,3]],"interval":3600,"db1Histo":[[1387742400000,68]]},"lpHisto":1,"map":{"src":{"RU":1},"dst":{"CA":1}},"count":1},{"dbHisto":5287,"paHisto":46,"name":"hosttaggertest1","graph":{"interval":3600,"pa1Histo":[[1386003600000,26]],"db1Histo":[[1386003600000,486]],"xmax":1482552000000,"lpHisto":[[1386003600000,3]],"pa2Histo":[[1386003600000,20]],"db2Histo":[[1386003600000,4801]],"xmin":1335956400000},"lpHisto":3,"map":{"dst":{"US":3},"src":{"US":3}},"count":3},{"lpHisto":3,"count":3,"map":{"dst":{"US":3},"src":{"US":3}},"paHisto":46,"dbHisto":5287,"graph":{"xmax":1482552000000,"pa1Histo":[[1386003600000,26]],"interval":3600,"db1Histo":[[1386003600000,486]],"pa2Histo":[[1386003600000,20]],"xmin":1335956400000,"db2Histo":[[1386003600000,4801]],"lpHisto":[[1386003600000,3]]},"name":"hosttaggertest2"},{"paHisto":2,"dbHisto":0,"name":"iptaggertest1","graph":{"pa2Histo":[[1335956400000,0]],"db2Histo":[[1335956400000,0]],"xmin":1335956400000,"lpHisto":[[1335956400000,1]],"xmax":1482552000000,"interval":3600,"pa1Histo":[[1335956400000,2]],"db1Histo":[[1335956400000,0]]},"lpHisto":1,"count":1,"map":{"src":{},"dst":{}}},{"lpHisto":1,"map":{"src":{},"dst":{}},"count":1,"paHisto":2,"graph":{"db1Histo":[[1335956400000,0]],"interval":3600,"pa1Histo":[[1335956400000,2]],"xmax":1482552000000,"lpHisto":[[1335956400000,1]],"db2Histo":[[1335956400000,0]],"xmin":1335956400000,"pa2Histo":[[1335956400000,0]]},"dbHisto":0,"name":"iptaggertest2"},{"map":{"dst":{},"src":{}},"count":1,"lpHisto":1,"paHisto":2,"graph":{"interval":3600,"pa1Histo":[[1335956400000,2]],"db1Histo":[[1335956400000,0]],"xmax":1482552000000,"lpHisto":[[1335956400000,1]],"pa2Histo":[[1335956400000,0]],"xmin":1335956400000,"db2Histo":[[1335956400000,0]]},"dbHisto":0,"name":"ipwise"},{"dbHisto":68,"paHisto":4,"name":"ipwisecsv","graph":{"lpHisto":[[1387742400000,1]],"pa2Histo":[[1387742400000,1]],"db2Histo":[[1387742400000,0]],"xmin":1335956400000,"interval":3600,"pa1Histo":[[1387742400000,3]],"db1Histo":[[1387742400000,68]],"xmax":1482552000000},"count":1,"map":{"src":{"RU":1},"dst":{"CA":1}},"lpHisto":1},{"count":1,"map":{"src":{"RU":1},"dst":{"CA":1}},"lpHisto":1,"paHisto":4,"name":"srcip","dbHisto":68,"graph":{"lpHisto":[[1387742400000,1]],"pa2Histo":[[1387742400000,1]],"xmin":1335956400000,"db2Histo":[[1387742400000,0]],"interval":3600,"pa1Histo":[[1387742400000,3]],"db1Histo":[[1387742400000,68]],"xmax":1482552000000}},{"lpHisto":3,"count":3,"map":{"src":{"US":3},"dst":{"US":3}},"paHisto":46,"dbHisto":5287,"graph":{"lpHisto":[[1386003600000,3]],"xmin":1335956400000,"db2Histo":[[1386003600000,4801]],"pa2Histo":[[1386003600000,20]],"db1Histo":[[1386003600000,486]],"interval":3600,"pa1Histo":[[1386003600000,26]],"xmax":1482552000000},"name":"wisebyhost2"},{"paHisto":2,"name":"wisebyip2","dbHisto":0,"graph":{"lpHisto":[[1335956400000,1]],"pa2Histo":[[1335956400000,0]],"xmin":1335956400000,"db2Histo":[[1335956400000,0]],"pa1Histo":[[1335956400000,2]],"interval":3600,"db1Histo":[[1335956400000,0]],"xmax":1482552000000},"count":1,"map":{"dst":{},"src":{}},"lpHisto":1}]'), "items field: tags", { context => 3 }); + +#tags multi + $mjson = multiGet("/spigraph.json?date=-1&field=tags&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); + eq_or_diff($mjson->{map}, $json->{map}, "multi map field: tags"); + eq_or_diff($mjson->{graph}->{lpHisto}, $json->{graph}->{lpHisto}, "multi lpHisto field: tags"); + eq_or_diff($mjson->{graph}->{pa1Histo}, $json->{graph}->{pa1Histo}, "multi pa1Histo field: tags"); + eq_or_diff($mjson->{graph}->{pa2Histo}, $json->{graph}->{pa2Histo}, "multi pa2Histo field: tags"); + eq_or_diff($mjson->{graph}->{db1Histo}, $json->{graph}->{db1Histo}, "multi db1Histo field: tags"); + eq_or_diff($mjson->{graph}->{db2Histo}, $json->{graph}->{db2Histo}, "multi db2Histo field: tags"); + + my @mitems = sort({$a->{name} cmp $b->{name}} @{$json->{items}}); + eq_or_diff(\@items, \@items, "multi items field: tags"); + + +#srcIp + $json = viewerGet("/spigraph.json?date=-1&field=srcIp&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); + eq_or_diff($json->{map}, from_json('{"dst":{"US": 3, "CA": 1}, "src":{"US": 3, "RU":1}}'), "map field: srcIp"); + eq_or_diff($json->{graph}->{lpHisto}, from_json('[["1335956400000", 1], ["1386003600000", 3], [1387742400000, 1], [1482552000000, 1]]'), "lpHisto field: srcIp"); + eq_or_diff($json->{graph}->{pa1Histo}, from_json('[["1335956400000", 2], ["1386003600000", 26], [1387742400000, 3], [1482552000000, 3]]'), "pa1Histo field: srcIp"); + eq_or_diff($json->{graph}->{pa2Histo}, from_json('[["1335956400000", 0], ["1386003600000", 20], [1387742400000, 1], [1482552000000, 1]]'), "pa2Histo field: srcIp"); + eq_or_diff($json->{graph}->{db1Histo}, from_json('[["1335956400000", 0], ["1386003600000", 486], [1387742400000, 68], [1482552000000, 68]]'), "db1Histo field: srcIp"); + eq_or_diff($json->{graph}->{db2Histo}, from_json('[["1335956400000", 0], ["1386003600000", 4801], [1387742400000, 0], [1482552000000, 0]]'), "db2Histo field: srcIp"); my @items = sort({$a->{name} cmp $b->{name}} @{$json->{items}}); - eq_or_diff(\@items, from_json('[{"map":{"dst":{"CAN":1},"src":{"RUS":1}},"paHisto":42,"name":"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)","dbHisto":24346,"graph":{"xmax":1482552000000,"pa2Histo":[[1386788400000,21]],"pa1Histo":[[1386788400000,31]],"db2Histo":[[1386788400000,954]],"interval":3600,"lpHisto":[[1386788400000,1]],"xmin":1386003600000,"db1Histo":[[1386788400000,23392]]},"count":1,"lpHisto":1},{"lpHisto":3,"count":3,"graph":{"db2Histo":[[1386003600000,4801]],"interval":3600,"lpHisto":[[1386003600000,3]],"pa1Histo":[[1386003600000,26]],"xmax":1482552000000,"pa2Histo":[[1386003600000,20]],"xmin":1386003600000,"db1Histo":[[1386003600000,486]]},"dbHisto":5287,"name":"curl/7.24.0 (x86_64-apple-darwin12.0) libcurl/7.24.0 OpenSSL/0.9.8y zlib/1.2.5","map":{"src":{"USA":3},"dst":{"USA":3}},"paHisto":40}]'), "items field: rawua"); - eq_or_diff($json->{graph}->{lpHisto}, from_json('[["1386003600000", 3], ["1386788400000", 1], [1387742400000, 1], [1482552000000, 1]]'), "multi lpHisto field: rawua"); - eq_or_diff($json->{graph}->{pa1Histo}, from_json('[["1386003600000", 26], ["1386788400000", 31], [1387742400000, 3], [1482552000000, 3]]'), "multi pa1Histo field: rawua"); - eq_or_diff($json->{graph}->{pa2Histo}, from_json('[["1386003600000", 20], ["1386788400000", 21], [1387742400000, 1], [1482552000000, 1]]'), "multi pa2Histo field: rawua"); - eq_or_diff($json->{graph}->{db1Histo}, from_json('[["1386003600000", 486], ["1386788400000", 23392], [1387742400000, 68], [1482552000000, 68]]'), "multi db1Histo field: rawua"); - eq_or_diff($json->{graph}->{db2Histo}, from_json('[["1386003600000", 4801], ["1386788400000", 954], [1387742400000, 0], [1482552000000, 0]]'), "multi db2Histo field: rawua"); - -#rawua multi - $json = multiGet("/spigraph.json?date=-1&field=rawua&expression=" . uri_escape("file=$pwd/socks5-reverse.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); + eq_or_diff(\@items, from_json('[{"paHisto":4,"name":"10.0.0.1","count":1,"lpHisto":1,"map":{"src":{"RU":1},"dst":{"CA":1}},"dbHisto":68,"graph":{"lpHisto":[[1387742400000,1]],"xmin":1335956400000,"db2Histo":[[1387742400000,0]],"db1Histo":[[1387742400000,68]],"pa2Histo":[[1387742400000,1]],"xmax":1482552000000,"pa1Histo":[[1387742400000,3]],"interval":3600}},{"name":"10.10.10.10","paHisto":4,"count":1,"lpHisto":1,"map":{"dst":{},"src":{}},"graph":{"interval":3600,"pa1Histo":[[1482552000000,3]],"xmax":1482552000000,"pa2Histo":[[1482552000000,1]],"db2Histo":[[1482552000000,0]],"db1Histo":[[1482552000000,68]],"xmin":1335956400000,"lpHisto":[[1482552000000,1]]},"dbHisto":68},{"graph":{"db2Histo":[[1386003600000,4801]],"db1Histo":[[1386003600000,486]],"lpHisto":[[1386003600000,3]],"xmin":1335956400000,"pa1Histo":[[1386003600000,26]],"interval":3600,"xmax":1482552000000,"pa2Histo":[[1386003600000,20]]},"dbHisto":5287,"lpHisto":3,"count":3,"map":{"dst":{"US":3},"src":{"US":3}},"name":"10.180.156.185","paHisto":46},{"graph":{"xmax":1482552000000,"pa2Histo":[[1335956400000,0]],"pa1Histo":[[1335956400000,2]],"interval":3600,"lpHisto":[[1335956400000,1]],"xmin":1335956400000,"db1Histo":[[1335956400000,0]],"db2Histo":[[1335956400000,0]]},"dbHisto":0,"map":{"dst":{},"src":{}},"lpHisto":1,"count":1,"name":"192.168.177.160","paHisto":2}]'), "items field: srcIp", { context => 3 }); + cmp_ok ($json->{recordsTotal}, '>=', 194); + cmp_ok ($json->{recordsFiltered}, '==', 6); + +#srcIp multi + $mjson = multiGet("/spigraph.json?date=-1&field=srcIp&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); + eq_or_diff($mjson->{map}, $json->{map}, "multi map field: srcIp"); + eq_or_diff($mjson->{graph}->{lpHisto}, $json->{graph}->{lpHisto}, "multi lpHisto field: srcIp"); + eq_or_diff($mjson->{graph}->{pa1Histo}, $json->{graph}->{pa1Histo}, "multi pa1Histo field: srcIp"); + eq_or_diff($mjson->{graph}->{pa2Histo}, $json->{graph}->{pa2Histo}, "multi pa2Histo field: srcIp"); + eq_or_diff($mjson->{graph}->{db1Histo}, $json->{graph}->{db1Histo}, "multi db1Histo field: srcIp"); + eq_or_diff($mjson->{graph}->{db2Histo}, $json->{graph}->{db2Histo}, "multi db2Histo field: srcIp"); + eq_or_diff($mjson->{items}, $json->{items}, "multi items field: srcIp"); + + my @mitems = sort({$a->{name} cmp $b->{name}} @{$json->{items}}); + eq_or_diff(\@items, \@items, "multi items field: srcIp"); + + SKIP: { + skip "Upgrade test", 15 if ($ENV{MOLOCH_REINDEX_TEST}); # reindex doesn't have requestHeader + +#http.requestHeader + $json = viewerGet("/spigraph.json?date=-1&field=http.requestHeader&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); + eq_or_diff($json->{map}, from_json('{"dst":{"US": 3, "CA": 1}, "src":{"US": 3, "RU":1}}'), "map field: http.requestHeader"); + eq_or_diff($json->{graph}->{lpHisto}, from_json('[["1335956400000", 1], ["1386003600000", 3], [1387742400000, 1], [1482552000000, 1]]'), "lpHisto field: h1"); + eq_or_diff($json->{graph}->{pa1Histo}, from_json('[["1335956400000", 2], ["1386003600000", 26], [1387742400000, 3], [1482552000000, 3]]'), "pa1Histo field: h1"); + eq_or_diff($json->{graph}->{pa2Histo}, from_json('[["1335956400000", 0], ["1386003600000", 20], [1387742400000, 1], [1482552000000, 1]]'), "pa2Histo field: h1"); + eq_or_diff($json->{graph}->{db1Histo}, from_json('[["1335956400000", 0], ["1386003600000", 486], [1387742400000, 68], [1482552000000, 68]]'), "db1Histo field: h1"); + eq_or_diff($json->{graph}->{db2Histo}, from_json('[["1335956400000", 0], ["1386003600000", 4801], [1387742400000, 0], [1482552000000, 0]]'), "db2Histo field: h1"); + my @items = sort({$a->{name} cmp $b->{name}} @{$json->{items}}); + eq_or_diff(\@items, from_json('[{"name":"accept","count":3,"dbHisto":5287,"map":{"dst":{"US":3},"src":{"US":3}},"graph":{"pa2Histo":[[1386003600000,20]],"pa1Histo":[[1386003600000,26]],"interval":3600,"lpHisto":[[1386003600000,3]],"xmax":1482552000000,"db1Histo":[[1386003600000,486]],"xmin":1335956400000,"db2Histo":[[1386003600000,4801]]},"paHisto":46,"lpHisto":3},{"lpHisto":3,"name":"host","count":3,"map":{"dst":{"US":3},"src":{"US":3}},"dbHisto":5287,"graph":{"db2Histo":[[1386003600000,4801]],"lpHisto":[[1386003600000,3]],"xmax":1482552000000,"db1Histo":[[1386003600000,486]],"xmin":1335956400000,"pa2Histo":[[1386003600000,20]],"pa1Histo":[[1386003600000,26]],"interval":3600},"paHisto":46},{"lpHisto":3,"name":"user-agent","count":3,"dbHisto":5287,"graph":{"interval":3600,"pa2Histo":[[1386003600000,20]],"pa1Histo":[[1386003600000,26]],"db1Histo":[[1386003600000,486]],"xmin":1335956400000,"lpHisto":[[1386003600000,3]],"xmax":1482552000000,"db2Histo":[[1386003600000,4801]]},"map":{"src":{"US":3},"dst":{"US":3}},"paHisto":46}]'), "items field: http.requestHeader", { context => 3 }); + cmp_ok ($json->{recordsTotal}, '>=', 194); + cmp_ok ($json->{recordsFiltered}, '==', 6); + +#http.requestHeader multi + $mjson = multiGet("/spigraph.json?date=-1&field=http.requestHeader&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); + eq_or_diff($mjson->{map}, $json->{map}, "multi map field: http.requestHeader"); + eq_or_diff($mjson->{graph}->{lpHisto}, $json->{graph}->{lpHisto}, "multi lpHisto field: http.requestHeader"); + eq_or_diff($mjson->{graph}->{pa1Histo}, $json->{graph}->{pa1Histo}, "multi pa1Histo field: http.requestHeader"); + eq_or_diff($mjson->{graph}->{pa2Histo}, $json->{graph}->{pa2Histo}, "multi pa2Histo field: http.requestHeader"); + eq_or_diff($mjson->{graph}->{db1Histo}, $json->{graph}->{db1Histo}, "multi db1Histo field: http.requestHeader"); + eq_or_diff($mjson->{graph}->{db2Histo}, $json->{graph}->{db2Histo}, "multi db2Histo field: http.requestHeader"); + eq_or_diff($mjson->{items}, $json->{items}, "multi items field: http.requestHeader"); + + my @mitems = sort({$a->{name} cmp $b->{name}} @{$json->{items}}); + eq_or_diff(\@items, \@items, "multi items field: http.requestHeader"); + } + + +#http.useragent + $json = viewerGet("/spigraph.json?date=-1&field=http.useragent&expression=" . uri_escape("file=$pwd/socks5-reverse.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); my @items = sort({$a->{name} cmp $b->{name}} @{$json->{items}}); - eq_or_diff(\@items, from_json('[{"map":{"dst":{"CAN":1},"src":{"RUS":1}},"paHisto":42,"name":"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)","dbHisto":24346,"graph":{"xmax":1482552000000,"pa2Histo":[[1386788400000,21]],"pa1Histo":[[1386788400000,31]],"db2Histo":[[1386788400000,954]],"interval":3600,"lpHisto":[[1386788400000,1]],"xmin":1386003600000,"db1Histo":[[1386788400000,23392]]},"count":1,"lpHisto":1},{"lpHisto":3,"count":3,"graph":{"db2Histo":[[1386003600000,4801]],"interval":3600,"lpHisto":[[1386003600000,3]],"pa1Histo":[[1386003600000,26]],"xmax":1482552000000,"pa2Histo":[[1386003600000,20]],"xmin":1386003600000,"db1Histo":[[1386003600000,486]]},"dbHisto":5287,"name":"curl/7.24.0 (x86_64-apple-darwin12.0) libcurl/7.24.0 OpenSSL/0.9.8y zlib/1.2.5","map":{"src":{"USA":3},"dst":{"USA":3}},"paHisto":40}]'), "multi items field: rawua"); + eq_or_diff(\@items, from_json('[{"map":{"dst":{"CA":1},"src":{"RU":1}},"paHisto":52,"name":"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)","dbHisto":24346,"graph":{"xmax":1482552000000,"pa2Histo":[[1386788400000,21]],"pa1Histo":[[1386788400000,31]],"db2Histo":[[1386788400000,954]],"interval":3600,"lpHisto":[[1386788400000,1]],"xmin":1386003600000,"db1Histo":[[1386788400000,23392]]},"count":1,"lpHisto":1},{"lpHisto":3,"count":3,"graph":{"db2Histo":[[1386003600000,4801]],"interval":3600,"lpHisto":[[1386003600000,3]],"pa1Histo":[[1386003600000,26]],"xmax":1482552000000,"pa2Histo":[[1386003600000,20]],"xmin":1386003600000,"db1Histo":[[1386003600000,486]]},"dbHisto":5287,"name":"curl/7.24.0 (x86_64-apple-darwin12.0) libcurl/7.24.0 OpenSSL/0.9.8y zlib/1.2.5","map":{"src":{"US":3},"dst":{"US":3}},"paHisto":46}]'), "items field: http.useragent", { context => 3 }); + eq_or_diff($json->{graph}->{lpHisto}, from_json('[["1386003600000", 3], ["1386788400000", 1], [1387742400000, 1], [1482552000000, 1]]'), "multi lpHisto field: http.useragent"); + eq_or_diff($json->{graph}->{pa1Histo}, from_json('[["1386003600000", 26], ["1386788400000", 31], [1387742400000, 3], [1482552000000, 3]]'), "multi pa1Histo field: http.useragent"); + eq_or_diff($json->{graph}->{pa2Histo}, from_json('[["1386003600000", 20], ["1386788400000", 21], [1387742400000, 1], [1482552000000, 1]]'), "multi pa2Histo field: http.useragent"); + eq_or_diff($json->{graph}->{db1Histo}, from_json('[["1386003600000", 486], ["1386788400000", 23392], [1387742400000, 68], [1482552000000, 68]]'), "multi db1Histo field: http.useragent"); + eq_or_diff($json->{graph}->{db2Histo}, from_json('[["1386003600000", 4801], ["1386788400000", 954], [1387742400000, 0], [1482552000000, 0]]'), "multi db2Histo field: http.useragent"); + cmp_ok ($json->{recordsTotal}, '>=', 194); + cmp_ok ($json->{recordsFiltered}, '==', 6); + +#http.useragent multi + $mjson = multiGet("/spigraph.json?date=-1&field=http.useragent&expression=" . uri_escape("file=$pwd/socks5-reverse.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); + my @mitems = sort({$a->{name} cmp $b->{name}} @{$json->{items}}); + eq_or_diff(\@items, \@items, "multi items field: http.useragent"); diff --git a/tests/api-spiview.t b/tests/api-spiview.t index f0f273ee34..91ac51ecf2 100644 --- a/tests/api-spiview.t +++ b/tests/api-spiview.t @@ -7,21 +7,22 @@ use Test::Differences; use Data::Dumper; use strict; -my $pwd = getcwd() . "/pcap"; +my $pwd = "*/pcap"; +my $fpwd = getcwd() . "/pcap"; # bigendian pcap file tests - my $json = viewerGet("/spiview.json?date=-1&facets=1&spi=a1,a2,pr,fileand&expression=" . uri_escape("file=$pwd/bigendian.pcap")); - my $mjson = multiGet("/spiview.json?date=-1&facets=1&spi=a1,a2,pr,fileand&expression=" . uri_escape("file=$pwd/bigendian.pcap")); - my $djson = multiGet("/spiview.json?startTime=1332734457&stopTime=1389743152&facets=1&spi=a1,a2,pr,fileand&expression=" . uri_escape("file=$pwd/bigendian.pcap")); + my $json = viewerGet("/spiview.json?date=-1&facets=1&spi=srcIp,dstIp,ipProtocol,fileand&expression=" . uri_escape("file=$pwd/bigendian.pcap")); + my $mjson = multiGet("/spiview.json?date=-1&facets=1&spi=srcIp,dstIp,ipProtocol,fileand&expression=" . uri_escape("file=$pwd/bigendian.pcap")); + my $djson = multiGet("/spiview.json?startTime=1332734457&stopTime=1389743152&facets=1&spi=srcIp,dstIp,ipProtocol,fileand&expression=" . uri_escape("file=$pwd/bigendian.pcap")); eq_or_diff($json->{map}, from_json('{"src": {}, "dst":{}}'), "map bigendian"); eq_or_diff($json->{protocols}, from_json('{"icmp": 1}'), "protocols bigendian"); eq_or_diff($json->{graph}, from_json('{"xmin":null,"pa2Histo":[[1335956400000,0]],"pa1Histo":[[1335956400000,2]],"xmax":null,"db2Histo":[[1335956400000,0]],"interval":3600,"lpHisto":[[1335956400000,1]],"db1Histo":[[1335956400000,0]]}'), "graph bigendian"); eq_or_diff($djson->{graph}, from_json('{"xmin":1332734457000,"pa2Histo":[[1335956400000,0]],"pa1Histo":[[1335956400000,2]],"db2Histo":[[1335956400000,0]],"interval":3600,"xmax":1389743152000,"lpHisto":[[1335956400000,1]],"db1Histo":[[1335956400000,0]]}'), "date graph bigendian"); - eq_or_diff($json->{spi}->{a1}, from_json('{"doc_count_error_upper_bound": 0, "sum_other_doc_count": 0, "buckets":[{"doc_count":1, "key":3232280992}]}'), "bigendian a1"); - eq_or_diff($json->{spi}->{a2}, from_json('{"doc_count_error_upper_bound": 0, "sum_other_doc_count": 0, "buckets":[{"doc_count":1, "key":171969329}]}'), "bigendian a2"); - eq_or_diff($json->{spi}->{pr}, from_json('{"doc_count_error_upper_bound": 0, "sum_other_doc_count": 0, "buckets":[{"doc_count":1, "key":"icmp"}]}'), "bigendian pr"); - eq_or_diff($json->{spi}->{fileand}, from_json(qq({"doc_count_error_upper_bound": 0, "sum_other_doc_count": 0, "buckets":[{"doc_count":1, "key":"$pwd/bigendian.pcap"}]})), "bigendian fileand"); + eq_or_diff($json->{spi}->{srcIp}, from_json('{"doc_count_error_upper_bound": 0, "sum_other_doc_count": 0, "buckets":[{"doc_count":1, "key":"192.168.177.160"}]}'), "bigendian srcIp"); + eq_or_diff($json->{spi}->{dstIp}, from_json('{"doc_count_error_upper_bound": 0, "sum_other_doc_count": 0, "buckets":[{"doc_count":1, "key":"10.64.11.49"}]}'), "bigendian dstIp"); + eq_or_diff($json->{spi}->{ipProtocol}, from_json('{"doc_count_error_upper_bound": 0, "sum_other_doc_count": 0, "buckets":[{"doc_count":1, "key":"icmp"}]}'), "bigendian ipProtocol"); + eq_or_diff($json->{spi}->{fileand}, from_json(qq({"doc_count_error_upper_bound": 0, "sum_other_doc_count": 0, "buckets":[{"doc_count":1, "key":"$fpwd/bigendian.pcap"}]})), "bigendian fileand"); is ($json->{health}->{number_of_data_nodes}, 1, "Correct health number_of_data_nodes bigendian"); is ($mjson->{health}->{number_of_data_nodes}, 2, "Correct health number_of_data_nodes multi bigendian"); @@ -35,14 +36,14 @@ my $pwd = getcwd() . "/pcap"; eq_or_diff($json, $djson, "single doesn't match date", { context => 3 }); # bigendian pcap file tests no facets - $json = viewerGet("/spiview.json?date=-1&spi=a1,a2,pr&expression=" . uri_escape("file=$pwd/bigendian.pcap")); - $mjson = multiGet("/spiview.json?date=-1&spi=a1,a2,pr&expression=" . uri_escape("file=$pwd/bigendian.pcap")); + $json = viewerGet("/spiview.json?date=-1&spi=srcIp,dstIp,ipProtocol&expression=" . uri_escape("file=$pwd/bigendian.pcap")); + $mjson = multiGet("/spiview.json?date=-1&spi=srcIp,dstIp,ipProtocol&expression=" . uri_escape("file=$pwd/bigendian.pcap")); is (!exists $json->{map}, 1, "map bigendian no facets"); is (!exists $json->{graph}, 1, "graph bigendian no facets"); - eq_or_diff($json->{spi}->{a1}, from_json('{"doc_count_error_upper_bound": 0, "sum_other_doc_count": 0, "buckets":[{"doc_count":1, "key":3232280992}]}'), "bigendian a1 no facets"); - eq_or_diff($json->{spi}->{a2}, from_json('{"doc_count_error_upper_bound": 0, "sum_other_doc_count": 0, "buckets":[{"doc_count":1, "key":171969329}]}'), "bigendian a2 no facets"); - eq_or_diff($json->{spi}->{pr}, from_json('{"doc_count_error_upper_bound": 0, "sum_other_doc_count": 0, "buckets":[{"doc_count":1, "key":"icmp"}]}'), "bigendian pr no facets"); + eq_or_diff($json->{spi}->{srcIp}, from_json('{"doc_count_error_upper_bound": 0, "sum_other_doc_count": 0, "buckets":[{"doc_count":1, "key":"192.168.177.160"}]}'), "bigendian srcIp no facets"); + eq_or_diff($json->{spi}->{dstIp}, from_json('{"doc_count_error_upper_bound": 0, "sum_other_doc_count": 0, "buckets":[{"doc_count":1, "key":"10.64.11.49"}]}'), "bigendian dstIp no facets"); + eq_or_diff($json->{spi}->{ipProtocol}, from_json('{"doc_count_error_upper_bound": 0, "sum_other_doc_count": 0, "buckets":[{"doc_count":1, "key":"icmp"}]}'), "bigendian ipProtocol no facets"); is ($json->{health}->{number_of_data_nodes}, 1, "Correct health number_of_data_nodes bigendian no facets"); is ($mjson->{health}->{number_of_data_nodes}, 2, "Correct health number_of_data_nodes multi bigendian no facets"); @@ -52,10 +53,10 @@ my $pwd = getcwd() . "/pcap"; eq_or_diff($json, $mjson, "single doesn't match multi", { context => 3 }); # Check facets short - $json = viewerGet("/spiview.json?startTime=1386004308&stopTime=1386004400&facets=1&spi=a1,a2,pr,fileand&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); - $mjson = multiGet("/spiview.json?startTime=1386004308&stopTime=1386004400&facets=1&spi=a1,a2,pr,fileand&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); + $json = viewerGet("/spiview.json?startTime=1386004308&stopTime=1386004400&facets=1&spi=srcIp,dstIp,ipProtocol,fileand&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); + $mjson = multiGet("/spiview.json?startTime=1386004308&stopTime=1386004400&facets=1&spi=srcIp,dstIp,ipProtocol,fileand&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); - eq_or_diff($json->{map}, from_json('{"src":{"USA": 3}, "dst":{"USA": 3}}'), "map short"); + eq_or_diff($json->{map}, from_json('{"src":{"US": 3}, "dst":{"US": 3}}'), "map short"); eq_or_diff($json->{protocols}, from_json('{"http": 3, "socks": 3, "tcp": 3}'), "protocols short"); eq_or_diff($json->{graph}->{lpHisto}, from_json('[["1386004309000", 1], ["1386004312000", 1], [1386004317000, 1]]'), "lpHisto short"); eq_or_diff($json->{graph}->{pa1Histo}, from_json('[["1386004309000", 8], ["1386004312000", 8], [1386004317000, 10]]'), "pa1Histo short"); @@ -66,20 +67,20 @@ my $pwd = getcwd() . "/pcap"; is ($json->{graph}->{interval}, 1, "correct interval short"); is ($json->{graph}->{xmax}, 1386004400000, "correct xmax short"); is ($json->{graph}->{xmin}, 1386004308000, "correct xmin short"); - eq_or_diff($json->{spi}->{a1}, from_json('{"doc_count_error_upper_bound": 0, "sum_other_doc_count": 0, "buckets":[{"doc_count":3, "key":179608761}]}'), "short a1"); - eq_or_diff($json->{spi}->{a2}, from_json('{"doc_count_error_upper_bound": 0, "sum_other_doc_count": 0, "buckets":[{"doc_count":3, "key":179608825}]}'), "short a2"); - eq_or_diff($json->{spi}->{pr}, from_json('{"doc_count_error_upper_bound": 0, "sum_other_doc_count": 0, "buckets":[{"doc_count":3, "key":"tcp"}]}'), "short pr"); - eq_or_diff($json->{spi}->{fileand}, from_json(qq({"doc_count_error_upper_bound": 0, "sum_other_doc_count": 0, "buckets":[{"doc_count":3, "key":"$pwd/socks-http-example.pcap"}]})), "bigendian fileand"); + eq_or_diff($json->{spi}->{srcIp}, from_json('{"doc_count_error_upper_bound": 0, "sum_other_doc_count": 0, "buckets":[{"doc_count":3, "key":"10.180.156.185"}]}'), "short srcIp"); + eq_or_diff($json->{spi}->{dstIp}, from_json('{"doc_count_error_upper_bound": 0, "sum_other_doc_count": 0, "buckets":[{"doc_count":3, "key":"10.180.156.249"}]}'), "short dstIp"); + eq_or_diff($json->{spi}->{ipProtocol}, from_json('{"doc_count_error_upper_bound": 0, "sum_other_doc_count": 0, "buckets":[{"doc_count":3, "key":"tcp"}]}'), "short ipProtocol"); + eq_or_diff($json->{spi}->{fileand}, from_json(qq({"doc_count_error_upper_bound": 0, "sum_other_doc_count": 0, "buckets":[{"doc_count":3, "key":"$fpwd/socks-http-example.pcap"}]})), "bigendian fileand"); delete $json->{health}; delete $mjson->{health}; eq_or_diff($json, $mjson, "single doesn't match multi", { context => 3 }); # Check facets medium - $json = viewerGet("/spiview.json?startTime=1386004308&stopTime=1386349908&facets=1&spi=a1,a2,pr&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); - $mjson = multiGet("/spiview.json?startTime=1386004308&stopTime=1386349908&facets=1&spi=a1,a2,pr&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); + $json = viewerGet("/spiview.json?startTime=1386004308&stopTime=1386349908&facets=1&spi=srcIp,dstIp,ipProtocol&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); + $mjson = multiGet("/spiview.json?startTime=1386004308&stopTime=1386349908&facets=1&spi=srcIp,dstIp,ipProtocol&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); - eq_or_diff($json->{map}, from_json('{"src":{"USA": 3}, "dst":{"USA": 3}}'), "map medium"); + eq_or_diff($json->{map}, from_json('{"src":{"US": 3}, "dst":{"US": 3}}'), "map medium"); eq_or_diff($json->{protocols}, from_json('{"http": 3, "socks": 3, "tcp": 3}'), "protocols medium"); eq_or_diff($json->{graph}->{lpHisto}, from_json('[["1386004260000", 3]]'), "lpHisto medium"); eq_or_diff($json->{graph}->{pa1Histo}, from_json('[["1386004260000", 26]]'), "pa1Histo medium"); @@ -90,28 +91,28 @@ my $pwd = getcwd() . "/pcap"; is ($json->{graph}->{interval}, 60, "correct interval medium"); is ($json->{graph}->{xmax}, 1386349908000, "correct xmax medium"); is ($json->{graph}->{xmin}, 1386004308000, "correct xmin medium"); - eq_or_diff($json->{spi}->{a1}, from_json('{"doc_count_error_upper_bound": 0, "sum_other_doc_count": 0, - "buckets":[{"doc_count":3, "key":179608761}]}'), "medium a1"); - eq_or_diff($json->{spi}->{a2}, from_json('{"doc_count_error_upper_bound": 0, "sum_other_doc_count": 0, - "buckets":[{"doc_count":3, "key":179608825}]}'), "medium a2"); - eq_or_diff($json->{spi}->{pr}, from_json('{"doc_count_error_upper_bound": 0, "sum_other_doc_count": 0, - "buckets":[{"doc_count":3, "key":"tcp"}]}'), "medium pr"); + eq_or_diff($json->{spi}->{srcIp}, from_json('{"doc_count_error_upper_bound": 0, "sum_other_doc_count": 0, + "buckets":[{"doc_count":3, "key":"10.180.156.185"}]}'), "medium srcIp"); + eq_or_diff($json->{spi}->{dstIp}, from_json('{"doc_count_error_upper_bound": 0, "sum_other_doc_count": 0, + "buckets":[{"doc_count":3, "key":"10.180.156.249"}]}'), "medium dstIp"); + eq_or_diff($json->{spi}->{ipProtocol}, from_json('{"doc_count_error_upper_bound": 0, "sum_other_doc_count": 0, + "buckets":[{"doc_count":3, "key":"tcp"}]}'), "medium ipProtocol"); delete $json->{health}; delete $mjson->{health}; eq_or_diff($json, $mjson, "single doesn't match multi", { context => 3 }); # Check facets ALL - $json = viewerGet("/spiview.json?date=-1&facets=1&spi=a1,a2,pr,fileand,ta:3,hh1&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); - $mjson = multiGet("/spiview.json?date=-1&facets=1&spi=a1,a2,pr,fileand,ta:3,hh1&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); - $djson = viewerGet("/spiview.json?startTime=1332734457&stopTime=1482563001&facets=1&spi=a1,a2,pr,fileand,ta:3,hh1&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); + $json = viewerGet("/spiview.json?date=-1&facets=1&spi=srcIp,dstIp,ipProtocol,fileand,tags:5,http.requestHeader&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); + $mjson = multiGet("/spiview.json?date=-1&facets=1&spi=srcIp,dstIp,ipProtocol,fileand,tags:5,http.requestHeader&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); + $djson = viewerGet("/spiview.json?startTime=1332734457&stopTime=1482563001&facets=1&spi=srcIp,dstIp,ipProtocol,fileand,tags:5,http.requestHeader&expression=" . uri_escape("file=$pwd/bigendian.pcap|file=$pwd/socks-http-example.pcap|file=$pwd/bt-tcp.pcap")); # Sort alpha since counts are the same and could come back in random order - @{$json->{spi}->{hh1}->{buckets}} = sort({$a->{key} cmp $b->{key}} @{$json->{spi}->{hh1}->{buckets}}); - @{$mjson->{spi}->{hh1}->{buckets}} = sort({$a->{key} cmp $b->{key}} @{$mjson->{spi}->{hh1}->{buckets}}); - @{$djson->{spi}->{hh1}->{buckets}} = sort({$a->{key} cmp $b->{key}} @{$djson->{spi}->{hh1}->{buckets}}); + @{$json->{spi}->{"http.requestHeader"}->{buckets}} = sort({$a->{key} cmp $b->{key}} @{$json->{spi}->{"http.requestHeader"}->{buckets}}); + @{$mjson->{spi}->{"http.requestHeader"}->{buckets}} = sort({$a->{key} cmp $b->{key}} @{$mjson->{spi}->{"http.requestHeader"}->{buckets}}); + @{$djson->{spi}->{"http.requestHeader"}->{buckets}} = sort({$a->{key} cmp $b->{key}} @{$djson->{spi}->{"http.requestHeader"}->{buckets}}); - eq_or_diff($json->{map}, from_json('{"dst":{"USA": 3, "CAN": 1}, "src":{"USA": 3, "RUS":1}}'), "map ALL"); + eq_or_diff($json->{map}, from_json('{"dst":{"US": 3, "CA": 1}, "src":{"US": 3, "RU":1}}'), "map ALL"); eq_or_diff($json->{protocols}, from_json('{"tcp": 5, "http": 3, "socks": 3, "bittorrent": 2, "icmp": 1}'), "protocols ALL"); eq_or_diff($json->{graph}->{lpHisto}, from_json('[["1335956400000", 1], ["1386003600000", 3], [1387742400000, 1], [1482552000000,1]]'), "lpHisto ALL"); eq_or_diff($json->{graph}->{pa1Histo}, from_json('[["1335956400000", 2], ["1386003600000", 26], [1387742400000, 3], [1482552000000,3]]'), "pa1Histo ALL"); @@ -123,28 +124,31 @@ my $pwd = getcwd() . "/pcap"; is ($json->{recordsFiltered}, 6, "records ALL"); is ($json->{graph}->{interval}, 3600, "correct interval ALL"); - eq_or_diff($json->{spi}->{a1}, from_json('{"doc_count_error_upper_bound": 0, "sum_other_doc_count": 0, - "buckets":[{"doc_count":3, "key":179608761},{"doc_count":1, "key":167772161}, {"doc_count":1, "key":168430090}, {"doc_count":1, "key":3232280992}]}'), "ALL a1"); - eq_or_diff($json->{spi}->{a2}, from_json('{"doc_count_error_upper_bound": 0, "sum_other_doc_count": 0, - "buckets":[{"doc_count":3, "key":179608825}, {"doc_count":1, "key":167772162}, {"doc_count":1, "key":168495883}, {"doc_count":1, "key":171969329}]}'), "ALL a2"); - eq_or_diff($json->{spi}->{pr}, from_json('{"doc_count_error_upper_bound": 0, "sum_other_doc_count": 0, - "buckets":[{"doc_count":5, "key":"tcp"}, {"doc_count":1, "key":"icmp"}]}'), "ALL pr"); - eq_or_diff($json->{spi}->{fileand}, from_json(qq({"doc_count_error_upper_bound": 0, "sum_other_doc_count": 0, "buckets":[{"doc_count":3, "key":"$pwd/socks-http-example.pcap"}, {"doc_count":2, "key":"$pwd/bt-tcp.pcap"},{"doc_count":1, "key":"$pwd/bigendian.pcap"}]})), "bigendian fileand"); + eq_or_diff($json->{spi}->{srcIp}, from_json('{"doc_count_error_upper_bound": 0, "sum_other_doc_count": 0, + "buckets":[{"doc_count":3, "key":"10.180.156.185"},{"doc_count":1, "key":"10.0.0.1"}, {"doc_count":1, "key":"10.10.10.10"}, {"doc_count":1, "key":"192.168.177.160"}]}'), "ALL srcIp"); + eq_or_diff($json->{spi}->{dstIp}, from_json('{"doc_count_error_upper_bound": 0, "sum_other_doc_count": 0, + "buckets":[{"doc_count":3, "key":"10.180.156.249"}, {"doc_count":1, "key":"10.0.0.2"}, {"doc_count":1, "key":"10.11.11.11"}, {"doc_count":1, "key":"10.64.11.49"}]}'), "ALL dstIp"); + eq_or_diff($json->{spi}->{ipProtocol}, from_json('{"doc_count_error_upper_bound": 0, "sum_other_doc_count": 0, + "buckets":[{"doc_count":5, "key":"tcp"}, {"doc_count":1, "key":"icmp"}]}'), "ALL ipProtocol"); + eq_or_diff($json->{spi}->{fileand}, from_json(qq({"doc_count_error_upper_bound": 0, "sum_other_doc_count": 0, "buckets":[{"doc_count":3, "key":"$fpwd/socks-http-example.pcap"}, {"doc_count":2, "key":"$fpwd/bt-tcp.pcap"},{"doc_count":1, "key":"$fpwd/bigendian.pcap"}]})), "bigendian fileand"); - my @buckets = sort {$a->{key} cmp $b->{key}} @{$json->{spi}->{ta}->{buckets}}; - $json->{spi}->{ta}->{buckets} = \@buckets; + my @buckets = sort {$a->{key} cmp $b->{key}} @{$json->{spi}->{tags}->{buckets}}; + $json->{spi}->{tags}->{buckets} = \@buckets; - my @mbuckets = sort {$a->{key} cmp $b->{key}} @{$mjson->{spi}->{ta}->{buckets}}; - $mjson->{spi}->{ta}->{buckets} = \@mbuckets; + my @mbuckets = sort {$a->{key} cmp $b->{key}} @{$mjson->{spi}->{tags}->{buckets}}; + $mjson->{spi}->{tags}->{buckets} = \@mbuckets; - my @dbuckets = sort {$a->{key} cmp $b->{key}} @{$djson->{spi}->{ta}->{buckets}}; - $djson->{spi}->{ta}->{buckets} = \@dbuckets; + my @dbuckets = sort {$a->{key} cmp $b->{key}} @{$djson->{spi}->{tags}->{buckets}}; + $djson->{spi}->{tags}->{buckets} = \@dbuckets; - eq_or_diff($json->{spi}->{ta}, from_json('{"doc_count_error_upper_bound": 0, "sum_other_doc_count": 14, - "buckets":[{"doc_count":3, "key":"byhost2"},{"doc_count":3, "key":"hosttaggertest1"},{"doc_count":3, "key":"hosttaggertest2"}]}'), "ALL ta"); + eq_or_diff($json->{spi}->{tags}, from_json('{"doc_count_error_upper_bound": 0, "sum_other_doc_count": 8, + "buckets":[{"doc_count":3, "key":"byhost2"},{"doc_count":3, "key":"domainwise"},{"doc_count":3, "key":"hosttaggertest1"},{"doc_count":3, "key":"hosttaggertest2"},{"doc_count":3, "key":"wisebyhost2"}]}'), "ALL ta"); - eq_or_diff($json->{spi}->{hh1}, from_json('{"doc_count_error_upper_bound": 0, "sum_other_doc_count": 0, - "buckets":[{"doc_count":3, "key":"accept"},{"doc_count":3, "key":"host"}, {"doc_count":3, "key":"user-agent"}]}'), "ALL hh1"); + SKIP: { + skip "Upgrade test", 1 if ($ENV{MOLOCH_REINDEX_TEST}); # reindex doesn't have http.requestHeader + eq_or_diff($json->{spi}->{"http.requestHeader"}, from_json('{"doc_count_error_upper_bound": 0, "sum_other_doc_count": 0, + "buckets":[{"doc_count":3, "key":"accept"},{"doc_count":3, "key":"host"}, {"doc_count":3, "key":"user-agent"}]}'), "ALL http.requestHeader"); + } delete $json->{health}; delete $mjson->{health}; diff --git a/tests/api-stats.t b/tests/api-stats.t index 0e517932bd..1abee8c1f7 100644 --- a/tests/api-stats.t +++ b/tests/api-stats.t @@ -7,7 +7,7 @@ use JSON; use Test::Differences; use strict; -my $pwd = getcwd() . "/pcap"; +my $pwd = "*/pcap"; my $token = getTokenCookie(); my $test1Token = getTokenCookie("test1"); @@ -68,7 +68,7 @@ my $test1Token = getTokenCookie("test1"); $result = viewerPostToken("/esshard/exclude/ip/1.2.3.4", "", $token); eq_or_diff($result, from_json('{"success": true, "text": "Excluded"}'), "esshard: exclude ip"); - $result = viewerPostToken("/esshard/exclude/node/thenode", "", $token); + $result = viewerPostToken("/esshard/exclude/name/thenode", "", $token); eq_or_diff($result, from_json('{"success": true, "text": "Excluded"}'), "esshard: exclude node"); $result = viewerPostToken("/esshard/exclude/foobar/1.2.3.4", "", $token); @@ -87,7 +87,7 @@ my $test1Token = getTokenCookie("test1"); $result = viewerPostToken("/esshard/include/ip/1.2.3.4", "", $token); eq_or_diff($result, from_json('{"success": true, "text": "Included"}'), "esshard: include ip"); - $result = viewerPostToken("/esshard/include/node/thenode", "", $token); + $result = viewerPostToken("/esshard/include/name/thenode", "", $token); eq_or_diff($result, from_json('{"success": true, "text": "Included"}'), "esshard: include node"); $result = viewerPostToken("/esshard/include/foobar/1.2.3.4", "", $token); @@ -99,4 +99,3 @@ my $test1Token = getTokenCookie("test1"); $shards = viewerGet("/esshard/list"); eq_or_diff($shards->{nodeExcludes}, [], "esshard: nodeExcludes empty"); eq_or_diff($shards->{ipExcludes}, [], "esshard: ipExcludes empty"); - diff --git a/tests/api-tagging.t b/tests/api-tagging.t index 84f1458296..3d3cc7c8a9 100644 --- a/tests/api-tagging.t +++ b/tests/api-tagging.t @@ -1,4 +1,4 @@ -use Test::More tests => 42; +use Test::More tests => 36; use Cwd; use URI::Escape; use MolochTest; @@ -6,7 +6,7 @@ use JSON; use Data::Dumper; use strict; -my $pwd = getcwd() . "/pcap"; +my $pwd = "*/pcap"; my $json; @@ -19,19 +19,17 @@ my $json; # adding/removing tags test expression viewerPost("/addTags?date=-1&expression=file=$pwd/socks-http-example.pcap", "tags=TAGTEST1"); esGet("/_refresh"); - $json = countTest(3, "date=-1&fields=ta,tags-term,tacnt&expression=" . uri_escape("tags==TAGTEST1")); + $json = countTest(3, "date=-1&fields=tags,tagsCnt&expression=" . uri_escape("tags==TAGTEST1")); foreach my $item (@{$json->{data}}) { - is (scalar @{$item->{ta}}, scalar @{$item->{"tags-term"}}, "add: ta and tags-term match"); - is ($item->{tacnt}, scalar @{$item->{"tags-term"}}, "add: tacnt and array size match"); + is ($item->{tagsCnt}, scalar @{$item->{"tags"}}, "add: tagsCnt and array size match"); } viewerPost("/removeTags?date=-1&expression=file=$pwd/socks-http-example.pcap", "tags=TAGTEST1"); esGet("/_refresh"); countTest(0, "date=-1&expression=" . uri_escape("tags==TAGTEST1")); - $json = countTest(3, "date=-1&fields=ta,tags-term,tacnt&expression=" . uri_escape("file=$pwd/socks-http-example.pcap && tags==domainwise")); + $json = countTest(3, "date=-1&fields=tags,tagsCnt&expression=" . uri_escape("file=$pwd/socks-http-example.pcap && tags==domainwise")); foreach my $item (@{$json->{data}}) { - is (scalar @{$item->{ta}}, scalar @{$item->{"tags-term"}}, "remove: ta and tags-term match"); - is ($item->{tacnt}, scalar @{$item->{"tags-term"}}, "remove: tacnt and array size match"); + is ($item->{tagsCnt}, scalar @{$item->{"tags"}}, "remove: tagsCnt and array size match"); } # adding/removing tags test ids - remove doesn't work on ES 2.4 diff --git a/tests/api-unique.t b/tests/api-unique.t index e77b21dbe1..d2d8788ee9 100644 --- a/tests/api-unique.t +++ b/tests/api-unique.t @@ -1,4 +1,4 @@ -use Test::More tests => 28; +use Test::More tests => 30; use Cwd; use URI::Escape; use MolochTest; @@ -22,8 +22,8 @@ my ($param, $multi) = @_; return join("\n", @lines) . "\n"; } -my $pwd = getcwd() . "/pcap"; -my $filestr = "(file=$pwd/socks-http-example.pcap||file=$pwd/socks-http-pass.pcap||file=$pwd/socks-https-example.pcap||file=$pwd/socks5-http-302.pcap||file=$pwd/socks5-rdp.pcap||file=$pwd/socks5-reverse.pcap||file=$pwd/socks5-smtp-503.pcap)"; +my $pwd = "*/pcap"; +my $filestr = "(file=$pwd/socks-http-example.pcap||file=$pwd/socks-http-pass.pcap||file=$pwd/socks-https-example.pcap||file=$pwd/socks5-http-302.pcap||file=$pwd/socks5-rdp.pcap||file=$pwd/socks5-reverse.pcap||file=$pwd/socks5-smtp-503.pcap||file=$pwd/v6-http.pcap)"; my $files = uri_escape($filestr); @@ -31,57 +31,63 @@ my $files = uri_escape($filestr); # my $txt = get(""); my $mtxt = get("", 1); -is ($txt, "Missing field or exp parameter\n", "unique.txt no field parameter"); +is ($txt, "Missing field or exp parameter\n", "unique.txt node field parameter"); eq_or_diff($mtxt, $txt, "single doesn't match multi", { context => 3 }); # -$txt = get("date=-1&field=no"); -$mtxt = get("date=-1&field=no", 1); +$txt = get("date=-1&field=node"); +$mtxt = get("date=-1&field=node", 1); eq_or_diff($txt, "test\n", "Nodes", { context => 3 }); eq_or_diff($mtxt, $txt, "single doesn't match multi", { context => 3 }); # -$txt = get("date=-1&field=no&autocomplete=1&expression=" . uri_escape("node=te*")); -$mtxt = get("date=-1&field=no&autocomplete=1&expression=" . uri_escape("node=te*"), 1); +$txt = get("date=-1&field=node&autocomplete=1&expression=" . uri_escape("node=te*")); +$mtxt = get("date=-1&field=node&autocomplete=1&expression=" . uri_escape("node=te*"), 1); eq_or_diff($txt, "[\"test\"]\n", "Autocomplete Nodes", { context => 3 }); eq_or_diff($mtxt, "[]\n", "Multi Autocomplete Nodes", { context => 3 }); # -$txt = get("date=-1&field=no&expression=$files&counts=1"); -$mtxt = get("date=-1&field=no&expression=$files&counts=1", 1); -eq_or_diff($txt, "test, 13\n", "Nodes count", { context => 3 }); +$txt = get("date=-1&field=node&expression=$files&counts=1"); +$mtxt = get("date=-1&field=node&expression=$files&counts=1", 1); +eq_or_diff($txt, "test, 19\n", "Nodes count", { context => 3 }); eq_or_diff($mtxt, $txt, "single doesn't match multi", { context => 3 }); # -$txt = get("date=-1&field=a1&expression=$files&counts=1"); -$mtxt = get("date=-1&field=a1&expression=$files&counts=1", 1); -eq_or_diff($txt, +$txt = get("date=-1&field=srcIp&expression=$files&counts=1"); +$mtxt = get("date=-1&field=srcIp&expression=$files&counts=1", 1); +eq_or_diff($txt, "10.0.0.1, 2 10.0.0.2, 1 10.0.0.3, 1 10.180.156.185, 9 +2001:6f8:102d:0:1033:c4c:7e57:b19e, 1 +2001:6f8:102d:0:2d0:9ff:fee3:e8de, 1 +::, 1 +fe80::211:25ff:fe82:95b5, 2 +fe80::2d0:9ff:fee3:e8de, 1 ", "ip count", { context => 3 }); eq_or_diff($mtxt, $txt, "single doesn't match multi", { context => 3 }); # -$txt = get("date=-1&field=a1&autocomplete=1&expression=" . uri_escape("$filestr && ip.src=10.180")); -$mtxt = get("date=-1&field=a1&autocomplete=1&expression=" . uri_escape("$filestr && ip.src=10.180"), 1); +$txt = get("date=-1&field=srcIp&autocomplete=1&expression=" . uri_escape("$filestr && ip.src=10.180")); +$mtxt = get("date=-1&field=srcIp&autocomplete=1&expression=" . uri_escape("$filestr && ip.src=10.180"), 1); eq_or_diff($txt, "[\"10.180.156.185\"]\n", "Autocomplete IPs", { context => 3 }); eq_or_diff($mtxt, "[]\n", "Multi Autocomplete IPs", { context => 3 }); # -$txt = get("date=-1&field=ta&expression=$files&counts=1"); -$mtxt = get("date=-1&field=ta&expression=$files&counts=1", 1); -eq_or_diff($txt, +$txt = get("date=-1&field=tags&expression=$files&counts=1"); +$mtxt = get("date=-1&field=tags&expression=$files&counts=1", 1); +eq_or_diff($txt, "byhost2, 7 byip1, 1 +byip2, 1 domainwise, 7 dstip, 4 hosttaggertest1, 7 hosttaggertest2, 7 -iptaggertest1, 1 -iptaggertest2, 1 +iptaggertest1, 2 +iptaggertest2, 2 ipwise, 1 ipwisecsv, 4 smtp:authlogin, 1 @@ -93,76 +99,110 @@ wisebyip1, 1 eq_or_diff($mtxt, $txt, "single doesn't match multi", { context => 3 }); # -$txt = get("date=-1&field=hh1&expression=$files&counts=1"); -$mtxt = get("date=-1&field=hh1&expression=$files&counts=1", 1); -eq_or_diff($txt, -"accept, 6 -accept-encoding, 2 -accept-language, 1 +$txt = get("date=-1&field=http.requestHeader&expression=$files&counts=1"); +$mtxt = get("date=-1&field=http.requestHeader&expression=$files&counts=1", 1); + +SKIP: { + skip "Upgrade test", 1 if ($ENV{MOLOCH_REINDEX_TEST}); # reindex doesn't have http.has-header +eq_or_diff($txt, +"accept, 7 +accept-encoding, 3 +accept-language, 2 connection, 1 cookie, 2 -host, 6 +host, 7 referer, 1 -user-agent, 6 +user-agent, 7 ", "http header count", { context => 3 }); +} eq_or_diff($mtxt, $txt, "single doesn't match multi", { context => 3 }); # -$txt = get("date=-1&field=hmd5&expression=$files"); -$mtxt = get("date=-1&field=hmd5&expression=$files", 1); +$txt = get("date=-1&field=http.md5&expression=$files"); +$mtxt = get("date=-1&field=http.md5&expression=$files", 1); eq_or_diff($txt, "09b9c392dc1f6e914cea287cb6be34b0 2069181ae704855f29caf964ca52ec49 222315d36e1313774cb1c2f0eb06864f +27cb95a0c4fff954073bc23328021b96 b0cecae354b9eab1f04f70e46a612cb1 ", "http md5", { context => 3 }); eq_or_diff($mtxt, $txt, "single doesn't match multi", { context => 3 }); # -$txt = get("date=-1&field=hmd5&autocomplete=1&expression=" . uri_escape("$filestr && http.md5=2*")); -$mtxt = get("date=-1&field=hmd5&autocomplete=1&expression=" . uri_escape("$filestr && http.md5=2*"), 1); -eq_or_diff($txt, "[\"2069181ae704855f29caf964ca52ec49\",\"222315d36e1313774cb1c2f0eb06864f\",\"b0cecae354b9eab1f04f70e46a612cb1\"]\n", "Autocomplete HTTP md5s", { context => 3 }); +$txt = get("date=-1&field=http.md5&autocomplete=1&expression=" . uri_escape("$filestr && http.md5=2*")); +$mtxt = get("date=-1&field=http.md5&autocomplete=1&expression=" . uri_escape("$filestr && http.md5=2*"), 1); +eq_or_diff($txt, "[\"2069181ae704855f29caf964ca52ec49\",\"222315d36e1313774cb1c2f0eb06864f\",\"27cb95a0c4fff954073bc23328021b96\",\"b0cecae354b9eab1f04f70e46a612cb1\"]\n", "Autocomplete HTTP md5s", { context => 3 }); eq_or_diff($mtxt, "[]\n", "Multi Autocomplete HTTP md5s", { context => 3 }); # -$txt = get("date=-1&field=hmd5&expression=$files&counts=1"); -$mtxt = get("date=-1&field=hmd5&expression=$files&counts=1", 1); +$txt = get("date=-1&field=http.md5&expression=$files&counts=1"); +$mtxt = get("date=-1&field=http.md5&expression=$files&counts=1", 1); eq_or_diff($txt, "09b9c392dc1f6e914cea287cb6be34b0, 4 2069181ae704855f29caf964ca52ec49, 1 222315d36e1313774cb1c2f0eb06864f, 1 +27cb95a0c4fff954073bc23328021b96, 1 b0cecae354b9eab1f04f70e46a612cb1, 1 ", "http md5 count", { context => 3 }); eq_or_diff($mtxt, $txt, "single doesn't match multi", { context => 3 }); # -$txt = get("date=-1&field=rawus&expression=$files&counts=0"); -$mtxt = get("date=-1&field=rawus&expression=$files&counts=0", 1); +$txt = get("date=-1&field=http.uri&expression=$files&counts=0"); +$mtxt = get("date=-1&field=http.uri&expression=$files&counts=0", 1); eq_or_diff($txt, -"//www.example.com/ -//www.google.com/ -//www.google.com/search?client=firefox&rls=en&q=Sheepskin%20Boots&start=0&num=10&hl=en&gl=us&uule=xxxxxxxxxxxxxxxxxxxxxxxxxxxx -//www.google.com/search?client=firefox&rls=en&q=Sheepskin%20Boots&start=10&num=10&hl=en&gl=us&uule=xxxxxxxxxxxxxxxxxxxxxxxxxxxx +"cl-1985.ham-01.de.sixxs.net/ +www.example.com/ +www.google.com/ +www.google.com/search?client=firefox&rls=en&q=Sheepskin%20Boots&start=0&num=10&hl=en&gl=us&uule=xxxxxxxxxxxxxxxxxxxxxxxxxxxx +www.google.com/search?client=firefox&rls=en&q=Sheepskin%20Boots&start=10&num=10&hl=en&gl=us&uule=xxxxxxxxxxxxxxxxxxxxxxxxxxxx ", "http uri", { context => 3 }); eq_or_diff($mtxt, $txt, "single doesn't match multi", { context => 3 }); # -$txt = get("date=-1&field=rawus&expression=$files&counts=1"); -$mtxt = get("date=-1&field=rawus&expression=$files&counts=1", 1); +$txt = get("date=-1&field=http.uri&expression=$files&counts=1"); +$mtxt = get("date=-1&field=http.uri&expression=$files&counts=1", 1); eq_or_diff($txt, -"//www.example.com/, 4 -//www.google.com/, 1 -//www.google.com/search?client=firefox&rls=en&q=Sheepskin%20Boots&start=0&num=10&hl=en&gl=us&uule=xxxxxxxxxxxxxxxxxxxxxxxxxxxx, 1 -//www.google.com/search?client=firefox&rls=en&q=Sheepskin%20Boots&start=10&num=10&hl=en&gl=us&uule=xxxxxxxxxxxxxxxxxxxxxxxxxxxx, 1 +"cl-1985.ham-01.de.sixxs.net/, 1 +www.example.com/, 4 +www.google.com/, 1 +www.google.com/search?client=firefox&rls=en&q=Sheepskin%20Boots&start=0&num=10&hl=en&gl=us&uule=xxxxxxxxxxxxxxxxxxxxxxxxxxxx, 1 +www.google.com/search?client=firefox&rls=en&q=Sheepskin%20Boots&start=10&num=10&hl=en&gl=us&uule=xxxxxxxxxxxxxxxxxxxxxxxxxxxx, 1 ", "http uri", { context => 3 }); eq_or_diff($mtxt, $txt, "single doesn't match multi", { context => 3 }); # -$txt = get("date=-1&field=rawua&expression=$files&counts=0"); -$mtxt = get("date=-1&field=rawua&expression=$files&counts=0", 1); +$txt = get("date=-1&field=http.useragent&expression=$files&counts=0"); +$mtxt = get("date=-1&field=http.useragent&expression=$files&counts=0", 1); eq_or_diff($txt, -"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322) +"Lynx/2.8.6rel.2 libwww-FM/2.14 SSL-MM/1.4.1 OpenSSL/0.9.8b +Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322) Mozilla/5.0 (Windows NT 5.1; rv:25.0) Gecko/20100101 Firefox/25.0 curl/7.24.0 (x86_64-apple-darwin12.0) libcurl/7.24.0 OpenSSL/0.9.8y zlib/1.2.5 ", "http user agent", {context => 3 }); eq_or_diff($mtxt, $txt, "single doesn't match multi", { context => 3 }); + +# +$txt = get("date=-1&field=ip.src:srcPort&expression=$files&counts=1"); +$mtxt = get("date=-1&field=ip.src:srcPort&expression=$files&counts=1", 1); +eq_or_diff($txt, +"10.0.0.1:1637, 1 +10.0.0.1:54263, 1 +10.0.0.2:53709, 1 +10.0.0.3:2276, 1 +10.180.156.185:53533, 1 +10.180.156.185:53534, 1 +10.180.156.185:53535, 1 +10.180.156.185:53554, 1 +10.180.156.185:53555, 1 +10.180.156.185:53556, 1 +10.180.156.185:54068, 1 +10.180.156.185:54069, 1 +10.180.156.185:54072, 1 +2001:6f8:102d:0:1033:c4c:7e57:b19e.5353, 1 +2001:6f8:102d:0:2d0:9ff:fee3:e8de.59201, 1 +::.0, 1 +fe80::211:25ff:fe82:95b5.0, 2 +fe80::2d0:9ff:fee3:e8de.0, 1 +", "ip count", { context => 3 }); +eq_or_diff($mtxt, $txt, "single doesn't match multi", { context => 3 }); diff --git a/tests/api-users.t b/tests/api-users.t index 0bb1fbcbc7..7769683916 100644 --- a/tests/api-users.t +++ b/tests/api-users.t @@ -1,4 +1,4 @@ -use Test::More tests => 53; +use Test::More tests => 71; use Cwd; use URI::Escape; use MolochTest; @@ -7,7 +7,7 @@ use Test::Differences; use Data::Dumper; use strict; -my $pwd = getcwd() . "/pcap"; +my $pwd = "*/pcap"; my $token = getTokenCookie(); my $token2 = getTokenCookie2(); @@ -55,6 +55,38 @@ my $pwd = getcwd() . "/pcap"; is (@{$users->{data}}, 2, "Check second add #2"); eq_or_diff($users->{data}->[1], from_json('{"createEnabled": false, "userId": "test2", "removeEnabled": false, "expression": "", "headerAuthEnabled": false, "userName": "UserName2", "id": "test2", "emailSearch": false, "enabled": true, "webEnabled": false}', {relaxed => 1}), "Test User Add", { context => 3 }); +# Filter + $users = viewerPost("/user/list", "filter=test"); + is (@{$users->{data}}, 2, "filter both"); + is ($users->{recordsTotal}, 2); + is ($users->{recordsFiltered}, 2); + + $users = viewerPost("/user/list", "filter=test1"); + is (@{$users->{data}}, 1, "filter one"); + is ($users->{recordsTotal}, 2); + is ($users->{recordsFiltered}, 1); + +# start, length + $users = viewerPost("/user/list", "start=0&length=2"); + is (@{$users->{data}}, 2, "start=0&length=2"); + is ($users->{recordsTotal}, 2); + is ($users->{recordsFiltered}, 2); + + $users = viewerPost("/user/list", "start=1&length=2"); + is (@{$users->{data}}, 1, "start=1&length=2"); + is ($users->{recordsTotal}, 2); + is ($users->{recordsFiltered}, 2); + + $users = viewerPost("/user/list", "start=0&length=1"); + is (@{$users->{data}}, 1, "start=1&length=1"); + is ($users->{recordsTotal}, 2); + is ($users->{recordsFiltered}, 2); + + $users = viewerPost("/user/list", "start=0&length=100000"); + is (@{$users->{data}}, 0, "start=0&length=100000"); + is ($users->{recordsTotal}, 0); + is ($users->{recordsFiltered}, 0); + # Update User Shared Server $json = viewerPostToken2("/user/update", '{"userId":"test2","userName":"UserNameUpdated2", "enabled":true, "removeEnabled":false, "headerAuthEnabled":true, "expression":"foo", "emailSearch":true, "webEnabled":true, "createEnabled":true}', $token2); @@ -81,21 +113,21 @@ my $pwd = getcwd() . "/pcap"; my $info = viewerGet("/user/columns?molochRegressionUser=test1"); eq_or_diff($info, from_json("[]"), "column: empty"); - $info = viewerPostToken("/user/columns/create?molochRegressionUser=test1", '{"name": "column1", "columns": ["a1","a2"], "order": [["lp", "asc"]]}', $test1Token); + $info = viewerPostToken("/user/columns/create?molochRegressionUser=test1", '{"name": "column1", "columns": ["a1","dstIp"], "order": [["lp", "asc"]]}', $test1Token); ok($info->{success}, "column: create success"); is($info->{name}, "column1", "column: create name"); $info = viewerGet("/user/columns?molochRegressionUser=test1"); - eq_or_diff($info, from_json('[{"name":"column1","order":[["lp","asc"]],"columns":["a1","a2"]}]'), "column: 1 item"); + eq_or_diff($info, from_json('[{"name":"column1","order":[["lastPacket","asc"]],"columns":["srcIp","dstIp"]}]'), "column: 1 item"); $info = viewerGet("/user/columns?molochRegressionUser=anonymous&userId=test1"); - eq_or_diff($info, from_json('[{"name":"column1","order":[["lp","asc"]],"columns":["a1","a2"]}]'), "column: 1 item admin"); + eq_or_diff($info, from_json('[{"name":"column1","order":[["lastPacket","asc"]],"columns":["srcIp","dstIp"]}]'), "column: 1 item admin"); $info = viewerPostToken("/user/columns/delete?molochRegressionUser=test1", 'name=fred', $test1Token); ok(! $info->{success}, "column: delete not found"); $info = viewerGet("/user/columns?molochRegressionUser=test1"); - eq_or_diff($info, from_json('[{"name":"column1","order":[["lp","asc"]],"columns":["a1","a2"]}]'), "column: 1 item"); + eq_or_diff($info, from_json('[{"name":"column1","order":[["lastPacket","asc"]],"columns":["srcIp","dstIp"]}]'), "column: 1 item"); $info = viewerPostToken("/user/columns/delete?molochRegressionUser=test1", 'name=column1', $test1Token); ok($info->{success}, "column: delete found"); @@ -112,21 +144,21 @@ my $pwd = getcwd() . "/pcap"; $info = viewerGet("/user/spiview/fields?molochRegressionUser=test1"); eq_or_diff($info, from_json("[]"), "spiview fields: empty"); - $info = viewerPostToken("/user/spiview/fields/create?molochRegressionUser=test1", '{"name": "sfields1", "fields": ["a1","a2"]}', $test1Token); + $info = viewerPostToken("/user/spiview/fields/create?molochRegressionUser=test1", '{"name": "sfields1", "fields": ["srcIp","dstIp"]}', $test1Token); ok($info->{success}, "spiview fields: create success"); is($info->{name}, "sfields1", "spiview fields: create name"); $info = viewerGet("/user/spiview/fields?molochRegressionUser=test1"); - eq_or_diff($info, from_json('[{"name":"sfields1","fields":["a1","a2"]}]'), "spiview fields: 1 item"); + eq_or_diff($info, from_json('[{"name":"sfields1","fields":["srcIp","dstIp"]}]'), "spiview fields: 1 item"); $info = viewerGet("/user/spiview/fields?molochRegressionUser=anonymous&userId=test1"); - eq_or_diff($info, from_json('[{"name":"sfields1","fields":["a1","a2"]}]'), "spiview fields: 1 item admin"); + eq_or_diff($info, from_json('[{"name":"sfields1","fields":["srcIp","dstIp"]}]'), "spiview fields: 1 item admin"); $info = viewerPostToken("/user/spiview/fields/delete?molochRegressionUser=test1", 'name=fred', $test1Token); ok(!$info->{success}, "spiview fields: delete not found"); $info = viewerGet("/user/spiview/fields?molochRegressionUser=test1"); - eq_or_diff($info, from_json('[{"name":"sfields1","fields":["a1","a2"]}]'), "spiview fields: 1 item"); + eq_or_diff($info, from_json('[{"name":"sfields1","fields":["srcIp","dstIp"]}]'), "spiview fields: 1 item"); $info = viewerPostToken("/user/spiview/fields/delete?molochRegressionUser=test1", 'name=sfields1', $test1Token); ok($info->{success}, "spiview fields: delete found"); diff --git a/tests/cert.t b/tests/cert.t index 604a7cc69e..49765f2f2d 100644 --- a/tests/cert.t +++ b/tests/cert.t @@ -4,7 +4,7 @@ use URI::Escape; use MolochTest; use strict; -my $pwd = getcwd() . "/pcap"; +my $pwd = "*/pcap"; my $files = "(file=$pwd/openssl-ssl3.pcap||file=$pwd/openssl-tls1.pcap||file=$pwd/https3-301-get.pcap)"; # cert.alt tests @@ -25,10 +25,10 @@ my $files = "(file=$pwd/openssl-ssl3.pcap||file=$pwd/openssl-tls1.pcap||file=$pw countTest(0, "date=-1&expression=" . uri_escape("$files&&cert.issuer.cn==\"google internet authority g3\"")); # cert.issuer.cn - countTest(2, "date=-1&expression=" . uri_escape("$files&&cert.issuer.on==\"Google Inc\"")); - countTest(0, "date=-1&expression=" . uri_escape("$files&&cert.issuer.on==\"Foo Inc\"")); - countTest(2, "date=-1&expression=" . uri_escape("$files&&cert.issuer.on==\"Google\"")); - countTest(0, "date=-1&expression=" . uri_escape("$files&&cert.issuer.on==\"Foo\"")); + countTest(2, "date=-1&expression=" . uri_escape("$files&&cert.issuer.on==\"*Google Inc*\"")); + countTest(0, "date=-1&expression=" . uri_escape("$files&&cert.issuer.on==\"*Foo Inc*\"")); + countTest(2, "date=-1&expression=" . uri_escape("$files&&cert.issuer.on==\"*Google*\"")); + countTest(0, "date=-1&expression=" . uri_escape("$files&&cert.issuer.on==\"*Foo*\"")); # cert.notafter countTest(2, "date=-1&expression=" . uri_escape("$files&&cert.notafter==\"2018/08/21 00:00:00\"")); @@ -51,7 +51,7 @@ my $files = "(file=$pwd/openssl-ssl3.pcap||file=$pwd/openssl-tls1.pcap||file=$pw # cert.subject.cn countTest(2, "date=-1&expression=" . uri_escape("$files&&cert.subject.on==\"Google Inc\"")); countTest(0, "date=-1&expression=" . uri_escape("$files&&cert.subject.on==\"Foo Inc\"")); - countTest(2, "date=-1&expression=" . uri_escape("$files&&cert.subject.on==\"Google\"")); + countTest(2, "date=-1&expression=" . uri_escape("$files&&cert.subject.on==\"*Google*\"")); countTest(0, "date=-1&expression=" . uri_escape("$files&&cert.subject.on==\"Foo\"")); # cert.validfor diff --git a/tests/config.test.ini b/tests/config.test.ini index 7f65ebea19..9f5520baa5 100644 --- a/tests/config.test.ini +++ b/tests/config.test.ini @@ -7,7 +7,10 @@ geoipFile = GeoIP.dat geoipASNFile = GeoIPASNum.dat geoip6File = GeoIPv6.dat geoipASN6File = GeoIPASNumv6.dat +geoLite2ASN=GeoLite2-ASN.mmdb +geoLite2Country=GeoLite2-Country.mmdb rirFile = ipv4-address-space.csv +ouiFile = oui.txt parsersDir = ../capture/parsers;parsers pluginsDir = plugins;../tests/plugins;../capture/plugins yara=rules.yara @@ -21,6 +24,7 @@ viewPort=8123 viewerPlugins=wise.js icmpTimeout=60 rulesFiles=rules.yaml +supportSha256=true packetThreads=2 magicMode=basic @@ -60,6 +64,7 @@ prefix=tests passwordSecret= regressionTests=true plugins=test.so;tagger.so +interface=en0 [test2] viewPort=8124 @@ -89,12 +94,13 @@ location=type:string [headers-email] x-priority=type:integer +x-elnk-trace=type:lotermfield [override-ips] -10.0.0.1=tag:srcip;asn:AS0000 This is neat;country:RUS -10.0.0.2=tag:dstip;asn:AS0001 Cool Beans!;rir:TEST;country:CAN +10.0.0.1=tag:srcip;asn:AS0000 This is neat;country:RU +10.0.0.2=tag:dstip;asn:AS0001 Cool Beans!;rir:TEST;country:CA 10.0.0.3=asn:AS0002 Hmm!@#$%^&*() -10.180/16=country:USA +10.180/16=country:US [moloch-clusters] test2=url:http://localhost:8124;passwordSecret:password;name:Test2 @@ -134,6 +140,12 @@ tags=ja3wise type=ja3 format=tagger +[file:sha256] +file=../../../tests/sha256.wise +tags=sha256wise +type=sha256 +format=tagger + [file:domain] file=../../../tests/domain.wise tags=domainwise diff --git a/tests/dhcp.t b/tests/dhcp.t new file mode 100644 index 0000000000..0d8f265675 --- /dev/null +++ b/tests/dhcp.t @@ -0,0 +1,25 @@ +use Test::More tests => 12; +use Cwd; +use URI::Escape; +use MolochTest; +use strict; + +my $pwd = "*/pcap"; +my $files = "(file=$pwd/wireshark-dhcp.pcap)"; + +countTest(2, "date=-1&expression=" . uri_escape("$files&&protocols==dhcp")); + +# dhcp.type +countTest(1, "date=-1&expression=" . uri_escape("$files&&dhcp.type==REQUEST")); + +# dhcp.mac +countTest(2, "date=-1&expression=" . uri_escape("$files&&dhcp.mac==\"00:0b:82:01:fc:42\"")); + +# dhcp.oui +countTest(2, "date=-1&expression=" . uri_escape("$files&&dhcp.oui==\"Grandstream*\"")); + +# dhcp.host +countTest(0, "date=-1&expression=" . uri_escape("$files&&dhcp.host==*")); + +# dhcp.id +countTest(2, "date=-1&expression=" . uri_escape("$files&&dhcp.id==3d1d")); diff --git a/tests/dns.t b/tests/dns.t index 4392ed3a1e..bce9927bad 100644 --- a/tests/dns.t +++ b/tests/dns.t @@ -1,10 +1,10 @@ -use Test::More tests => 70; +use Test::More tests => 76; use Cwd; use URI::Escape; use MolochTest; use strict; -my $pwd = getcwd() . "/pcap"; +my $pwd = "*/pcap"; countTest(4, "date=-1&expression=" . uri_escape("(file=$pwd/dns-udp.pcap||file=$pwd/dns-mx.pcap)&&protocols==dns")); @@ -47,3 +47,11 @@ countTest(4, "date=-1&expression=" . uri_escape("(file=$pwd/dns-udp.pcap||file=$ countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/dns-udp.pcap||file=$pwd/dns-mx.pcap)&&dns.host==*hub.com")); countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/dns-udp.pcap||file=$pwd/dns-mx.pcap)&&dns.host==/.*hub.com/")); countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/dns-udp.pcap||file=$pwd/dns-mx.pcap)&&dns.host!=/.*hub.com/")); + +# dns ip v6 tests + SKIP: { + skip "Upgrade test", 6 if ($ENV{MOLOCH_REINDEX_TEST}); # reindex doesn't have ipv6 dns + countTest(2, "date=-1&expression=" . uri_escape("file=$pwd/v6.pcap&&dns.ip==3ffe:501:410::2c0:dfff:fe47:33e")); + countTest(2, "date=-1&expression=" . uri_escape("file=$pwd/v6.pcap&&dns.ip==3ffe:501:410:0:2c0:dfff:fe47:33e")); + countTest(16, "date=-1&expression=" . uri_escape("file=$pwd/v6.pcap&&ip==3ffe:501:410:0:2c0:dfff:fe47:33e")); + } diff --git a/tests/email.t b/tests/email.t index 75bc90cb26..028c7df328 100644 --- a/tests/email.t +++ b/tests/email.t @@ -4,22 +4,22 @@ use URI::Escape; use MolochTest; use strict; -my $pwd = getcwd() . "/pcap"; +my $pwd = "*/pcap"; my $files = "(file=$pwd/smtp-subject-8859-b.pcap||file=$pwd/smtp-data-250.pcap||file=$pwd/smtp-originating.pcap||file=$pwd/smtp-zip.pcap||file=$pwd/smtp-subject-multi-nospace.pcap||file=$pwd/smtp-subject-utf8-q.pcap)"; countTest(6, "date=-1&expression=" . uri_escape("$files&&protocols==smtp")); # asn.email countTest(1, "date=-1&expression=" . uri_escape("$files&&asn.email==\"AS0001 Cool Beans!\"")); - countTest(1, "date=-1&expression=" . uri_escape("$files&&asn.email==\"AS0001\"")); - countTest(1, "date=-1&expression=" . uri_escape("$files&&asn.email==\"aS0001\"")); + countTest(1, "date=-1&expression=" . uri_escape("$files&&asn.email==\"AS0001*\"")); + countTest(0, "date=-1&expression=" . uri_escape("$files&&asn.email==\"aS0001*\"")); # ip.email countTest(2, "date=-1&expression=" . uri_escape("$files&&ip.email==10.0.0.4")); # country.email - countTest(1, "date=-1&expression=" . uri_escape("$files&&country.email==USA")); - countTest(1, "date=-1&expression=" . uri_escape("$files&&country.email==usa")); + countTest(1, "date=-1&expression=" . uri_escape("$files&&country.email==US")); + countTest(1, "date=-1&expression=" . uri_escape("$files&&country.email==us")); # rir.email countTest(1, "date=-1&expression=" . uri_escape("$files&&rir.email==ARIN")); @@ -45,10 +45,13 @@ countTest(6, "date=-1&expression=" . uri_escape("$files&&protocols==smtp")); countTest(0, "date=-1&expression=" . uri_escape("$files&&email.fn==\"A.zip\"")); countTest(1, "date=-1&expression=" . uri_escape("$files&&email.fn.cnt==1")); +SKIP: { + skip "Upgrade test", 6 if ($ENV{MOLOCH_REINDEX_TEST}); # reindex doesn't have email.has-header # email.has-header countTest(6, "date=-1&expression=" . uri_escape("$files&&email.has-header==\"to\"")); countTest(6, "date=-1&expression=" . uri_escape("$files&&email.has-header==\"To\"")); countTest(1, "date=-1&expression=" . uri_escape("$files&&email.has-header.cnt==3")); +} # email.bodymagic countTest(1, "date=-1&expression=" . uri_escape("$files&&email.bodymagic==\"application/zip\"")); @@ -77,8 +80,8 @@ countTest(6, "date=-1&expression=" . uri_escape("$files&&protocols==smtp")); # email.x-mailer countTest(1, "date=-1&expression=" . uri_escape("$files&&email.x-mailer==\"Mutt/1.5.20 (2009-12-10)\"")); - countTest(1, "date=-1&expression=" . uri_escape("$files&&email.x-mailer==\"mutt/1.5.20 (2009-12-10)\"")); - countTest(1, "date=-1&expression=" . uri_escape("$files&&email.x-mailer==Mutt")); + countTest(0, "date=-1&expression=" . uri_escape("$files&&email.x-mailer==\"mutt/1.5.20 (2009-12-10)\"")); + countTest(1, "date=-1&expression=" . uri_escape("$files&&email.x-mailer==Mutt*")); countTest(3, "date=-1&expression=" . uri_escape("$files&&email.x-mailer.cnt==1")); # host.email diff --git a/tests/email.tagger2.json b/tests/email.tagger2.json index b60858b4ae..c948a7642a 100644 --- a/tests/email.tagger2.json +++ b/tests/email.tagger2.json @@ -1,10 +1,10 @@ -#field:tagger.str;kind:lotermfield;count:true;friendly:Str;db:tagger.str-term;help:Help String;shortcut:0 +#field:tagger.str;kind:lotermfield;count:true;friendly:Str;db:tagger.str;help:Help String;shortcut:0 #field:tagger.int;kind:integer;count:true;friendly:Int;db:tagger.int;help:Help Int;shortcut:1 #field:tags;shortcut:2 -#view:if (session.tagger.str-term) +#view:if (session.tagger.str) #view: div.sessionDetailMeta.bold Tagger #view: dl.sessionDetailMeta.bold Tagger -#view: +arrayList(session.tagger, 'str-term', 'Str', 'tagger.str') +#view: +arrayList(session.tagger, 'str', 'Str', 'tagger.str') 12345678@aol.com;email.dst=added1;2=srcmatch;0=house;tagger.str=boat;dontSaveSPI=0 xxxxx-xxxx@xxxx.xxx.xx.jp;email.src=added2;tags=dstmatch;1=1;tagger.int=3 diff --git a/tests/email.wise b/tests/email.wise index d1c7058124..3f431b3fdc 100644 --- a/tests/email.wise +++ b/tests/email.wise @@ -1,11 +1,11 @@ -#field:wise.str;kind:lotermfield;count:true;friendly:Str;db:wise.str-term;help:Help String;shortcut:0 +#field:wise.str;kind:lotermfield;count:true;friendly:Str;db:wise.str;help:Help String;shortcut:0 #field:wise.int;kind:integer;count:true;friendly:Int;db:wise.int;help:Help Int;shortcut:1 #field:tags;shortcut:2 #field:email.dst #view:if (session.wise) #view: div.sessionDetailMeta.bold Wise #view: dl.sessionDetailMeta -#view: +arrayList(session.wise, 'str-term', 'Str', 'wise.str') +#view: +arrayList(session.wise, 'str', 'Str', 'wise.str') 12345678@aol.com;email.dst=wiseadded1;2=wisesrcmatch;0=house;wise.str=boat xxxxx-xxxx@xxxx.xxx.xx.jp;email.src=wiseadded2;tags=wisedstmatch;1=1;wise.int=3 fudge@fudge.com diff --git a/tests/general.t b/tests/general.t index aa23e66211..e675335db4 100644 --- a/tests/general.t +++ b/tests/general.t @@ -1,10 +1,10 @@ -use Test::More tests => 537; +use Test::More tests => 581; use Cwd; use URI::Escape; use MolochTest; use strict; -my $pwd = getcwd() . "/pcap"; +my $pwd = "*/pcap"; # Regex missing backslash tests errTest("date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap)&&http.uri==/js/xxxxxx/")); @@ -34,8 +34,8 @@ my $pwd = getcwd() . "/pcap"; countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&asn==\"AS0000 This is neat\"")); countTest(0, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&asn==\"AS0000 This is bad\"")); countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&asn.src==\"AS0001 Cool Beans!\"")); - countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&asn.src==\"Cool\"")); - countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&asn==\"Cool\"")); + countTest(0, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&asn.src==\"Cool\"")); + countTest(0, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&asn==\"Cool\"")); countTest(0, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&asn==\"Coo\"")); countTest(0, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&asn==\"Coo*\"")); countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&asn==\"*Cool*\"")); @@ -46,32 +46,32 @@ my $pwd = getcwd() . "/pcap"; countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-tcp.pcap&&test.ip.asn==/.*nea.*/")); countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-tcp.pcap&&test.ip.asn==*nea*")); # country tests - countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&country.src==CAN")); - countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&country.src==can")); - countTest(2, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&country.src!=CAN")); - countTest(2, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&country.src!=can")); - countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&country==CAN")); - countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&country==can")); - countTest(2, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&country!=CAN")); - countTest(2, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&country!=can")); + countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&country.src==CA")); + countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&country.src==ca")); + countTest(2, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&country.src!=CA")); + countTest(2, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&country.src!=ca")); + countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&country==CA")); + countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&country==ca")); + countTest(2, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&country!=CA")); + countTest(2, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&country!=ca")); countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&country.src==/CA.*/")); countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&country.src==/ca.*/")); countTest(2, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&country.src!=/CA.*/")); countTest(2, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&country.src!=/ca.*/")); - countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&country.src==*AN")); - countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&country.src==*an")); - countTest(2, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&country.src!=*AN")); - countTest(2, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&country.src!=*an")); - countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-tcp.pcap&&country==RUS")); - countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-tcp.pcap&&test.ip.country==RUS")); - countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-tcp.pcap&&country==Rus")); - countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-tcp.pcap&&test.ip.country==Rus")); - countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-tcp.pcap&&country!=RUS")); - countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-tcp.pcap&&test.ip.country!=RUS")); - countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-tcp.pcap&&country==/.*US/")); - countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-tcp.pcap&&test.ip.country==/.*US/")); - countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-tcp.pcap&&country==*US")); - countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-tcp.pcap&&test.ip.country==*US")); + countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&country.src==*A")); + countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&country.src==*a")); + countTest(2, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&country.src!=*A")); + countTest(2, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&country.src!=*a")); + countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-tcp.pcap&&country==RU")); + countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-tcp.pcap&&test.ip.country==RU")); + countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-tcp.pcap&&country==Ru")); + countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-tcp.pcap&&test.ip.country==Ru")); + countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-tcp.pcap&&country!=RU")); + countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-tcp.pcap&&test.ip.country!=RU")); + countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-tcp.pcap&&country==/.*U/")); + countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-tcp.pcap&&test.ip.country==/.*U/")); + countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-tcp.pcap&&country==*U")); + countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-tcp.pcap&&test.ip.country==*U")); # rir tests countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&rir==\"TEST\"")); countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&rir==\"test\"")); @@ -101,6 +101,8 @@ my $pwd = getcwd() . "/pcap"; countTest(2, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&ip.dst!=10.0.0.1")); countTest(3, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&ip.dst=10.0.0.0/24")); countTest(3, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&ip.dst=[10.0.0.0/24]")); + countTest(3, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&ip.dst=10.0.0/24")); + countTest(3, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&ip.dst=[10.0.0/24]")); countTest(3, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&ip.dst=10.0.0")); countTest(0, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&ip.dst=0")); countTest(3, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&ip.dst!=0")); @@ -110,6 +112,22 @@ my $pwd = getcwd() . "/pcap"; countTest(2, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&ip.dst=[10.0.0.1,10.0.0.3]")); countTest(2, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&ip.dst=[10.0.0.1/32,10.0.0.3/32]")); countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&ip=[10.0.0.1/32]")); + countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&ip.src=10.0.0.2:")); + countTest(2, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&ip.src!=10.0.0.2:")); + +# ipv6 tests + countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/v6-http.pcap&&ip.dst=2001:6f8:900:7c0::2")); + countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/v6-http.pcap&&ip.dst=2001:6f8:900:7c0:0:0:0:2")); + countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/v6-http.pcap&&ip.dst=2001:6f8:900:7c0::2.80")); + countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/v6-http.pcap&&ip.src=.59201")); + countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/v6-http.pcap&&ip.dst=2001:6f8:900:7c0::2.")); + countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/v6-http.pcap&&ip.dst=2001:6f8:900:7c0:0:0:0:2.")); + +# ipv6 all tests + countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/v6-http.pcap&&ip=2001:6f8:900:7c0::2")); + countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/v6-http.pcap&&ip=2001:6f8:900:7c0:0:0:0:2")); + countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/v6-http.pcap&&ip=2001:6f8:900:7c0::2.80")); + countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/v6-http.pcap&&ip=.59201")); # ip boundary tests countTest(0, "date=-1&expression=" . uri_escape("file=$pwd/ip-boundaries.pcap&&ip.src=0.0.0.0")); @@ -149,7 +167,7 @@ my $pwd = getcwd() . "/pcap"; countTest(0, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&ip.dst=10.0.0.2:50758")); countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&ip=10.0.0.2:50759")); countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&ip.src=[10.0.0.2:50759]")); - countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&ip=[10.0.0.2:50759/32]")); + countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&ip=[10.0.0.2/32:50759]")); # port tests countTest(1, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&port.src=50759")); countTest(2, "date=-1&expression=" . uri_escape("file=$pwd/bt-udp.pcap&&port.src!=50759")); @@ -310,6 +328,18 @@ if (0) { countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/dns-flags0110.pcap||file=$pwd/dns-dnskey.pcap)&&mac=/00:.*/")); countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/dns-flags0110.pcap||file=$pwd/dns-dnskey.pcap)&&mac=[00:23:04:17:9b:00,00:1a:e3:dc:2e:c0]")); +# oui.src tests + countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/dns-flags0110.pcap||file=$pwd/dns-dnskey.pcap)&&oui.src=\"Cisco Systems, Inc\"")); + countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/dns-flags0110.pcap||file=$pwd/dns-dnskey.pcap)&&oui.src=Cisco*")); + countTest(0, "date=-1&expression=" . uri_escape("(file=$pwd/dns-flags0110.pcap||file=$pwd/dns-dnskey.pcap)&&oui.src=\"cisco Systems, Inc\"")); + countTest(0, "date=-1&expression=" . uri_escape("(file=$pwd/dns-flags0110.pcap||file=$pwd/dns-dnskey.pcap)&&oui.src=cisco*")); + +# oui.dst tests + countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/dns-flags0110.pcap||file=$pwd/dns-dnskey.pcap)&&oui.dst=\"Juniper Networks\"")); + countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/dns-flags0110.pcap||file=$pwd/dns-dnskey.pcap)&&oui.dst=Juniper*")); + countTest(0, "date=-1&expression=" . uri_escape("(file=$pwd/dns-flags0110.pcap||file=$pwd/dns-dnskey.pcap)&&oui.dst=\"juniper Networks\"")); + countTest(0, "date=-1&expression=" . uri_escape("(file=$pwd/dns-flags0110.pcap||file=$pwd/dns-dnskey.pcap)&&oui.dst=juniper*")); + #starttime countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/dns-flags0110.pcap||file=$pwd/socks5-reverse.pcap)&&starttime==\"2014/02/26 10:27:57\"")); countTest(0, "date=-1&expression=" . uri_escape("(file=$pwd/dns-flags0110.pcap||file=$pwd/socks5-reverse.pcap)&&starttime==\"2014/02/26 10:27:58\"")); diff --git a/tests/http.t b/tests/http.t index 2ccc1e9eb3..921798b504 100644 --- a/tests/http.t +++ b/tests/http.t @@ -1,10 +1,10 @@ -use Test::More tests => 308; +use Test::More tests => 264; use Cwd; use URI::Escape; use MolochTest; use strict; -my $pwd = getcwd() . "/pcap"; +my $pwd = "*/pcap"; # http.host tests countTest(3, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/socks-http-example.pcap)&&http.host==www.example.com")); countTest(3, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/socks-http-example.pcap)&&http.host==*.example.com")); @@ -27,40 +27,22 @@ my $pwd = getcwd() . "/pcap"; countTest(0, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/http-500-head.pcap)&&http.method!=[GET,HEAD]")); countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/http-500-head.pcap)&&http.method==/.*E.*/")); # http.uri tests - countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/http-500-head.pcap)&&http.uri==//samples.example.com/UpdataConfig.dat")); - countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/http-500-head.pcap)&&http.uri==//samples.example.com")); - countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/http-500-head.pcap)&&http.uri==UpdataConfig.dat")); + countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/http-500-head.pcap)&&http.uri==samples.example.com/UpdataConfig.dat")); + countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/http-500-head.pcap)&&http.uri==samples.example.com*")); + countTest(0, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/http-500-head.pcap)&&http.uri==UpdataConfig.dat")); countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/http-500-head.pcap)&&http.uri==*Config.dat")); countTest(0, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/http-500-head.pcap)&&http.uri==Config.dat")); countTest(0, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/http-500-head.pcap)&&http.uri==*config.dat")); countTest(0, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/http-500-head.pcap)&&http.uri==config.dat")); - countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/http-500-head.pcap)&&http.uri==a.zip")); + countTest(0, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/http-500-head.pcap)&&http.uri==a.zip")); countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/http-500-head.pcap)&&http.uri==/.*a.zip/")); countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/http-500-head.pcap)&&http.uri==/.*a.zip/")); -# http.uri slash tests - tokeniezd field - countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap)&&http.uri==/js/xxxxxx.js")); - countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap)&&http.uri==//js/xxxxxx.js")); - countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap)&&http.uri==js/xxxxxx")); +# http.uri.path slash tests countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap)&&http.uri==/.*\\/js\\/xxxxxx.js/")); - countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap)&&http.uri==/*/js/xxxxxx.j*")); - countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap)&&http.uri==[/js/xxxxxx.js]")); - countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap)&&http.uri==[//js/xxxxxx.js]")); - countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap)&&http.uri==[js/xxxxxx]")); + countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap)&&http.uri==*/js/xxxxxx.j*")); countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap)&&http.uri==[/.*js\\/xxxxxx.js/]")); countTest(0, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap)&&http.uri==[\"/.*js\\/xxxxxx.js/\"]")); - countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap)&&http.uri==[/*/js/xxxxxx.j*]")); - countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap)&&http.uri==[\"/\\/js\\/xxxxxx.js/\"]")); - countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap||file=$pwd/http-content-zip.pcap)&&http.uri==[a.zip,/js/xxxxxx.js]")); - countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap||file=$pwd/http-content-zip.pcap)&&http.uri==[a.zip,//js/xxxxxx.js]")); - countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap||file=$pwd/http-content-zip.pcap)&&http.uri==[a.zip,js/xxxxxx]")); - countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap||file=$pwd/http-content-zip.pcap)&&http.uri==[a.zip,/.*js\\/xxxxxx.js/]")); - countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap||file=$pwd/http-content-zip.pcap)&&http.uri==[a.zip,/*/js/xxxxxx.j*]")); - countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap||file=$pwd/http-content-zip.pcap)&&http.uri==[a.zip , /js/xxxxxx.js]")); - countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap||file=$pwd/http-content-zip.pcap)&&http.uri==[a.zip , //js/xxxxxx.js]")); - countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap||file=$pwd/http-content-zip.pcap)&&http.uri==[a.zip , js/xxxxxx]")); - countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap||file=$pwd/http-content-zip.pcap)&&http.uri==[a.zip , /.*js\\/xxxxxx.js/]")); - countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap||file=$pwd/http-content-zip.pcap)&&http.uri==[a.zip , /*/js/xxxxxx.j*]")); -# http.uri.path slash tests - not tokenized + countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap)&&http.uri==[*/js/xxxxxx.j*]")); countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap)&&http.uri.path==/js/xxxxxx.js")); countTest(0, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap)&&http.uri.path==//js/xxxxxx.js")); countTest(0, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap)&&http.uri.path==js/xxxxxx")); @@ -90,6 +72,9 @@ my $pwd = getcwd() . "/pcap"; countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap||file=$pwd/http-content-zip.pcap)&&http.uri.path==[/\\/js\\/.*.js/,*a.zip*]")); countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap||file=$pwd/http-content-zip.pcap)&&http.uri.path==[/js/xxxxxx.js*,*a.zip*]")); countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-no-length.pcap||file=$pwd/http-content-zip.pcap)&&http.uri.path==[//js/xxxxxx.js*,*a.zip*]")); + +SKIP: { + skip "Upgrade test", 42 if ($ENV{MOLOCH_REINDEX_TEST}); # reindex doesn't have http.has-header # http.hasheader, http.hasheader.src, http.hasheader.dst tests countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/socks5-reverse.pcap)&&http.hasheader==server")); countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/socks5-reverse.pcap)&&http.hasheader.dst==server")); @@ -118,6 +103,8 @@ my $pwd = getcwd() . "/pcap"; countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/socks5-reverse.pcap)&&http.hasheader==[content-length]")); countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/socks5-reverse.pcap)&&http.hasheader.dst==[content-length]")); countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/socks5-reverse.pcap)&&http.hasheader.src==[accept-encoding]")); +} + # http.version tests countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/socks5-reverse.pcap)&&http.version==1.1")); countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/socks5-reverse.pcap)&&http.version.src==1.1")); @@ -136,17 +123,13 @@ my $pwd = getcwd() . "/pcap"; countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/socks5-reverse.pcap)&&http.version.dst==/1.*/")); # http.user-agent tests countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/socks5-reverse.pcap)&&http.user-agent==\"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)\"")); - countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/socks5-reverse.pcap)&&http.user-agent!=\"Mozilla/4.0\"")); + countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/socks5-reverse.pcap)&&http.user-agent!=\"*Mozilla/4.0*\"")); countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/socks5-reverse.pcap)&&http.user-agent==\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36\"")); - countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/socks5-reverse.pcap)&&http.user-agent!=\"Mozilla/5.0\"")); - countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/socks5-reverse.pcap)&&http.user-agent==Mozilla")); - countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/socks5-reverse.pcap)&&http.user-agent==mozilla")); + countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/socks5-reverse.pcap)&&http.user-agent!=\"*Mozilla/5.0*\"")); countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/socks5-reverse.pcap)&&http.user-agent==*Mozilla*")); countTest(0, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/socks5-reverse.pcap)&&http.user-agent==*mozilla*")); countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/socks5-reverse.pcap)&&http.user-agent==/.*Mozilla.*/")); countTest(0, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/socks5-reverse.pcap)&&http.user-agent==/.*mozilla.*/")); - countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/socks5-reverse.pcap)&&http.user-agent==[Mozilla]")); - countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/socks5-reverse.pcap)&&http.user-agent==[mozilla]")); # http.md5 tests countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/socks5-reverse.pcap)&&http.md5=40be8f5100e9beabab293c9d7bacaff0")); countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/socks5-reverse.pcap)&&http.md5=40Be8f5100e9beabab293c9d7bacaff0")); @@ -177,8 +160,7 @@ my $pwd = getcwd() . "/pcap"; # http.referer countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-gzip.pcap||file=$pwd/socks5-reverse.pcap)&&http.referer==EXISTS!")); - countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-gzip.pcap||file=$pwd/socks5-reverse.pcap)&&http.referer==search")); - countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-gzip.pcap||file=$pwd/socks5-reverse.pcap)&&http.referer==search")); + countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-gzip.pcap||file=$pwd/socks5-reverse.pcap)&&http.referer==*search*")); countTest(0, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-gzip.pcap||file=$pwd/socks5-reverse.pcap)&&http.referer==notfound")); countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-gzip.pcap||file=$pwd/socks5-reverse.pcap)&&http.referer!=notfound")); countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-gzip.pcap||file=$pwd/socks5-reverse.pcap)&&http.referer==/.*id=xxx.*/")); diff --git a/tests/irc.t b/tests/irc.t index 161ef1c7d6..52a4d7c9c8 100644 --- a/tests/irc.t +++ b/tests/irc.t @@ -4,7 +4,7 @@ use URI::Escape; use MolochTest; use strict; -my $pwd = getcwd() . "/pcap"; +my $pwd = "*/pcap"; my $files = "file=$pwd/irc.pcap"; countTest(1, "date=-1&expression=" . uri_escape("$files&&protocols==irc")); diff --git a/tests/mysql.t b/tests/mysql.t index d6026c38e6..0a5e7746a3 100644 --- a/tests/mysql.t +++ b/tests/mysql.t @@ -4,7 +4,7 @@ use URI::Escape; use MolochTest; use strict; -my $pwd = getcwd() . "/pcap"; +my $pwd = "*/pcap"; my $files = "(file=$pwd/mysql-allow.pcap||file=$pwd/mysql-deny.pcap)"; countTest(2, "date=-1&expression=" . uri_escape("$files&&protocols==mysql")); diff --git a/tests/parliament.t b/tests/parliament.t new file mode 100644 index 0000000000..c7ddd54203 --- /dev/null +++ b/tests/parliament.t @@ -0,0 +1,112 @@ +use Test::More tests => 25; +use Cwd; +use URI::Escape; +use MolochTest; +use Data::Dumper; +use JSON; +use Test::Differences; +use strict; + +my $result; + + +# Get parliament, empty +$result = parliamentGet("/parliament/api/parliament"); +eq_or_diff($result, from_json('{"groups": []}')); + + +# Set first password +$result = parliamentPut("/parliament/api/auth/update", '{"newPassword": "test"}'); +ok(exists $result->{token}); +delete $result->{token}; +eq_or_diff($result, from_json('{"success":true,"text":"Here\'s your new token!"}')); + +# Try and change without current password +$result = parliamentPut("/parliament/api/auth/update", '{"newPassword": "test2"}'); +eq_or_diff($result, from_json('{"success":false,"text":"You must provide your current password"}')); + +# Try and change wrong current password +$result = parliamentPut("/parliament/api/auth/update", '{"newPassword": "test2", "currentPassword": "wrong"}'); +eq_or_diff($result, from_json('{"success":false,"text":"Authentication failed."}')); + +# Change password right +$result = parliamentPut("/parliament/api/auth/update", '{"newPassword": "test2", "currentPassword": "test"}'); +my $token = $result->{token}; +ok(exists $result->{token}); +delete $result->{token}; +eq_or_diff($result, from_json('{"success":true,"text":"Here\'s your new token!"}')); + + + +# Create group no title no token +$result = parliamentPost("/parliament/api/groups", '{}'); +eq_or_diff($result, from_json('{"tokenError":true,"success":false,"text":"Permission Denied: No token provided."}')); + +# Create group no title wrong token +$result = parliamentPost("/parliament/api/groups", '{"token": "token"}'); +eq_or_diff($result, from_json('{"tokenError":true,"success":false,"text":"Permission Denied: Failed to authenticate token. Try logging in again."}')); + +# Create group no title +$result = parliamentPost("/parliament/api/groups", '{"token": "' . $token . '"}'); +eq_or_diff($result, from_json('{"success":false,"text":"A group must have a title"}')); + +# Create group +$result = parliamentPost("/parliament/api/groups", '{"token": "' . $token . '", "title": "the title"}'); +eq_or_diff($result, from_json('{"success":true,"text":"Successfully added new group.", "group": {"clusters": [], "id": 0, "title": "the title"}}')); + +# Get parliament no token +$result = parliamentGet("/parliament/api/parliament"); +eq_or_diff($result, from_json('{"groups": [{"clusters": [], "id": 0, "title": "the title"}]}')); + +# Get settings no token +$result = parliamentGet("/parliament/api/settings"); +eq_or_diff($result, from_json('{"tokenError":true,"success":false,"text":"Permission Denied: No token provided."}')); + +# Get settings bad token +$result = parliamentGetToken("/parliament/api/settings", "token"); +eq_or_diff($result, from_json('{"tokenError":true,"success":false,"text":"Permission Denied: Failed to authenticate token. Try logging in again."}')); + +# Get settings good token +$result = parliamentGetToken("/parliament/api/settings", $token); +ok (exists $result->{notifiers}); + +# Create second group +$result = parliamentPost("/parliament/api/groups", '{"token": "' . $token . '", "title": "the second title", "description": "description for 2"}'); +eq_or_diff($result, from_json('{"success":true,"text":"Successfully added new group.", "group": {"clusters": [], "id": 1, "title": "the second title", "description": "description for 2"}}')); + +# Get parliament +$result = parliamentGet("/parliament/api/parliament"); +eq_or_diff($result, from_json('{"groups": [{"clusters": [], "id": 0, "title": "the title"}, {"clusters": [], "description": "description for 2", "id": 1, "title": "the second title"}]}')); + + +# Update second group no token +$result = parliamentPut("/parliament/api/groups/1", '{"title": "UP the second title", "description": "UP description for 2"}'); +eq_or_diff($result, from_json('{"tokenError":true,"success":false,"text":"Permission Denied: No token provided."}')); + +# Update second group bad token +$result = parliamentPut("/parliament/api/groups/1", '{"token": "token", "title": "UP the second title", "description": "UP description for 2"}'); +eq_or_diff($result, from_json('{"tokenError":true,"success":false,"text":"Permission Denied: Failed to authenticate token. Try logging in again."}')); + +# Update second group +$result = parliamentPut("/parliament/api/groups/1", '{"token": "' . $token . '", "title": "UP the second title", "description": "UP description for 2"}'); +eq_or_diff($result, from_json('{"success":true,"text":"Successfully updated the requested group."}')); + +# Get parliament +$result = parliamentGet("/parliament/api/parliament"); +eq_or_diff($result, from_json('{"groups": [{"clusters": [], "id": 0, "title": "the title"}, {"clusters": [], "description": "UP description for 2", "id": 1, "title": "UP the second title"}]}')); + +# Delete second group no token +$result = parliamentDelete("/parliament/api/groups/1"); +eq_or_diff($result, from_json('{"tokenError":true,"success":false,"text":"Permission Denied: No token provided."}')); + +# Delete second group bad token +$result = parliamentDeleteToken("/parliament/api/groups/1", "token"); +eq_or_diff($result, from_json('{"tokenError":true,"success":false,"text":"Permission Denied: Failed to authenticate token. Try logging in again."}')); + +# Delete second group +$result = parliamentDeleteToken("/parliament/api/groups/1", $token); +eq_or_diff($result, from_json('{"success":true,"text":"Successfully removed the requested group."}')); + +# Get parliament after delete +$result = parliamentGet("/parliament/api/parliament"); +eq_or_diff($result, from_json('{"groups": [{"clusters": [], "id": 0, "title": "the title"}]}')); diff --git a/tests/pcap/CVE-2018-6794.pcap b/tests/pcap/CVE-2018-6794.pcap new file mode 100644 index 0000000000000000000000000000000000000000..3ce5450f8608272e451cc98f1331b6f7dce94b40 GIT binary patch literal 7413 zcmeI1eM}o=9LJv)SfJ9y8IAalJd18C(B8Ea3KVo87EoS>yp0&+Jjz3Rp zxM(u|7-mkGItLq@Q?|vI$zpWboLS=35jKtSB`zlZVIq>4EHN=z@b^5|-k{J+Ok9Md zmptU@wfDK_dA`5jeSgpK&87Dzj3^o1K9Z3EK1^D7J^kFrCiEEeX+!B*1Gf&K`j!jG zn2J0I3 zyvo}CeGJRbSQ#ZFeZ2@(H8s`q4$gsd?RLDQ+H5KhN(zw_Tho9{+*l#~3NHr)QM&z2 z4Uv3`-wjuoO=i6LxXVVho>MdBwQjm6wIfWH~@~lD2A5 z;q#pZoYRFZ)m2TkHC7xDI|;5N-p*haF87CmU4(ZzIXmafbrx|Bdm(NVI)soIxeV?v z`$9rHu{|M$iMJ<2Yz?HlhlG_dOvG#|^LmL4BV)Szm9BtQfLn`Rflh?)r)P5dW6yR4 zin|}Nb46CMOYjlCkF?2=t{_W3Yc`+FzAud8Z63)RY!@Y;8~X-D*^1jqM?g?uan#$} zY6QvG11L8R_1GGkt)ZUC4TxlGZd74bBC$>Z4MG|u!mMtPjIkhjX1Tnr53(THv?zCk z5IPE!>HF0Hi4!E_EJ!|EPH9m??Ewgqp+u0Z*&w0v{^&~6dj=e0@?K2HyAGcL$3Wge zDNWCj$8lq6m{#DCV?1r>#+`JRT!w( zsaxONP1*FItU6%Dck2j?j&3PSVJvX#(o=WHt)oMW@78gq`2`yqxb@9#>Lk@^cJEHp zbL%&l2(2+9T!&@5LfzU?u2TLeO4rNOH(92h*C=Y?m3R=Z2e+PrT#z3&ptDej(YB%( zJ0Toq@_whut7fViBv6c53L`$-A@qvgpv1wSbzYZ6E3egSDD`ai(psv4bdV4Uxv^hS zWH-;RR)+FgN%fzXTvWs#wixs>{VkU1&C5kxXkeKRB^Q4QAxhbGGDR;Gj<8Vpb~$BU z4FxEBSOcN3@QLm|d4h#Pr6%J3%yIX?;m4_MQ0KMPE_YrU7F)m4+b6fqQ|G-a z6^K!{+@Jm1fhU3Lq0altj;JM3);3_pcV3lc`XGi?%~;^PKkvCi&Kn(CeCLfb%_D4R z;Jl-MFP&zc{yuq)uf(Mpuf;#W)eQG}L`E{dSJ|1QHcuk`$eKaTX(aym(_kYgE z6A^Eb@9`U{dO3QW z$vdmbn|RRDw~K432A&2H&urFxVfYRc@jXq%M2@bX96b&4PW__0IeC}KdqtBsk)vxZ zN6&(Yy*avXq^7BeWmTGpEJs^2*}n$vMTp!$Xa*=Vh7`Ts?kPrz-QuX#HSBh0fkY`= ujk=W6j8dVEMcpV>uo@i*SBF(f#;uKdV>!bp9*x4na-os!yn({o5&8$Qll370 literal 0 HcmV?d00001 diff --git a/tests/pcap/CVE-2018-6794.test b/tests/pcap/CVE-2018-6794.test new file mode 100644 index 0000000000..3a0447d2ff --- /dev/null +++ b/tests/pcap/CVE-2018-6794.test @@ -0,0 +1,426 @@ +{ + "sessions2" : [ + { + "body" : { + "dstBytes" : 259, + "dstDataBytes" : 85, + "dstIp" : "192.168.235.136", + "dstMac" : [ + "00:0c:29:79:fd:94" + ], + "dstMacCnt" : 1, + "dstOui" : [ + "VMware, Inc." + ], + "dstOuiCnt" : 1, + "dstPackets" : 3, + "dstPayload8" : "485454502f312e31", + "dstPort" : 8089, + "dstRIR" : "ARIN", + "fileId" : [], + "firstPacket" : 1516186786703, + "http" : { + "clientVersion" : [ + "1.1" + ], + "clientVersionCnt" : 1, + "host" : [ + "192.168.235.136", + "192.168.235.136:8089" + ], + "hostCnt" : 2, + "md5" : [ + "ba9be40a2bf7d245d3f332338530dad2" + ], + "md5Cnt" : 1, + "method" : [ + "GET" + ], + "methodCnt" : 1, + "path" : [ + "/" + ], + "pathCnt" : 1, + "requestHeader" : [ + "accept", + "user-agent", + "accept-encoding", + "connection", + "host", + "upgrade-insecure-requests", + "accept-language" + ], + "requestHeaderCnt" : 7, + "responseHeader" : [ + "content-type", + "content-length" + ], + "responseHeaderCnt" : 2, + "serverVersion" : [ + "1.1" + ], + "serverVersionCnt" : 1, + "sha256" : [ + "9a4b621075fc02ffdc6a5bd0eb5744880d580b71e8e998647e98413e437ae6d0" + ], + "sha256Cnt" : 1, + "statuscode" : [ + 200 + ], + "statuscodeCnt" : 1, + "uri" : [ + "192.168.235.136:8089/" + ], + "uriCnt" : 1, + "useragent" : [ + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.108 Safari/537.36" + ], + "useragentCnt" : 1 + }, + "ipProtocol" : 6, + "lastPacket" : 1516186805600, + "length" : 18898, + "node" : "test", + "packetLen" : [ + 82, + 76, + 155, + 76, + 70, + 479, + 70, + 479, + 479, + 479, + 479, + 479, + 70 + ], + "packetPos" : [ + 24, + 106, + 182, + 337, + 413, + 483, + 962, + 1032, + 1511, + 2449, + 5269, + 6201, + 7133 + ], + "protocol" : [ + "http", + "tcp" + ], + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 3006, + "srcDataBytes" : 409, + "srcIp" : "192.168.235.1", + "srcMac" : [ + "00:50:56:c0:00:02" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "VMware, Inc." + ], + "srcOuiCnt" : 1, + "srcPackets" : 10, + "srcPayload8" : "474554202f204854", + "srcPort" : 53649, + "srcRIR" : "ARIN", + "tcpflags" : { + "ack" : 1, + "dstZero" : 0, + "fin" : 7, + "psh" : 7, + "rst" : 1, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, + "timestamp" : "SET", + "totBytes" : 3265, + "totDataBytes" : 494, + "totPackets" : 13 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-180117", + "_type" : "session" + } + } + }, + { + "body" : { + "dstBytes" : 259, + "dstDataBytes" : 85, + "dstIp" : "192.168.235.136", + "dstMac" : [ + "00:0c:29:79:fd:94" + ], + "dstMacCnt" : 1, + "dstOui" : [ + "VMware, Inc." + ], + "dstOuiCnt" : 1, + "dstPackets" : 3, + "dstPayload8" : "485454502f312e31", + "dstPort" : 8089, + "dstRIR" : "ARIN", + "fileId" : [], + "firstPacket" : 1516186788618, + "http" : { + "clientVersion" : [ + "1.1" + ], + "clientVersionCnt" : 1, + "host" : [ + "192.168.235.136", + "192.168.235.136:8089" + ], + "hostCnt" : 2, + "md5" : [ + "ba9be40a2bf7d245d3f332338530dad2" + ], + "md5Cnt" : 1, + "method" : [ + "GET" + ], + "methodCnt" : 1, + "path" : [ + "/favicon.ico" + ], + "pathCnt" : 1, + "request-referer" : [ + "http://192.168.235.136:8089/" + ], + "request-refererCnt" : 1, + "requestHeader" : [ + "accept", + "user-agent", + "referer", + "accept-encoding", + "connection", + "host", + "accept-language" + ], + "requestHeaderCnt" : 7, + "responseHeader" : [ + "content-type", + "content-length" + ], + "responseHeaderCnt" : 2, + "serverVersion" : [ + "1.1" + ], + "serverVersionCnt" : 1, + "sha256" : [ + "9a4b621075fc02ffdc6a5bd0eb5744880d580b71e8e998647e98413e437ae6d0" + ], + "sha256Cnt" : 1, + "statuscode" : [ + 200 + ], + "statuscodeCnt" : 1, + "uri" : [ + "192.168.235.136:8089/favicon.ico" + ], + "uriCnt" : 1, + "useragent" : [ + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.108 Safari/537.36" + ], + "useragentCnt" : 1 + }, + "ipProtocol" : 6, + "lastPacket" : 1516186807982, + "length" : 19364, + "node" : "test", + "packetLen" : [ + 82, + 76, + 155, + 76, + 70, + 453, + 70, + 453, + 453, + 453, + 453, + 453, + 70 + ], + "packetPos" : [ + 1990, + 2072, + 2148, + 2303, + 2379, + 2928, + 3381, + 3451, + 4363, + 4816, + 5748, + 6680, + 7343 + ], + "protocol" : [ + "http", + "tcp" + ], + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 2850, + "srcDataBytes" : 383, + "srcIp" : "192.168.235.1", + "srcMac" : [ + "00:50:56:c0:00:02" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "VMware, Inc." + ], + "srcOuiCnt" : 1, + "srcPackets" : 10, + "srcPayload8" : "474554202f666176", + "srcPort" : 53656, + "srcRIR" : "ARIN", + "tcpflags" : { + "ack" : 1, + "dstZero" : 0, + "fin" : 7, + "psh" : 7, + "rst" : 1, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, + "timestamp" : "SET", + "totBytes" : 3109, + "totDataBytes" : 468, + "totPackets" : 13 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-180117", + "_type" : "session" + } + } + }, + { + "body" : { + "dstBytes" : 259, + "dstDataBytes" : 85, + "dstIp" : "192.168.235.136", + "dstMac" : [ + "00:0c:29:79:fd:94" + ], + "dstMacCnt" : 1, + "dstOui" : [ + "VMware, Inc." + ], + "dstOuiCnt" : 1, + "dstPackets" : 3, + "dstPayload8" : "485454502f312e31", + "dstPort" : 8089, + "dstRIR" : "ARIN", + "fileId" : [], + "firstPacket" : 1516186789703, + "http" : { + "md5" : [ + "ba9be40a2bf7d245d3f332338530dad2" + ], + "md5Cnt" : 1, + "responseHeader" : [ + "content-type", + "content-length" + ], + "responseHeaderCnt" : 2, + "serverVersion" : [ + "1.1" + ], + "serverVersionCnt" : 1, + "sha256" : [ + "9a4b621075fc02ffdc6a5bd0eb5744880d580b71e8e998647e98413e437ae6d0" + ], + "sha256Cnt" : 1, + "statuscode" : [ + 200 + ], + "statuscodeCnt" : 1 + }, + "ipProtocol" : 6, + "lastPacket" : 1516186807328, + "length" : 17625, + "node" : "test", + "packetLen" : [ + 82, + 76, + 155, + 76, + 70, + 70, + 70 + ], + "packetPos" : [ + 3904, + 3986, + 4062, + 4217, + 4293, + 7203, + 7273 + ], + "protocol" : [ + "http", + "tcp" + ], + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 228, + "srcDataBytes" : 0, + "srcIp" : "192.168.235.1", + "srcMac" : [ + "00:50:56:c0:00:02" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "VMware, Inc." + ], + "srcOuiCnt" : 1, + "srcPackets" : 4, + "srcPort" : 53648, + "srcRIR" : "ARIN", + "tcpflags" : { + "ack" : 1, + "dstZero" : 0, + "fin" : 2, + "psh" : 1, + "rst" : 1, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, + "timestamp" : "SET", + "totBytes" : 487, + "totDataBytes" : 85, + "totPackets" : 7 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-180117", + "_type" : "session" + } + } + } + ] +} + diff --git a/tests/pcap/aerospike.test b/tests/pcap/aerospike.test index 80d206baf6..3ddcc9ec97 100644 --- a/tests/pcap/aerospike.test +++ b/tests/pcap/aerospike.test @@ -1,13 +1,30 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "fb1" : "0201000000000023", - "tacnt" : 3, - "a1" : "10.0.0.1", - "by1" : 588, - "db1" : 86, - "psl" : [ + "dstASN" : "AS0001 Cool Beans!", + "dstBytes" : 198, + "dstDataBytes" : 0, + "dstGEO" : "CA", + "dstIp" : "10.0.0.2", + "dstMac" : [ + "2c:6b:f5:d6:17:c5" + ], + "dstMacCnt" : 1, + "dstOui" : [ + "Juniper Networks" + ], + "dstOuiCnt" : 1, + "dstPackets" : 3, + "dstPort" : 3000, + "dstRIR" : "TEST", + "fileId" : [], + "firstPacket" : 1466097898472, + "ipProtocol" : 6, + "lastPacket" : 1466097973153, + "length" : 74681, + "node" : "test", + "packetLen" : [ 98, 94, 86, @@ -19,66 +36,7 @@ 76, 76 ], - "db2" : 0, - "timestamp" : "SET", - "lastPacket" : 1466097973153, - "no" : "test", - "by2" : 198, - "mac2-term-cnt" : 1, - "prot-term" : [ - "aerospike", - "tcp" - ], - "prot-term-cnt" : 2, - "test" : { - "ip-asn" : [ - "AS0000 This is neat" - ], - "number" : [ - 33554442 - ], - "ip-geo" : [ - "RUS" - ], - "string" : [ - "16777226:53226,33554442:3000" - ], - "ip-rir" : [ - "" - ], - "ip" : [ - 167772161 - ] - }, - "ipSrc" : "10.0.0.1", - "pr" : 6, - "g2" : "CAN", - "as1" : "AS0000 This is neat", - "tcpflags" : { - "psh" : 2, - "syn" : 1, - "ack" : 3, - "rst" : 2, - "urg" : 0, - "syn-ack" : 1, - "fin" : 1 - }, - "a2" : "10.0.0.2", - "vlan-cnt" : 1, - "mac1-term-cnt" : 1, - "sl" : 74681, - "ta" : [ - "acked-unseen-segment-src", - "dstip", - "srcip" - ], - "pa1" : 7, - "portDst" : 3000, - "vlan" : [ - 50 - ], - "pa" : 10, - "ps" : [ + "packetPos" : [ 24, 122, 216, @@ -90,69 +48,119 @@ 818, 894 ], - "fp" : 1466097898, - "ss" : 1, - "by" : 786, - "rir2" : "TEST", - "p2" : 3000, - "g1" : "RUS", - "firstPacket" : 1466097898472, - "ipDst" : "10.0.0.2", - "fs" : [], - "p1" : 53226, - "as2" : "AS0001 Cool Beans!", - "fpd" : 1466097898472, - "lpd" : 1466097973153, - "mac1-term" : [ + "protocol" : [ + "aerospike", + "tcp" + ], + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcASN" : "AS0000 This is neat", + "srcBytes" : 588, + "srcDataBytes" : 86, + "srcGEO" : "RU", + "srcIp" : "10.0.0.1", + "srcMac" : [ "2c:6b:f5:d6:17:cc" ], - "tags-term" : [ + "srcMacCnt" : 1, + "srcOui" : [ + "Juniper Networks" + ], + "srcOuiCnt" : 1, + "srcPackets" : 7, + "srcPayload8" : "0201000000000023", + "srcPort" : 53226, + "tags" : [ "acked-unseen-segment-src", - "srcip", - "dstip" + "dstip", + "srcip" ], - "db" : 86, - "portSrc" : 53226, - "mac2-term" : [ - "2c:6b:f5:d6:17:c5" + "tagsCnt" : 3, + "tcpflags" : { + "ack" : 3, + "dstZero" : 0, + "fin" : 1, + "psh" : 2, + "rst" : 2, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, + "test" : { + "ASN" : [ + "AS0000 This is neat" + ], + "GEO" : [ + "RU" + ], + "RIR" : [ + "" + ], + "ip" : [ + "10.0.0.1" + ], + "number" : [ + 33554442 + ], + "string.snow" : [ + "16777226:53226,33554442:3000" + ] + }, + "timestamp" : "SET", + "totBytes" : 786, + "totDataBytes" : 86, + "totPackets" : 10, + "vlan" : [ + 50 ], - "lp" : 1466097973, - "pa2" : 3 + "vlanCnt" : 1 }, "header" : { "index" : { - "_index" : "tests_sessions-160616", + "_index" : "tests_sessions2-160616", "_type" : "session" } } }, { "body" : { - "fb2" : "020100000000004e", - "mac2-term" : [ + "dstASN" : "AS0002 Hmm!@#$%^&*()", + "dstBytes" : 444, + "dstDataBytes" : 86, + "dstIp" : "10.0.0.3", + "dstMac" : [ "00:22:83:3f:17:c5", "2c:6b:f5:d6:17:c5" ], - "pa2" : 5, - "lp" : 1466101718, - "portSrc" : 43178, - "db" : 108, - "fpd" : 1466101718441, - "lpd" : 1466101718908, - "mac1-term" : [ - "00:22:83:3f:17:cc", - "2c:6b:f5:d6:17:cc" + "dstMacCnt" : 2, + "dstOui" : [ + "Juniper Networks" ], - "ipDst" : "10.0.0.3", + "dstOuiCnt" : 1, + "dstPackets" : 5, + "dstPayload8" : "020100000000004e", + "dstPort" : 3000, + "fileId" : [], "firstPacket" : 1466101718441, - "as2" : "AS0002 Hmm!@#$%^&*()", - "fs" : [], - "p1" : 43178, - "p2" : 3000, - "ss" : 1, - "by" : 894, - "fp" : 1466101718, - "ps" : [ + "ipProtocol" : 6, + "lastPacket" : 1466101718908, + "length" : 467, + "node" : "test", + "packetLen" : [ + 94, + 94, + 86, + 108, + 86, + 172, + 86, + 86, + 86, + 86, + 86 + ], + "packetPos" : [ 970, 1064, 1158, @@ -165,59 +173,50 @@ 1868, 1954 ], - "pa" : 11, - "vlan" : [ - 50 + "protocol" : [ + "aerospike", + "tcp" + ], + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 450, + "srcDataBytes" : 22, + "srcIp" : "10.0.0.4", + "srcMac" : [ + "00:22:83:3f:17:cc", + "2c:6b:f5:d6:17:cc" ], - "sl" : 467, - "portDst" : 3000, - "pa1" : 6, - "mac1-term-cnt" : 2, + "srcMacCnt" : 2, + "srcOui" : [ + "Juniper Networks" + ], + "srcOuiCnt" : 1, + "srcPackets" : 6, + "srcPayload8" : "020100000000000e", + "srcPort" : 43178, "tcpflags" : { - "psh" : 2, - "syn" : 1, "ack" : 5, - "urg" : 0, - "rst" : 0, + "dstZero" : 0, "fin" : 2, - "syn-ack" : 1 + "psh" : 2, + "rst" : 0, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 }, - "a2" : "10.0.0.3", - "vlan-cnt" : 1, - "ipSrc" : "10.0.0.4", - "pr" : 6, - "mac2-term-cnt" : 2, - "prot-term-cnt" : 2, - "prot-term" : [ - "aerospike", - "tcp" - ], - "lastPacket" : 1466101718908, - "no" : "test", - "by2" : 444, - "db1" : 22, - "by1" : 450, - "a1" : "10.0.0.4", "timestamp" : "SET", - "psl" : [ - 94, - 94, - 86, - 108, - 86, - 172, - 86, - 86, - 86, - 86, - 86 + "totBytes" : 894, + "totDataBytes" : 108, + "totPackets" : 11, + "vlan" : [ + 50 ], - "db2" : 86, - "fb1" : "020100000000000e" + "vlanCnt" : 1 }, "header" : { "index" : { - "_index" : "tests_sessions-160616", + "_index" : "tests_sessions2-160616", "_type" : "session" } } diff --git a/tests/pcap/bigendian.test b/tests/pcap/bigendian.test index fbe03024b2..0674cdb844 100644 --- a/tests/pcap/bigendian.test +++ b/tests/pcap/bigendian.test @@ -1,68 +1,70 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "a2" : "10.64.11.49", - "ipSrc" : "192.168.177.160", - "prot-term-cnt" : 1, - "fp" : 1335958313, - "by1" : 196, - "pa" : 2, - "icmpType" : [ - 8 + "dstBytes" : 0, + "dstDataBytes" : 0, + "dstIp" : "10.64.11.49", + "dstMac" : [ + "00:00:5e:00:01:b1" ], - "portDst" : 0, - "prot-term" : [ - "icmp" + "dstMacCnt" : 1, + "dstOui" : [ + "ICANN, IANA Department" ], - "portSrc" : 0, - "pr" : 1, - "ipDst" : "10.64.11.49", - "timestamp" : "SET", - "db" : 0, - "fpd" : 1335958313152, - "p1" : 0, - "mac2-term-cnt" : 1, - "fs" : [], - "lpd" : 1335958317529, - "pa2" : 0, - "pa1" : 2, - "ps" : [ + "dstOuiCnt" : 1, + "dstPackets" : 0, + "dstPort" : 0, + "fileId" : [], + "firstPacket" : 1335958313152, + "icmp" : { + "code" : [ + 0 + ], + "type" : [ + 8 + ] + }, + "ipProtocol" : 1, + "lastPacket" : 1335958317529, + "length" : 4376, + "node" : "test", + "packetLen" : [ + 114, + 114 + ], + "packetPos" : [ 24, 138 ], - "no" : "test", - "mac1-term" : [ - "00:21:28:05:29:ba" - ], - "by2" : 0, - "lp" : 1335958317, - "db2" : 0, - "psl" : [ - 114, - 114 + "protocol" : [ + "icmp" ], - "mac2-term" : [ - "00:00:5e:00:01:b1" + "protocolCnt" : 1, + "segmentCnt" : 1, + "srcBytes" : 196, + "srcDataBytes" : 0, + "srcIp" : "192.168.177.160", + "srcMac" : [ + "00:21:28:05:29:ba" ], - "sl" : 4376, - "ss" : 1, - "p2" : 0, - "icmpCode" : [ - 0 + "srcMacCnt" : 1, + "srcOui" : [ + "Oracle Corporation" ], - "a1" : "192.168.177.160", - "rir1" : "ARIN", - "firstPacket" : 1335958313152, - "by" : 196, - "db1" : 0, - "mac1-term-cnt" : 1, - "lastPacket" : 1335958317529 + "srcOuiCnt" : 1, + "srcPackets" : 2, + "srcPort" : 0, + "srcRIR" : "ARIN", + "timestamp" : "SET", + "totBytes" : 196, + "totDataBytes" : 0, + "totPackets" : 2 }, "header" : { "index" : { - "_type" : "session", - "_index" : "tests_sessions-120502" + "_index" : "tests_sessions2-120502", + "_type" : "session" } } } diff --git a/tests/pcap/bt-tcp.test b/tests/pcap/bt-tcp.test index 93389d432b..910493ed30 100644 --- a/tests/pcap/bt-tcp.test +++ b/tests/pcap/bt-tcp.test @@ -1,190 +1,191 @@ { - "sessions" : [ + "sessions2" : [ { - "header" : { - "index" : { - "_index" : "tests_sessions-131222", - "_type" : "session" - } - }, "body" : { - "tcpflags" : { - "syn" : 1, - "psh" : 1, - "rst" : 0, - "syn-ack" : 1, - "fin" : 0, - "urg" : 0, - "ack" : 1 - }, - "prot-term" : [ + "dstASN" : "AS0001 Cool Beans!", + "dstBytes" : 66, + "dstDataBytes" : 0, + "dstGEO" : "CA", + "dstIp" : "10.0.0.2", + "dstMac" : [ + "00:00:5e:00:01:02", + "00:1d:b5:ce:ef:c0" + ], + "dstMacCnt" : 2, + "dstOui" : [ + "ICANN, IANA Department", + "Juniper Networks" + ], + "dstOuiCnt" : 2, + "dstPackets" : 1, + "dstPort" : 26001, + "dstRIR" : "TEST", + "fileId" : [], + "firstPacket" : 1387744084182, + "ipProtocol" : 6, + "lastPacket" : 1387744084305, + "length" : 123, + "node" : "test", + "packetLen" : [ + 82, + 82, + 76, + 138 + ], + "packetPos" : [ + 24, + 106, + 188, + 264 + ], + "protocol" : [ "tcp", "bittorrent" ], - "db1" : 68, - "mac1-term" : [ + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcASN" : "AS0000 This is neat", + "srcBytes" : 248, + "srcDataBytes" : 68, + "srcGEO" : "RU", + "srcIp" : "10.0.0.1", + "srcMac" : [ "00:0f:f7:76:82:80" ], - "mac1-term-cnt" : 1, - "pa" : 4, - "ta" : [ + "srcMacCnt" : 1, + "srcOui" : [ + "Cisco Systems, Inc" + ], + "srcOuiCnt" : 1, + "srcPackets" : 3, + "srcPayload8" : "13426974546f7272", + "srcPort" : 1203, + "tags" : [ "dstip", "srcip" ], - "fp" : 1387744084, - "ipDst" : "10.0.0.2", + "tagsCnt" : 2, + "tcpflags" : { + "ack" : 1, + "dstZero" : 0, + "fin" : 0, + "psh" : 1, + "rst" : 0, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, "test" : { - "number" : [ - 33554442 + "ASN" : [ + "AS0000 This is neat" ], - "string" : [ - "16777226:1203,33554442:26001" + "GEO" : [ + "RU" ], - "ip-rir" : [ + "RIR" : [ "" ], - "ip-asn" : [ - "AS0000 This is neat" - ], "ip" : [ - 167772161 + "10.0.0.1" ], - "ip-geo" : [ - "RUS" + "number" : [ + 33554442 + ], + "string.snow" : [ + "16777226:1203,33554442:26001" ] }, - "psl" : [ - 82, - 82, - 76, - 138 - ], - "tacnt" : 2, - "p2" : 26001, - "db" : 68, - "fs" : [], - "g2" : "CAN", - "lpd" : 1387744084305, - "pr" : 6, - "pa1" : 3, - "by2" : 66, - "a1" : "10.0.0.1", - "prot-term-cnt" : 2, - "p1" : 1203, - "lp" : 1387744084, - "by" : 314, - "no" : "test", - "sl" : 123, - "as1" : "AS0000 This is neat", - "a2" : "10.0.0.2", - "tags-term" : [ - "srcip", - "dstip" - ], - "mac2-term" : [ - "00:00:5e:00:01:02", - "00:1d:b5:ce:ef:c0" - ], - "portDst" : 26001, - "ps" : [ - 24, - 106, - 188, - 264 - ], - "mac2-term-cnt" : 2, - "fpd" : 1387744084182, - "fb1" : "13426974546f7272", - "lastPacket" : 1387744084305, - "by1" : 248, - "portSrc" : 1203, - "g1" : "RUS", - "db2" : 0, "timestamp" : "SET", - "as2" : "AS0001 Cool Beans!", - "firstPacket" : 1387744084182, - "ss" : 1, - "ipSrc" : "10.0.0.1", - "rir2" : "TEST", - "pa2" : 1 - } - }, - { + "totBytes" : 314, + "totDataBytes" : 68, + "totPackets" : 4 + }, "header" : { "index" : { - "_type" : "session", - "_index" : "tests_sessions-161224" + "_index" : "tests_sessions2-131222", + "_type" : "session" } - }, + } + }, + { "body" : { - "lpd" : 1482553479744, - "fs" : [], - "pa1" : 3, - "by2" : 82, - "pr" : 6, - "vlan" : [ - 300 - ], - "a1" : "10.10.10.10", - "lp" : 1482553479, - "p1" : 43890, - "prot-term-cnt" : 2, - "mac1-term" : [ - "2c:6b:f5:d6:17:c5" + "dstBytes" : 82, + "dstDataBytes" : 0, + "dstIp" : "10.11.11.11", + "dstMac" : [ + "00:1b:17:00:02:30" ], - "db1" : 68, - "tcpflags" : { - "syn" : 1, - "psh" : 1, - "rst" : 0, - "ack" : 1, - "syn-ack" : 1, - "urg" : 0, - "fin" : 0 - }, - "prot-term" : [ - "tcp", - "bittorrent" + "dstMacCnt" : 1, + "dstOui" : [ + "Palo Alto Networks" ], - "ipDst" : "10.11.11.11", - "fp" : 1482553479, - "pa" : 4, - "mac1-term-cnt" : 1, - "db" : 68, - "p2" : 15365, - "psl" : [ + "dstOuiCnt" : 1, + "dstPackets" : 1, + "dstPort" : 15365, + "fileId" : [], + "firstPacket" : 1482553479723, + "ipProtocol" : 6, + "lastPacket" : 1482553479744, + "length" : 21, + "node" : "test", + "packetLen" : [ 94, 98, 86, 154 ], - "lastPacket" : 1482553479744, - "fb1" : "4253594e43000000", - "timestamp" : "SET", - "firstPacket" : 1482553479723, - "db2" : 0, - "portSrc" : 43890, - "by1" : 286, - "pa2" : 1, - "ipSrc" : "10.10.10.10", - "ss" : 1, - "no" : "test", - "sl" : 21, - "vlan-cnt" : 1, - "by" : 368, - "a2" : "10.11.11.11", - "mac2-term" : [ - "00:1b:17:00:02:30" - ], - "mac2-term-cnt" : 1, - "fpd" : 1482553479723, - "ps" : [ + "packetPos" : [ 402, 496, 594, 680 ], - "portDst" : 15365 + "protocol" : [ + "tcp", + "bittorrent" + ], + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 286, + "srcDataBytes" : 68, + "srcIp" : "10.10.10.10", + "srcMac" : [ + "2c:6b:f5:d6:17:c5" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Juniper Networks" + ], + "srcOuiCnt" : 1, + "srcPackets" : 3, + "srcPayload8" : "4253594e43000000", + "srcPort" : 43890, + "tcpflags" : { + "ack" : 1, + "dstZero" : 0, + "fin" : 0, + "psh" : 1, + "rst" : 0, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, + "timestamp" : "SET", + "totBytes" : 368, + "totDataBytes" : 68, + "totPackets" : 4, + "vlan" : [ + 300 + ], + "vlanCnt" : 1 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-161224", + "_type" : "session" + } } } ] diff --git a/tests/pcap/bt-udp.test b/tests/pcap/bt-udp.test index 7684ca0002..ef40efab81 100644 --- a/tests/pcap/bt-udp.test +++ b/tests/pcap/bt-udp.test @@ -1,194 +1,190 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "lp" : 1387253713, - "no" : "test", - "a1" : "10.0.0.2", - "tacnt" : 2, - "as2" : "AS0000 This is neat", - "pa" : 1, - "prot-term" : [ - "udp", - "bittorrent" + "dstASN" : "AS0000 This is neat", + "dstBytes" : 0, + "dstDataBytes" : 0, + "dstGEO" : "RU", + "dstIp" : "10.0.0.1", + "dstMac" : [ + "00:10:db:ff:26:00" ], - "psl" : [ + "dstMacCnt" : 1, + "dstOui" : [ + "Juniper Networks" + ], + "dstOuiCnt" : 1, + "dstPackets" : 0, + "dstPort" : 3207, + "fileId" : [], + "firstPacket" : 1387253713030, + "ipProtocol" : 17, + "lastPacket" : 1387253713030, + "length" : 0, + "node" : "test", + "packetLen" : [ 161 ], - "ipDst" : "10.0.0.1", - "by2" : 0, - "by1" : 145, - "by" : 145, - "mac1-term" : [ + "packetPos" : [ + 24 + ], + "protocol" : [ + "udp", + "bittorrent" + ], + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcASN" : "AS0001 Cool Beans!", + "srcBytes" : 145, + "srcDataBytes" : 137, + "srcGEO" : "CA", + "srcIp" : "10.0.0.2", + "srcMac" : [ "78:fe:3d:11:21:f2" ], - "prot-term-cnt" : 2, - "db1" : 137, - "mac2-term-cnt" : 1, - "ta" : [ + "srcMacCnt" : 1, + "srcOui" : [ + "Juniper Networks" + ], + "srcOuiCnt" : 1, + "srcPackets" : 1, + "srcPayload8" : "64313a6164323a69", + "srcPort" : 50759, + "srcRIR" : "TEST", + "tags" : [ "dstip", "srcip" ], - "ipSrc" : "10.0.0.2", - "rir1" : "TEST", - "pr" : 17, - "g2" : "RUS", - "tags-term" : [ - "srcip", - "dstip" - ], - "ss" : 1, - "pa2" : 0, - "ps" : [ - 24 - ], - "db2" : 0, - "fs" : [], + "tagsCnt" : 2, "timestamp" : "SET", - "p1" : 50759, - "sl" : 0, - "pa1" : 1, - "lastPacket" : 1387253713030, - "fp" : 1387253713, - "p2" : 3207, - "mac2-term" : [ - "00:10:db:ff:26:00" - ], - "as1" : "AS0001 Cool Beans!", - "fb1" : "64313a6164323a69", - "a2" : "10.0.0.1", - "db" : 137, - "firstPacket" : 1387253713030, - "mac1-term-cnt" : 1, - "lpd" : 1387253713030, - "g1" : "CAN", - "portDst" : 3207, - "fpd" : 1387253713030, - "portSrc" : 50759 + "totBytes" : 145, + "totDataBytes" : 137, + "totPackets" : 1 }, "header" : { "index" : { - "_index" : "tests_sessions-131217", + "_index" : "tests_sessions2-131217", "_type" : "session" } } }, { "body" : { - "mac2-term-cnt" : 1, - "db1" : 321, - "prot-term-cnt" : 2, - "ipSrc" : "10.0.0.4", - "pr" : 17, - "as2" : "AS0002 Hmm!@#$%^&*()", - "no" : "test", - "a1" : "10.0.0.4", - "lp" : 1387253793, - "psl" : [ + "dstASN" : "AS0002 Hmm!@#$%^&*()", + "dstBytes" : 0, + "dstDataBytes" : 0, + "dstIp" : "10.0.0.3", + "dstMac" : [ + "00:00:5e:00:01:03" + ], + "dstMacCnt" : 1, + "dstOui" : [ + "ICANN, IANA Department" + ], + "dstOuiCnt" : 1, + "dstPackets" : 0, + "dstPort" : 12074, + "fileId" : [], + "firstPacket" : 1387253793904, + "ipProtocol" : 17, + "lastPacket" : 1387253793904, + "length" : 0, + "node" : "test", + "packetLen" : [ 345 ], - "prot-term" : [ + "packetPos" : [ + 185 + ], + "protocol" : [ "udp", "bittorrent" ], - "pa" : 1, - "mac1-term" : [ + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 329, + "srcDataBytes" : 321, + "srcIp" : "10.0.0.4", + "srcMac" : [ "00:10:db:ff:26:00" ], - "by2" : 0, - "by1" : 329, - "by" : 329, - "ipDst" : "10.0.0.3", - "fb1" : "64313a7264323a69", - "p2" : 12074, - "mac2-term" : [ - "00:00:5e:00:01:03" - ], - "a2" : "10.0.0.3", - "db" : 321, - "portDst" : 12074, - "firstPacket" : 1387253793904, - "mac1-term-cnt" : 1, - "lpd" : 1387253793904, - "portSrc" : 44102, - "fpd" : 1387253793904, - "ss" : 1, - "fs" : [], - "db2" : 0, - "ps" : [ - 185 + "srcMacCnt" : 1, + "srcOui" : [ + "Juniper Networks" ], - "pa2" : 0, - "sl" : 0, - "p1" : 44102, + "srcOuiCnt" : 1, + "srcPackets" : 1, + "srcPayload8" : "64313a7264323a69", + "srcPort" : 44102, "timestamp" : "SET", - "fp" : 1387253793, - "lastPacket" : 1387253793904, - "pa1" : 1 + "totBytes" : 329, + "totDataBytes" : 321, + "totPackets" : 1 }, "header" : { "index" : { - "_index" : "tests_sessions-131217", + "_index" : "tests_sessions2-131217", "_type" : "session" } } }, { "body" : { - "ss" : 1, - "db2" : 0, - "fs" : [], - "pa2" : 0, - "ps" : [ - 530 - ], - "sl" : 0, - "timestamp" : "SET", - "p1" : 47061, - "pa1" : 1, - "fp" : 1387257610, - "lastPacket" : 1387257610963, - "fb1" : "64313a71393a6669", - "mac2-term" : [ + "dstBytes" : 0, + "dstDataBytes" : 0, + "dstIp" : "10.0.0.5", + "dstMac" : [ "00:00:5e:00:01:03" ], - "p2" : 20551, - "db" : 328, - "a2" : "10.0.0.5", - "portDst" : 20551, - "mac1-term-cnt" : 1, + "dstMacCnt" : 1, + "dstOui" : [ + "ICANN, IANA Department" + ], + "dstOuiCnt" : 1, + "dstPackets" : 0, + "dstPort" : 20551, + "fileId" : [], "firstPacket" : 1387257610963, - "lpd" : 1387257610963, - "portSrc" : 47061, - "fpd" : 1387257610963, - "a1" : "10.0.0.6", - "no" : "test", - "lp" : 1387257610, - "psl" : [ + "ipProtocol" : 17, + "lastPacket" : 1387257610963, + "length" : 0, + "node" : "test", + "packetLen" : [ 352 ], - "pa" : 1, - "prot-term" : [ + "packetPos" : [ + 530 + ], + "protocol" : [ "udp", "bittorrent" ], - "mac1-term" : [ + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 336, + "srcDataBytes" : 328, + "srcIp" : "10.0.0.6", + "srcMac" : [ "00:10:db:ff:26:00" ], - "ipDst" : "10.0.0.5", - "by1" : 336, - "by2" : 0, - "by" : 336, - "mac2-term-cnt" : 1, - "prot-term-cnt" : 2, - "db1" : 328, - "ipSrc" : "10.0.0.6", - "pr" : 17 + "srcMacCnt" : 1, + "srcOui" : [ + "Juniper Networks" + ], + "srcOuiCnt" : 1, + "srcPackets" : 1, + "srcPayload8" : "64313a71393a6669", + "srcPort" : 47061, + "timestamp" : "SET", + "totBytes" : 336, + "totDataBytes" : 328, + "totPackets" : 1 }, "header" : { "index" : { - "_type" : "session", - "_index" : "tests_sessions-131217" + "_index" : "tests_sessions2-131217", + "_type" : "session" } } } diff --git a/tests/pcap/cassandra1.test b/tests/pcap/cassandra1.test index 62840b3d7e..19db2ce3cb 100644 --- a/tests/pcap/cassandra1.test +++ b/tests/pcap/cassandra1.test @@ -1,84 +1,87 @@ { - "sessions" : [ + "sessions2" : [ { - "header" : { - "index" : { - "_index" : "tests_sessions-160616", - "_type" : "session" - } - }, "body" : { - "db1" : 41, - "tcpflags" : { - "fin" : 0, - "syn-ack" : 1, - "syn" : 1, - "psh" : 1, - "urg" : 0, - "rst" : 0, - "ack" : 1 - }, - "ipDst" : "10.176.171.11", - "prot-term-cnt" : 3, - "portDst" : 9160, - "ps" : [ - 24, - 122, - 216, - 302 + "dstBytes" : 78, + "dstDataBytes" : 0, + "dstIp" : "10.176.171.11", + "dstMac" : [ + "2c:6b:f5:d6:17:c5" ], - "fb1" : "0000002580010001", - "prot-term" : [ - "cassandra", - "tcp", - "thrift" + "dstMacCnt" : 1, + "dstOui" : [ + "Juniper Networks" ], + "dstOuiCnt" : 1, + "dstPackets" : 1, + "dstPort" : 9160, + "fileId" : [], "firstPacket" : 1466085103886, - "by2" : 78, - "pr" : 6, + "ipProtocol" : 6, "lastPacket" : 1466085103897, - "p2" : 9160, - "pa" : 4, - "psl" : [ + "length" : 10, + "node" : "test", + "packetLen" : [ 98, 94, 86, 127 ], - "fp" : 1466085103, - "a2" : "10.176.171.11", - "lp" : 1466085103, - "timestamp" : "SET", - "mac1-term-cnt" : 2, - "mac1-term" : [ + "packetPos" : [ + 24, + 122, + 216, + 302 + ], + "protocol" : [ + "cassandra", + "tcp", + "thrift" + ], + "protocolCnt" : 3, + "segmentCnt" : 1, + "srcBytes" : 263, + "srcDataBytes" : 41, + "srcIp" : "10.10.10.10", + "srcMac" : [ "00:1b:17:00:02:30", "2c:6b:f5:d6:17:cc" ], - "ss" : 1, - "fpd" : 1466085103886, - "by" : 341, - "db2" : 0, - "by1" : 263, - "db" : 41, - "p1" : 56391, - "fs" : [], - "mac2-term-cnt" : 1, - "no" : "test", - "vlan-cnt" : 2, - "sl" : 10, - "pa1" : 3, - "pa2" : 1, - "ipSrc" : "10.10.10.10", - "portSrc" : 56391, - "mac2-term" : [ - "2c:6b:f5:d6:17:c5" + "srcMacCnt" : 2, + "srcOui" : [ + "Juniper Networks", + "Palo Alto Networks" ], - "a1" : "10.10.10.10", + "srcOuiCnt" : 2, + "srcPackets" : 3, + "srcPayload8" : "0000002580010001", + "srcPort" : 56391, + "tcpflags" : { + "ack" : 1, + "dstZero" : 0, + "fin" : 0, + "psh" : 1, + "rst" : 0, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, + "timestamp" : "SET", + "totBytes" : 341, + "totDataBytes" : 41, + "totPackets" : 4, "vlan" : [ 50, 300 ], - "lpd" : 1466085103897 + "vlanCnt" : 2 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-160616", + "_type" : "session" + } } } ] diff --git a/tests/pcap/dns-dnskey.test b/tests/pcap/dns-dnskey.test index 20271e4422..676bf7b51b 100644 --- a/tests/pcap/dns-dnskey.test +++ b/tests/pcap/dns-dnskey.test @@ -1,116 +1,113 @@ { - "sessions" : [ + "sessions2" : [ { - "header" : { - "index" : { - "_index" : "tests_sessions-140226", - "_type" : "session" - } - }, "body" : { - "rir2" : "ARIN", - "mac1-term" : [ - "00:1a:e3:dc:2e:c0" + "dns" : { + "host" : [ + "" + ], + "hostCnt" : 1, + "opcode" : [ + "QUERY" + ], + "opcodeCnt" : 1, + "qc" : [ + "IN" + ], + "qcCnt" : 1, + "qt" : [ + "DNSKEY" + ], + "qtCnt" : 1 + }, + "dstASN" : "AS15169 Google LLC", + "dstBytes" : 0, + "dstDataBytes" : 0, + "dstGEO" : "US", + "dstIp" : "8.8.8.8", + "dstMac" : [ + "00:19:e2:ba:2f:c1" ], - "tacnt" : 1, - "ipDst" : "8.8.8.8", - "tags-term" : [ - "srcip" + "dstMacCnt" : 1, + "dstOui" : [ + "Juniper Networks" ], - "as2" : "AS15169 Google LLC", - "ss" : 1, - "prot-term-cnt" : 2, - "pa1" : 1, - "mac2-term-cnt" : 1, - "fb1" : "f376011000010000", - "portDst" : 53, - "mac2-term" : [ - "00:19:e2:ba:2f:c1" + "dstOuiCnt" : 1, + "dstPackets" : 0, + "dstPort" : 53, + "dstRIR" : "ARIN", + "fileId" : [], + "firstPacket" : 1393428477365, + "ipProtocol" : 17, + "lastPacket" : 1393428477365, + "length" : 0, + "node" : "test", + "packetLen" : [ + 90 + ], + "packetPos" : [ + 24 ], - "lp" : 1393428477, - "by2" : 0, - "no" : "test", - "by" : 74, - "prot-term" : [ + "protocol" : [ "udp", "dns" ], - "g1" : "RUS", - "lastPacket" : 1393428477365, - "lpd" : 1393428477365, - "db" : 66, - "timestamp" : "SET", - "db2" : 0, - "fpd" : 1393428477365, - "fs" : [], - "dnshocnt" : 1, - "a2" : "8.8.8.8", - "ps" : [ - 24 + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcASN" : "AS0000 This is neat", + "srcBytes" : 74, + "srcDataBytes" : 66, + "srcGEO" : "RU", + "srcIp" : "10.0.0.1", + "srcMac" : [ + "00:1a:e3:dc:2e:c0" ], - "sl" : 0, - "firstPacket" : 1393428477365, - "dns" : { - "qt-term-cnt" : 1, - "qt-term" : [ - "DNSKEY" - ], - "opcode-term" : [ - "QUERY" - ], - "qc-term-cnt" : 1, - "opcode-term-cnt" : 1, - "qc-term" : [ - "IN" - ] - }, - "fp" : 1393428477, - "vlan" : [ - 500 + "srcMacCnt" : 1, + "srcOui" : [ + "Cisco Systems, Inc" ], + "srcOuiCnt" : 1, + "srcPackets" : 1, + "srcPayload8" : "f376011000010000", + "srcPort" : 53869, + "tags" : [ + "srcip" + ], + "tagsCnt" : 1, "test" : { - "string" : [ - "16777226:53869,134744072:53" + "ASN" : [ + "AS0000 This is neat" ], - "ip-geo" : [ - "RUS" + "GEO" : [ + "RU" + ], + "RIR" : [ + "" ], "ip" : [ - 167772161 + "10.0.0.1" ], "number" : [ 134744072 ], - "ip-rir" : [ - "" - ], - "ip-asn" : [ - "AS0000 This is neat" + "string.snow" : [ + "16777226:53869,134744072:53" ] }, - "pr" : 17, - "ta" : [ - "srcip" - ], - "mac1-term-cnt" : 1, - "a1" : "10.0.0.1", - "pa2" : 0, - "pa" : 1, - "ipSrc" : "10.0.0.1", - "db1" : 66, - "p1" : 53869, - "g2" : "USA", - "by1" : 74, - "dnsho" : [ - "" - ], - "p2" : 53, - "vlan-cnt" : 1, - "psl" : [ - 90 + "timestamp" : "SET", + "totBytes" : 74, + "totDataBytes" : 66, + "totPackets" : 1, + "vlan" : [ + 500 ], - "as1" : "AS0000 This is neat", - "portSrc" : 53869 + "vlanCnt" : 1 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-140226", + "_type" : "session" + } } } ] diff --git a/tests/pcap/dns-error.test b/tests/pcap/dns-error.test index 4355f09ead..4e4295ba4a 100644 --- a/tests/pcap/dns-error.test +++ b/tests/pcap/dns-error.test @@ -1,121 +1,118 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "tags-term" : [ - "srcip", - "dstip" - ], - "pa2" : 1, - "g2" : "CAN", - "portDst" : 53, - "fs" : [], - "mac1-term-cnt" : 1, - "by1" : 72, - "portSrc" : 56329, - "g1" : "RUS", - "mac1-term" : [ - "00:13:72:c4:f1:e1" - ], - "lastPacket" : 1394587409103, - "pa1" : 1, - "dnshocnt" : 1, - "as2" : "AS0001 Cool Beans!", - "prot-term-cnt" : 2, - "mac2-term-cnt" : 2, - "by2" : 147, - "mac2-term" : [ + "dns" : { + "host" : [ + "no.such.host" + ], + "hostCnt" : 1, + "opcode" : [ + "QUERY" + ], + "opcodeCnt" : 1, + "qc" : [ + "IN" + ], + "qcCnt" : 1, + "qt" : [ + "A" + ], + "qtCnt" : 1, + "status" : [ + "NXDOMAIN" + ], + "statusCnt" : 1 + }, + "dstASN" : "AS0001 Cool Beans!", + "dstBytes" : 147, + "dstDataBytes" : 139, + "dstGEO" : "CA", + "dstIp" : "10.0.0.2", + "dstMac" : [ "00:00:0c:07:ac:01", "00:0e:d6:0b:98:80" ], - "ipSrc" : "10.0.0.1", - "timestamp" : "SET", - "ta" : [ - "dstip", - "srcip" + "dstMacCnt" : 2, + "dstOui" : [ + "Cisco", + "Cisco Systems, Inc" ], - "psl" : [ + "dstOuiCnt" : 2, + "dstPackets" : 1, + "dstPayload8" : "a9ba858300010000", + "dstPort" : 53, + "dstRIR" : "TEST", + "fileId" : [], + "firstPacket" : 1394587409097, + "ipProtocol" : 17, + "lastPacket" : 1394587409103, + "length" : 5, + "node" : "test", + "packetLen" : [ 88, 163 ], - "db2" : 139, - "by" : 219, - "ipDst" : "10.0.0.2", - "fb1" : "a9ba010000010000", - "a1" : "10.0.0.1", + "packetPos" : [ + 24, + 112 + ], + "protocol" : [ + "udp", + "dns" + ], + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcASN" : "AS0000 This is neat", + "srcBytes" : 72, + "srcDataBytes" : 64, + "srcGEO" : "RU", + "srcIp" : "10.0.0.1", + "srcMac" : [ + "00:13:72:c4:f1:e1" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Dell Inc." + ], + "srcOuiCnt" : 1, + "srcPackets" : 1, + "srcPayload8" : "a9ba010000010000", + "srcPort" : 56329, + "tags" : [ + "dstip", + "srcip" + ], + "tagsCnt" : 2, "test" : { - "ip-asn" : [ + "ASN" : [ "AS0000 This is neat" ], - "string" : [ - "16777226:56329,33554442:53" - ], - "number" : [ - 33554442 - ], - "ip" : [ - 167772161 + "GEO" : [ + "RU" ], - "ip-rir" : [ + "RIR" : [ "" ], - "ip-geo" : [ - "RUS" - ] - }, - "tacnt" : 2, - "dnsho" : [ - "no.such.host" - ], - "dns" : { - "qt-term-cnt" : 1, - "qc-term-cnt" : 1, - "status-term" : [ - "NXDOMAIN" - ], - "opcode-term-cnt" : 1, - "opcode-term" : [ - "QUERY" + "ip" : [ + "10.0.0.1" ], - "status-term-cnt" : 1, - "qc-term" : [ - "IN" + "number" : [ + 33554442 ], - "qt-term" : [ - "A" + "string.snow" : [ + "16777226:56329,33554442:53" ] }, - "firstPacket" : 1394587409097, - "p1" : 56329, - "p2" : 53, - "lp" : 1394587409, - "ps" : [ - 24, - 112 - ], - "as1" : "AS0000 This is neat", - "prot-term" : [ - "udp", - "dns" - ], - "rir2" : "TEST", - "no" : "test", - "pr" : 17, - "db" : 203, - "sl" : 5, - "fpd" : 1394587409097, - "a2" : "10.0.0.2", - "db1" : 64, - "fp" : 1394587409, - "fb2" : "a9ba858300010000", - "pa" : 2, - "ss" : 1, - "lpd" : 1394587409103 + "timestamp" : "SET", + "totBytes" : 219, + "totDataBytes" : 203, + "totPackets" : 2 }, "header" : { "index" : { - "_type" : "session", - "_index" : "tests_sessions-140312" + "_index" : "tests_sessions2-140312", + "_type" : "session" } } } diff --git a/tests/pcap/dns-flags0000.test b/tests/pcap/dns-flags0000.test index c058203872..291e0cf6ee 100644 --- a/tests/pcap/dns-flags0000.test +++ b/tests/pcap/dns-flags0000.test @@ -1,101 +1,98 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "a1" : "10.0.0.2", - "ta" : [ - "dstip", - "srcip" - ], - "db1" : 74, - "by1" : 82, - "fb1" : "b2b5000000010000", - "db2" : 74, - "rir1" : "TEST", - "psl" : [ - 98, - 98 - ], - "pa" : 2, - "fpd" : 1393422583830, - "fb2" : "b2b5800500010000", - "mac2-term-cnt" : 1, - "sl" : 102, - "prot-term" : [ - "udp", - "dns" - ], - "portSrc" : 61912, - "dnsho" : [ - "flashservice.adobe.com" - ], - "fp" : 1393422583, - "prot-term-cnt" : 2, - "timestamp" : "SET", - "no" : "test", - "mac2-term" : [ - "00:10:db:ff:26:00" - ], - "as2" : "AS0000 This is neat", - "lastPacket" : 1393422583932, - "tags-term" : [ - "srcip", - "dstip" - ], - "pa2" : 1, - "ps" : [ - 24, - 122 - ], "dns" : { - "qt-term" : [ - "A" + "host" : [ + "flashservice.adobe.com" ], - "qc-term-cnt" : 1, - "qt-term-cnt" : 1, - "status-term" : [ - "REFUSED" + "hostCnt" : 1, + "opcode" : [ + "QUERY" ], - "qc-term" : [ + "opcodeCnt" : 1, + "qc" : [ "IN" ], - "status-term-cnt" : 1, - "opcode-term" : [ - "QUERY" + "qcCnt" : 1, + "qt" : [ + "A" ], - "opcode-term-cnt" : 1 + "qtCnt" : 1, + "status" : [ + "REFUSED" + ], + "statusCnt" : 1 }, - "g2" : "RUS", - "portDst" : 53, - "as1" : "AS0001 Cool Beans!", - "fs" : [], - "a2" : "10.0.0.1", - "pr" : 17, - "p2" : 53, - "g1" : "CAN", - "by" : 164, - "db" : 148, - "pa1" : 1, - "mac1-term-cnt" : 2, - "dnshocnt" : 1, - "lp" : 1393422583, - "ipSrc" : "10.0.0.2", - "mac1-term" : [ + "dstASN" : "AS0000 This is neat", + "dstBytes" : 82, + "dstDataBytes" : 74, + "dstGEO" : "RU", + "dstIp" : "10.0.0.1", + "dstMac" : [ + "00:10:db:ff:26:00" + ], + "dstMacCnt" : 1, + "dstOui" : [ + "Juniper Networks" + ], + "dstOuiCnt" : 1, + "dstPackets" : 1, + "dstPayload8" : "b2b5800500010000", + "dstPort" : 53, + "fileId" : [], + "firstPacket" : 1393422583830, + "ipProtocol" : 17, + "lastPacket" : 1393422583932, + "length" : 102, + "node" : "test", + "packetLen" : [ + 98, + 98 + ], + "packetPos" : [ + 24, + 122 + ], + "protocol" : [ + "udp", + "dns" + ], + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcASN" : "AS0001 Cool Beans!", + "srcBytes" : 82, + "srcDataBytes" : 74, + "srcGEO" : "CA", + "srcIp" : "10.0.0.2", + "srcMac" : [ "00:00:5e:00:01:03", "88:e0:f3:f1:91:f2" ], - "ipDst" : "10.0.0.1", - "ss" : 1, - "tacnt" : 2, - "lpd" : 1393422583932, - "p1" : 61912, - "firstPacket" : 1393422583830, - "by2" : 82 + "srcMacCnt" : 2, + "srcOui" : [ + "ICANN, IANA Department", + "Juniper Networks" + ], + "srcOuiCnt" : 2, + "srcPackets" : 1, + "srcPayload8" : "b2b5000000010000", + "srcPort" : 61912, + "srcRIR" : "TEST", + "tags" : [ + "dstip", + "srcip" + ], + "tagsCnt" : 2, + "timestamp" : "SET", + "totBytes" : 164, + "totDataBytes" : 148, + "totPackets" : 2 }, "header" : { "index" : { - "_type" : "session", - "_index" : "tests_sessions-140226" + "_index" : "tests_sessions2-140226", + "_type" : "session" } } } diff --git a/tests/pcap/dns-flags0110.test b/tests/pcap/dns-flags0110.test index a185d09753..8dd0981402 100644 --- a/tests/pcap/dns-flags0110.test +++ b/tests/pcap/dns-flags0110.test @@ -1,124 +1,121 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "asdnsip" : [ - "AS41231 Canonical Group Limited", - "AS41231 Canonical Group Limited", - "AS41231 Canonical Group Limited" - ], - "rir1" : "ARIN", - "g1" : "USA", - "rirdnsip" : [ - "RIPE", - "RIPE", - "RIPE" - ], - "fb1" : "5aae819000010003", - "dnsip" : [ - "91.189.89.88", - "91.189.90.40", - "91.189.90.41" - ], - "lp" : 1393428477, - "mac1-term" : [ - "00:19:e2:b1:ef:c6", - "00:19:e2:ba:2f:c1" - ], "dns" : { - "qc-term" : [ - "IN" + "ASN" : [ + "AS41231 Canonical Group Limited", + "AS41231 Canonical Group Limited", + "AS41231 Canonical Group Limited" + ], + "GEO" : [ + "GB", + "GB", + "GB" + ], + "RIR" : [ + "RIPE", + "RIPE", + "RIPE" + ], + "host" : [ + "start.ubuntu.com" + ], + "hostCnt" : 1, + "ip" : [ + "91.189.90.40", + "91.189.90.41", + "91.189.89.88" ], - "opcode-term" : [ + "ipCnt" : 3, + "opcode" : [ "QUERY" ], - "qt-term-cnt" : 1, - "status-term" : [ - "NOERROR" + "opcodeCnt" : 1, + "qc" : [ + "IN" ], - "status-term-cnt" : 1, - "opcode-term-cnt" : 1, - "qt-term" : [ + "qcCnt" : 1, + "qt" : [ "A" ], - "qc-term-cnt" : 1 + "qtCnt" : 1, + "status" : [ + "NOERROR" + ], + "statusCnt" : 1 }, - "prot-term-cnt" : 2, - "a2" : "10.0.0.1", - "timestamp" : "SET", - "pa2" : 1, - "psl" : [ + "dstASN" : "AS0000 This is neat", + "dstBytes" : 91, + "dstDataBytes" : 83, + "dstGEO" : "RU", + "dstIp" : "10.0.0.1", + "dstMac" : [ + "00:1a:e3:dc:2e:c0", + "00:23:04:17:9b:00" + ], + "dstMacCnt" : 2, + "dstOui" : [ + "Cisco Systems, Inc" + ], + "dstOuiCnt" : 1, + "dstPackets" : 1, + "dstPayload8" : "5aae011000010000", + "dstPort" : 62928, + "fileId" : [], + "firstPacket" : 1393428477363, + "ipProtocol" : 17, + "lastPacket" : 1393428477327, + "length" : 4294967260, + "node" : "test", + "packetLen" : [ 151, 107 ], - "prot-term" : [ + "packetPos" : [ + 24, + 175 + ], + "protocol" : [ "udp", "dns" ], - "dnshocnt" : 1, - "no" : "test", - "db" : 210, - "db1" : 127, - "g2" : "RUS", - "db2" : 83, - "p1" : 53, - "fpd" : 1393428477363, - "gdnsip" : [ - "GBR", - "GBR", - "GBR" - ], - "fb2" : "5aae011000010000", - "tacnt" : 1, - "as1" : "AS15169 Google LLC", - "lpd" : 1393428477327, - "p2" : 62928, - "vlan-cnt" : 1, - "by1" : 135, - "dnsho" : [ - "start.ubuntu.com" - ], - "vlan" : [ - 500 - ], - "as2" : "AS0000 This is neat", - "portSrc" : 53, - "fp" : 1393428477, - "portDst" : 62928, - "mac2-term-cnt" : 2, - "pa" : 2, - "by2" : 91, - "pa1" : 1, - "tags-term" : [ - "srcip" + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcASN" : "AS15169 Google LLC", + "srcBytes" : 135, + "srcDataBytes" : 127, + "srcGEO" : "US", + "srcIp" : "8.8.8.8", + "srcMac" : [ + "00:19:e2:b1:ef:c6", + "00:19:e2:ba:2f:c1" ], - "ps" : [ - 24, - 175 + "srcMacCnt" : 2, + "srcOui" : [ + "Juniper Networks" ], - "a1" : "8.8.8.8", - "sl" : 4294967260, - "ta" : [ + "srcOuiCnt" : 1, + "srcPackets" : 1, + "srcPayload8" : "5aae819000010003", + "srcPort" : 53, + "srcRIR" : "ARIN", + "tags" : [ "srcip" ], - "pr" : 17, - "by" : 226, - "dnsipcnt" : 3, - "firstPacket" : 1393428477363, - "ipSrc" : "8.8.8.8", - "ipDst" : "10.0.0.1", - "lastPacket" : 1393428477327, - "fs" : [], - "ss" : 1, - "mac2-term" : [ - "00:1a:e3:dc:2e:c0", - "00:23:04:17:9b:00" + "tagsCnt" : 1, + "timestamp" : "SET", + "totBytes" : 226, + "totDataBytes" : 210, + "totPackets" : 2, + "vlan" : [ + 500 ], - "mac1-term-cnt" : 2 + "vlanCnt" : 1 }, "header" : { "index" : { - "_index" : "tests_sessions-140226", + "_index" : "tests_sessions2-140226", "_type" : "session" } } diff --git a/tests/pcap/dns-mx.test b/tests/pcap/dns-mx.test index 8873739f78..cd839358a4 100644 --- a/tests/pcap/dns-mx.test +++ b/tests/pcap/dns-mx.test @@ -1,90 +1,91 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "lpd" : 1386104997055, - "fpd" : 1386104996973, - "db2" : 208, - "portSrc" : 51427, - "no" : "test", - "prot-term" : [ - "udp", - "dns" + "dns" : { + "host" : [ + "cluster5.us.messagelabs.com", + "mx.com", + "cluster5a.us.messagelabs.com" + ], + "hostCnt" : 3, + "opcode" : [ + "QUERY" + ], + "opcodeCnt" : 1, + "qc" : [ + "IN" + ], + "qcCnt" : 1, + "qt" : [ + "MX" + ], + "qtCnt" : 1, + "status" : [ + "NOERROR" + ], + "statusCnt" : 1 + }, + "dstBytes" : 216, + "dstDataBytes" : 208, + "dstIp" : "10.2.95.39", + "dstMac" : [ + "00:00:0c:07:ac:01", + "00:d0:2b:d1:76:00" + ], + "dstMacCnt" : 2, + "dstOui" : [ + "Cisco", + "Jetcell, Inc." ], - "pa" : 2, - "prot-term-cnt" : 2, - "fs" : [], - "psl" : [ + "dstOuiCnt" : 2, + "dstPackets" : 1, + "dstPayload8" : "4c17818000010002", + "dstPort" : 53, + "fileId" : [], + "firstPacket" : 1386104996973, + "ipProtocol" : 17, + "lastPacket" : 1386104997055, + "length" : 83, + "node" : "test", + "packetLen" : [ 82, 232 ], - "by1" : 66, - "dnsho" : [ - "cluster5.us.messagelabs.com", - "mx.com", - "cluster5a.us.messagelabs.com" - ], - "ps" : [ + "packetPos" : [ 24, 106 ], - "fb1" : "4c17010000010000", - "a2" : "10.2.95.39", - "pr" : 17, - "mac1-term" : [ + "protocol" : [ + "udp", + "dns" + ], + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 66, + "srcDataBytes" : 58, + "srcGEO" : "US", + "srcIp" : "10.180.156.185", + "srcMac" : [ "00:1f:5b:ff:51:cb" ], - "dnshocnt" : 3, - "pa1" : 1, - "by" : 282, - "db" : 266, - "fb2" : "4c17818000010002", - "ss" : 1, - "portDst" : 53, - "a1" : "10.180.156.185", - "mac2-term-cnt" : 2, - "sl" : 83, - "mac2-term" : [ - "00:00:0c:07:ac:01", - "00:d0:2b:d1:76:00" + "srcMacCnt" : 1, + "srcOui" : [ + "Apple, Inc." ], - "ipSrc" : "10.180.156.185", - "dns" : { - "qt-term-cnt" : 1, - "opcode-term-cnt" : 1, - "opcode-term" : [ - "QUERY" - ], - "status-term-cnt" : 1, - "status-term" : [ - "NOERROR" - ], - "qc-term-cnt" : 1, - "qt-term" : [ - "MX" - ], - "qc-term" : [ - "IN" - ] - }, - "mac1-term-cnt" : 1, - "fp" : 1386104996, - "lp" : 1386104997, - "g1" : "USA", - "firstPacket" : 1386104996973, - "lastPacket" : 1386104997055, - "db1" : 58, + "srcOuiCnt" : 1, + "srcPackets" : 1, + "srcPayload8" : "4c17010000010000", + "srcPort" : 51427, "timestamp" : "SET", - "ipDst" : "10.2.95.39", - "by2" : 216, - "pa2" : 1, - "p1" : 51427, - "p2" : 53 + "totBytes" : 282, + "totDataBytes" : 266, + "totPackets" : 2 }, "header" : { "index" : { - "_type" : "session", - "_index" : "tests_sessions-131203" + "_index" : "tests_sessions2-131203", + "_type" : "session" } } } diff --git a/tests/pcap/dns-notify.test b/tests/pcap/dns-notify.test index 0cfa325277..f0d35d0e05 100644 --- a/tests/pcap/dns-notify.test +++ b/tests/pcap/dns-notify.test @@ -1,94 +1,95 @@ { - "sessions" : [ + "sessions2" : [ { - "header" : { - "index" : { - "_type" : "session", - "_index" : "tests_sessions-130923" - } - }, "body" : { - "db2" : 65, - "mac2-term" : [ - "52:54:00:d9:83:b3" - ], - "by1" : 165, - "pa" : 2, - "g1" : "FRA", - "portDst" : 53, - "pa1" : 1, - "lastPacket" : 1379970300840, - "ss" : 1, - "portSrc" : 55612, - "fpd" : 1379970300840, - "pa2" : 1, - "a1" : "217.70.190.232", - "rir2" : "ARIN", - "sl" : 0, - "ipSrc" : "217.70.190.232", - "db" : 222, - "mac2-term-cnt" : 1, - "fb2" : "9668a40000010000", - "rir1" : "RIPE", - "g2" : "USA", - "db1" : 157, - "lpd" : 1379970300840, - "as1" : "AS29169 GANDI SAS", - "as2" : "AS46636 NatCoWeb Corp.", - "mac1-term" : [ - "00:23:33:69:4d:00", - "00:00:5e:00:01:01" - ], - "prot-term-cnt" : 2, - "fb1" : "9668240000010001", "dns" : { - "qc-term" : [ + "host" : [ + "bortzmeyer.42" + ], + "hostCnt" : 1, + "opcode" : [ + "NOTIFY" + ], + "opcodeCnt" : 1, + "qc" : [ "IN" ], - "qt-term-cnt" : 1, - "opcode-term-cnt" : 1, - "qt-term" : [ + "qcCnt" : 1, + "qt" : [ "SOA" ], - "qc-term-cnt" : 1, - "status-term-cnt" : 1, - "opcode-term" : [ - "NOTIFY" - ], - "status-term" : [ + "qtCnt" : 1, + "status" : [ "NOERROR" - ] + ], + "statusCnt" : 1 }, - "no" : "test", - "p2" : 53, - "fs" : [], - "a2" : "204.62.14.153", - "dnshocnt" : 1, - "by" : 238, - "ipDst" : "204.62.14.153", + "dstASN" : "AS46636 NatCoWeb Corp.", + "dstBytes" : 73, + "dstDataBytes" : 65, + "dstGEO" : "US", + "dstIp" : "204.62.14.153", + "dstMac" : [ + "52:54:00:d9:83:b3" + ], + "dstMacCnt" : 1, + "dstOui" : [ + "Realtek (UpTech? also reported)" + ], + "dstOuiCnt" : 1, + "dstPackets" : 1, + "dstPayload8" : "9668a40000010000", + "dstPort" : 53, + "dstRIR" : "ARIN", + "fileId" : [], "firstPacket" : 1379970300840, - "mac1-term-cnt" : 2, - "by2" : 73, - "psl" : [ + "ipProtocol" : 17, + "lastPacket" : 1379970300840, + "length" : 0, + "node" : "test", + "packetLen" : [ 181, 89 ], - "dnsho" : [ - "bortzmeyer.42" + "packetPos" : [ + 24, + 205 ], - "lp" : 1379970300, - "prot-term" : [ + "protocol" : [ "udp", "dns" ], - "fp" : 1379970300, - "timestamp" : "SET", - "ps" : [ - 24, - 205 + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcASN" : "AS203476 GANDI SAS", + "srcBytes" : 165, + "srcDataBytes" : 157, + "srcGEO" : "FR", + "srcIp" : "217.70.190.232", + "srcMac" : [ + "00:00:5e:00:01:01", + "00:23:33:69:4d:00" + ], + "srcMacCnt" : 2, + "srcOui" : [ + "Cisco Systems, Inc", + "ICANN, IANA Department" ], - "p1" : 55612, - "pr" : 17 + "srcOuiCnt" : 2, + "srcPackets" : 1, + "srcPayload8" : "9668240000010001", + "srcPort" : 55612, + "srcRIR" : "RIPE", + "timestamp" : "SET", + "totBytes" : 238, + "totDataBytes" : 222, + "totPackets" : 2 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-130923", + "_type" : "session" + } } } ] diff --git a/tests/pcap/dns-tcp.test b/tests/pcap/dns-tcp.test index 0108635553..a735423478 100644 --- a/tests/pcap/dns-tcp.test +++ b/tests/pcap/dns-tcp.test @@ -1,143 +1,105 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "firstPacket" : 1385482078892, - "lp" : 1385482078, - "fb2" : "01545e6381800001", - "dnsipcnt" : 11, - "fs" : [], - "no" : "test", - "ipDst" : "10.2.95.39", - "lastPacket" : 1385482078899, - "portSrc" : 49342, - "by1" : 438, - "timestamp" : "SET", - "by2" : 614, - "ps" : [ - 24, - 118, - 208, - 290, - 402, - 484, - 908, - 990, - 1072, - 1154 - ], - "pa" : 10, - "mac2-term" : [ - "00:00:0c:07:ac:01", - "00:0e:d6:0b:98:80" - ], - "prot-term" : [ - "dns", - "tcp" - ], - "ipSrc" : "10.180.156.141", - "dnshocnt" : 1, "dns" : { - "qt-term" : [ - "A" + "ASN" : [ + "AS15169 Google LLC", + "AS15169 Google LLC", + "AS15169 Google LLC", + "AS15169 Google LLC", + "AS15169 Google LLC", + "AS15169 Google LLC", + "AS15169 Google LLC", + "AS15169 Google LLC", + "AS15169 Google LLC", + "AS15169 Google LLC", + "AS15169 Google LLC" + ], + "GEO" : [ + "US", + "US", + "US", + "US", + "US", + "US", + "US", + "US", + "US", + "US", + "US" ], - "status-term-cnt" : 1, - "qc-term-cnt" : 1, - "opcode-term" : [ + "RIR" : [ + "ARIN", + "ARIN", + "ARIN", + "ARIN", + "ARIN", + "ARIN", + "ARIN", + "ARIN", + "ARIN", + "ARIN", + "ARIN" + ], + "host" : [ + "google.com" + ], + "hostCnt" : 1, + "ip" : [ + "74.125.228.32", + "74.125.228.33", + "74.125.228.34", + "74.125.228.35", + "74.125.228.36", + "74.125.228.37", + "74.125.228.38", + "74.125.228.39", + "74.125.228.40", + "74.125.228.41", + "74.125.228.46" + ], + "ipCnt" : 11, + "opcode" : [ "QUERY" ], - "qt-term-cnt" : 1, - "qc-term" : [ + "opcodeCnt" : 1, + "qc" : [ "IN" ], - "opcode-term-cnt" : 1, - "status-term" : [ + "qcCnt" : 1, + "qt" : [ + "A" + ], + "qtCnt" : 1, + "status" : [ "NOERROR" - ] - }, - "db2" : 342, - "mac1-term-cnt" : 1, - "lpd" : 1385482078899, - "db" : 372, - "fb1" : "001c5e6301000001", - "by" : 1052, - "dnsip" : [ - "74.125.228.40", - "74.125.228.46", - "74.125.228.33", - "74.125.228.35", - "74.125.228.37", - "74.125.228.39", - "74.125.228.41", - "74.125.228.32", - "74.125.228.34", - "74.125.228.36", - "74.125.228.38" - ], - "tcpflags" : { - "ack" : 4, - "syn" : 1, - "urg" : 0, - "syn-ack" : 1, - "psh" : 2, - "rst" : 0, - "fin" : 2 + ], + "statusCnt" : 1 }, - "pr" : 6, - "pa2" : 4, - "mac1-term" : [ - "00:1f:5b:ff:51:cb" - ], - "sl" : 7, - "mac2-term-cnt" : 2, - "ss" : 1, - "rirdnsip" : [ - "ARIN", - "ARIN", - "ARIN", - "ARIN", - "ARIN", - "ARIN", - "ARIN", - "ARIN", - "ARIN", - "ARIN", - "ARIN" - ], - "a2" : "10.2.95.39", - "pa1" : 6, - "portDst" : 53, - "p1" : 49342, - "asdnsip" : [ - "AS15169 Google LLC", - "AS15169 Google LLC", - "AS15169 Google LLC", - "AS15169 Google LLC", - "AS15169 Google LLC", - "AS15169 Google LLC", - "AS15169 Google LLC", - "AS15169 Google LLC", - "AS15169 Google LLC", - "AS15169 Google LLC", - "AS15169 Google LLC" + "dstBytes" : 614, + "dstDataBytes" : 342, + "dstIp" : "10.2.95.39", + "dstMac" : [ + "00:00:0c:07:ac:01", + "00:0e:d6:0b:98:80" ], - "p2" : 53, - "gdnsip" : [ - "USA", - "USA", - "USA", - "USA", - "USA", - "USA", - "USA", - "USA", - "USA", - "USA", - "USA" + "dstMacCnt" : 2, + "dstOui" : [ + "Cisco", + "Cisco Systems, Inc" ], - "fpd" : 1385482078892, - "a1" : "10.180.156.141", - "psl" : [ + "dstOuiCnt" : 2, + "dstPackets" : 4, + "dstPayload8" : "01545e6381800001", + "dstPort" : 53, + "fileId" : [], + "firstPacket" : 1385482078892, + "ipProtocol" : 6, + "lastPacket" : 1385482078899, + "length" : 7, + "node" : "test", + "packetLen" : [ 94, 90, 82, @@ -149,31 +111,138 @@ 82, 82 ], - "db1" : 30, - "fp" : 1385482078, - "dnsho" : [ - "google.com" + "packetPos" : [ + 24, + 118, + 208, + 290, + 402, + 484, + 908, + 990, + 1072, + 1154 + ], + "protocol" : [ + "dns", + "tcp" + ], + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 438, + "srcDataBytes" : 30, + "srcGEO" : "US", + "srcIp" : "10.180.156.141", + "srcMac" : [ + "00:1f:5b:ff:51:cb" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Apple, Inc." ], - "g1" : "USA", - "prot-term-cnt" : 2 + "srcOuiCnt" : 1, + "srcPackets" : 6, + "srcPayload8" : "001c5e6301000001", + "srcPort" : 49342, + "tcpflags" : { + "ack" : 4, + "dstZero" : 0, + "fin" : 2, + "psh" : 2, + "rst" : 0, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, + "timestamp" : "SET", + "totBytes" : 1052, + "totDataBytes" : 372, + "totPackets" : 10 }, "header" : { "index" : { - "_type" : "session", - "_index" : "tests_sessions-131126" + "_index" : "tests_sessions2-131126", + "_type" : "session" } } }, { "body" : { - "prot-term-cnt" : 2, - "fp" : 1385482080, - "dnsho" : [ - "aol.com" + "dns" : { + "ASN" : [ + "AS1668 AOL Transit Data Network", + "AS1668 AOL Transit Data Network", + "AS1668 AOL Transit Data Network", + "AS1668 AOL Transit Data Network", + "AS1668 AOL Transit Data Network" + ], + "GEO" : [ + "US", + "US", + "US", + "US", + "US" + ], + "RIR" : [ + "ARIN", + "ARIN", + "ARIN", + "ARIN", + "ARIN" + ], + "host" : [ + "aol.com" + ], + "hostCnt" : 1, + "ip" : [ + "64.12.89.186", + "205.188.101.58", + "205.188.100.58", + "207.200.74.38", + "64.12.79.57" + ], + "ipCnt" : 5, + "opcode" : [ + "QUERY" + ], + "opcodeCnt" : 1, + "qc" : [ + "IN" + ], + "qcCnt" : 1, + "qt" : [ + "A" + ], + "qtCnt" : 1, + "status" : [ + "NOERROR" + ], + "statusCnt" : 1 + }, + "dstBytes" : 530, + "dstDataBytes" : 258, + "dstIp" : "10.2.95.39", + "dstMac" : [ + "00:00:0c:07:ac:01", + "00:0e:d6:0b:98:80" + ], + "dstMacCnt" : 2, + "dstOui" : [ + "Cisco", + "Cisco Systems, Inc" ], - "g1" : "USA", - "a1" : "10.180.156.141", - "psl" : [ + "dstOuiCnt" : 2, + "dstPackets" : 4, + "dstPayload8" : "0100e2ec81800001", + "dstPort" : 53, + "fileId" : [], + "firstPacket" : 1385482080084, + "ipProtocol" : 6, + "lastPacket" : 1385482080086, + "length" : 2, + "node" : "test", + "packetLen" : [ 94, 90, 82, @@ -185,97 +254,7 @@ 82, 82 ], - "db1" : 27, - "fpd" : 1385482080084, - "p2" : 53, - "gdnsip" : [ - "USA", - "USA", - "USA", - "USA", - "USA" - ], - "asdnsip" : [ - "AS1668 AOL Transit Data Network", - "AS1668 AOL Transit Data Network", - "AS1668 AOL Transit Data Network", - "AS1668 AOL Transit Data Network", - "AS1668 AOL Transit Data Network" - ], - "portDst" : 53, - "p1" : 49343, - "pa1" : 6, - "mac1-term" : [ - "00:1f:5b:ff:51:cb" - ], - "sl" : 2, - "ss" : 1, - "mac2-term-cnt" : 2, - "rirdnsip" : [ - "ARIN", - "ARIN", - "ARIN", - "ARIN", - "ARIN" - ], - "a2" : "10.2.95.39", - "by" : 965, - "dnsip" : [ - "64.12.89.186", - "207.200.74.38", - "205.188.100.58", - "64.12.79.57", - "205.188.101.58" - ], - "tcpflags" : { - "rst" : 0, - "psh" : 2, - "fin" : 2, - "urg" : 0, - "syn-ack" : 1, - "syn" : 1, - "ack" : 4 - }, - "pr" : 6, - "pa2" : 4, - "db" : 285, - "fb1" : "0019e2ec01000001", - "lpd" : 1385482080086, - "mac2-term" : [ - "00:00:0c:07:ac:01", - "00:0e:d6:0b:98:80" - ], - "prot-term" : [ - "dns", - "tcp" - ], - "ipSrc" : "10.180.156.141", - "dnshocnt" : 1, - "dns" : { - "status-term" : [ - "NOERROR" - ], - "qt-term-cnt" : 1, - "qc-term" : [ - "IN" - ], - "opcode-term-cnt" : 1, - "opcode-term" : [ - "QUERY" - ], - "qc-term-cnt" : 1, - "status-term-cnt" : 1, - "qt-term" : [ - "A" - ] - }, - "mac1-term-cnt" : 1, - "db2" : 258, - "by1" : 435, - "timestamp" : "SET", - "portSrc" : 49343, - "by2" : 530, - "ps" : [ + "packetPos" : [ 1236, 1330, 1420, @@ -287,20 +266,47 @@ 2197, 2279 ], - "pa" : 10, - "no" : "test", - "ipDst" : "10.2.95.39", - "lastPacket" : 1385482080086, - "fs" : [], - "firstPacket" : 1385482080084, - "lp" : 1385482080, - "fb2" : "0100e2ec81800001", - "dnsipcnt" : 5 + "protocol" : [ + "dns", + "tcp" + ], + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 435, + "srcDataBytes" : 27, + "srcGEO" : "US", + "srcIp" : "10.180.156.141", + "srcMac" : [ + "00:1f:5b:ff:51:cb" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Apple, Inc." + ], + "srcOuiCnt" : 1, + "srcPackets" : 6, + "srcPayload8" : "0019e2ec01000001", + "srcPort" : 49343, + "tcpflags" : { + "ack" : 4, + "dstZero" : 0, + "fin" : 2, + "psh" : 2, + "rst" : 0, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, + "timestamp" : "SET", + "totBytes" : 965, + "totDataBytes" : 285, + "totPackets" : 10 }, "header" : { "index" : { - "_type" : "session", - "_index" : "tests_sessions-131126" + "_index" : "tests_sessions2-131126", + "_type" : "session" } } } diff --git a/tests/pcap/dns-udp.test b/tests/pcap/dns-udp.test index ece1a09b7e..78ede75316 100644 --- a/tests/pcap/dns-udp.test +++ b/tests/pcap/dns-udp.test @@ -1,278 +1,280 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "mac1-term-cnt" : 1, - "dnsip" : [ - "192.30.252.128" - ], - "firstPacket" : 1385400647217, - "prot-term" : [ - "udp", - "dns" - ], - "db1" : 66, - "dnsipcnt" : 1, - "psl" : [ - 90, - 270 - ], - "p1" : 62563, - "pa2" : 1, - "ipDst" : "10.2.95.39", - "sl" : 0, - "a2" : "10.2.95.39", - "gdnsip" : [ - "USA" - ], - "by2" : 254, - "dnsho" : [ - "www.github.com", - "github.com" - ], - "dnshocnt" : 2, - "fp" : 1385400647, - "fb1" : "e039010000010000", - "db" : 312, "dns" : { - "qt-term-cnt" : 1, - "qt-term" : [ - "A" + "ASN" : [ + "AS36459 GitHub, Inc." ], - "opcode-term-cnt" : 1, - "qc-term" : [ - "IN" + "GEO" : [ + "US" + ], + "RIR" : [ + "ARIN" ], - "status-term-cnt" : 1, - "opcode-term" : [ + "host" : [ + "www.github.com", + "github.com" + ], + "hostCnt" : 2, + "ip" : [ + "192.30.252.128" + ], + "ipCnt" : 1, + "opcode" : [ "QUERY" ], - "status-term" : [ + "opcodeCnt" : 1, + "qc" : [ + "IN" + ], + "qcCnt" : 1, + "qt" : [ + "A" + ], + "qtCnt" : 1, + "status" : [ "NOERROR" ], - "qc-term-cnt" : 1 + "statusCnt" : 1 }, - "timestamp" : "SET", - "by" : 328, - "portSrc" : 62563, - "by1" : 74, - "no" : "test", - "p2" : 53, - "rirdnsip" : [ - "ARIN" + "dstBytes" : 254, + "dstDataBytes" : 246, + "dstIp" : "10.2.95.39", + "dstMac" : [ + "00:00:0c:07:ac:01", + "00:d0:2b:d1:76:00" ], - "fb2" : "e039818000010002", - "a1" : "10.180.156.141", - "lp" : 1385400647, - "mac1-term" : [ - "00:1f:5b:ff:51:cb" + "dstMacCnt" : 2, + "dstOui" : [ + "Cisco", + "Jetcell, Inc." ], - "pr" : 17, - "pa" : 2, - "prot-term-cnt" : 2, - "db2" : 246, - "fs" : [], + "dstOuiCnt" : 2, + "dstPackets" : 1, + "dstPayload8" : "e039818000010002", + "dstPort" : 53, + "fileId" : [], + "firstPacket" : 1385400647217, + "ipProtocol" : 17, "lastPacket" : 1385400647218, - "ss" : 1, - "portDst" : 53, - "lpd" : 1385400647218, - "ps" : [ + "length" : 0, + "node" : "test", + "packetLen" : [ + 90, + 270 + ], + "packetPos" : [ 24, 114 ], - "fpd" : 1385400647217, - "asdnsip" : [ - "AS36459 GitHub, Inc." + "protocol" : [ + "udp", + "dns" ], - "pa1" : 1, - "ipSrc" : "10.180.156.141", - "g1" : "USA", - "mac2-term" : [ - "00:00:0c:07:ac:01", - "00:d0:2b:d1:76:00" + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 74, + "srcDataBytes" : 66, + "srcGEO" : "US", + "srcIp" : "10.180.156.141", + "srcMac" : [ + "00:1f:5b:ff:51:cb" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Apple, Inc." ], - "mac2-term-cnt" : 2 + "srcOuiCnt" : 1, + "srcPackets" : 1, + "srcPayload8" : "e039010000010000", + "srcPort" : 62563, + "timestamp" : "SET", + "totBytes" : 328, + "totDataBytes" : 312, + "totPackets" : 2 }, "header" : { "index" : { - "_type" : "session", - "_index" : "tests_sessions-131125" + "_index" : "tests_sessions2-131125", + "_type" : "session" } } }, { - "header" : { - "index" : { - "_index" : "tests_sessions-131125", - "_type" : "session" - } - }, "body" : { - "db" : 312, - "fb1" : "e039010000010000", "dns" : { - "opcode-term-cnt" : 1, - "qc-term" : [ + "ASN" : [ + "AS36459 GitHub, Inc." + ], + "GEO" : [ + "US" + ], + "RIR" : [ + "ARIN" + ], + "host" : [ + "www.github.com", + "github.com" + ], + "hostCnt" : 2, + "ip" : [ + "192.30.252.130" + ], + "ipCnt" : 1, + "opcode" : [ + "QUERY" + ], + "opcodeCnt" : 1, + "qc" : [ "IN" ], - "status-term-cnt" : 1, - "qt-term-cnt" : 1, - "qt-term" : [ + "qcCnt" : 1, + "qt" : [ "A" ], - "status-term" : [ + "qtCnt" : 1, + "status" : [ "NOERROR" ], - "qc-term-cnt" : 1, - "opcode-term" : [ - "QUERY" - ] + "statusCnt" : 1 }, - "portSrc" : 62416, - "by1" : 74, - "timestamp" : "SET", - "by" : 328, - "by2" : 254, - "a2" : "10.178.8.71", - "gdnsip" : [ - "USA" + "dstBytes" : 254, + "dstDataBytes" : 246, + "dstIp" : "10.178.8.71", + "dstMac" : [ + "00:00:0c:07:ac:01", + "00:0e:d6:0b:98:80" ], - "fp" : 1385400648, - "dnsho" : [ - "www.github.com", - "github.com" + "dstMacCnt" : 2, + "dstOui" : [ + "Cisco", + "Cisco Systems, Inc" ], - "dnshocnt" : 2, - "psl" : [ + "dstOuiCnt" : 2, + "dstPackets" : 1, + "dstPayload8" : "e039818000010002", + "dstPort" : 53, + "fileId" : [], + "firstPacket" : 1385400648218, + "ipProtocol" : 17, + "lastPacket" : 1385400648228, + "length" : 9, + "node" : "test", + "packetLen" : [ 90, 270 ], - "ipDst" : "10.178.8.71", - "sl" : 9, - "pa2" : 1, - "p1" : 62416, - "firstPacket" : 1385400648218, - "prot-term" : [ - "udp", - "dns" - ], - "mac1-term-cnt" : 1, - "dnsip" : [ - "192.30.252.130" - ], - "dnsipcnt" : 1, - "db1" : 66, - "asdnsip" : [ - "AS36459 GitHub, Inc." - ], - "ps" : [ + "packetPos" : [ 384, 474 ], - "fpd" : 1385400648218, - "g1" : "USA", - "ipSrc" : "10.180.156.141", - "mac2-term-cnt" : 2, - "mac2-term" : [ - "00:00:0c:07:ac:01", - "00:0e:d6:0b:98:80" - ], - "pa1" : 1, - "db2" : 246, - "fs" : [], - "lastPacket" : 1385400648228, - "portDst" : 53, - "lpd" : 1385400648228, - "ss" : 1, - "prot-term-cnt" : 2, - "p2" : 53, - "rirdnsip" : [ - "ARIN" + "protocol" : [ + "udp", + "dns" ], - "fb2" : "e039818000010002", - "no" : "test", - "mac1-term" : [ + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 74, + "srcDataBytes" : 66, + "srcGEO" : "US", + "srcIp" : "10.180.156.141", + "srcMac" : [ "00:1f:5b:ff:51:cb" ], - "lp" : 1385400648, - "pr" : 17, - "pa" : 2, - "a1" : "10.180.156.141" + "srcMacCnt" : 1, + "srcOui" : [ + "Apple, Inc." + ], + "srcOuiCnt" : 1, + "srcPackets" : 1, + "srcPayload8" : "e039010000010000", + "srcPort" : 62416, + "timestamp" : "SET", + "totBytes" : 328, + "totDataBytes" : 312, + "totPackets" : 2 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-131125", + "_type" : "session" + } } }, { "body" : { - "psl" : [ - 112 - ], - "ipDst" : "10.10.10.11", - "sl" : 0, - "pa2" : 0, - "p1" : 5353, - "firstPacket" : 1482767159342, - "prot-term" : [ - "udp", - "dns" - ], - "mac1-term-cnt" : 1, - "db1" : 88, - "fb1" : "caa6001000010000", - "db" : 88, "dns" : { - "opcode-term-cnt" : 1, - "qc-term" : [ - "IN" + "host" : [ + "10.100.10.10.in-addr.arpa" ], - "qc-term-cnt" : 1, - "qt-term-cnt" : 1, - "opcode-term" : [ + "hostCnt" : 1, + "opcode" : [ "QUERY" ], - "qt-term" : [ + "opcodeCnt" : 1, + "qc" : [ + "IN" + ], + "qcCnt" : 1, + "qt" : [ "ANY" - ] + ], + "qtCnt" : 1 }, - "portSrc" : 5353, - "by1" : 96, - "timestamp" : "SET", - "by" : 96, - "by2" : 0, - "a2" : "10.10.10.11", - "fp" : 1482767159, - "dnsho" : [ - "10.100.10.10.in-addr.arpa" + "dstBytes" : 0, + "dstDataBytes" : 0, + "dstIp" : "10.10.10.11", + "dstMac" : [ + "20:4e:71:c5:11:c0" ], - "dnshocnt" : 1, - "prot-term-cnt" : 2, - "p2" : 53, - "no" : "test", - "lp" : 1482767159, - "mac1-term" : [ - "ac:4b:c8:4c:9f:c1" + "dstMacCnt" : 1, + "dstOui" : [ + "Juniper Networks" ], - "pr" : 17, - "pa" : 1, - "a1" : "10.10.10.10", - "ps" : [ + "dstOuiCnt" : 1, + "dstPackets" : 0, + "dstPort" : 53, + "fileId" : [], + "firstPacket" : 1482767159342, + "ipProtocol" : 17, + "lastPacket" : 1482767159342, + "length" : 0, + "node" : "test", + "packetLen" : [ + 112 + ], + "packetPos" : [ 744 ], - "fpd" : 1482767159342, - "ipSrc" : "10.10.10.10", - "mac2-term" : [ - "20:4e:71:c5:11:c0" + "protocol" : [ + "udp", + "dns" ], - "mac2-term-cnt" : 1, - "pa1" : 1, - "fs" : [], - "db2" : 0, - "lastPacket" : 1482767159342, - "lpd" : 1482767159342, - "portDst" : 53, - "ss" : 1 + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 96, + "srcDataBytes" : 88, + "srcIp" : "10.10.10.10", + "srcMac" : [ + "ac:4b:c8:4c:9f:c1" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Juniper Networks" + ], + "srcOuiCnt" : 1, + "srcPackets" : 1, + "srcPayload8" : "caa6001000010000", + "srcPort" : 5353, + "timestamp" : "SET", + "totBytes" : 96, + "totDataBytes" : 88, + "totPackets" : 1 }, "header" : { "index" : { - "_index" : "tests_sessions-161226", + "_index" : "tests_sessions2-161226", "_type" : "session" } } diff --git a/tests/pcap/dns-update.test b/tests/pcap/dns-update.test index e14e485548..5b02bc0114 100644 --- a/tests/pcap/dns-update.test +++ b/tests/pcap/dns-update.test @@ -1,131 +1,127 @@ { - "sessions" : [ + "sessions2" : [ { - "header" : { - "index" : { - "_type" : "session", - "_index" : "tests_sessions-160702" - } - }, "body" : { - "no" : "test", - "mac1-term" : [ - "3c:8a:b0:6f:27:c5", - "3c:8a:b0:6e:77:c5" - ], - "ipSrc" : "10.0.0.1", - "mac1-term-cnt" : 2, - "lpd" : 1467464485007, - "dnsip" : [ - "10.0.0.1" - ], - "by" : 340, - "mac2-term-cnt" : 1, - "sl" : 24, - "tacnt" : 2, "dns" : { - "status-term" : [ - "NOTIMPL" + "ASN" : [ + "AS0000 This is neat" + ], + "GEO" : [ + "RU" + ], + "RIR" : [ + "" + ], + "host" : [ + "xxxxxxxxxxxxxx.xxxxxxx.xxxx.xxxxxxxxx.com" + ], + "hostCnt" : 1, + "ip" : [ + "10.0.0.1" ], - "opcode-term" : [ + "ipCnt" : 1, + "opcode" : [ "UPDATE" ], - "status-term-cnt" : 1, - "opcode-term-cnt" : 1 + "opcodeCnt" : 1, + "status" : [ + "NOTIMPL" + ], + "statusCnt" : 1 }, - "pa" : 2, - "psl" : [ + "dstASN" : "AS0001 Cool Beans!", + "dstBytes" : 170, + "dstDataBytes" : 162, + "dstGEO" : "CA", + "dstIp" : "10.0.0.2", + "dstMac" : [ + "00:1b:17:00:03:24" + ], + "dstMacCnt" : 1, + "dstOui" : [ + "Palo Alto Networks" + ], + "dstOuiCnt" : 1, + "dstPackets" : 1, + "dstPayload8" : "6919a80400010000", + "dstPort" : 53, + "dstRIR" : "TEST", + "fileId" : [], + "firstPacket" : 1467464484984, + "ipProtocol" : 17, + "lastPacket" : 1467464485007, + "length" : 24, + "node" : "test", + "packetLen" : [ 186, 186 ], - "lastPacket" : 1467464485007, - "ipDst" : "10.0.0.2", - "a2" : "10.0.0.2", - "prot-term" : [ + "packetPos" : [ + 24, + 210 + ], + "protocol" : [ "udp", "dns" ], - "pa1" : 1, - "as2" : "AS0001 Cool Beans!", - "p2" : 53, - "pr" : 17, - "fs" : [], - "dnsho" : [ - "xxxxxxxxxxxxxx.xxxxxxx.xxxx.xxxxxxxxx.com" + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcASN" : "AS0000 This is neat", + "srcBytes" : 170, + "srcDataBytes" : 162, + "srcGEO" : "RU", + "srcIp" : "10.0.0.1", + "srcMac" : [ + "3c:8a:b0:6e:77:c5", + "3c:8a:b0:6f:27:c5" ], - "tags-term" : [ - "srcip", - "dstip" - ], - "db2" : 162, - "fb1" : "6919280000010001", - "prot-term-cnt" : 2, - "db" : 324, - "rir2" : "TEST", - "dnsipcnt" : 1, - "lp" : 1467464485, - "db1" : 162, - "vlan" : [ - 100 + "srcMacCnt" : 2, + "srcOui" : [ + "Juniper Networks" ], - "by2" : 170, - "vlan-cnt" : 1, - "as1" : "AS0000 This is neat", - "firstPacket" : 1467464484984, - "dnshocnt" : 1, - "gdnsip" : [ - "RUS" - ], - "portSrc" : 51031, - "g2" : "CAN", - "fpd" : 1467464484984, - "asdnsip" : [ - "AS0000 This is neat" + "srcOuiCnt" : 1, + "srcPackets" : 1, + "srcPayload8" : "6919280000010001", + "srcPort" : 51031, + "tags" : [ + "dstip", + "srcip" ], - "fb2" : "6919a80400010000", - "pa2" : 1, - "p1" : 51031, - "a1" : "10.0.0.1", - "g1" : "RUS", - "timestamp" : "SET", + "tagsCnt" : 2, "test" : { - "ip" : [ - 167772161 - ], - "ip-asn" : [ + "ASN" : [ "AS0000 This is neat" ], - "ip-rir" : [ - "" + "GEO" : [ + "RU" ], - "string" : [ - "16777226:51031,33554442:53" + "RIR" : [ + "" ], - "ip-geo" : [ - "RUS" + "ip" : [ + "10.0.0.1" ], "number" : [ 33554442 + ], + "string.snow" : [ + "16777226:51031,33554442:53" ] }, - "by1" : 170, - "ta" : [ - "dstip", - "srcip" - ], - "rirdnsip" : [ - "" - ], - "ps" : [ - 24, - 210 + "timestamp" : "SET", + "totBytes" : 340, + "totDataBytes" : 324, + "totPackets" : 2, + "vlan" : [ + 100 ], - "portDst" : 53, - "fp" : 1467464484, - "ss" : 1, - "mac2-term" : [ - "00:1b:17:00:03:24" - ] + "vlanCnt" : 1 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-160702", + "_type" : "session" + } } } ] diff --git a/tests/pcap/dns-wiresharkrepo.test b/tests/pcap/dns-wiresharkrepo.test index 076783bc00..5336428099 100644 --- a/tests/pcap/dns-wiresharkrepo.test +++ b/tests/pcap/dns-wiresharkrepo.test @@ -1,48 +1,59 @@ { - "sessions" : [ + "sessions2" : [ { - "header" : { - "index" : { - "_type" : "session", - "_index" : "tests_sessions-050330" - } - }, "body" : { - "db" : 2028, - "by2" : 1328, - "fp" : 1112172466, - "ss" : 1, - "mac2-term-cnt" : 1, - "rir2" : "ARIN", - "dnsho" : [ - "www.isc.org", - "www.example.notginh", - "smtp1.google.com", - "www.google.com", - "www.netbsd.org", - "google.com", - "smtp6.google.com", - "smtp5.google.com", - "smtp4.google.com", - "104.9.192.66.in-addr.arpa", - "smtp3.google.com", - "www.example.com", - "smtp2.google.com", - "www.l.google.com" - ], - "asdnsip" : [ - "AS1280 Internet Systems Consortium, Inc.", - "AS1280 Internet Systems Consortium, Inc." - ], - "pa1" : 12, - "db1" : 796, - "gdnsip" : [ - "USA", - "USA" - ], "dns" : { - "qc-term-cnt" : 1, - "qt-term" : [ + "ASN" : [ + "AS1280 Internet Systems Consortium, Inc.", + "AS1280 Internet Systems Consortium, Inc.", + "AS1280 Internet Systems Consortium, Inc.", + "AS1280 Internet Systems Consortium, Inc." + ], + "GEO" : [ + "US", + "US", + "US", + "US" + ], + "RIR" : [ + "ARIN", + "", + "", + "ARIN" + ], + "host" : [ + "www.isc.org", + "www.example.notginh", + "smtp1.google.com", + "www.google.com", + "www.netbsd.org", + "google.com", + "smtp6.google.com", + "smtp5.google.com", + "smtp4.google.com", + "104.9.192.66.in-addr.arpa", + "smtp3.google.com", + "www.example.com", + "smtp2.google.com", + "www.l.google.com" + ], + "hostCnt" : 14, + "ip" : [ + "204.152.190.12", + "2001:04f8:0004:0007:02e0:81ff:fe52:9a6b", + "2001:04f8:0000:0002:0000:0000:0000:000d", + "204.152.184.88" + ], + "ipCnt" : 4, + "opcode" : [ + "QUERY" + ], + "opcodeCnt" : 1, + "qc" : [ + "IN" + ], + "qcCnt" : 1, + "qt" : [ "AAAA", "A", "MX", @@ -51,28 +62,61 @@ "TXT", "ANY" ], - "opcode-term" : [ - "QUERY" - ], - "opcode-term-cnt" : 1, - "status-term-cnt" : 2, - "qc-term" : [ - "IN" - ], - "qt-term-cnt" : 7, - "status-term" : [ + "qtCnt" : 7, + "status" : [ "NXDOMAIN", "NOERROR" - ] + ], + "statusCnt" : 2 }, - "fs" : [], - "p2" : 53, - "ipSrc" : "192.168.170.8", - "fb1" : "1032010000010000", - "mac1-term-cnt" : 1, - "p1" : 32795, - "dnsipcnt" : 2, - "ps" : [ + "dstBytes" : 1328, + "dstDataBytes" : 1232, + "dstIp" : "192.168.170.20", + "dstMac" : [ + "00:c0:9f:32:41:8c" + ], + "dstMacCnt" : 1, + "dstOui" : [ + "Quanta Computer Inc" + ], + "dstOuiCnt" : 1, + "dstPackets" : 12, + "dstPayload8" : "1032818000010001", + "dstPort" : 53, + "dstRIR" : "ARIN", + "fileId" : [], + "firstPacket" : 1112172466496, + "ipProtocol" : 17, + "lastPacket" : 1112172737733, + "length" : 271237, + "node" : "test", + "packetLen" : [ + 86, + 114, + 86, + 314, + 86, + 86, + 101, + 145, + 90, + 106, + 90, + 118, + 90, + 118, + 90, + 110, + 92, + 92, + 91, + 91, + 95, + 95, + 87, + 131 + ], + "packetPos" : [ 24, 110, 224, @@ -98,685 +142,649 @@ 2410, 2497 ], - "prot-term" : [ + "protocol" : [ "udp", "dns" ], - "dnsip" : [ - "204.152.190.12", - "204.152.184.88" - ], - "no" : "test", - "ipDst" : "192.168.170.20", - "mac2-term" : [ - "00:c0:9f:32:41:8c" - ], - "by" : 2220, - "dnshocnt" : 14, - "rirdnsip" : [ - "ARIN", - "ARIN" - ], - "a2" : "192.168.170.20", - "db2" : 1232, - "portDst" : 53, - "pr" : 17, - "a1" : "192.168.170.8", - "lpd" : 1112172737733, - "sl" : 271237, - "by1" : 892, - "lp" : 1112172737, - "mac1-term" : [ + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 892, + "srcDataBytes" : 796, + "srcIp" : "192.168.170.8", + "srcMac" : [ "00:e0:18:b1:0c:ad" ], - "pa" : 24, - "firstPacket" : 1112172466496, - "portSrc" : 32795, - "fb2" : "1032818000010001", - "prot-term-cnt" : 2, - "lastPacket" : 1112172737733, - "fpd" : 1112172466496, - "timestamp" : "SET", - "psl" : [ - 86, - 114, - 86, - 314, - 86, - 86, - 101, - 145, - 90, - 106, - 90, - 118, - 90, - 118, - 90, - 110, - 92, - 92, - 91, - 91, - 95, - 95, - 87, - 131 + "srcMacCnt" : 1, + "srcOui" : [ + "Asustek" ], - "rir1" : "ARIN", - "pa2" : 12 - } - }, - { + "srcOuiCnt" : 1, + "srcPackets" : 12, + "srcPayload8" : "1032010000010000", + "srcPort" : 32795, + "srcRIR" : "ARIN", + "timestamp" : "SET", + "totBytes" : 2220, + "totDataBytes" : 2028, + "totPackets" : 24 + }, "header" : { "index" : { - "_type" : "session", - "_index" : "tests_sessions-050330" + "_index" : "tests_sessions2-050330", + "_type" : "session" } - }, + } + }, + { "body" : { - "ipDst" : "192.168.170.20", - "mac2-term" : [ + "dns" : { + "host" : [ + "1.0.0.127.in-addr.arpa" + ], + "hostCnt" : 1, + "opcode" : [ + "QUERY" + ], + "opcodeCnt" : 1, + "qc" : [ + "IN" + ], + "qcCnt" : 1, + "qt" : [ + "PTR" + ], + "qtCnt" : 1, + "status" : [ + "NOERROR" + ], + "statusCnt" : 1 + }, + "dstBytes" : 105, + "dstDataBytes" : 97, + "dstIp" : "192.168.170.20", + "dstMac" : [ "00:c0:9f:32:41:8c" ], - "by" : 187, - "dnshocnt" : 1, - "a2" : "192.168.170.20", - "db2" : 97, - "portDst" : 53, - "pr" : 17, - "a1" : "192.168.170.8", - "sl" : 0, - "lpd" : 1112172737737, - "lp" : 1112172737, - "by1" : 82, - "mac1-term" : [ - "00:e0:18:b1:0c:ad" + "dstMacCnt" : 1, + "dstOui" : [ + "Quanta Computer Inc" ], - "pa" : 2, - "portSrc" : 32796, - "fb2" : "5a53858000010001", + "dstOuiCnt" : 1, + "dstPackets" : 1, + "dstPayload8" : "5a53858000010001", + "dstPort" : 53, + "dstRIR" : "ARIN", + "fileId" : [], "firstPacket" : 1112172737737, - "prot-term-cnt" : 2, + "ipProtocol" : 17, "lastPacket" : 1112172737737, - "fpd" : 1112172737737, - "timestamp" : "SET", - "psl" : [ + "length" : 0, + "node" : "test", + "packetLen" : [ 98, 121 ], - "rir1" : "ARIN", - "pa2" : 1, - "by2" : 105, - "db" : 171, - "fp" : 1112172737, - "ss" : 1, - "mac2-term-cnt" : 1, - "rir2" : "ARIN", - "dnsho" : [ - "1.0.0.127.in-addr.arpa" - ], - "pa1" : 1, - "db1" : 74, - "dns" : { - "qt-term-cnt" : 1, - "qc-term" : [ - "IN" - ], - "status-term-cnt" : 1, - "opcode-term-cnt" : 1, - "status-term" : [ - "NOERROR" - ], - "qc-term-cnt" : 1, - "opcode-term" : [ - "QUERY" - ], - "qt-term" : [ - "PTR" - ] - }, - "p2" : 53, - "fs" : [], - "ipSrc" : "192.168.170.8", - "mac1-term-cnt" : 1, - "fb1" : "5a53010000010000", - "p1" : 32796, - "ps" : [ + "packetPos" : [ 2628, 2726 ], - "prot-term" : [ + "protocol" : [ "udp", "dns" ], - "no" : "test" + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 82, + "srcDataBytes" : 74, + "srcIp" : "192.168.170.8", + "srcMac" : [ + "00:e0:18:b1:0c:ad" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Asustek" + ], + "srcOuiCnt" : 1, + "srcPackets" : 1, + "srcPayload8" : "5a53010000010000", + "srcPort" : 32796, + "srcRIR" : "ARIN", + "timestamp" : "SET", + "totBytes" : 187, + "totDataBytes" : 171, + "totPackets" : 2 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-050330", + "_type" : "session" + } } }, { "body" : { - "lp" : 1112172737, - "by1" : 67, - "pa" : 2, - "portSrc" : 32797, - "firstPacket" : 1112172737740, - "fb2" : "208a818000010004", - "mac1-term" : [ - "00:e0:18:b1:0c:ad" - ], - "prot-term-cnt" : 2, - "timestamp" : "SET", - "lastPacket" : 1112172737758, - "fpd" : 1112172737740, - "psl" : [ - 83, - 182 - ], - "rir1" : "ARIN", - "pa2" : 1, - "by" : 233, - "mac2-term" : [ - "00:c0:9f:32:41:8c" - ], - "ipDst" : "192.168.170.20", - "dnshocnt" : 1, - "a2" : "192.168.170.20", - "db2" : 158, - "pr" : 17, - "portDst" : 53, - "a1" : "192.168.170.8", - "sl" : 18, - "lpd" : 1112172737758, "dns" : { - "status-term" : [ - "NOERROR" + "host" : [ + "isc.org" + ], + "hostCnt" : 1, + "opcode" : [ + "QUERY" ], - "qc-term" : [ + "opcodeCnt" : 1, + "qc" : [ "IN" ], - "status-term-cnt" : 1, - "opcode-term-cnt" : 1, - "qt-term-cnt" : 1, - "qt-term" : [ + "qcCnt" : 1, + "qt" : [ "NS" ], - "opcode-term" : [ - "QUERY" + "qtCnt" : 1, + "status" : [ + "NOERROR" ], - "qc-term-cnt" : 1 + "statusCnt" : 1 }, - "ipSrc" : "192.168.170.8", - "p2" : 53, - "fs" : [], - "fb1" : "208a010000010000", - "mac1-term-cnt" : 1, - "ps" : [ + "dstBytes" : 166, + "dstDataBytes" : 158, + "dstIp" : "192.168.170.20", + "dstMac" : [ + "00:c0:9f:32:41:8c" + ], + "dstMacCnt" : 1, + "dstOui" : [ + "Quanta Computer Inc" + ], + "dstOuiCnt" : 1, + "dstPackets" : 1, + "dstPayload8" : "208a818000010004", + "dstPort" : 53, + "dstRIR" : "ARIN", + "fileId" : [], + "firstPacket" : 1112172737740, + "ipProtocol" : 17, + "lastPacket" : 1112172737758, + "length" : 18, + "node" : "test", + "packetLen" : [ + 83, + 182 + ], + "packetPos" : [ 2847, 3075 ], - "p1" : 32797, - "prot-term" : [ + "protocol" : [ "udp", "dns" ], - "no" : "test", - "by2" : 166, - "db" : 217, - "fp" : 1112172737, - "mac2-term-cnt" : 1, - "ss" : 1, - "rir2" : "ARIN", - "dnsho" : [ - "isc.org" - ], - "pa1" : 1, - "db1" : 59 + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 67, + "srcDataBytes" : 59, + "srcIp" : "192.168.170.8", + "srcMac" : [ + "00:e0:18:b1:0c:ad" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Asustek" + ], + "srcOuiCnt" : 1, + "srcPackets" : 1, + "srcPayload8" : "208a010000010000", + "srcPort" : 32797, + "srcRIR" : "ARIN", + "timestamp" : "SET", + "totBytes" : 233, + "totDataBytes" : 217, + "totPackets" : 2 }, "header" : { "index" : { - "_type" : "session", - "_index" : "tests_sessions-050330" + "_index" : "tests_sessions2-050330", + "_type" : "session" } } }, { - "header" : { - "index" : { - "_type" : "session", - "_index" : "tests_sessions-050330" - } - }, "body" : { - "dnshocnt" : 1, - "a2" : "217.13.4.24", - "db2" : 121, - "ipDst" : "217.13.4.24", - "mac2-term" : [ + "dns" : { + "host" : [ + "_ldap._tcp.default-first-site-name._sites.dc._msdcs.utelsystems.local" + ], + "hostCnt" : 1, + "opcode" : [ + "QUERY" + ], + "opcodeCnt" : 1, + "qc" : [ + "IN" + ], + "qcCnt" : 1, + "qt" : [ + "SRV" + ], + "qtCnt" : 1, + "status" : [ + "NXDOMAIN" + ], + "statusCnt" : 1 + }, + "dstASN" : "AS15659 NextGenTel AS", + "dstBytes" : 129, + "dstDataBytes" : 121, + "dstGEO" : "NO", + "dstIp" : "217.13.4.24", + "dstMac" : [ "00:12:a9:00:32:23" ], - "by" : 258, - "g2" : "NOR", - "sl" : 19, - "lpd" : 1112172737775, - "portDst" : 53, - "pr" : 17, - "a1" : "192.168.170.56", - "prot-term-cnt" : 2, - "fpd" : 1112172737755, - "lastPacket" : 1112172737775, - "timestamp" : "SET", - "by1" : 129, - "lp" : 1112172737, - "mac1-term" : [ - "00:60:08:45:e4:55" + "dstMacCnt" : 1, + "dstOui" : [ + "3Com Ltd" ], - "pa" : 2, - "fb2" : "326e858300010000", - "portSrc" : 1707, + "dstOuiCnt" : 1, + "dstPackets" : 1, + "dstPayload8" : "326e858300010000", + "dstPort" : 53, + "dstRIR" : "RIPE", + "fileId" : [], "firstPacket" : 1112172737755, - "rir1" : "ARIN", - "pa2" : 1, - "psl" : [ + "ipProtocol" : 17, + "lastPacket" : 1112172737775, + "length" : 19, + "node" : "test", + "packetLen" : [ 145, 145 ], - "fp" : 1112172737, - "ss" : 1, - "mac2-term-cnt" : 1, - "by2" : 129, - "db" : 242, - "pa1" : 1, - "db1" : 121, - "rir2" : "RIPE", - "dnsho" : [ - "_ldap._tcp.default-first-site-name._sites.dc._msdcs.utelsystems.local" + "packetPos" : [ + 2930, + 3257 ], - "dns" : { - "qc-term-cnt" : 1, - "opcode-term" : [ - "QUERY" - ], - "qt-term" : [ - "SRV" - ], - "qt-term-cnt" : 1, - "opcode-term-cnt" : 1, - "qc-term" : [ - "IN" - ], - "status-term-cnt" : 1, - "status-term" : [ - "NXDOMAIN" - ] - }, - "fs" : [], - "as2" : "AS15659 NextGenTel AS", - "p2" : 53, - "ipSrc" : "192.168.170.56", - "prot-term" : [ + "protocol" : [ "udp", "dns" ], - "no" : "test", - "fb1" : "326e010000010000", - "mac1-term-cnt" : 1, - "p1" : 1707, - "ps" : [ - 2930, - 3257 - ] - } - }, - { + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 129, + "srcDataBytes" : 121, + "srcIp" : "192.168.170.56", + "srcMac" : [ + "00:60:08:45:e4:55" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "3Com" + ], + "srcOuiCnt" : 1, + "srcPackets" : 1, + "srcPayload8" : "326e010000010000", + "srcPort" : 1707, + "srcRIR" : "ARIN", + "timestamp" : "SET", + "totBytes" : 258, + "totDataBytes" : 242, + "totPackets" : 2 + }, "header" : { "index" : { - "_type" : "session", - "_index" : "tests_sessions-050330" + "_index" : "tests_sessions2-050330", + "_type" : "session" } - }, + } + }, + { "body" : { - "no" : "test", - "prot-term" : [ - "udp", - "dns" - ], - "p1" : 1708, - "ps" : [ - 3402, - 3516 - ], - "fb1" : "f161010000010000", - "mac1-term-cnt" : 1, - "p2" : 53, - "fs" : [], - "as2" : "AS15659 NextGenTel AS", - "ipSrc" : "192.168.170.56", "dns" : { - "qc-term-cnt" : 1, - "opcode-term" : [ - "QUERY" + "host" : [ + "_ldap._tcp.dc._msdcs.utelsystems.local" ], - "qt-term" : [ - "SRV" + "hostCnt" : 1, + "opcode" : [ + "QUERY" ], - "qt-term-cnt" : 1, - "opcode-term-cnt" : 1, - "qc-term" : [ + "opcodeCnt" : 1, + "qc" : [ "IN" ], - "status-term-cnt" : 1, - "status-term" : [ + "qcCnt" : 1, + "qt" : [ + "SRV" + ], + "qtCnt" : 1, + "status" : [ "NXDOMAIN" - ] + ], + "statusCnt" : 1 }, - "db1" : 90, - "pa1" : 1, - "rir2" : "RIPE", - "dnsho" : [ - "_ldap._tcp.dc._msdcs.utelsystems.local" - ], - "ss" : 1, - "mac2-term-cnt" : 1, - "fp" : 1112172737, - "db" : 180, - "by2" : 98, - "pa2" : 1, - "rir1" : "ARIN", - "psl" : [ + "dstASN" : "AS15659 NextGenTel AS", + "dstBytes" : 98, + "dstDataBytes" : 90, + "dstGEO" : "NO", + "dstIp" : "217.13.4.24", + "dstMac" : [ + "00:12:a9:00:32:23" + ], + "dstMacCnt" : 1, + "dstOui" : [ + "3Com Ltd" + ], + "dstOuiCnt" : 1, + "dstPackets" : 1, + "dstPayload8" : "f161858300010000", + "dstPort" : 53, + "dstRIR" : "RIPE", + "fileId" : [], + "firstPacket" : 1112172737776, + "ipProtocol" : 17, + "lastPacket" : 1112172737793, + "length" : 17, + "node" : "test", + "packetLen" : [ 114, 114 ], - "lastPacket" : 1112172737793, - "fpd" : 1112172737776, - "timestamp" : "SET", - "prot-term-cnt" : 2, - "mac1-term" : [ + "packetPos" : [ + 3402, + 3516 + ], + "protocol" : [ + "udp", + "dns" + ], + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 98, + "srcDataBytes" : 90, + "srcIp" : "192.168.170.56", + "srcMac" : [ "00:60:08:45:e4:55" ], - "pa" : 2, - "fb2" : "f161858300010000", - "portSrc" : 1708, - "firstPacket" : 1112172737776, - "by1" : 98, - "lp" : 1112172737, - "lpd" : 1112172737793, - "sl" : 17, - "g2" : "NOR", - "a1" : "192.168.170.56", - "portDst" : 53, - "pr" : 17, - "db2" : 90, - "a2" : "217.13.4.24", - "dnshocnt" : 1, - "ipDst" : "217.13.4.24", - "mac2-term" : [ - "00:12:a9:00:32:23" + "srcMacCnt" : 1, + "srcOui" : [ + "3Com" ], - "by" : 196 - } - }, - { + "srcOuiCnt" : 1, + "srcPackets" : 1, + "srcPayload8" : "f161010000010000", + "srcPort" : 1708, + "srcRIR" : "ARIN", + "timestamp" : "SET", + "totBytes" : 196, + "totDataBytes" : 180, + "totPackets" : 2 + }, "header" : { "index" : { - "_type" : "session", - "_index" : "tests_sessions-050330" + "_index" : "tests_sessions2-050330", + "_type" : "session" } - }, + } + }, + { "body" : { - "db2" : 132, - "dnshocnt" : 1, - "a2" : "217.13.4.24", - "by" : 280, - "mac2-term" : [ - "00:12:a9:00:32:23" - ], - "ipDst" : "217.13.4.24", - "lpd" : 1112172737813, - "sl" : 19, - "g2" : "NOR", - "a1" : "192.168.170.56", - "pr" : 17, - "portDst" : 53, - "timestamp" : "SET", - "lastPacket" : 1112172737813, - "fpd" : 1112172737794, - "prot-term-cnt" : 2, - "pa" : 2, - "firstPacket" : 1112172737794, - "portSrc" : 1709, - "fb2" : "8361858300010000", - "mac1-term" : [ - "00:60:08:45:e4:55" - ], - "by1" : 140, - "lp" : 1112172737, - "pa2" : 1, - "rir1" : "ARIN", - "psl" : [ - 156, - 156 - ], - "mac2-term-cnt" : 1, - "ss" : 1, - "fp" : 1112172737, - "db" : 264, - "by2" : 140, - "db1" : 132, - "pa1" : 1, - "rir2" : "RIPE", - "dnsho" : [ - "_ldap._tcp.05b5292b-34b8-4fb7-85a3-8beef5fd2069.domains._msdcs.utelsystems.local" - ], - "ipSrc" : "192.168.170.56", - "fs" : [], - "p2" : 53, - "as2" : "AS15659 NextGenTel AS", "dns" : { - "opcode-term" : [ + "host" : [ + "_ldap._tcp.05b5292b-34b8-4fb7-85a3-8beef5fd2069.domains._msdcs.utelsystems.local" + ], + "hostCnt" : 1, + "opcode" : [ "QUERY" ], - "qt-term" : [ + "opcodeCnt" : 1, + "qc" : [ + "IN" + ], + "qcCnt" : 1, + "qt" : [ "SRV" ], - "qc-term-cnt" : 1, - "status-term" : [ + "qtCnt" : 1, + "status" : [ "NXDOMAIN" ], - "qt-term-cnt" : 1, - "qc-term" : [ - "IN" - ], - "status-term-cnt" : 1, - "opcode-term-cnt" : 1 + "statusCnt" : 1 }, - "no" : "test", - "prot-term" : [ - "udp", - "dns" + "dstASN" : "AS15659 NextGenTel AS", + "dstBytes" : 140, + "dstDataBytes" : 132, + "dstGEO" : "NO", + "dstIp" : "217.13.4.24", + "dstMac" : [ + "00:12:a9:00:32:23" + ], + "dstMacCnt" : 1, + "dstOui" : [ + "3Com Ltd" + ], + "dstOuiCnt" : 1, + "dstPackets" : 1, + "dstPayload8" : "8361858300010000", + "dstPort" : 53, + "dstRIR" : "RIPE", + "fileId" : [], + "firstPacket" : 1112172737794, + "ipProtocol" : 17, + "lastPacket" : 1112172737813, + "length" : 19, + "node" : "test", + "packetLen" : [ + 156, + 156 ], - "ps" : [ + "packetPos" : [ 3630, 3786 ], - "p1" : 1709, - "mac1-term-cnt" : 1, - "fb1" : "8361010000010000" - } - }, - { + "protocol" : [ + "udp", + "dns" + ], + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 140, + "srcDataBytes" : 132, + "srcIp" : "192.168.170.56", + "srcMac" : [ + "00:60:08:45:e4:55" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "3Com" + ], + "srcOuiCnt" : 1, + "srcPackets" : 1, + "srcPayload8" : "8361010000010000", + "srcPort" : 1709, + "srcRIR" : "ARIN", + "timestamp" : "SET", + "totBytes" : 280, + "totDataBytes" : 264, + "totPackets" : 2 + }, "header" : { "index" : { - "_index" : "tests_sessions-050330", + "_index" : "tests_sessions2-050330", "_type" : "session" } - }, + } + }, + { "body" : { - "ipSrc" : "192.168.170.56", - "fs" : [], - "p2" : 53, - "as2" : "AS15659 NextGenTel AS", "dns" : { - "status-term" : [ - "NXDOMAIN" + "host" : [ + "grimm.utelsystems.local" + ], + "hostCnt" : 1, + "opcode" : [ + "QUERY" ], - "qc-term" : [ + "opcodeCnt" : 1, + "qc" : [ "IN" ], - "status-term-cnt" : 1, - "opcode-term-cnt" : 1, - "qt-term-cnt" : 1, - "qt-term" : [ + "qcCnt" : 1, + "qt" : [ "A" ], - "opcode-term" : [ - "QUERY" + "qtCnt" : 1, + "status" : [ + "NXDOMAIN" ], - "qc-term-cnt" : 1 + "statusCnt" : 1 }, - "ps" : [ + "dstASN" : "AS15659 NextGenTel AS", + "dstBytes" : 83, + "dstDataBytes" : 75, + "dstGEO" : "NO", + "dstIp" : "217.13.4.24", + "dstMac" : [ + "00:12:a9:00:32:23" + ], + "dstMacCnt" : 1, + "dstOui" : [ + "3Com Ltd" + ], + "dstOuiCnt" : 1, + "dstPackets" : 1, + "dstPayload8" : "d060858300010000", + "dstPort" : 53, + "dstRIR" : "RIPE", + "fileId" : [], + "firstPacket" : 1112172737915, + "ipProtocol" : 17, + "lastPacket" : 1112172737932, + "length" : 16, + "node" : "test", + "packetLen" : [ + 99, + 99 + ], + "packetPos" : [ 3942, 4041 ], - "p1" : 1710, - "mac1-term-cnt" : 1, - "fb1" : "d060010000010000", - "no" : "test", - "prot-term" : [ + "protocol" : [ "udp", "dns" ], - "db" : 150, - "by2" : 83, - "mac2-term-cnt" : 1, - "ss" : 1, - "fp" : 1112172737, - "dnsho" : [ - "grimm.utelsystems.local" - ], - "rir2" : "RIPE", - "db1" : 75, - "pa1" : 1, - "portSrc" : 1710, - "pa" : 2, - "fb2" : "d060858300010000", - "firstPacket" : 1112172737915, - "mac1-term" : [ + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 83, + "srcDataBytes" : 75, + "srcIp" : "192.168.170.56", + "srcMac" : [ "00:60:08:45:e4:55" ], - "by1" : 83, - "lp" : 1112172737, - "timestamp" : "SET", - "fpd" : 1112172737915, - "lastPacket" : 1112172737932, - "prot-term-cnt" : 2, - "psl" : [ - 99, - 99 + "srcMacCnt" : 1, + "srcOui" : [ + "3Com" ], - "pa2" : 1, - "rir1" : "ARIN", - "by" : 166, - "ipDst" : "217.13.4.24", - "mac2-term" : [ - "00:12:a9:00:32:23" - ], - "db2" : 75, - "dnshocnt" : 1, - "a2" : "217.13.4.24", - "a1" : "192.168.170.56", - "pr" : 17, - "portDst" : 53, - "lpd" : 1112172737932, - "sl" : 16, - "g2" : "NOR" - } - }, - { + "srcOuiCnt" : 1, + "srcPackets" : 1, + "srcPayload8" : "d060010000010000", + "srcPort" : 1710, + "srcRIR" : "ARIN", + "timestamp" : "SET", + "totBytes" : 166, + "totDataBytes" : 150, + "totPackets" : 2 + }, "header" : { "index" : { - "_type" : "session", - "_index" : "tests_sessions-050330" + "_index" : "tests_sessions2-050330", + "_type" : "session" } - }, + } + }, + { "body" : { - "a2" : "217.13.4.24", - "dnshocnt" : 1, - "db2" : 75, - "ipDst" : "217.13.4.24", - "mac2-term" : [ + "dns" : { + "host" : [ + "grimm.utelsystems.local" + ], + "hostCnt" : 1, + "opcode" : [ + "QUERY" + ], + "opcodeCnt" : 1, + "qc" : [ + "IN" + ], + "qcCnt" : 1, + "qt" : [ + "A" + ], + "qtCnt" : 1, + "status" : [ + "NXDOMAIN" + ], + "statusCnt" : 1 + }, + "dstASN" : "AS15659 NextGenTel AS", + "dstBytes" : 83, + "dstDataBytes" : 75, + "dstGEO" : "NO", + "dstIp" : "217.13.4.24", + "dstMac" : [ "00:12:a9:00:32:23" ], - "by" : 166, - "g2" : "NOR", - "sl" : 18, - "lpd" : 1112172745375, - "portDst" : 53, - "pr" : 17, - "a1" : "192.168.170.56", - "prot-term-cnt" : 2, - "lastPacket" : 1112172745375, - "fpd" : 1112172745357, - "timestamp" : "SET", - "by1" : 83, - "lp" : 1112172745, - "mac1-term" : [ - "00:60:08:45:e4:55" + "dstMacCnt" : 1, + "dstOui" : [ + "3Com Ltd" ], - "fb2" : "7663858300010000", - "pa" : 2, - "portSrc" : 1711, + "dstOuiCnt" : 1, + "dstPackets" : 1, + "dstPayload8" : "7663858300010000", + "dstPort" : 53, + "dstRIR" : "RIPE", + "fileId" : [], "firstPacket" : 1112172745357, - "rir1" : "ARIN", - "pa2" : 1, - "psl" : [ + "ipProtocol" : 17, + "lastPacket" : 1112172745375, + "length" : 18, + "node" : "test", + "packetLen" : [ 99, 99 ], - "fp" : 1112172745, - "ss" : 1, - "mac2-term-cnt" : 1, - "by2" : 83, - "db" : 150, - "pa1" : 1, - "db1" : 75, - "rir2" : "RIPE", - "dnsho" : [ - "grimm.utelsystems.local" + "packetPos" : [ + 4140, + 4239 ], - "dns" : { - "qt-term" : [ - "A" - ], - "opcode-term" : [ - "QUERY" - ], - "qc-term-cnt" : 1, - "status-term" : [ - "NXDOMAIN" - ], - "opcode-term-cnt" : 1, - "status-term-cnt" : 1, - "qc-term" : [ - "IN" - ], - "qt-term-cnt" : 1 - }, - "p2" : 53, - "fs" : [], - "as2" : "AS15659 NextGenTel AS", - "ipSrc" : "192.168.170.56", - "prot-term" : [ + "protocol" : [ "udp", "dns" ], - "no" : "test", - "fb1" : "7663010000010000", - "mac1-term-cnt" : 1, - "p1" : 1711, - "ps" : [ - 4140, - 4239 - ] + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 83, + "srcDataBytes" : 75, + "srcIp" : "192.168.170.56", + "srcMac" : [ + "00:60:08:45:e4:55" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "3Com" + ], + "srcOuiCnt" : 1, + "srcPackets" : 1, + "srcPayload8" : "7663010000010000", + "srcPort" : 1711, + "srcRIR" : "ARIN", + "timestamp" : "SET", + "totBytes" : 166, + "totDataBytes" : 150, + "totPackets" : 2 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-050330", + "_type" : "session" + } } } ] diff --git a/tests/pcap/fbzero-android.test b/tests/pcap/fbzero-android.test index 6ed47c1c86..3578f7b590 100644 --- a/tests/pcap/fbzero-android.test +++ b/tests/pcap/fbzero-android.test @@ -1,39 +1,77 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "a2" : "31.13.74.1", - "g2" : "IRL", - "timestamp" : "SET", - "lastPacket" : 1479719902505, - "prot-term-cnt" : 3, - "ipDst" : "31.13.74.1", - "quic" : { - "host-termcnt" : 1, - "host-term" : [ - "graph.facebook.com" - ] - }, - "by1" : 6462, - "portSrc" : 39584, - "portDst" : 443, - "sl" : 22965, - "as2" : "AS32934 Facebook, Inc.", - "mac1-term-cnt" : 1, - "mac2-term" : [ + "dstASN" : "AS32934 Facebook, Inc.", + "dstBytes" : 5678, + "dstDataBytes" : 4416, + "dstGEO" : "IE", + "dstIp" : "31.13.74.1", + "dstMac" : [ "4c:02:89:0a:d2:76" ], - "fp" : 1479719879, - "a1" : "192.168.1.111", - "ipSrc" : "192.168.1.111", - "db2" : 4416, - "fs" : [], - "by2" : 5678, + "dstMacCnt" : 1, + "dstOui" : [ + "Lex Computech Co., Ltd" + ], + "dstOuiCnt" : 1, + "dstPackets" : 19, + "dstPayload8" : "3151545630300000", + "dstPort" : 443, + "dstRIR" : "RIPE", + "fileId" : [], "firstPacket" : 1479719879541, - "pr" : 6, - "fpd" : 1479719879541, - "p2" : 443, - "ps" : [ + "ipProtocol" : 6, + "lastPacket" : 1479719902505, + "length" : 22965, + "node" : "test", + "packetLen" : [ + 90, + 90, + 82, + 289, + 1355, + 82, + 220, + 159, + 82, + 82, + 1236, + 142, + 1430, + 280, + 82, + 1176, + 82, + 120, + 82, + 318, + 82, + 124, + 124, + 82, + 644, + 120, + 82, + 383, + 82, + 458, + 120, + 82, + 348, + 82, + 460, + 120, + 82, + 292, + 82, + 894, + 120, + 82, + 338, + 82 + ], + "packetPos" : [ 24, 114, 204, @@ -79,90 +117,54 @@ 12448, 12786 ], - "db1" : 4804, - "pa" : 44, - "p1" : 39584, - "by" : 12140, - "ss" : 1, - "db" : 9220, - "rir2" : "RIPE", - "pa1" : 25, - "tcpflags" : { - "syn-ack" : 1, - "urg" : 0, - "fin" : 0, - "psh" : 24, - "ack" : 18, - "syn" : 1, - "rst" : 0 - }, - "no" : "test", - "fb1" : "3151545630c60000", - "pa2" : 19, - "lpd" : 1479719902505, - "fb2" : "3151545630300000", - "mac2-term-cnt" : 1, - "mac1-term" : [ - "88:28:b3:b3:7c:41" - ], - "prot-term" : [ + "protocol" : [ "fbzero", "quic", "tcp" ], - "rir1" : "ARIN", - "psl" : [ - 90, - 90, - 82, - 289, - 1355, - 82, - 220, - 159, - 82, - 82, - 1236, - 142, - 1430, - 280, - 82, - 1176, - 82, - 120, - 82, - 318, - 82, - 124, - 124, - 82, - 644, - 120, - 82, - 383, - 82, - 458, - 120, - 82, - 348, - 82, - 460, - 120, - 82, - 292, - 82, - 894, - 120, - 82, - 338, - 82 + "protocolCnt" : 3, + "quic" : { + "host" : [ + "graph.facebook.com" + ], + "hostCnt" : 1 + }, + "segmentCnt" : 1, + "srcBytes" : 6462, + "srcDataBytes" : 4804, + "srcIp" : "192.168.1.111", + "srcMac" : [ + "88:28:b3:b3:7c:41" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Huawei Technologies Co.,Ltd" ], - "lp" : 1479719902 + "srcOuiCnt" : 1, + "srcPackets" : 25, + "srcPayload8" : "3151545630c60000", + "srcPort" : 39584, + "srcRIR" : "ARIN", + "tcpflags" : { + "ack" : 18, + "dstZero" : 0, + "fin" : 0, + "psh" : 24, + "rst" : 0, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, + "timestamp" : "SET", + "totBytes" : 12140, + "totDataBytes" : 9220, + "totPackets" : 44 }, "header" : { "index" : { - "_type" : "session", - "_index" : "tests_sessions-161121" + "_index" : "tests_sessions2-161121", + "_type" : "session" } } } diff --git a/tests/pcap/ftp.test b/tests/pcap/ftp.test index 1771392c30..84ce57689c 100644 --- a/tests/pcap/ftp.test +++ b/tests/pcap/ftp.test @@ -1,45 +1,31 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "ps" : [ - 24, - 114, - 204, - 286, - 401, - 483, - 581, - 663, - 779, - 883 - ], - "ipSrc" : "10.10.30.26", - "lp" : 1454301531, - "mac2-term" : [ + "dstASN" : "AS4385 Rochester Institute of Technology", + "dstBytes" : 428, + "dstDataBytes" : 90, + "dstGEO" : "US", + "dstIp" : "129.21.171.72", + "dstMac" : [ "58:f3:9c:f7:2f:9f" ], - "a2" : "129.21.171.72", - "lastPacket" : 1454301531009, - "ss" : 1, - "fpd" : 1454301530932, - "no" : "test", - "user" : [ - "anonymous" + "dstMacCnt" : 1, + "dstOui" : [ + "Cisco Systems, Inc" ], - "timestamp" : "SET", - "fs" : [], - "portDst" : 21, - "tcpflags" : { - "psh" : 5, - "ack" : 3, - "urg" : 0, - "syn-ack" : 1, - "rst" : 0, - "fin" : 0, - "syn" : 1 - }, - "psl" : [ + "dstOuiCnt" : 1, + "dstPackets" : 5, + "dstPayload8" : "3232302057656c63", + "dstPort" : 21, + "dstRIR" : "ARIN", + "fileId" : [], + "firstPacket" : 1454301530932, + "ipProtocol" : 6, + "lastPacket" : 1454301531009, + "length" : 78, + "node" : "test", + "packetLen" : [ 90, 90, 82, @@ -51,46 +37,62 @@ 104, 105 ], - "as2" : "AS4385 Rochester Institute of Technology", - "mac2-term-cnt" : 1, - "by2" : 428, - "pa" : 10, - "fp" : 1454301530, - "ipDst" : "129.21.171.72", - "mac1-term-cnt" : 1, - "mac1-term" : [ - "00:08:e3:ff:fc:28" + "packetPos" : [ + 24, + 114, + 204, + 286, + 401, + 483, + 581, + 663, + 779, + 883 ], - "a1" : "10.10.30.26", - "lpd" : 1454301531009, - "prot-term-cnt" : 2, - "pa2" : 5, - "p1" : 43958, - "pr" : 6, - "rir2" : "ARIN", - "by" : 804, - "db" : 128, - "fb2" : "3232302057656c63", - "portSrc" : 43958, - "firstPacket" : 1454301530932, - "fb1" : "5553455220616e6f", - "sl" : 78, - "by1" : 376, - "db1" : 38, - "g2" : "USA", - "usercnt" : 1, - "pa1" : 5, - "db2" : 90, - "prot-term" : [ + "protocol" : [ "ftp", "tcp" ], - "p2" : 21 + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 376, + "srcDataBytes" : 38, + "srcIp" : "10.10.30.26", + "srcMac" : [ + "00:08:e3:ff:fc:28" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Cisco Systems, Inc" + ], + "srcOuiCnt" : 1, + "srcPackets" : 5, + "srcPayload8" : "5553455220616e6f", + "srcPort" : 43958, + "tcpflags" : { + "ack" : 3, + "dstZero" : 0, + "fin" : 0, + "psh" : 5, + "rst" : 0, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, + "timestamp" : "SET", + "totBytes" : 804, + "totDataBytes" : 128, + "totPackets" : 10, + "user" : [ + "anonymous" + ], + "userCnt" : 1 }, "header" : { "index" : { - "_type" : "session", - "_index" : "tests_sessions-160201" + "_index" : "tests_sessions2-160201", + "_type" : "session" } } } diff --git a/tests/pcap/gre-sample.test b/tests/pcap/gre-sample.test index 483a3b9821..156636d1f1 100644 --- a/tests/pcap/gre-sample.test +++ b/tests/pcap/gre-sample.test @@ -1,41 +1,68 @@ { - "sessions" : [ + "sessions2" : [ { - "header" : { - "index" : { - "_index" : "tests_sessions-030611", - "_type" : "session" - } - }, "body" : { - "prot-term-cnt" : 2, - "pa" : 10, - "g1" : "USA", - "rir2" : "ARIN", - "timestamp" : "SET", - "ipSrc" : "66.59.111.190", + "dstBytes" : 742, + "dstDataBytes" : 0, + "dstIp" : "172.28.2.3", + "dstMac" : [ + "00:c0:ca:14:b0:52" + ], + "dstMacCnt" : 1, + "dstOui" : [ + "Alfa, Inc." + ], + "dstOuiCnt" : 1, + "dstPackets" : 6, + "dstPort" : 0, + "dstRIR" : "ARIN", + "fileId" : [], "firstPacket" : 1055289968793, - "by1" : 488, - "greip-asn" : [ + "greASN" : [ "---", "AS11590 Cumberland Technologies International" ], - "pr" : 1, - "lpd" : 1055289992101, - "greip-rir" : [ + "greGEO" : [ + "---", + "US" + ], + "greIp" : [ + "172.27.1.66", + "66.59.109.137" + ], + "greIpCnt" : 2, + "greRIR" : [ "ARIN", "ARIN" ], - "db2" : 0, - "icmpType" : [ - 8, - 0, - 3 + "icmp" : { + "code" : [ + 0, + 3 + ], + "type" : [ + 8, + 0, + 3 + ] + }, + "ipProtocol" : 1, + "lastPacket" : 1055289992101, + "length" : 23309, + "node" : "test", + "packetLen" : [ + 138, + 138, + 138, + 138, + 138, + 138, + 138, + 138, + 143, + 143 ], - "a2" : "172.28.2.3", - "fs" : [], - "pa1" : 4, - "ps" : [ + "packetPos" : [ 24, 162, 300, @@ -47,192 +74,191 @@ 6474, 6732 ], - "portSrc" : 0, - "lp" : 1055289992, - "rir1" : "ARIN", - "fp" : 1055289968, - "no" : "test", - "greip" : [ - 2887450946, - 1111190921 - ], - "mac1-term-cnt" : 1, - "prot-term" : [ + "protocol" : [ "icmp", "gre" ], - "greip-geo" : [ - "---", - "USA" - ], - "by2" : 742, - "ss" : 1, - "psl" : [ - 138, - 138, - 138, - 138, - 138, - 138, - 138, - 138, - 143, - 143 - ], - "greip-cnt" : 2, - "as1" : "AS11590 Cumberland Technologies International", - "p1" : 0, - "db1" : 0, - "portDst" : 0, - "db" : 0, - "pa2" : 6, - "mac2-term-cnt" : 1, - "by" : 1230, - "fpd" : 1055289968793, - "lastPacket" : 1055289992101, - "sl" : 23309, - "mac1-term" : [ + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcASN" : "AS11590 Cumberland Technologies International", + "srcBytes" : 488, + "srcDataBytes" : 0, + "srcGEO" : "US", + "srcIp" : "66.59.111.190", + "srcMac" : [ "00:02:2d:56:4a:fd" ], - "a1" : "66.59.111.190", - "icmpCode" : [ - 0, - 3 + "srcMacCnt" : 1, + "srcOui" : [ + "Agere Systems" ], - "mac2-term" : [ - "00:c0:ca:14:b0:52" - ], - "ipDst" : "172.28.2.3", - "p2" : 0 - } - }, - { + "srcOuiCnt" : 1, + "srcPackets" : 4, + "srcPort" : 0, + "srcRIR" : "ARIN", + "timestamp" : "SET", + "totBytes" : 1230, + "totDataBytes" : 0, + "totPackets" : 10 + }, "header" : { "index" : { - "_type" : "session", - "_index" : "tests_sessions-030611" + "_index" : "tests_sessions2-030611", + "_type" : "session" } - }, + } + }, + { "body" : { - "portDst" : 123, - "mac2-term-cnt" : 1, - "pa2" : 1, - "db" : 212, - "by" : 228, - "fpd" : 1055289973849, - "fb1" : "230b06ef00000000", - "lastPacket" : 1055289973923, - "as2" : "AS3 Massachusetts Institute of Technology", - "sl" : 74, - "mac1-term" : [ - "00:02:2d:56:4a:fd" - ], - "ipDst" : "18.26.4.105", - "mac2-term" : [ + "dstASN" : "AS3 Massachusetts Institute of Technology", + "dstBytes" : 114, + "dstDataBytes" : 106, + "dstGEO" : "US", + "dstIp" : "18.26.4.105", + "dstMac" : [ "00:c0:ca:14:b0:52" ], - "p2" : 123, - "a1" : "66.59.111.190", - "portSrc" : 123, - "ps" : [ + "dstMacCnt" : 1, + "dstOui" : [ + "Alfa, Inc." + ], + "dstOuiCnt" : 1, + "dstPackets" : 1, + "dstPayload8" : "240206ee000003bc", + "dstPort" : 123, + "dstRIR" : "ARIN", + "fileId" : [], + "firstPacket" : 1055289973849, + "greASN" : [ + "---", + "AS11590 Cumberland Technologies International" + ], + "greGEO" : [ + "---", + "US" + ], + "greIp" : [ + "172.27.1.66", + "66.59.109.137" + ], + "greIpCnt" : 2, + "greRIR" : [ + "ARIN", + "ARIN" + ], + "ipProtocol" : 17, + "lastPacket" : 1055289973923, + "length" : 74, + "node" : "test", + "packetLen" : [ + 130, + 130 + ], + "packetPos" : [ 1128, 1258 ], - "lp" : 1055289973, - "rir1" : "ARIN", - "greip" : [ - 2887450946, - 1111190921 - ], - "fp" : 1055289973, - "no" : "test", - "prot-term" : [ + "protocol" : [ "udp", "gre", "ntp" ], - "mac1-term-cnt" : 1, - "by2" : 114, - "g2" : "USA", - "greip-geo" : [ - "---", - "USA" - ], - "psl" : [ - 130, - 130 + "protocolCnt" : 3, + "segmentCnt" : 1, + "srcASN" : "AS11590 Cumberland Technologies International", + "srcBytes" : 114, + "srcDataBytes" : 106, + "srcGEO" : "US", + "srcIp" : "66.59.111.190", + "srcMac" : [ + "00:02:2d:56:4a:fd" ], - "ss" : 1, - "db1" : 106, - "p1" : 123, - "as1" : "AS11590 Cumberland Technologies International", - "greip-cnt" : 2, - "greip-rir" : [ - "ARIN", - "ARIN" + "srcMacCnt" : 1, + "srcOui" : [ + "Agere Systems" ], - "lpd" : 1055289973923, - "db2" : 106, - "a2" : "18.26.4.105", - "pa1" : 1, - "fs" : [], - "pa" : 2, - "fb2" : "240206ee000003bc", - "prot-term-cnt" : 3, - "g1" : "USA", - "rir2" : "ARIN", - "firstPacket" : 1055289973849, - "ipSrc" : "66.59.111.190", + "srcOuiCnt" : 1, + "srcPackets" : 1, + "srcPayload8" : "230b06ef00000000", + "srcPort" : 123, + "srcRIR" : "ARIN", "timestamp" : "SET", - "by1" : 114, - "pr" : 17, - "greip-asn" : [ - "---", - "AS11590 Cumberland Technologies International" - ] + "totBytes" : 228, + "totDataBytes" : 212, + "totPackets" : 2 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-030611", + "_type" : "session" + } } }, { "body" : { - "prot-term-cnt" : 3, - "pa" : 22, - "fb2" : "5353482d312e3939", - "tcpflags" : { - "syn" : 1, - "rst" : 0, - "ack" : 10, - "fin" : 2, - "urg" : 0, - "psh" : 8, - "syn-ack" : 1 - }, - "g1" : "USA", - "sshvercnt" : 2, - "rir2" : "ARIN", - "ipSrc" : "66.59.111.190", - "timestamp" : "SET", + "dstBytes" : 2579, + "dstDataBytes" : 1671, + "dstIp" : "172.28.2.3", + "dstMac" : [ + "00:c0:ca:14:b0:52" + ], + "dstMacCnt" : 1, + "dstOui" : [ + "Alfa, Inc." + ], + "dstOuiCnt" : 1, + "dstPackets" : 10, + "dstPayload8" : "5353482d312e3939", + "dstPort" : 22, + "dstRIR" : "ARIN", + "fileId" : [], "firstPacket" : 1055289978756, - "by1" : 2040, - "greip-asn" : [ + "greASN" : [ "---", "AS11590 Cumberland Technologies International" ], - "pr" : 6, - "lpd" : 1055289981914, - "greip-rir" : [ + "greGEO" : [ + "---", + "US" + ], + "greIp" : [ + "172.27.1.66", + "66.59.109.137" + ], + "greIpCnt" : 2, + "greRIR" : [ "ARIN", "ARIN" ], - "db2" : 1671, - "a2" : "172.28.2.3", - "fs" : [], - "pa1" : 12, - "sshver" : [ - "ssh-2.0-openssh_3.6.1p1", - "ssh-1.99-openssh_3.1p1" + "ipProtocol" : 6, + "lastPacket" : 1055289981914, + "length" : 3157, + "node" : "test", + "packetLen" : [ + 114, + 114, + 106, + 129, + 106, + 130, + 106, + 594, + 594, + 106, + 106, + 130, + 106, + 530, + 106, + 522, + 106, + 842, + 106, + 106, + 106, + 106 ], - "portSrc" : 40264, - "ps" : [ + "packetPos" : [ 1388, 1502, 1616, @@ -256,355 +282,335 @@ 6147, 6253 ], - "lp" : 1055289981, - "rir1" : "ARIN", - "fp" : 1055289978, - "no" : "test", - "greip" : [ - 2887450946, - 1111190921 - ], - "sshkeycnt" : 1, - "mac1-term-cnt" : 1, - "prot-term" : [ + "protocol" : [ "ssh", "gre", "tcp" ], - "greip-geo" : [ - "---", - "USA" - ], - "by2" : 2579, - "sshkey" : [ - "AAAAB3NzaC1yc2EAAAABIwAAAIEApsEyRdoxEZ/Xmehe9Oo6s0uuroriAKOqim32L2V9DZbzt101U41iUHjI8ESaANQZd73O7uop7tIfK6ZMq/cshMcuuI1YxqYxCBuCFLsiomlYq7+7cURJEClVtSyPmtijvhnaJKAlN4P3iXZDCnUQkKrWC+XhBZT8E5DxSSIRAhk=" - ], - "psl" : [ - 114, - 114, - 106, - 129, - 106, - 130, - 106, - 594, - 594, - 106, - 106, - 130, - 106, - 530, - 106, - 522, - 106, - 842, - 106, - 106, - 106, - 106 - ], - "ss" : 1, - "greip-cnt" : 2, - "as1" : "AS11590 Cumberland Technologies International", - "p1" : 40264, - "db1" : 952, - "portDst" : 22, - "db" : 2623, - "pa2" : 10, - "mac2-term-cnt" : 1, - "fpd" : 1055289978756, - "by" : 4619, - "lastPacket" : 1055289981914, - "fb1" : "5353482d322e302d", - "sl" : 3157, - "mac1-term" : [ + "protocolCnt" : 3, + "segmentCnt" : 1, + "srcASN" : "AS11590 Cumberland Technologies International", + "srcBytes" : 2040, + "srcDataBytes" : 952, + "srcGEO" : "US", + "srcIp" : "66.59.111.190", + "srcMac" : [ "00:02:2d:56:4a:fd" ], - "a1" : "66.59.111.190", - "mac2-term" : [ - "00:c0:ca:14:b0:52" - ], - "p2" : 22, - "ipDst" : "172.28.2.3" + "srcMacCnt" : 1, + "srcOui" : [ + "Agere Systems" + ], + "srcOuiCnt" : 1, + "srcPackets" : 12, + "srcPayload8" : "5353482d322e302d", + "srcPort" : 40264, + "srcRIR" : "ARIN", + "ssh" : { + "key" : [ + "AAAAB3NzaC1yc2EAAAABIwAAAIEApsEyRdoxEZ/Xmehe9Oo6s0uuroriAKOqim32L2V9DZbzt101U41iUHjI8ESaANQZd73O7uop7tIfK6ZMq/cshMcuuI1YxqYxCBuCFLsiomlYq7+7cURJEClVtSyPmtijvhnaJKAlN4P3iXZDCnUQkKrWC+XhBZT8E5DxSSIRAhk=" + ], + "keyCnt" : 1, + "version" : [ + "ssh-2.0-openssh_3.6.1p1", + "ssh-1.99-openssh_3.1p1" + ], + "versionCnt" : 2 + }, + "tcpflags" : { + "ack" : 10, + "dstZero" : 0, + "fin" : 2, + "psh" : 8, + "rst" : 0, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, + "timestamp" : "SET", + "totBytes" : 4619, + "totDataBytes" : 2623, + "totPackets" : 22 }, "header" : { "index" : { - "_type" : "session", - "_index" : "tests_sessions-030611" + "_index" : "tests_sessions2-030611", + "_type" : "session" } } }, { - "header" : { - "index" : { - "_type" : "session", - "_index" : "tests_sessions-030611" - } - }, "body" : { - "mac1-term" : [ - "00:02:2d:56:4a:fd" - ], - "a1" : "66.59.111.190", - "mac2-term" : [ + "dns" : { + "host" : [ + "www.gleeble.org" + ], + "hostCnt" : 1, + "opcode" : [ + "QUERY" + ], + "opcodeCnt" : 1, + "qc" : [ + "IN" + ], + "qcCnt" : 1, + "qt" : [ + "ANY" + ], + "qtCnt" : 1 + }, + "dstBytes" : 0, + "dstDataBytes" : 0, + "dstIp" : "172.28.2.3", + "dstMac" : [ "00:c0:ca:14:b0:52" ], - "p2" : 53, - "ipDst" : "172.28.2.3", - "sl" : 5001, - "fpd" : 1055289987055, - "by" : 198, - "lastPacket" : 1055289992056, - "fb1" : "bdaa010000010000", - "portDst" : 53, - "db" : 182, - "mac2-term-cnt" : 1, - "pa2" : 0, - "greip-cnt" : 2, - "p1" : 37675, - "as1" : "AS11590 Cumberland Technologies International", - "db1" : 182, - "greip-geo" : [ + "dstMacCnt" : 1, + "dstOui" : [ + "Alfa, Inc." + ], + "dstOuiCnt" : 1, + "dstPackets" : 0, + "dstPort" : 53, + "dstRIR" : "ARIN", + "fileId" : [], + "firstPacket" : 1055289987055, + "greASN" : [ "---", - "USA" + "AS11590 Cumberland Technologies International" + ], + "greGEO" : [ + "---", + "US" + ], + "greIp" : [ + "172.27.1.66", + "66.59.109.137" ], - "by2" : 0, - "psl" : [ + "greIpCnt" : 2, + "greRIR" : [ + "ARIN", + "ARIN" + ], + "ipProtocol" : 17, + "lastPacket" : 1055289992056, + "length" : 5001, + "node" : "test", + "packetLen" : [ 115, 115 ], - "ss" : 1, - "no" : "test", - "fp" : 1055289987, - "greip" : [ - 2887450946, - 1111190921 + "packetPos" : [ + 6359, + 6617 ], - "mac1-term-cnt" : 1, - "prot-term" : [ + "protocol" : [ "udp", "dns", "gre" ], - "dnshocnt" : 1, - "ps" : [ - 6359, - 6617 - ], - "portSrc" : 37675, - "lp" : 1055289992, - "rir1" : "ARIN", - "a2" : "172.28.2.3", - "fs" : [], - "pa1" : 2, - "lpd" : 1055289992056, - "greip-rir" : [ - "ARIN", - "ARIN" + "protocolCnt" : 3, + "segmentCnt" : 1, + "srcASN" : "AS11590 Cumberland Technologies International", + "srcBytes" : 198, + "srcDataBytes" : 182, + "srcGEO" : "US", + "srcIp" : "66.59.111.190", + "srcMac" : [ + "00:02:2d:56:4a:fd" ], - "db2" : 0, - "dns" : { - "qt-term-cnt" : 1, - "opcode-term-cnt" : 1, - "qc-term-cnt" : 1, - "qt-term" : [ - "ANY" - ], - "qc-term" : [ - "IN" - ], - "opcode-term" : [ - "QUERY" - ] - }, - "greip-asn" : [ - "---", - "AS11590 Cumberland Technologies International" + "srcMacCnt" : 1, + "srcOui" : [ + "Agere Systems" ], - "pr" : 17, + "srcOuiCnt" : 1, + "srcPackets" : 2, + "srcPayload8" : "bdaa010000010000", + "srcPort" : 37675, + "srcRIR" : "ARIN", "timestamp" : "SET", - "ipSrc" : "66.59.111.190", - "firstPacket" : 1055289987055, - "by1" : 198, - "g1" : "USA", - "rir2" : "ARIN", - "prot-term-cnt" : 3, - "pa" : 2, - "dnsho" : [ - "www.gleeble.org" - ] - } - }, - { + "totBytes" : 198, + "totDataBytes" : 182, + "totPackets" : 2 + }, "header" : { "index" : { - "_index" : "tests_sessions-030611", + "_index" : "tests_sessions2-030611", "_type" : "session" } - }, + } + }, + { "body" : { - "pr" : 17, - "greip-asn" : [ + "dstASN" : "AS11590 Cumberland Technologies International", + "dstBytes" : 114, + "dstDataBytes" : 106, + "dstGEO" : "US", + "dstIp" : "66.59.111.182", + "dstMac" : [ + "00:c0:ca:14:b0:52" + ], + "dstMacCnt" : 1, + "dstOui" : [ + "Alfa, Inc." + ], + "dstOuiCnt" : 1, + "dstPackets" : 1, + "dstPayload8" : "240306ef000008cf", + "dstPort" : 123, + "dstRIR" : "ARIN", + "fileId" : [], + "firstPacket" : 1055289992849, + "greASN" : [ "---", "AS11590 Cumberland Technologies International" ], - "firstPacket" : 1055289992849, - "timestamp" : "SET", - "ipSrc" : "66.59.111.190", - "by1" : 114, - "g1" : "USA", - "rir2" : "ARIN", - "fb2" : "240306ef000008cf", - "pa" : 2, - "prot-term-cnt" : 3, - "a2" : "66.59.111.182", - "pa1" : 1, - "fs" : [], - "greip-rir" : [ + "greGEO" : [ + "---", + "US" + ], + "greIp" : [ + "172.27.1.66", + "66.59.109.137" + ], + "greIpCnt" : 2, + "greRIR" : [ "ARIN", "ARIN" ], - "lpd" : 1055289992905, - "db2" : 106, - "db1" : 106, - "p1" : 123, - "as1" : "AS11590 Cumberland Technologies International", - "greip-cnt" : 2, - "g2" : "USA", - "by2" : 114, - "greip-geo" : [ - "---", - "USA" - ], - "psl" : [ + "ipProtocol" : 17, + "lastPacket" : 1055289992905, + "length" : 56, + "node" : "test", + "packetLen" : [ 130, 130 ], - "ss" : 1, - "greip" : [ - 2887450946, - 1111190921 + "packetPos" : [ + 6875, + 7005 ], - "fp" : 1055289992, - "no" : "test", - "prot-term" : [ + "protocol" : [ "udp", "gre", "ntp" ], - "mac1-term-cnt" : 1, - "ps" : [ - 6875, - 7005 - ], - "lp" : 1055289992, - "portSrc" : 123, - "rir1" : "ARIN", - "mac1-term" : [ + "protocolCnt" : 3, + "segmentCnt" : 1, + "srcASN" : "AS11590 Cumberland Technologies International", + "srcBytes" : 114, + "srcDataBytes" : 106, + "srcGEO" : "US", + "srcIp" : "66.59.111.190", + "srcMac" : [ "00:02:2d:56:4a:fd" ], - "mac2-term" : [ - "00:c0:ca:14:b0:52" + "srcMacCnt" : 1, + "srcOui" : [ + "Agere Systems" ], - "p2" : 123, - "ipDst" : "66.59.111.182", - "a1" : "66.59.111.190", - "as2" : "AS11590 Cumberland Technologies International", - "sl" : 56, - "fpd" : 1055289992849, - "by" : 228, - "fb1" : "230b06ef00000000", - "lastPacket" : 1055289992905, - "portDst" : 123, - "pa2" : 1, - "mac2-term-cnt" : 1, - "db" : 212 - } - }, - { + "srcOuiCnt" : 1, + "srcPackets" : 1, + "srcPayload8" : "230b06ef00000000", + "srcPort" : 123, + "srcRIR" : "ARIN", + "timestamp" : "SET", + "totBytes" : 228, + "totDataBytes" : 212, + "totPackets" : 2 + }, "header" : { "index" : { - "_type" : "session", - "_index" : "tests_sessions-030611" + "_index" : "tests_sessions2-030611", + "_type" : "session" } - }, + } + }, + { "body" : { - "g1" : "USA", - "rir2" : "ARIN", - "pa" : 2, - "fb2" : "240206f5000005b6", - "prot-term-cnt" : 3, - "pr" : 17, - "greip-asn" : [ + "dstASN" : "AS10755 Dartmouth College", + "dstBytes" : 114, + "dstDataBytes" : 106, + "dstGEO" : "US", + "dstIp" : "129.170.17.4", + "dstMac" : [ + "00:c0:ca:14:b0:52" + ], + "dstMacCnt" : 1, + "dstOui" : [ + "Alfa, Inc." + ], + "dstOuiCnt" : 1, + "dstPackets" : 1, + "dstPayload8" : "240206f5000005b6", + "dstPort" : 123, + "dstRIR" : "ARIN", + "fileId" : [], + "firstPacket" : 1055289996849, + "greASN" : [ "---", "AS11590 Cumberland Technologies International" ], - "firstPacket" : 1055289996849, - "ipSrc" : "66.59.111.190", - "timestamp" : "SET", - "by1" : 114, - "greip-rir" : [ + "greGEO" : [ + "---", + "US" + ], + "greIp" : [ + "172.27.1.66", + "66.59.109.137" + ], + "greIpCnt" : 2, + "greRIR" : [ "ARIN", "ARIN" ], - "lpd" : 1055289996921, - "db2" : 106, - "a2" : "129.170.17.4", - "pa1" : 1, - "fs" : [], - "greip" : [ - 2887450946, - 1111190921 - ], - "fp" : 1055289996, - "no" : "test", - "prot-term" : [ - "udp", - "gre", - "ntp" + "ipProtocol" : 17, + "lastPacket" : 1055289996921, + "length" : 72, + "node" : "test", + "packetLen" : [ + 130, + 130 ], - "mac1-term-cnt" : 1, - "ps" : [ + "packetPos" : [ 7135, 7265 ], - "portSrc" : 123, - "lp" : 1055289996, - "rir1" : "ARIN", - "as1" : "AS11590 Cumberland Technologies International", - "db1" : 106, - "p1" : 123, - "greip-cnt" : 2, - "by2" : 114, - "g2" : "USA", - "greip-geo" : [ - "---", - "USA" - ], - "psl" : [ - 130, - 130 + "protocol" : [ + "udp", + "gre", + "ntp" ], - "ss" : 1, - "by" : 228, - "fpd" : 1055289996849, - "fb1" : "230b06ef00000000", - "lastPacket" : 1055289996921, - "portDst" : 123, - "pa2" : 1, - "mac2-term-cnt" : 1, - "db" : 212, - "mac1-term" : [ + "protocolCnt" : 3, + "segmentCnt" : 1, + "srcASN" : "AS11590 Cumberland Technologies International", + "srcBytes" : 114, + "srcDataBytes" : 106, + "srcGEO" : "US", + "srcIp" : "66.59.111.190", + "srcMac" : [ "00:02:2d:56:4a:fd" ], - "ipDst" : "129.170.17.4", - "mac2-term" : [ - "00:c0:ca:14:b0:52" + "srcMacCnt" : 1, + "srcOui" : [ + "Agere Systems" ], - "p2" : 123, - "a1" : "66.59.111.190", - "as2" : "AS10755 Dartmouth College", - "sl" : 72 + "srcOuiCnt" : 1, + "srcPackets" : 1, + "srcPayload8" : "230b06ef00000000", + "srcPort" : 123, + "srcRIR" : "ARIN", + "timestamp" : "SET", + "totBytes" : 228, + "totDataBytes" : 212, + "totPackets" : 2 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-030611", + "_type" : "session" + } } } ] diff --git a/tests/pcap/http-301-get.test b/tests/pcap/http-301-get.test index 921170b121..2674744a7b 100644 --- a/tests/pcap/http-301-get.test +++ b/tests/pcap/http-301-get.test @@ -1,37 +1,92 @@ { - "sessions" : [ + "sessions2" : [ { - "header" : { - "index" : { - "_type" : "session", - "_index" : "tests_sessions-131125" - } - }, "body" : { - "hh2" : [ - "http:header:connection", - "http:header:content-length", - "http:header:location" + "dstASN" : "AS36459 GitHub, Inc.", + "dstBytes" : 313, + "dstDataBytes" : 107, + "dstGEO" : "US", + "dstIp" : "192.30.252.130", + "dstMac" : [ + "00:00:0c:07:ac:01", + "00:d0:2b:d1:76:00" + ], + "dstMacCnt" : 2, + "dstOui" : [ + "Cisco", + "Jetcell, Inc." ], - "rir2" : "ARIN", - "db1" : 145, - "hocnt" : 1, - "pa1" : 5, + "dstOuiCnt" : 2, + "dstPackets" : 3, + "dstPayload8" : "485454502f312e31", + "dstPort" : 80, + "dstRIR" : "ARIN", + "fileId" : [], + "firstPacket" : 1385394928482, "http" : { - "statuscode-cnt" : 1, + "clientVersion" : [ + "1.1" + ], + "clientVersionCnt" : 1, + "host" : [ + "www.github.com" + ], + "hostCnt" : 1, + "method" : [ + "GET" + ], + "methodCnt" : 1, + "path" : [ + "/" + ], + "pathCnt" : 1, + "requestHeader" : [ + "accept", + "user-agent", + "host" + ], + "requestHeaderCnt" : 3, + "response-location" : [ + "https://www.github.com/" + ], + "responseHeader" : [ + "content-length", + "connection", + "location" + ], + "responseHeaderCnt" : 3, + "serverVersion" : [ + "1.1" + ], + "serverVersionCnt" : 1, "statuscode" : [ 301 ], - "method-term" : [ - "GET" + "statuscodeCnt" : 1, + "uri" : [ + "www.github.com/" + ], + "uriCnt" : 1, + "useragent" : [ + "curl/7.24.0 (x86_64-apple-darwin12.0) libcurl/7.24.0 OpenSSL/0.9.8y zlib/1.2.5" ], - "method-term-cnt" : 1 + "useragentCnt" : 1 }, - "g2" : "USA", - "ipSrc" : "10.180.156.141", - "p2" : 80, - "hpathcnt" : 1, - "ps" : [ + "ipProtocol" : 6, + "lastPacket" : 1385394928608, + "length" : 125, + "node" : "test", + "packetLen" : [ + 94, + 90, + 82, + 227, + 189, + 82, + 82, + 82 + ], + "packetPos" : [ 24, 118, 208, @@ -41,102 +96,48 @@ 788, 870 ], - "pa2" : 3, - "lastPacket" : 1385394928608, - "prot-term-cnt" : 2, - "mac2-term-cnt" : 2, - "g1" : "USA", - "by1" : 487, - "sl" : 125, - "hsvercnt" : 1, - "portDst" : 80, - "hdrs" : { - "hres-location" : [ - "https://www.github.com/" - ] - }, - "db" : 252, - "pr" : 6, - "hh2cnt" : 3, + "protocol" : [ + "http", + "tcp" + ], + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 487, + "srcDataBytes" : 145, + "srcGEO" : "US", + "srcIp" : "10.180.156.141", + "srcMac" : [ + "00:1f:5b:ff:51:cb" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Apple, Inc." + ], + "srcOuiCnt" : 1, + "srcPackets" : 5, + "srcPayload8" : "474554202f204854", + "srcPort" : 62341, "tcpflags" : { + "ack" : 3, + "dstZero" : 0, + "fin" : 2, "psh" : 1, "rst" : 0, - "ack" : 3, + "srcZero" : 0, + "syn" : 1, "syn-ack" : 1, - "urg" : 0, - "fin" : 2, - "syn" : 1 + "urg" : 0 }, - "hdver" : [ - "1.1" - ], - "mac1-term" : [ - "00:1f:5b:ff:51:cb" - ], - "hh1cnt" : 3, - "us" : [ - "//www.github.com/" - ], - "a1" : "10.180.156.141", - "mac1-term-cnt" : 1, - "as2" : "AS36459 GitHub, Inc.", - "lpd" : 1385394928608, - "p1" : 62341, - "prot-term" : [ - "http", - "tcp" - ], - "firstPacket" : 1385394928482, - "hdvercnt" : 1, - "db2" : 107, - "no" : "test", - "fb1" : "474554202f204854", "timestamp" : "SET", - "by" : 800, - "fb2" : "485454502f312e31", - "pa" : 8, - "portSrc" : 62341, - "fp" : 1385394928, - "uacnt" : 1, - "uscnt" : 1, - "hpath" : [ - "/" - ], - "ipDst" : "192.30.252.130", - "ss" : 1, - "by2" : 313, - "lp" : 1385394928, - "psl" : [ - 94, - 90, - 82, - 227, - 189, - 82, - 82, - 82 - ], - "hh1" : [ - "http:header:accept", - "http:header:host", - "http:header:user-agent" - ], - "ua" : [ - "curl/7.24.0 (x86_64-apple-darwin12.0) libcurl/7.24.0 OpenSSL/0.9.8y zlib/1.2.5" - ], - "fs" : [], - "fpd" : 1385394928482, - "hsver" : [ - "1.1" - ], - "mac2-term" : [ - "00:00:0c:07:ac:01", - "00:d0:2b:d1:76:00" - ], - "ho" : [ - "www.github.com" - ], - "a2" : "192.30.252.130" + "totBytes" : 800, + "totDataBytes" : 252, + "totPackets" : 8 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-131125", + "_type" : "session" + } } } ] diff --git a/tests/pcap/http-500-head.test b/tests/pcap/http-500-head.test index 85d7b88b08..091be8efd3 100644 --- a/tests/pcap/http-500-head.test +++ b/tests/pcap/http-500-head.test @@ -1,43 +1,76 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "portDst" : 80, - "sl" : 775, - "timestamp" : "SET", - "hh2cnt" : 4, - "a1" : "10.172.10.16", - "lpd" : 1361891421481, - "p1" : 49323, - "no" : "test", - "tcpflags" : { - "psh" : 2, - "rst" : 0, - "urg" : 0, - "syn" : 1, - "syn-ack" : 1, - "fin" : 2, - "ack" : 4 - }, - "firstPacket" : 1361891420707, - "lp" : 1361891421, - "hpathcnt" : 1, - "db" : 312, - "hh1cnt" : 4, - "by1" : 457, - "db1" : 157, - "hh2" : [ - "http:header:connection", - "http:header:content-type", - "http:header:date", - "http:header:server" - ], - "ipDst" : "10.156.206.202", - "mac2-term" : [ + "dstBytes" : 455, + "dstDataBytes" : 155, + "dstIp" : "10.156.206.202", + "dstMac" : [ "00:10:db:ff:26:00" ], - "pr" : 6, - "psl" : [ + "dstMacCnt" : 1, + "dstOui" : [ + "Juniper Networks" + ], + "dstOuiCnt" : 1, + "dstPackets" : 5, + "dstPayload8" : "485454502f312e31", + "dstPort" : 80, + "fileId" : [], + "firstPacket" : 1361891420707, + "http" : { + "clientVersion" : [ + "1.1" + ], + "clientVersionCnt" : 1, + "host" : [ + "samples.example.com" + ], + "hostCnt" : 1, + "method" : [ + "HEAD" + ], + "methodCnt" : 1, + "path" : [ + "/UpdataConfig.dat" + ], + "pathCnt" : 1, + "requestHeader" : [ + "accept", + "user-agent", + "connection", + "host" + ], + "requestHeaderCnt" : 4, + "responseHeader" : [ + "content-type", + "date", + "connection", + "server" + ], + "responseHeaderCnt" : 4, + "serverVersion" : [ + "1.1" + ], + "serverVersionCnt" : 1, + "statuscode" : [ + 500 + ], + "statuscodeCnt" : 1, + "uri" : [ + "samples.example.com/UpdataConfig.dat" + ], + "uriCnt" : 1, + "useragent" : [ + "Mozilla/4.0 (compatible;+MSIE+8.0;+Windows+NT+5.1)" + ], + "useragentCnt" : 1 + }, + "ipProtocol" : 6, + "lastPacket" : 1361891421481, + "length" : 775, + "node" : "test", + "packetLen" : [ 82, 82, 76, @@ -49,57 +82,7 @@ 76, 76 ], - "prot-term-cnt" : 2, - "uacnt" : 1, - "hdvercnt" : 1, - "http" : { - "statuscode" : [ - 500 - ], - "statuscode-cnt" : 1, - "method-term-cnt" : 1, - "method-term" : [ - "HEAD" - ] - }, - "hsvercnt" : 1, - "hpath" : [ - "/UpdataConfig.dat" - ], - "hsver" : [ - "1.1" - ], - "by" : 912, - "prot-term" : [ - "http", - "tcp" - ], - "ipSrc" : "10.172.10.16", - "pa1" : 5, - "us" : [ - "//samples.example.com/UpdataConfig.dat" - ], - "by2" : 455, - "ua" : [ - "Mozilla/4.0 (compatible;+MSIE+8.0;+Windows+NT+5.1)" - ], - "a2" : "10.156.206.202", - "fb2" : "485454502f312e31", - "ss" : 1, - "pa" : 10, - "mac1-term" : [ - "78:fe:3d:11:21:f2", - "00:00:5e:00:01:03" - ], - "lastPacket" : 1361891421481, - "pa2" : 5, - "portSrc" : 49323, - "p2" : 80, - "ho" : [ - "samples.example.com" - ], - "fs" : [], - "ps" : [ + "packetPos" : [ 24, 106, 188, @@ -111,28 +94,48 @@ 944, 1020 ], - "mac1-term-cnt" : 2, - "uscnt" : 1, - "mac2-term-cnt" : 1, - "hh1" : [ - "http:header:accept", - "http:header:connection", - "http:header:host", - "http:header:user-agent" + "protocol" : [ + "http", + "tcp" ], - "fp" : 1361891420, - "hocnt" : 1, - "fpd" : 1361891420707, - "fb1" : "48454144202f5570", - "hdver" : [ - "1.1" + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 457, + "srcDataBytes" : 157, + "srcIp" : "10.172.10.16", + "srcMac" : [ + "00:00:5e:00:01:03", + "78:fe:3d:11:21:f2" ], - "db2" : 155 + "srcMacCnt" : 2, + "srcOui" : [ + "ICANN, IANA Department", + "Juniper Networks" + ], + "srcOuiCnt" : 2, + "srcPackets" : 5, + "srcPayload8" : "48454144202f5570", + "srcPort" : 49323, + "tcpflags" : { + "ack" : 4, + "dstZero" : 0, + "fin" : 2, + "psh" : 2, + "rst" : 0, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, + "timestamp" : "SET", + "totBytes" : 912, + "totDataBytes" : 312, + "totPackets" : 10 }, "header" : { "index" : { - "_type" : "session", - "_index" : "tests_sessions-130226" + "_index" : "tests_sessions2-130226", + "_type" : "session" } } } diff --git a/tests/pcap/http-basicauth.test b/tests/pcap/http-basicauth.test index c92e6d00db..2ab47d3359 100644 --- a/tests/pcap/http-basicauth.test +++ b/tests/pcap/http-basicauth.test @@ -1,79 +1,128 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "hh1" : [ - "http:header:accept", - "http:header:accept-encoding", - "http:header:accept-language", - "http:header:authorization", - "http:header:connection", - "http:header:cookie", - "http:header:host", - "http:header:if-modified-since", - "http:header:referer", - "http:header:user-agent" - ], - "uacnt" : 1, - "by1" : 884, - "by" : 1311, - "huser-term" : [ - "userrrrr" - ], - "pr" : 6, - "hat-term" : [ - "basic" - ], - "mac2-term-cnt" : 2, - "mac1-term-cnt" : 1, - "fs" : [], - "db2" : 253, - "hdver" : [ - "1.1" - ], - "ta" : [ - "dstip", - "srcip" - ], - "a1" : "10.0.0.2", - "mac1-term" : [ - "00:0f:f7:76:82:80" - ], - "p2" : 2082, - "g1" : "CAN", - "sl" : 1953, - "hckey-term" : [ - "langedit", - "lang", - "cprelogin", - "cpsession", - "zzzzzzzzzzzzzzzzzz" - ], - "hh1cnt" : 10, - "pa2" : 3, - "fb1" : "474554202f787878", - "hckey-term-cnt" : 5, - "hpath" : [ - "/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx/xxx/xxxxxxxxxxxxxxxxxxxx.js" + "dstASN" : "AS0000 This is neat", + "dstBytes" : 427, + "dstDataBytes" : 253, + "dstGEO" : "RU", + "dstIp" : "10.0.0.1", + "dstMac" : [ + "00:00:5e:00:01:02", + "00:1d:b5:ce:ef:c0" ], - "huser-termcnt" : 1, - "fp" : 1414604109, - "us" : [ - "//10.000.000.001:1234/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx/xxx/xxxxxxxxxxxxxxxxxxxx.js" + "dstMacCnt" : 2, + "dstOui" : [ + "ICANN, IANA Department", + "Juniper Networks" ], + "dstOuiCnt" : 2, + "dstPackets" : 3, + "dstPayload8" : "485454502f312e31", + "dstPort" : 2082, + "fileId" : [], + "firstPacket" : 1414604109610, "http" : { + "authType" : [ + "basic" + ], + "authTypeCnt" : 1, + "clientVersion" : [ + "1.1" + ], + "clientVersionCnt" : 1, + "cookieKey" : [ + "langedit", + "lang", + "cprelogin", + "cpsession", + "zzzzzzzzzzzzzzzzzz" + ], + "cookieKeyCnt" : 5, + "cookieValue" : [ + "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", + "auto", + "no" + ], + "cookieValueCnt" : 3, + "host" : [ + "10.000.000.001:1234", + "10.000.000.001" + ], + "hostCnt" : 2, + "method" : [ + "GET" + ], + "methodCnt" : 1, + "path" : [ + "/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx/xxx/xxxxxxxxxxxxxxxxxxxx.js" + ], + "pathCnt" : 1, + "request-authorization" : [ + "Basic dXNlcnJycnI6cGFzc3dvcmRkZGRk" + ], + "request-authorizationCnt" : 1, + "request-referer" : [ + "http://10.000.000.001:1234/zzzzzzzzzzzzzzzz/zzzzzzzz/zz/zzzzzzzzz/index.html" + ], + "request-refererCnt" : 1, + "requestHeader" : [ + "accept", + "user-agent", + "referer", + "accept-encoding", + "connection", + "host", + "cookie", + "if-modified-since", + "accept-language", + "authorization" + ], + "requestHeaderCnt" : 10, + "responseHeader" : [ + "expires", + "keep-alive", + "cache-control", + "date", + "connection", + "x-keep-alive-count", + "server" + ], + "responseHeaderCnt" : 7, + "serverVersion" : [ + "1.1" + ], + "serverVersionCnt" : 1, "statuscode" : [ 304 ], - "statuscode-cnt" : 1 + "statuscodeCnt" : 1, + "uri" : [ + "10.000.000.001:1234/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx/xxx/xxxxxxxxxxxxxxxxxxxx.js" + ], + "uriCnt" : 1, + "user" : [ + "userrrrr" + ], + "userCnt" : 1, + "useragent" : [ + "Mozilla/5.0 (Windows NT 5.1; rv:32.0) Gecko/20100101 Firefox/32.0" + ], + "useragentCnt" : 1 }, - "as2" : "AS0000 This is neat", - "ho" : [ - "10.000.000.001:1234", - "10.000.000.001" + "ipProtocol" : 6, + "lastPacket" : 1414604111563, + "length" : 1953, + "node" : "test", + "packetLen" : [ + 94, + 76, + 76, + 762, + 76, + 323 ], - "hpathcnt" : 1, - "ps" : [ + "packetPos" : [ 24, 118, 194, @@ -81,102 +130,54 @@ 1032, 1108 ], - "firstPacket" : 1414604109610, - "as1" : "AS0001 Cool Beans!", - "psl" : [ - 94, - 76, - 76, - 762, - 76, - 323 + "protocol" : [ + "http", + "tcp" ], - "db" : 945, - "hsver" : [ - "1.1" + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcASN" : "AS0001 Cool Beans!", + "srcBytes" : 884, + "srcDataBytes" : 692, + "srcGEO" : "CA", + "srcIp" : "10.0.0.2", + "srcMac" : [ + "00:0f:f7:76:82:80" ], - "portSrc" : 1087, - "portDst" : 2082, - "a2" : "10.0.0.1", - "hdrs" : { - "hreq-referercnt" : 1, - "hreq-referer" : [ - "http://10.000.000.001:1234/zzzzzzzzzzzzzzzz/zzzzzzzz/zz/zzzzzzzzz/index.html" - ], - "hreq-authorization" : [ - "Basic dXNlcnJycnI6cGFzc3dvcmRkZGRk" - ], - "hreq-authorizationcnt" : 1 - }, + "srcMacCnt" : 1, + "srcOui" : [ + "Cisco Systems, Inc" + ], + "srcOuiCnt" : 1, + "srcPackets" : 3, + "srcPayload8" : "474554202f787878", + "srcPort" : 1087, + "srcRIR" : "TEST", + "tags" : [ + "dstip", + "srcip" + ], + "tagsCnt" : 2, "tcpflags" : { + "ack" : 2, + "dstZero" : 0, "fin" : 0, "psh" : 2, - "urg" : 0, "rst" : 0, - "syn-ack" : 1, + "srcZero" : 0, "syn" : 1, - "ack" : 2 + "syn-ack" : 1, + "urg" : 0 }, - "hcval-term-cnt" : 3, - "fpd" : 1414604109610, - "hat-termcnt" : 1, - "hh2" : [ - "http:header:cache-control", - "http:header:connection", - "http:header:date", - "http:header:expires", - "http:header:keep-alive", - "http:header:server", - "http:header:x-keep-alive-count" - ], - "prot-term-cnt" : 2, - "ua" : [ - "Mozilla/5.0 (Windows NT 5.1; rv:32.0) Gecko/20100101 Firefox/32.0" - ], - "hcval-term" : [ - "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", - "auto", - "no" - ], - "hh2cnt" : 7, - "lpd" : 1414604111563, - "tacnt" : 2, - "db1" : 692, - "hocnt" : 2, - "ipSrc" : "10.0.0.2", - "no" : "test", - "hsvercnt" : 1, - "lastPacket" : 1414604111563, - "prot-term" : [ - "http", - "tcp" - ], - "p1" : 1087, - "g2" : "RUS", - "tags-term" : [ - "srcip", - "dstip" - ], - "rir1" : "TEST", - "mac2-term" : [ - "00:00:5e:00:01:02", - "00:1d:b5:ce:ef:c0" - ], - "pa1" : 3, - "lp" : 1414604111, - "pa" : 6, "timestamp" : "SET", - "by2" : 427, - "hdvercnt" : 1, - "ss" : 1, - "fb2" : "485454502f312e31", - "uscnt" : 1, - "ipDst" : "10.0.0.1" + "totBytes" : 1311, + "totDataBytes" : 945, + "totPackets" : 6 }, "header" : { "index" : { - "_type" : "session", - "_index" : "tests_sessions-141029" + "_index" : "tests_sessions2-141029", + "_type" : "session" } } } diff --git a/tests/pcap/http-content-gzip.test b/tests/pcap/http-content-gzip.test index ab6fcc016f..50cd9b0b66 100644 --- a/tests/pcap/http-content-gzip.test +++ b/tests/pcap/http-content-gzip.test @@ -1,65 +1,130 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "hmd5cnt" : 1, - "db" : 895, - "hh2cnt" : 10, - "mac2-term-cnt" : 2, - "by" : 2093, - "p2" : 80, - "lpd" : 1407242225628, - "hckey-term" : [ - "xxxxxxxxxx" - ], - "db1" : 417, - "sl" : 916, - "g2" : "CAN", - "mac2-term" : [ + "dstASN" : "AS0001 Cool Beans!", + "dstBytes" : 1250, + "dstDataBytes" : 478, + "dstGEO" : "CA", + "dstIp" : "10.0.0.2", + "dstMac" : [ "00:00:5e:00:01:01", "00:26:88:d8:bf:c2" ], - "fpd" : 1407242224712, - "us" : [ - "//xxxxxxxxxxx.xxxxxxx.xxx/crossdomain.xml" - ], - "g1" : "RUS", - "uscnt" : 1, - "fp" : 1407242224, - "uacnt" : 1, - "fb2" : "485454502f312e31", - "as1" : "AS0000 This is neat", - "portDst" : 80, - "pa2" : 5, - "p1" : 50638, - "mac1-term-cnt" : 1, - "prot-term" : [ - "http", - "tcp" - ], - "a2" : "10.0.0.2", - "pa1" : 7, - "test" : { - "ip-geo" : [ - "RUS" + "dstMacCnt" : 2, + "dstOui" : [ + "ICANN, IANA Department", + "Juniper Networks" + ], + "dstOuiCnt" : 2, + "dstPackets" : 5, + "dstPayload8" : "485454502f312e31", + "dstPort" : 80, + "dstRIR" : "TEST", + "fileId" : [], + "firstPacket" : 1407242224712, + "http" : { + "bodyMagic" : [ + "application/x-gzip" ], - "string" : [ - "16777226:50638,33554442:80" + "bodyMagicCnt" : 1, + "clientVersion" : [ + "1.1" ], - "ip-asn" : [ - "AS0000 This is neat" + "clientVersionCnt" : 1, + "cookieKey" : [ + "xxxxxxxxxx" ], - "ip-rir" : [ - "" + "cookieKeyCnt" : 1, + "cookieValue" : [ + "xxx" ], - "number" : [ - 33554442 + "cookieValueCnt" : 1, + "host" : [ + "xxxxxxxxxxx.xxxxxxx.xxx" ], - "ip" : [ - 167772161 - ] + "hostCnt" : 1, + "md5" : [ + "5ff7b2c69c3b22826a717cd5ea4c9f32" + ], + "md5Cnt" : 1, + "method" : [ + "GET" + ], + "methodCnt" : 1, + "path" : [ + "/crossdomain.xml" + ], + "pathCnt" : 1, + "request-referer" : [ + "http://xx.xxxxx.xxx/xx?id=xxxxxxx&cb=xxxxxxxxxxxxx&referrer=xxxxxxx.xxx" + ], + "request-refererCnt" : 1, + "requestHeader" : [ + "accept", + "user-agent", + "referer", + "accept-encoding", + "if-none-match", + "connection", + "host", + "cookie", + "accept-language" + ], + "requestHeaderCnt" : 9, + "responseHeader" : [ + "expires", + "content-type", + "alternate-protocol", + "cache-control", + "content-length", + "etag", + "date", + "content-encoding", + "server", + "age" + ], + "responseHeaderCnt" : 10, + "serverVersion" : [ + "1.1" + ], + "serverVersionCnt" : 1, + "sha256" : [ + "5f814421f0d932b8b082cdc539469c85ae89a09d4b01ea41c1ff0424e184206a" + ], + "sha256Cnt" : 1, + "statuscode" : [ + 200 + ], + "statuscodeCnt" : 1, + "uri" : [ + "xxxxxxxxxxx.xxxxxxx.xxx/crossdomain.xml" + ], + "uriCnt" : 1, + "useragent" : [ + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36" + ], + "useragentCnt" : 1 }, - "ps" : [ + "ipProtocol" : 6, + "lastPacket" : 1407242225628, + "length" : 916, + "node" : "test", + "packetLen" : [ + 82, + 82, + 76, + 487, + 76, + 548, + 548, + 76, + 82, + 76, + 76, + 76 + ], + "packetPos" : [ 24, 106, 188, @@ -73,137 +138,73 @@ 2157, 2233 ], - "hh1" : [ - "http:header:accept", - "http:header:accept-encoding", - "http:header:accept-language", - "http:header:connection", - "http:header:cookie", - "http:header:host", - "http:header:if-none-match", - "http:header:referer", - "http:header:user-agent" + "protocol" : [ + "http", + "tcp" ], - "prot-term-cnt" : 2, - "fs" : [], - "mac1-term" : [ + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcASN" : "AS0000 This is neat", + "srcBytes" : 843, + "srcDataBytes" : 417, + "srcGEO" : "RU", + "srcIp" : "10.0.0.1", + "srcMac" : [ "00:0a:f3:31:90:00" ], + "srcMacCnt" : 1, + "srcOui" : [ + "Cisco Systems, Inc" + ], + "srcOuiCnt" : 1, + "srcPackets" : 7, + "srcPayload8" : "474554202f63726f", + "srcPort" : 50638, + "tags" : [ + "dstip", + "srcip" + ], + "tagsCnt" : 2, "tcpflags" : { - "syn" : 1, "ack" : 5, + "dstZero" : 0, + "fin" : 2, + "psh" : 3, "rst" : 0, + "srcZero" : 0, + "syn" : 1, "syn-ack" : 1, - "urg" : 0, - "fin" : 2, - "psh" : 3 + "urg" : 0 }, - "hdver" : [ - "1.1" - ], - "hcval-term-cnt" : 1, - "hcval-term" : [ - "xxx" - ], - "ipDst" : "10.0.0.2", - "ta" : [ - "dstip", - "srcip" - ], - "hpathcnt" : 1, - "as2" : "AS0001 Cool Beans!", - "ipSrc" : "10.0.0.1", - "rir2" : "TEST", - "hh2" : [ - "http:header:age", - "http:header:alternate-protocol", - "http:header:cache-control", - "http:header:content-encoding", - "http:header:content-length", - "http:header:content-type", - "http:header:date", - "http:header:etag", - "http:header:expires", - "http:header:server" - ], - "by1" : 843, - "tacnt" : 2, - "hh1cnt" : 9, - "http" : { - "method-term-cnt" : 1, - "statuscode" : [ - 200 + "test" : { + "ASN" : [ + "AS0000 This is neat" ], - "method-term" : [ - "GET" + "GEO" : [ + "RU" ], - "statuscode-cnt" : 1, - "bodymagic-term-cnt" : 1, - "bodymagic-term" : [ - "application/x-gzip" + "RIR" : [ + "" + ], + "ip" : [ + "10.0.0.1" + ], + "number" : [ + 33554442 + ], + "string.snow" : [ + "16777226:50638,33554442:80" ] }, - "hdvercnt" : 1, - "db2" : 478, - "hsvercnt" : 1, - "psl" : [ - 82, - 82, - 76, - 487, - 76, - 548, - 548, - 76, - 82, - 76, - 76, - 76 - ], - "lp" : 1407242225, - "ss" : 1, "timestamp" : "SET", - "ua" : [ - "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36" - ], - "no" : "test", - "pa" : 12, - "hpath" : [ - "/crossdomain.xml" - ], - "hmd5" : [ - "5ff7b2c69c3b22826a717cd5ea4c9f32" - ], - "a1" : "10.0.0.1", - "hckey-term-cnt" : 1, - "portSrc" : 50638, - "ho" : [ - "xxxxxxxxxxx.xxxxxxx.xxx" - ], - "firstPacket" : 1407242224712, - "tags-term" : [ - "srcip", - "dstip" - ], - "by2" : 1250, - "pr" : 6, - "hocnt" : 1, - "fb1" : "474554202f63726f", - "hdrs" : { - "hreq-referercnt" : 1, - "hreq-referer" : [ - "http://xx.xxxxx.xxx/xx?id=xxxxxxx&cb=xxxxxxxxxxxxx&referrer=xxxxxxx.xxx" - ] - }, - "lastPacket" : 1407242225628, - "hsver" : [ - "1.1" - ] + "totBytes" : 2093, + "totDataBytes" : 895, + "totPackets" : 12 }, "header" : { "index" : { - "_type" : "session", - "_index" : "tests_sessions-140805" + "_index" : "tests_sessions2-140805", + "_type" : "session" } } } diff --git a/tests/pcap/http-content-zip.test b/tests/pcap/http-content-zip.test index 75775cda82..acc619808b 100644 --- a/tests/pcap/http-content-zip.test +++ b/tests/pcap/http-content-zip.test @@ -1,51 +1,97 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "ipSrc" : "10.0.0.1", - "pr" : 6, - "ta" : [ - "dstip", - "srcip" - ], - "a2" : "10.0.0.2", - "ho" : [ - "xxxxxxxxxxxxx.xxx.com" - ], - "db1" : 362, - "no" : "test", - "timestamp" : "SET", - "ps" : [ - 24, - 118, - 208, - 290, - 734, - 816, - 1469, - 1551, - 1633, - 1715, - 1797, - 1879 - ], - "pa1" : 7, - "uacnt" : 1, - "p2" : 80, - "pa2" : 5, - "db2" : 571, - "by1" : 836, - "lpd" : 1388428585176, - "hmd5" : [ - "40be8f5100e9beabab293c9d7bacaff0" + "dstASN" : "AS0001 Cool Beans!", + "dstBytes" : 909, + "dstDataBytes" : 571, + "dstGEO" : "CA", + "dstIp" : "10.0.0.2", + "dstMac" : [ + "00:13:72:c4:f1:e1" ], - "by2" : 909, - "fb1" : "474554202f612e7a", - "hsver" : [ - "1.1" + "dstMacCnt" : 1, + "dstOui" : [ + "Dell Inc." ], - "a1" : "10.0.0.1", - "psl" : [ + "dstOuiCnt" : 1, + "dstPackets" : 5, + "dstPayload8" : "485454502f312e31", + "dstPort" : 80, + "dstRIR" : "TEST", + "fileId" : [], + "firstPacket" : 1388428585136, + "http" : { + "bodyMagic" : [ + "application/zip" + ], + "bodyMagicCnt" : 1, + "clientVersion" : [ + "1.1" + ], + "clientVersionCnt" : 1, + "host" : [ + "xxxxxxxxxxxxx.xxx.com" + ], + "hostCnt" : 1, + "md5" : [ + "40be8f5100e9beabab293c9d7bacaff0" + ], + "md5Cnt" : 1, + "method" : [ + "GET" + ], + "methodCnt" : 1, + "path" : [ + "/a.zip" + ], + "pathCnt" : 1, + "requestHeader" : [ + "accept", + "user-agent", + "accept-encoding", + "connection", + "host", + "accept-language" + ], + "requestHeaderCnt" : 6, + "responseHeader" : [ + "content-type", + "accept-ranges", + "content-length", + "etag", + "date", + "last-modified", + "connection", + "server" + ], + "responseHeaderCnt" : 8, + "serverVersion" : [ + "1.1" + ], + "serverVersionCnt" : 1, + "sha256" : [ + "61479904e443b354d4427a51b990c696f731e341e4c63328e07c1a92658ba591" + ], + "sha256Cnt" : 1, + "statuscode" : [ + 200 + ], + "statuscodeCnt" : 1, + "uri" : [ + "xxxxxxxxxxxxx.xxx.com/a.zip" + ], + "uriCnt" : 1, + "useragent" : [ + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36" + ], + "useragentCnt" : 1 + }, + "ipProtocol" : 6, + "lastPacket" : 1388428585176, + "length" : 39, + "node" : "test", + "packetLen" : [ 94, 90, 82, @@ -59,132 +105,89 @@ 82, 82 ], - "ipDst" : "10.0.0.2", - "hh1cnt" : 6, - "hpathcnt" : 1, - "ua" : [ - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36" - ], - "ss" : 1, - "hdver" : [ - "1.1" - ], - "us" : [ - "//xxxxxxxxxxxxx.xxx.com/a.zip" + "packetPos" : [ + 24, + 118, + 208, + 290, + 734, + 816, + 1469, + 1551, + 1633, + 1715, + 1797, + 1879 ], - "tags-term" : [ - "srcip", - "dstip" + "protocol" : [ + "http", + "tcp" ], - "mac1-term" : [ + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcASN" : "AS0000 This is neat", + "srcBytes" : 836, + "srcDataBytes" : 362, + "srcGEO" : "RU", + "srcIp" : "10.0.0.1", + "srcMac" : [ "00:00:0c:07:ac:01", "00:0e:d6:0b:98:80" ], - "portSrc" : 52925, - "mac2-term" : [ - "00:13:72:c4:f1:e1" + "srcMacCnt" : 2, + "srcOui" : [ + "Cisco", + "Cisco Systems, Inc" ], - "firstPacket" : 1388428585136, - "prot-term-cnt" : 2, - "hh1" : [ - "http:header:accept", - "http:header:accept-encoding", - "http:header:accept-language", - "http:header:connection", - "http:header:host", - "http:header:user-agent" - ], - "p1" : 52925, - "hpath" : [ - "/a.zip" + "srcOuiCnt" : 2, + "srcPackets" : 7, + "srcPayload8" : "474554202f612e7a", + "srcPort" : 52925, + "tags" : [ + "dstip", + "srcip" ], - "fp" : 1388428585, - "fb2" : "485454502f312e31", - "mac2-term-cnt" : 1, - "http" : { - "bodymagic-term" : [ - "application/zip" - ], - "statuscode-cnt" : 1, - "method-term-cnt" : 1, - "bodymagic-term-cnt" : 1, - "method-term" : [ - "GET" - ], - "statuscode" : [ - 200 - ] + "tagsCnt" : 2, + "tcpflags" : { + "ack" : 5, + "dstZero" : 0, + "fin" : 3, + "psh" : 2, + "rst" : 0, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 }, - "tacnt" : 2, - "hh2" : [ - "http:header:accept-ranges", - "http:header:connection", - "http:header:content-length", - "http:header:content-type", - "http:header:date", - "http:header:etag", - "http:header:last-modified", - "http:header:server" - ], - "by" : 1745, - "hmd5cnt" : 1, - "g2" : "CAN", - "g1" : "RUS", - "hocnt" : 1, - "prot-term" : [ - "http", - "tcp" - ], - "lastPacket" : 1388428585176, "test" : { - "ip-asn" : [ + "ASN" : [ "AS0000 This is neat" ], - "ip" : [ - 167772161 + "GEO" : [ + "RU" ], - "ip-geo" : [ - "RUS" - ], - "ip-rir" : [ + "RIR" : [ "" ], + "ip" : [ + "10.0.0.1" + ], "number" : [ 33554442 ], - "string" : [ + "string.snow" : [ "16777226:52925,33554442:80" ] }, - "portDst" : 80, - "hsvercnt" : 1, - "fs" : [], - "hh2cnt" : 8, - "lp" : 1388428585, - "hdvercnt" : 1, - "fpd" : 1388428585136, - "pa" : 12, - "as2" : "AS0001 Cool Beans!", - "as1" : "AS0000 This is neat", - "tcpflags" : { - "syn" : 1, - "rst" : 0, - "psh" : 2, - "syn-ack" : 1, - "urg" : 0, - "ack" : 5, - "fin" : 3 - }, - "sl" : 39, - "uscnt" : 1, - "rir2" : "TEST", - "db" : 933, - "mac1-term-cnt" : 2 + "timestamp" : "SET", + "totBytes" : 1745, + "totDataBytes" : 933, + "totPackets" : 12 }, "header" : { "index" : { - "_type" : "session", - "_index" : "tests_sessions-131230" + "_index" : "tests_sessions2-131230", + "_type" : "session" } } } diff --git a/tests/pcap/http-digestauth.test b/tests/pcap/http-digestauth.test index 10cbed7278..355071a861 100644 --- a/tests/pcap/http-digestauth.test +++ b/tests/pcap/http-digestauth.test @@ -1,159 +1,156 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "hdrs" : { - "hreq-authorization" : [ - "Digest username=\"user12345678@domain.xxxx\",realm=\"xxxxxxx\",nonce=\"xxxxxxxxxxxxxxxx\",uri=\"/xxxxxxxxxxxxxxxxxxxxxxxxxxx/xxxx/xxxxxxxxxxxxxxxxxxxxxxxx\",algorithm=\"MD5\",cnonce=\"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\",nc=00000003,qop=\"auth\",response=\"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\",opaque=\"\"" - ], - "hreq-authorizationcnt" : 1 - }, - "mac2-term" : [ + "dstASN" : "AS0001 Cool Beans!", + "dstBytes" : 62, + "dstDataBytes" : 0, + "dstGEO" : "CA", + "dstIp" : "10.0.0.2", + "dstMac" : [ "00:00:5e:00:01:01", "00:1d:b5:ce:ef:c0" ], - "hsvercnt" : 1, - "mac1-term" : [ - "00:0f:f7:76:82:80" - ], - "pa2" : 1, - "fpd" : 1414887555048, - "psl" : [ - 78, - 78, - 76, - 651 - ], - "us" : [ - "//xxxxxxx.xxxxxxxxxxx.xxx/xxxxxxxxxxxxxxxxxxxxxxxxxxx/xxxx/xxxxxxxxxxxxxxxxxxxxxxxx" + "dstMacCnt" : 2, + "dstOui" : [ + "ICANN, IANA Department", + "Juniper Networks" ], - "mac2-term-cnt" : 2, - "pa1" : 3, - "ipDst" : "10.0.0.2", - "by1" : 757, - "ua" : [ - "xxxxxxxxxxxxxxxxxx" - ], - "db1" : 581, - "tacnt" : 2, - "g2" : "CAN", - "test" : { - "ip-geo" : [ - "RUS" + "dstOuiCnt" : 2, + "dstPackets" : 1, + "dstPort" : 80, + "dstRIR" : "TEST", + "fileId" : [], + "firstPacket" : 1414887555048, + "http" : { + "authType" : [ + "digest" ], - "number" : [ - 33554442 + "authTypeCnt" : 1, + "clientVersion" : [ + "1.1" ], - "ip-asn" : [ - "AS0000 This is neat" + "clientVersionCnt" : 1, + "host" : [ + "xxxxxxx.xxxxxxxxxxx.xxx" ], - "ip-rir" : [ - "" + "hostCnt" : 1, + "method" : [ + "GET" ], - "ip" : [ - 167772161 + "methodCnt" : 1, + "path" : [ + "/xxxxxxxxxxxxxxxxxxxxxxxxxxx/xxxx/xxxxxxxxxxxxxxxxxxxxxxxx" ], - "string" : [ - "16777226:4411,33554442:80" - ] + "pathCnt" : 1, + "request-authorization" : [ + "Digest username=\"user12345678@domain.xxxx\",realm=\"xxxxxxx\",nonce=\"xxxxxxxxxxxxxxxx\",uri=\"/xxxxxxxxxxxxxxxxxxxxxxxxxxx/xxxx/xxxxxxxxxxxxxxxxxxxxxxxx\",algorithm=\"MD5\",cnonce=\"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\",nc=00000003,qop=\"auth\",response=\"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\",opaque=\"\"" + ], + "request-authorizationCnt" : 1, + "requestHeader" : [ + "pragma", + "user-agent", + "cache-control", + "xxxxxxxxxxxxxxxxxx", + "host", + "authorization" + ], + "requestHeaderCnt" : 6, + "uri" : [ + "xxxxxxx.xxxxxxxxxxx.xxx/xxxxxxxxxxxxxxxxxxxxxxxxxxx/xxxx/xxxxxxxxxxxxxxxxxxxxxxxx" + ], + "uriCnt" : 1, + "user" : [ + "user12345678@domain.xxxx" + ], + "userCnt" : 1, + "useragent" : [ + "xxxxxxxxxxxxxxxxxx" + ], + "useragentCnt" : 1 }, - "mac1-term-cnt" : 1, - "firstPacket" : 1414887555048, - "uacnt" : 1, - "lp" : 1414887555, - "timestamp" : "SET", - "prot-term" : [ - "http", - "tcp" - ], - "ss" : 1, - "p1" : 4411, - "fs" : [], - "sl" : 194, - "p2" : 80, - "pa" : 4, - "hpath" : [ - "/xxxxxxxxxxxxxxxxxxxxxxxxxxx/xxxx/xxxxxxxxxxxxxxxxxxxxxxxx" - ], - "as1" : "AS0000 This is neat", - "hat-termcnt" : 1, - "huser-term" : [ - "user12345678@domain.xxxx" + "ipProtocol" : 6, + "lastPacket" : 1414887555242, + "length" : 194, + "node" : "test", + "packetLen" : [ + 78, + 78, + 76, + 651 ], - "fb1" : "474554202f787878", - "ps" : [ + "packetPos" : [ 24, 102, 180, 256 ], + "protocol" : [ + "http", + "tcp" + ], + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcASN" : "AS0000 This is neat", + "srcBytes" : 757, + "srcDataBytes" : 581, + "srcGEO" : "RU", + "srcIp" : "10.0.0.1", + "srcMac" : [ + "00:0f:f7:76:82:80" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Cisco Systems, Inc" + ], + "srcOuiCnt" : 1, + "srcPackets" : 3, + "srcPayload8" : "474554202f787878", + "srcPort" : 4411, + "tags" : [ + "dstip", + "srcip" + ], + "tagsCnt" : 2, "tcpflags" : { - "rst" : 0, - "urg" : 0, + "ack" : 1, + "dstZero" : 0, "fin" : 0, "psh" : 1, + "rst" : 0, + "srcZero" : 0, "syn" : 1, "syn-ack" : 1, - "ack" : 1 + "urg" : 0 }, - "by" : 819, - "g1" : "RUS", - "huser-termcnt" : 1, - "a2" : "10.0.0.2", - "ta" : [ - "dstip", - "srcip" - ], - "lpd" : 1414887555242, - "http" : { - "method-term" : [ - "GET" + "test" : { + "ASN" : [ + "AS0000 This is neat" + ], + "GEO" : [ + "RU" + ], + "RIR" : [ + "" + ], + "ip" : [ + "10.0.0.1" + ], + "number" : [ + 33554442 ], - "method-term-cnt" : 1 + "string.snow" : [ + "16777226:4411,33554442:80" + ] }, - "ipSrc" : "10.0.0.1", - "pr" : 6, - "tags-term" : [ - "srcip", - "dstip" - ], - "uscnt" : 1, - "hsver" : [ - "1.1" - ], - "portDst" : 80, - "hh1" : [ - "http:header:authorization", - "http:header:cache-control", - "http:header:host", - "http:header:pragma", - "http:header:user-agent", - "http:header:xxxxxxxxxxxxxxxxxx" - ], - "as2" : "AS0001 Cool Beans!", - "by2" : 62, - "no" : "test", - "hh1cnt" : 6, - "db2" : 0, - "fp" : 1414887555, - "lastPacket" : 1414887555242, - "hocnt" : 1, - "hat-term" : [ - "digest" - ], - "db" : 581, - "rir2" : "TEST", - "a1" : "10.0.0.1", - "portSrc" : 4411, - "prot-term-cnt" : 2, - "hpathcnt" : 1, - "ho" : [ - "xxxxxxx.xxxxxxxxxxx.xxx" - ] + "timestamp" : "SET", + "totBytes" : 819, + "totDataBytes" : 581, + "totPackets" : 4 }, "header" : { "index" : { - "_index" : "tests_sessions-141102", + "_index" : "tests_sessions2-141102", "_type" : "session" } } diff --git a/tests/pcap/http-no-length.test b/tests/pcap/http-no-length.test index d9c98fdf53..cd5b36b6a4 100644 --- a/tests/pcap/http-no-length.test +++ b/tests/pcap/http-no-length.test @@ -1,169 +1,115 @@ { - "sessions" : [ + "sessions2" : [ { - "header" : { - "index" : { - "_type" : "session", - "_index" : "tests_sessions-100510" - } - }, "body" : { - "hdvercnt" : 1, - "pa" : 10, - "prot-term" : [ - "http", - "tcp" - ], - "p2" : 80, - "hckey-term-cnt" : 1, - "ss" : 1, - "as2" : "AS0001 Cool Beans!", - "hdver" : [ - "1.0" + "dstASN" : "AS0001 Cool Beans!", + "dstBytes" : 1026, + "dstDataBytes" : 744, + "dstGEO" : "CA", + "dstIp" : "10.0.0.2", + "dstMac" : [ + "00:c0:ca:30:eb:0c" ], - "hmd5" : [ - "9fb54a2726ca3cf54a82804d0e66d08a" + "dstMacCnt" : 1, + "dstOui" : [ + "Alfa, Inc." ], - "test" : { - "ip" : [ - 167772161 + "dstOuiCnt" : 1, + "dstPackets" : 5, + "dstPayload8" : "485454502f312e30", + "dstPort" : 80, + "dstRIR" : "TEST", + "fileId" : [], + "firstPacket" : 1273479692982, + "http" : { + "bodyMagic" : [ + "text/javascript" ], - "string" : [ - "16777226:50384,33554442:80" + "bodyMagicCnt" : 1, + "clientVersion" : [ + "1.1" ], - "ip-rir" : [ - "" + "clientVersionCnt" : 1, + "cookieKey" : [ + "trafic_ranking" ], - "number" : [ - 33554442 + "cookieKeyCnt" : 1, + "cookieValue" : [ + "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" ], - "ip-asn" : [ - "AS0000 This is neat" + "cookieValueCnt" : 1, + "host" : [ + "xxxxxxx.xxxxxx.xx" ], - "ip-geo" : [ - "RUS" - ] - }, - "fp" : 1273479692, - "pa1" : 5, - "p1" : 50384, - "hocnt" : 1, - "db2" : 744, - "as1" : "AS0000 This is neat", - "ipSrc" : "10.0.0.1", - "hpath" : [ - "/js/xxxxxx.js" - ], - "mac2-term" : [ - "00:c0:ca:30:eb:0c" - ], - "hh2cnt" : 10, - "hh1" : [ - "http:header:accept", - "http:header:accept-charset", - "http:header:accept-encoding", - "http:header:accept-language", - "http:header:connection", - "http:header:cookie", - "http:header:host", - "http:header:keep-alive", - "http:header:referer", - "http:header:user-agent" - ], - "portSrc" : 50384, - "ps" : [ - 24, - 106, - 188, - 258, - 780, - 850, - 937, - 1734, - 1804, - 1874 - ], - "by2" : 1026, - "lpd" : 1273479693105, - "hcval-term-cnt" : 1, - "db" : 1196, - "fb2" : "485454502f312e30", - "fb1" : "474554202f6a732f", - "hh1cnt" : 10, - "hsver" : [ - "1.1" - ], - "uacnt" : 1, - "mac2-term-cnt" : 1, - "g1" : "RUS", - "db1" : 452, - "sl" : 124, - "by" : 1760, - "pa2" : 5, - "timestamp" : "SET", - "mac1-term-cnt" : 1, - "a1" : "10.0.0.1", - "fs" : [], - "http" : { - "statuscode-cnt" : 1, - "bodymagic-term-cnt" : 1, + "hostCnt" : 1, + "md5" : [ + "9fb54a2726ca3cf54a82804d0e66d08a" + ], + "md5Cnt" : 1, + "method" : [ + "GET" + ], + "methodCnt" : 1, + "path" : [ + "/js/xxxxxx.js" + ], + "pathCnt" : 1, + "request-referer" : [ + "http://www.xxxxxxxx.com/xxxxxxxxxxxxxxx/xxxxxxxxxxxxxxxx.html" + ], + "request-refererCnt" : 1, + "requestHeader" : [ + "accept", + "accept-charset", + "user-agent", + "keep-alive", + "referer", + "accept-encoding", + "connection", + "host", + "cookie", + "accept-language" + ], + "requestHeaderCnt" : 10, + "responseHeader" : [ + "pragma", + "expires", + "content-type", + "cache-control", + "date", + "last-modified", + "connection", + "p3p", + "server", + "set-cookie" + ], + "responseHeaderCnt" : 10, + "serverVersion" : [ + "1.0" + ], + "serverVersionCnt" : 1, + "sha256" : [ + "be86ba67700e1e23003ee9899119995d2babebbf6d753aafdd4a63017e6cb7b7" + ], + "sha256Cnt" : 1, "statuscode" : [ 200 ], - "bodymagic-term" : [ - "text/javascript" + "statuscodeCnt" : 1, + "uri" : [ + "xxxxxxx.xxxxxx.xx/js/xxxxxx.js" ], - "method-term-cnt" : 1, - "method-term" : [ - "GET" - ] - }, - "rir2" : "TEST", - "hh2" : [ - "http:header:cache-control", - "http:header:connection", - "http:header:content-type", - "http:header:date", - "http:header:expires", - "http:header:last-modified", - "http:header:p3p", - "http:header:pragma", - "http:header:server", - "http:header:set-cookie" - ], - "hdrs" : { - "hreq-referercnt" : 1, - "hreq-referer" : [ - "http://www.xxxxxxxx.com/xxxxxxxxxxxxxxx/xxxxxxxxxxxxxxxx.html" - ] + "uriCnt" : 1, + "useragent" : [ + "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.3) Gecko/20100401 Firefox/3.6.3" + ], + "useragentCnt" : 1 }, - "portDst" : 80, - "tacnt" : 2, - "pr" : 6, - "ipDst" : "10.0.0.2", - "hpathcnt" : 1, - "hsvercnt" : 1, - "hmd5cnt" : 1, - "prot-term-cnt" : 2, - "firstPacket" : 1273479692982, - "ta" : [ - "dstip", - "srcip" - ], + "ipProtocol" : 6, "lastPacket" : 1273479693105, - "hckey-term" : [ - "trafic_ranking" - ], - "ho" : [ - "xxxxxxx.xxxxxx.xx" - ], - "a2" : "10.0.0.2", - "mac1-term" : [ - "00:16:44:a0:a0:7e" - ], - "lp" : 1273479693, - "g2" : "CAN", - "psl" : [ + "length" : 124, + "node" : "test", + "packetLen" : [ 82, 82, 70, @@ -175,32 +121,86 @@ 70, 70 ], - "no" : "test", - "hcval-term" : [ - "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + "packetPos" : [ + 24, + 106, + 188, + 258, + 780, + 850, + 937, + 1734, + 1804, + 1874 ], - "us" : [ - "//xxxxxxx.xxxxxx.xx/js/xxxxxx.js" + "protocol" : [ + "http", + "tcp" + ], + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcASN" : "AS0000 This is neat", + "srcBytes" : 734, + "srcDataBytes" : 452, + "srcGEO" : "RU", + "srcIp" : "10.0.0.1", + "srcMac" : [ + "00:16:44:a0:a0:7e" ], - "uscnt" : 1, - "tags-term" : [ - "srcip", - "dstip" + "srcMacCnt" : 1, + "srcOui" : [ + "LITE-ON Technology Corp." + ], + "srcOuiCnt" : 1, + "srcPackets" : 5, + "srcPayload8" : "474554202f6a732f", + "srcPort" : 50384, + "tags" : [ + "dstip", + "srcip" ], + "tagsCnt" : 2, "tcpflags" : { - "rst" : 0, - "fin" : 2, - "syn-ack" : 1, - "syn" : 1, "ack" : 4, + "dstZero" : 0, + "fin" : 2, "psh" : 3, + "rst" : 0, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, "urg" : 0 }, - "fpd" : 1273479692982, - "by1" : 734, - "ua" : [ - "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.3) Gecko/20100401 Firefox/3.6.3" - ] + "test" : { + "ASN" : [ + "AS0000 This is neat" + ], + "GEO" : [ + "RU" + ], + "RIR" : [ + "" + ], + "ip" : [ + "10.0.0.1" + ], + "number" : [ + 33554442 + ], + "string.snow" : [ + "16777226:50384,33554442:80" + ] + }, + "timestamp" : "SET", + "totBytes" : 1760, + "totDataBytes" : 1196, + "totPackets" : 10 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-100510", + "_type" : "session" + } } } ] diff --git a/tests/pcap/http-simple-get.test b/tests/pcap/http-simple-get.test index 195df26293..35564a5c79 100644 --- a/tests/pcap/http-simple-get.test +++ b/tests/pcap/http-simple-get.test @@ -1,112 +1,90 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "mac2-term" : [ + "dstBytes" : 5773, + "dstDataBytes" : 5237, + "dstGEO" : "US", + "dstIp" : "10.180.156.249", + "dstMac" : [ "00:13:72:c4:f1:e1" ], - "ua" : [ - "curl/7.24.0 (x86_64-apple-darwin12.0) libcurl/7.24.0 OpenSSL/0.9.8y zlib/1.2.5" + "dstMacCnt" : 1, + "dstOui" : [ + "Dell Inc." ], - "ps" : [ - 24, - 118, - 208, - 290, - 524, - 606, - 2136, - 3666, - 3748, - 5278, - 5360, - 6335, - 6417, - 6499, - 6581, - 6663 - ], - "pa2" : 8, + "dstOuiCnt" : 1, + "dstPackets" : 8, + "dstPayload8" : "485454502f312e31", + "dstPort" : 80, + "fileId" : [], + "firstPacket" : 1385391358382, "http" : { - "bodymagic-term" : [ + "bodyMagic" : [ "text/html" ], - "method-term" : [ + "bodyMagicCnt" : 1, + "clientVersion" : [ + "1.1" + ], + "clientVersionCnt" : 1, + "host" : [ + "xxxxxxxxxxxxx.xxx.com" + ], + "hostCnt" : 1, + "md5" : [ + "230e3b4387b64caf54a7487b4f726adb" + ], + "md5Cnt" : 1, + "method" : [ "GET" ], + "methodCnt" : 1, + "path" : [ + "/" + ], + "pathCnt" : 1, + "requestHeader" : [ + "accept", + "user-agent", + "host" + ], + "requestHeaderCnt" : 3, + "responseHeader" : [ + "content-type", + "accept-ranges", + "content-length", + "date", + "connection", + "server" + ], + "responseHeaderCnt" : 6, + "serverVersion" : [ + "1.1" + ], + "serverVersionCnt" : 1, + "sha256" : [ + "a5bae43656eb0c0c5db924a1764dd4631f58d3f0d2145333589521ea1d514ba5" + ], + "sha256Cnt" : 1, "statuscode" : [ 403 ], - "bodymagic-term-cnt" : 1, - "statuscode-cnt" : 1, - "method-term-cnt" : 1 + "statuscodeCnt" : 1, + "uri" : [ + "xxxxxxxxxxxxx.xxx.com/" + ], + "uriCnt" : 1, + "useragent" : [ + "curl/7.24.0 (x86_64-apple-darwin12.0) libcurl/7.24.0 OpenSSL/0.9.8y zlib/1.2.5" + ], + "useragentCnt" : 1 }, - "hh2cnt" : 6, - "ho" : [ - "xxxxxxxxxxxxx.xxx.com" - ], - "g2" : "USA", - "no" : "test", - "mac1-term" : [ - "00:1f:5b:ff:51:cb" - ], - "uacnt" : 1, - "ipSrc" : "10.180.156.141", - "pr" : 6, - "ss" : 1, - "hmd5cnt" : 1, - "fb2" : "485454502f312e31", - "db" : 5389, - "uscnt" : 1, - "prot-term-cnt" : 2, - "a2" : "10.180.156.249", - "db2" : 5237, - "hsver" : [ - "1.1" - ], - "fp" : 1385391358, - "hh1cnt" : 3, + "ipProtocol" : 6, "lastPacket" : 1385391358387, - "us" : [ - "//xxxxxxxxxxxxx.xxx.com/" - ], - "pa" : 16, - "by1" : 692, - "hh1" : [ - "http:header:accept", - "http:header:host", - "http:header:user-agent" - ], - "pa1" : 8, - "portDst" : 80, - "hsvercnt" : 1, - "by2" : 5773, - "hdver" : [ - "1.1" - ], - "hpathcnt" : 1, - "g1" : "USA", - "tcpflags" : { - "psh" : 2, - "rst" : 0, - "ack" : 10, - "syn" : 1, - "urg" : 0, - "fin" : 2, - "syn-ack" : 1 - }, - "hocnt" : 1, - "sl" : 5, - "hmd5" : [ - "230e3b4387b64caf54a7487b4f726adb" - ], - "firstPacket" : 1385391358382, - "a1" : "10.180.156.141", - "hdvercnt" : 1, - "p2" : 80, - "lpd" : 1385391358387, - "timestamp" : "SET", - "psl" : [ + "length" : 5, + "node" : "test", + "packetLen" : [ 94, 90, 82, @@ -124,36 +102,64 @@ 82, 82 ], - "by" : 6465, - "lp" : 1385391358, - "p1" : 61450, - "fpd" : 1385391358382, - "db1" : 152, - "mac1-term-cnt" : 1, - "portSrc" : 61450, - "ipDst" : "10.180.156.249", - "fb1" : "474554202f204854", - "hpath" : [ - "/" + "packetPos" : [ + 24, + 118, + 208, + 290, + 524, + 606, + 2136, + 3666, + 3748, + 5278, + 5360, + 6335, + 6417, + 6499, + 6581, + 6663 ], - "mac2-term-cnt" : 1, - "fs" : [], - "prot-term" : [ + "protocol" : [ "http", "tcp" ], - "hh2" : [ - "http:header:accept-ranges", - "http:header:connection", - "http:header:content-length", - "http:header:content-type", - "http:header:date", - "http:header:server" - ] + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 692, + "srcDataBytes" : 152, + "srcGEO" : "US", + "srcIp" : "10.180.156.141", + "srcMac" : [ + "00:1f:5b:ff:51:cb" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Apple, Inc." + ], + "srcOuiCnt" : 1, + "srcPackets" : 8, + "srcPayload8" : "474554202f204854", + "srcPort" : 61450, + "tcpflags" : { + "ack" : 10, + "dstZero" : 0, + "fin" : 2, + "psh" : 2, + "rst" : 0, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, + "timestamp" : "SET", + "totBytes" : 6465, + "totDataBytes" : 5389, + "totPackets" : 16 }, "header" : { "index" : { - "_index" : "tests_sessions-131125", + "_index" : "tests_sessions2-131125", "_type" : "session" } } diff --git a/tests/pcap/http-syn-ack.test b/tests/pcap/http-syn-ack.test index 03de71ea99..a7de7105ac 100644 --- a/tests/pcap/http-syn-ack.test +++ b/tests/pcap/http-syn-ack.test @@ -1,40 +1,113 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "hdvercnt" : 1, - "ipDst" : "10.0.0.33", - "hsvercnt" : 1, - "mac1-term-cnt" : 1, - "db1" : 427, - "fb1" : "474554202f312048", - "firstPacket" : 1467366160266, - "hh2cnt" : 11, - "ho" : [ - "js.navigator.io" - ], - "a2" : "10.0.0.33", - "hmd5cnt" : 1, - "us" : [ - "//js.navigator.io/1" + "dstBytes" : 12880, + "dstDataBytes" : 12252, + "dstIp" : "10.0.0.33", + "dstMac" : [ + "aa:aa:aa:aa:aa:aa" ], - "ss" : 1, - "hh1cnt" : 7, - "hh2" : [ - "http:header:access-control-allow-origin", - "http:header:cache-control", - "http:header:connection", - "http:header:content-encoding", - "http:header:content-type", - "http:header:date", - "http:header:expires", - "http:header:pragma", - "http:header:server", - "http:header:transfer-encoding", - "http:header:vary" + "dstMacCnt" : 1, + "dstPackets" : 11, + "dstPayload8" : "485454502f312e31", + "dstPort" : 80, + "fileId" : [], + "firstPacket" : 1467366160266, + "http" : { + "bodyMagic" : [ + "application/x-gzip" + ], + "bodyMagicCnt" : 1, + "clientVersion" : [ + "1.1" + ], + "clientVersionCnt" : 1, + "host" : [ + "js.navigator.io" + ], + "hostCnt" : 1, + "md5" : [ + "38bf34783fc26180631a614a88301cab" + ], + "md5Cnt" : 1, + "method" : [ + "GET" + ], + "methodCnt" : 1, + "path" : [ + "/1" + ], + "pathCnt" : 1, + "request-referer" : [ + "http://xxxxxxx.xxxxx.xx/news/xxxxxxx/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + ], + "request-refererCnt" : 1, + "requestHeader" : [ + "accept", + "user-agent", + "referer", + "accept-encoding", + "connection", + "host", + "accept-language" + ], + "requestHeaderCnt" : 7, + "responseHeader" : [ + "pragma", + "expires", + "content-type", + "transfer-encoding", + "vary", + "cache-control", + "access-control-allow-origin", + "date", + "content-encoding", + "connection", + "server" + ], + "responseHeaderCnt" : 11, + "serverVersion" : [ + "1.1" + ], + "serverVersionCnt" : 1, + "sha256" : [ + "ac518a69ca82f4efbd3ea94d4d423d843582771127ef6d214c3d7a45acf7be04" + ], + "sha256Cnt" : 1, + "statuscode" : [ + 200 + ], + "statuscodeCnt" : 1, + "uri" : [ + "js.navigator.io/1" + ], + "uriCnt" : 1, + "useragent" : [ + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" + ], + "useragentCnt" : 1 + }, + "ipProtocol" : 6, + "lastPacket" : 1467366160339, + "length" : 73, + "node" : "test", + "packetLen" : [ + 84, + 76, + 500, + 72, + 1532, + 1532, + 1532, + 1532, + 1532, + 1084, + 1532, + 1532, + 1092 ], - "lpd" : 1467366160339, - "ps" : [ + "packetPos" : [ 24, 108, 184, @@ -49,118 +122,41 @@ 11032, 12564 ], - "by" : 13424, - "db2" : 12252, - "prot-term" : [ + "protocol" : [ "http", "tcp" ], - "hpath" : [ - "/1" - ], - "pr" : 6, - "uacnt" : 1, - "lastPacket" : 1467366160339, - "prot-term-cnt" : 2, - "mac1-term" : [ - "aa:aa:aa:aa:aa:aa" - ], - "hh1" : [ - "http:header:accept", - "http:header:accept-encoding", - "http:header:accept-language", - "http:header:connection", - "http:header:host", - "http:header:referer", - "http:header:user-agent" - ], - "hmd5" : [ - "38bf34783fc26180631a614a88301cab" - ], - "no" : "test", - "pa1" : 2, - "fpd" : 1467366160266, - "mac2-term" : [ + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 544, + "srcDataBytes" : 427, + "srcIp" : "10.0.0.32", + "srcMac" : [ "aa:aa:aa:aa:aa:aa" ], - "by1" : 544, + "srcMacCnt" : 1, + "srcPackets" : 2, + "srcPayload8" : "474554202f312048", + "srcPort" : 10882, "tcpflags" : { - "rst" : 0, - "psh" : 3, - "urg" : 0, - "fin" : 0, "ack" : 9, + "dstZero" : 0, + "fin" : 0, + "psh" : 3, + "rst" : 0, + "srcZero" : 0, "syn" : 0, - "syn-ack" : 1 + "syn-ack" : 1, + "urg" : 0 }, - "pa2" : 11, - "hdrs" : { - "hreq-referer" : [ - "http://xxxxxxx.xxxxx.xx/news/xxxxxxx/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - ], - "hreq-referercnt" : 1 - }, - "hsver" : [ - "1.1" - ], - "by2" : 12880, - "sl" : 73, - "a1" : "10.0.0.32", - "fp" : 1467366160, "timestamp" : "SET", - "p1" : 10882, - "p2" : 80, - "hdver" : [ - "1.1" - ], - "hocnt" : 1, - "hpathcnt" : 1, - "db" : 12679, - "lp" : 1467366160, - "portSrc" : 10882, - "http" : { - "bodymagic-term" : [ - "application/x-gzip" - ], - "statuscode" : [ - 200 - ], - "statuscode-cnt" : 1, - "method-term" : [ - "GET" - ], - "method-term-cnt" : 1, - "bodymagic-term-cnt" : 1 - }, - "uscnt" : 1, - "portDst" : 80, - "psl" : [ - 84, - 76, - 500, - 72, - 1532, - 1532, - 1532, - 1532, - 1532, - 1084, - 1532, - 1532, - 1092 - ], - "fs" : [], - "pa" : 13, - "mac2-term-cnt" : 1, - "ua" : [ - "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" - ], - "fb2" : "485454502f312e31", - "ipSrc" : "10.0.0.32" + "totBytes" : 13424, + "totDataBytes" : 12679, + "totPackets" : 13 }, "header" : { "index" : { - "_index" : "tests_sessions-160701", + "_index" : "tests_sessions2-160701", "_type" : "session" } } diff --git a/tests/pcap/http-wrapped-header.test b/tests/pcap/http-wrapped-header.test index eda37184e3..da8cc9dc82 100644 --- a/tests/pcap/http-wrapped-header.test +++ b/tests/pcap/http-wrapped-header.test @@ -1,32 +1,186 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "tags-term" : [ - "srcip", - "dstip" + "dstASN" : "AS0001 Cool Beans!", + "dstBytes" : 1002, + "dstDataBytes" : 706, + "dstGEO" : "CA", + "dstIp" : "10.0.0.2", + "dstMac" : [ + "88:43:e1:94:fc:2d" + ], + "dstMacCnt" : 1, + "dstOui" : [ + "Cisco Systems, Inc" ], - "fb1" : "474554202f782f78", - "p1" : 61462, - "as2" : "AS0001 Cool Beans!", - "mac2-term-cnt" : 1, - "g2" : "CAN", - "a2" : "10.0.0.2", - "mac1-term-cnt" : 1, - "p2" : 80, - "hdrs" : { - "hreq-referer" : [ + "dstOuiCnt" : 1, + "dstPackets" : 5, + "dstPayload8" : "485454502f312e31", + "dstPort" : 80, + "dstRIR" : "TEST", + "fileId" : [], + "firstPacket" : 1404135320459, + "http" : { + "bodyMagic" : [ + "image/gif" + ], + "bodyMagicCnt" : 1, + "clientVersion" : [ + "1.1" + ], + "clientVersionCnt" : 1, + "cookieKey" : [ + "xxxx", + "autocomplete", + "xxxxxxxxxxxxxxxxxxx", + "lp", + "xxxxx", + "xxxxxx", + "rememberUn", + "xxxxxxxxxxxxxxxxxxxxxx", + "xxx.xxxxxxxxxx.xxxxxxxxxx" + ], + "cookieKeyCnt" : 9, + "cookieValue" : [ + "xxxx", + "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "ie", + "false", + "xxxxxx", + "true", + "xx", + "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx2xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "xxxxv1|xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "1" + ], + "cookieValueCnt" : 10, + "host" : [ + "xxxxx.xxxxxxxx.xxxxxxxxxx.xxx" + ], + "hostCnt" : 1, + "key" : [ + "v17", + "events", + "pageName", + "xxx", + "v1", + "ns", + "v2", + "x", + "products", + "AQE", + "vmf", + "c49", + "r", + "ce", + "g" + ], + "keyCnt" : 15, + "md5" : [ + "ad480fd0732d0f6f1a8b06359e3a42bb" + ], + "md5Cnt" : 1, + "method" : [ + "GET" + ], + "methodCnt" : 1, + "path" : [ + "/x/xx/xxxxxxxxxxxxxxxxxxx/x/xxxxxx/xxxxxxxxxxxxxxx" + ], + "pathCnt" : 1, + "request-referer" : [ "http://www.xxxxxxxxxx.xxx/xx/xxxxxxxxxxxxxxxxx/xxxxxxxxxxxxxxxxxxxxxx/xxxxxxxxxxxx.jsp" ], - "hreq-referercnt" : 1 + "request-refererCnt" : 1, + "requestHeader" : [ + "accept", + "user-agent", + "referer", + "accept-encoding", + "connection", + "host", + "cookie", + "accept-language" + ], + "requestHeaderCnt" : 8, + "responseHeader" : [ + "pragma", + "expires", + "content-type", + "xserver", + "keep-alive", + "vary", + "cache-control", + "access-control-allow-origin", + "content-length", + "etag", + "date", + "last-modified", + "connection", + "p3p", + "server", + "set-cookie", + "x-c" + ], + "responseHeaderCnt" : 17, + "serverVersion" : [ + "1.1" + ], + "serverVersionCnt" : 1, + "sha256" : [ + "a1ecbaed793a1f564c49c671f2dd0ce36f858534ef6d26b55783a06b884cc506" + ], + "sha256Cnt" : 1, + "statuscode" : [ + 200 + ], + "statuscodeCnt" : 1, + "uri" : [ + "xxxxx.xxxxxxxx.xxxxxxxxxx.xxx/x/xx/xxxxxxxxxxxxxxxxxxx/x/xxxxxx/xxxxxxxxxxxxxxx?xxx=1&xxx=1&x=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&vmf=xxxxxxxxxx.xxx.xxx.xxx&ce=UTF-8&ns=xxxxxxxxxx&pageName=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&g=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.jsp&r=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&events=xxxxxxxxxxxxxxxxxxxxxxxxxxx&products=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&v1=xxxxxxxxxxxxxxx&v2=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&v17=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx_cxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&c49=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&AQE=1" + ], + "uriCnt" : 1, + "useragent" : [ + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36" + ], + "useragentCnt" : 1, + "value" : [ + "xxxxxxxxxxxxxxxxxxxxxxxxxxx", + "xxxxxxxxxxxxxxx", + "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx_cxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "UTF-8", + "xxxxxxxxxx.xxx.xxx.xxx", + "xxxxxxxxxx", + "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "1", + "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.jsp" + ], + "valueCnt" : 14 }, - "mac2-term" : [ - "88:43:e1:94:fc:2d" + "ipProtocol" : 6, + "lastPacket" : 1404135330336, + "length" : 9877, + "node" : "test", + "packetLen" : [ + 78, + 78, + 76, + 1240, + 948, + 76, + 1240, + 265, + 76, + 776, + 76, + 76 ], - "a1" : "10.0.0.1", - "hcval-term-cnt" : 10, - "hckey-term-cnt" : 9, - "ps" : [ + "packetPos" : [ 24, 102, 180, @@ -40,226 +194,72 @@ 4877, 4953 ], - "mac1-term" : [ - "a4:93:4c:43:13:9b" - ], - "uscnt" : 1, - "prot-term" : [ + "protocol" : [ "http", "tcp" ], - "hmd5" : [ - "ad480fd0732d0f6f1a8b06359e3a42bb" - ], - "rir2" : "TEST", - "hckey-term" : [ - "xxxx", - "autocomplete", - "xxxxxxxxxxxxxxxxxxx", - "lp", - "xxxxx", - "xxxxxx", - "rememberUn", - "xxxxxxxxxxxxxxxxxxxxxx", - "xxx.xxxxxxxxxx.xxxxxxxxxx" + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcASN" : "AS0000 This is neat", + "srcBytes" : 3811, + "srcDataBytes" : 3413, + "srcGEO" : "RU", + "srcIp" : "10.0.0.1", + "srcMac" : [ + "a4:93:4c:43:13:9b" ], - "hh2" : [ - "http:header:access-control-allow-origin", - "http:header:cache-control", - "http:header:connection", - "http:header:content-length", - "http:header:content-type", - "http:header:date", - "http:header:etag", - "http:header:expires", - "http:header:keep-alive", - "http:header:last-modified", - "http:header:p3p", - "http:header:pragma", - "http:header:server", - "http:header:set-cookie", - "http:header:vary", - "http:header:x-c", - "http:header:xserver" + "srcMacCnt" : 1, + "srcOui" : [ + "Cisco Systems, Inc" ], - "portSrc" : 61462, - "as1" : "AS0000 This is neat", - "ho" : [ - "xxxxx.xxxxxxxx.xxxxxxxxxx.xxx" + "srcOuiCnt" : 1, + "srcPackets" : 7, + "srcPayload8" : "474554202f782f78", + "srcPort" : 61462, + "tags" : [ + "dstip", + "srcip" ], - "no" : "test", - "fb2" : "485454502f312e31", + "tagsCnt" : 2, + "tcpflags" : { + "ack" : 6, + "dstZero" : 0, + "fin" : 0, + "psh" : 3, + "rst" : 1, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, "test" : { - "number" : [ - 33554442 - ], - "ip" : [ - 167772161 - ], - "ip-asn" : [ + "ASN" : [ "AS0000 This is neat" ], - "ip-geo" : [ - "RUS" + "GEO" : [ + "RU" ], - "ip-rir" : [ + "RIR" : [ "" ], - "string" : [ + "ip" : [ + "10.0.0.1" + ], + "number" : [ + 33554442 + ], + "string.snow" : [ "16777226:61462,33554442:80" ] }, - "firstPacket" : 1404135320459, - "db2" : 706, - "tacnt" : 2, - "hsvercnt" : 1, - "hocnt" : 1, - "g1" : "RUS", - "hdvercnt" : 1, - "us" : [ - "//xxxxx.xxxxxxxx.xxxxxxxxxx.xxx/x/xx/xxxxxxxxxxxxxxxxxxx/x/xxxxxx/xxxxxxxxxxxxxxx?xxx=1&xxx=1&x=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&vmf=xxxxxxxxxx.xxx.xxx.xxx&ce=UTF-8&ns=xxxxxxxxxx&pageName=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&g=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.jsp&r=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&events=xxxxxxxxxxxxxxxxxxxxxxxxxxx&products=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&v1=xxxxxxxxxxxxxxx&v2=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&v17=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx_cxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&c49=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&AQE=1" - ], - "by1" : 3811, - "by2" : 1002, "timestamp" : "SET", - "hh1cnt" : 8, - "sl" : 9877, - "ipDst" : "10.0.0.2", - "ss" : 1, - "hpathcnt" : 1, - "hval" : [ - "xxxxxxxxxxxxxxxxxxxxxxxxxxx", - "xxxxxxxxxxxxxxx", - "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx_cxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", - "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", - "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", - "UTF-8", - "xxxxxxxxxx.xxx.xxx.xxx", - "xxxxxxxxxx", - "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", - "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", - "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", - "1", - "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", - "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.jsp" - ], - "hh2cnt" : 17, - "fs" : [], - "lp" : 1404135330, - "pa1" : 7, - "tcpflags" : { - "psh" : 3, - "urg" : 0, - "fin" : 0, - "syn-ack" : 1, - "ack" : 6, - "syn" : 1, - "rst" : 1 - }, - "prot-term-cnt" : 2, - "psl" : [ - 78, - 78, - 76, - 1240, - 948, - 76, - 1240, - 265, - 76, - 776, - 76, - 76 - ], - "hvalcnt" : 14, - "fp" : 1404135320, - "lpd" : 1404135330336, - "ua" : [ - "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36" - ], - "hmd5cnt" : 1, - "lastPacket" : 1404135330336, - "pa2" : 5, - "hsver" : [ - "1.1" - ], - "hcval-term" : [ - "xxxx", - "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", - "ie", - "false", - "xxxxxx", - "true", - "xx", - "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx2xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", - "xxxxv1|xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", - "1" - ], - "db" : 4119, - "fpd" : 1404135320459, - "ta" : [ - "dstip", - "srcip" - ], - "db1" : 3413, - "hkey" : [ - "v17", - "events", - "pageName", - "xxx", - "v1", - "ns", - "v2", - "x", - "products", - "AQE", - "vmf", - "c49", - "r", - "ce", - "g" - ], - "ipSrc" : "10.0.0.1", - "by" : 4813, - "portDst" : 80, - "hkeycnt" : 15, - "pa" : 12, - "hh1" : [ - "http:header:accept", - "http:header:accept-encoding", - "http:header:accept-language", - "http:header:connection", - "http:header:cookie", - "http:header:host", - "http:header:referer", - "http:header:user-agent" - ], - "uacnt" : 1, - "hdver" : [ - "1.1" - ], - "pr" : 6, - "hpath" : [ - "/x/xx/xxxxxxxxxxxxxxxxxxx/x/xxxxxx/xxxxxxxxxxxxxxx" - ], - "http" : { - "method-term" : [ - "GET" - ], - "method-term-cnt" : 1, - "statuscode" : [ - 200 - ], - "bodymagic-term" : [ - "image/gif" - ], - "statuscode-cnt" : 1, - "bodymagic-term-cnt" : 1 - } + "totBytes" : 4813, + "totDataBytes" : 4119, + "totPackets" : 12 }, "header" : { "index" : { - "_index" : "tests_sessions-140630", + "_index" : "tests_sessions2-140630", "_type" : "session" } } diff --git a/tests/pcap/http-xff.test b/tests/pcap/http-xff.test index c94ce508cd..e389b5901e 100644 --- a/tests/pcap/http-xff.test +++ b/tests/pcap/http-xff.test @@ -1,31 +1,124 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { + "dstASN" : "AS0001 Cool Beans!", + "dstBytes" : 863, + "dstDataBytes" : 569, + "dstGEO" : "CA", + "dstIp" : "10.0.0.2", + "dstMac" : [ + "00:00:5e:00:01:01", + "5c:5e:ab:b3:67:c2" + ], + "dstMacCnt" : 2, + "dstOui" : [ + "ICANN, IANA Department", + "Juniper Networks" + ], + "dstOuiCnt" : 2, + "dstPackets" : 5, + "dstPayload8" : "485454502f312e31", + "dstPort" : 80, + "dstRIR" : "TEST", + "fileId" : [], "firstPacket" : 1454513228371, - "portDst" : 80, - "lpd" : 1454513228516, - "mac1-term-cnt" : 1, "http" : { - "method-term-cnt" : 1, - "bodymagic-term-cnt" : 1, + "bodyMagic" : [ + "application/x-gzip" + ], + "bodyMagicCnt" : 1, + "clientVersion" : [ + "1.1" + ], + "clientVersionCnt" : 1, + "host" : [ + "www.xxxxxxxxx.xx" + ], + "hostCnt" : 1, + "key" : [ + "xx" + ], + "keyCnt" : 1, + "md5" : [ + "9daa848cbed13db71c8cce96e14a6d57" + ], + "md5Cnt" : 1, + "method" : [ + "GET" + ], + "methodCnt" : 1, + "path" : [ + "/xx/xxxxx" + ], + "pathCnt" : 1, + "request-referer" : [ + "http://www.xxxxx.xxx/xxxx/xxxxxxxxxxxx" + ], + "request-refererCnt" : 1, + "requestHeader" : [ + "user-agent", + "x-forwarded-for", + "referer", + "accept-encoding", + "connection", + "host" + ], + "requestHeaderCnt" : 6, + "responseHeader" : [ + "content-type", + "transfer-encoding", + "cf-ray", + "date", + "content-encoding", + "connection", + "server", + "set-cookie" + ], + "responseHeaderCnt" : 8, + "serverVersion" : [ + "1.1" + ], + "serverVersionCnt" : 1, + "sha256" : [ + "af205d1c0b8409f29764420aa9b74fc81216a509d4d242aab62699fbb204c127" + ], + "sha256Cnt" : 1, "statuscode" : [ 403 ], - "bodymagic-term" : [ - "application/x-gzip" + "statuscodeCnt" : 1, + "uri" : [ + "www.xxxxxxxxx.xx/xx/xxxxx?xx=xxxxxxxxxxxx" ], - "statuscode-cnt" : 1, - "method-term" : [ - "GET" + "uriCnt" : 1, + "useragent" : [ + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.125 Safari/537.36" + ], + "useragentCnt" : 1, + "value" : [ + "xxxxxxxxxxxx" + ], + "valueCnt" : 1, + "xffASN" : [ + "AS0002 Hmm!@#$%^&*()" + ], + "xffGEO" : [ + "---" + ], + "xffIp" : [ + "10.0.0.3" + ], + "xffIpCnt" : 1, + "xffRIR" : [ + "" ] }, - "ipSrc" : "10.0.0.1", - "timestamp" : "SET", - "hkeycnt" : 1, - "prot-term-cnt" : 2, - "mac2-term-cnt" : 2, - "psl" : [ + "ipProtocol" : 6, + "lastPacket" : 1454513228516, + "length" : 145, + "node" : "test", + "packetLen" : [ 90, 82, 76, @@ -38,74 +131,7 @@ 76, 76 ], - "db2" : 569, - "lp" : 1454513228, - "as1" : "AS0000 This is neat", - "hh1cnt" : 6, - "hmd5" : [ - "9daa848cbed13db71c8cce96e14a6d57" - ], - "asxff" : [ - "AS0002 Hmm!@#$%^&*()" - ], - "pa" : 11, - "hdver" : [ - "1.1" - ], - "test" : { - "ip-geo" : [ - "RUS" - ], - "ip-rir" : [ - "" - ], - "number" : [ - 33554442 - ], - "ip-asn" : [ - "AS0000 This is neat" - ], - "ip" : [ - 167772161 - ], - "string" : [ - "16777226:41954,33554442:80" - ] - }, - "by1" : 680, - "tcpflags" : { - "syn-ack" : 1, - "psh" : 3, - "urg" : 0, - "fin" : 1, - "ack" : 4, - "rst" : 1, - "syn" : 1 - }, - "ss" : 1, - "hh2cnt" : 8, - "xff" : [ - 167772163 - ], - "fb1" : "474554202f78782f", - "tags-term" : [ - "srcip", - "dstip" - ], - "gxff" : [ - "---" - ], - "uacnt" : 1, - "hdrs" : { - "hreq-referer" : [ - "http://www.xxxxx.xxx/xxxx/xxxxxxxxxxxx" - ], - "hreq-referercnt" : 1 - }, - "ho" : [ - "www.xxxxxxxxx.xx" - ], - "ps" : [ + "packetPos" : [ 24, 114, 196, @@ -118,97 +144,72 @@ 1591, 1667 ], - "db1" : 312, - "pa2" : 5, - "hh2" : [ - "http:header:cf-ray", - "http:header:connection", - "http:header:content-encoding", - "http:header:content-type", - "http:header:date", - "http:header:server", - "http:header:set-cookie", - "http:header:transfer-encoding" - ], - "fp" : 1454513228, - "fpd" : 1454513228371, - "hmd5cnt" : 1, - "hdvercnt" : 1, - "us" : [ - "//www.xxxxxxxxx.xx/xx/xxxxx?xx=xxxxxxxxxxxx" - ], - "lastPacket" : 1454513228516, - "ta" : [ - "dstip", - "srcip" - ], - "ua" : [ - "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.125 Safari/537.36" - ], - "sl" : 145, - "hvalcnt" : 1, - "portSrc" : 41954, - "fs" : [], - "mac2-term" : [ - "00:00:5e:00:01:01", - "5c:5e:ab:b3:67:c2" - ], - "hh1" : [ - "http:header:accept-encoding", - "http:header:connection", - "http:header:host", - "http:header:referer", - "http:header:user-agent", - "http:header:x-forwarded-for" - ], - "hpathcnt" : 1, - "prot-term" : [ + "protocol" : [ "http", "tcp" ], - "ipDst" : "10.0.0.2", - "fb2" : "485454502f312e31", - "by" : 1543, - "pa1" : 6, - "as2" : "AS0001 Cool Beans!", - "mac1-term" : [ + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcASN" : "AS0000 This is neat", + "srcBytes" : 680, + "srcDataBytes" : 312, + "srcGEO" : "RU", + "srcIp" : "10.0.0.1", + "srcMac" : [ "00:0a:f3:31:90:00" ], - "db" : 881, - "hocnt" : 1, - "a1" : "10.0.0.1", - "no" : "test", - "uscnt" : 1, - "a2" : "10.0.0.2", - "hkey" : [ - "xx" - ], - "p2" : 80, - "tacnt" : 2, - "hsver" : [ - "1.1" - ], - "g1" : "RUS", - "pr" : 6, - "xffscnt" : 1, - "rir2" : "TEST", - "p1" : 41954, - "hpath" : [ - "/xx/xxxxx" - ], - "hval" : [ - "xxxxxxxxxxxx" + "srcMacCnt" : 1, + "srcOui" : [ + "Cisco Systems, Inc" ], - "g2" : "CAN", - "rirxff" : [ - "" + "srcOuiCnt" : 1, + "srcPackets" : 6, + "srcPayload8" : "474554202f78782f", + "srcPort" : 41954, + "tags" : [ + "dstip", + "srcip" ], - "hsvercnt" : 1, - "by2" : 863 + "tagsCnt" : 2, + "tcpflags" : { + "ack" : 4, + "dstZero" : 0, + "fin" : 1, + "psh" : 3, + "rst" : 1, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, + "test" : { + "ASN" : [ + "AS0000 This is neat" + ], + "GEO" : [ + "RU" + ], + "RIR" : [ + "" + ], + "ip" : [ + "10.0.0.1" + ], + "number" : [ + 33554442 + ], + "string.snow" : [ + "16777226:41954,33554442:80" + ] + }, + "timestamp" : "SET", + "totBytes" : 1543, + "totDataBytes" : 881, + "totPackets" : 11 }, "header" : { "index" : { - "_index" : "tests_sessions-160203", + "_index" : "tests_sessions2-160203", "_type" : "session" } } diff --git a/tests/pcap/https-generalizedtime.test b/tests/pcap/https-generalizedtime.test index 9074c15467..52b4ea8992 100644 --- a/tests/pcap/https-generalizedtime.test +++ b/tests/pcap/https-generalizedtime.test @@ -1,89 +1,8 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "fs" : [], - "ho" : [ - "mail.yandex.com" - ], - "tacnt" : 2, - "portSrc" : 3627, - "mac1-term" : [ - "00:0f:f7:76:82:80" - ], - "ipDst" : "10.0.0.2", - "no" : "test", - "prot-term" : [ - "tlsrulestest", - "tls", - "tcp" - ], - "timestamp" : "SET", - "by2" : 5216, - "test" : { - "ip-asn" : [ - "AS0000 This is neat" - ], - "string" : [ - "16777226:3627,33554442:443" - ], - "ip" : [ - 167772161 - ], - "number" : [ - 33554442 - ], - "ip-rir" : [ - "" - ], - "ip-geo" : [ - "RUS" - ] - }, - "fpd" : 1421771550035, - "p1" : 3627, - "prot-term-cnt" : 3, - "lastPacket" : 1421771550224, - "pa2" : 3, - "as2" : "AS0001 Cool Beans!", - "sl" : 188, - "p2" : 443, - "tlsja3-termcnt" : 1, - "portDst" : 443, - "tlsver-term" : [ - "TLSv1.2" - ], - "tlscipher-term" : [ - "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256" - ], - "tlsver-termcnt" : 1, - "fp" : 1421771550, - "tags-term" : [ - "srcip", - "dstip" - ], - "by1" : 697, - "fb2" : "160303005b020000", - "mac2-term" : [ - "00:00:5e:00:01:02", - "00:1d:b5:ce:ef:c0" - ], - "fb1" : "1603010200010001", - "mac1-term-cnt" : 1, - "firstPacket" : 1421771550035, - "tlscipher-termcnt" : 1, - "tlssrcid-term" : [ - "51273b31b1b975c9276907197fbf6b855bc4387e341cea1d5cf727c086c0774b" - ], - "lp" : 1421771550, - "db" : 5553, - "as1" : "AS0000 This is neat", - "g2" : "CAN", - "db2" : 5036, - "tlsja3-term" : [ - "8ca01c116f7fa9e77c6e1800eac1bec2" - ], - "tls" : [ + "cert" : [ { "alt" : [ "mail.yandex.az", @@ -99,39 +18,70 @@ "mail.yandex.com", "mail.yandex.ru" ], - "iOn" : "Unizeto Technologies S.A.", - "iCn" : [ + "altCnt" : 12, + "hash" : "59:4d:51:83:75:d0:9c:29:82:26:58:5f:ec:83:7b:60:ce:33:ba:d8", + "issuerCN" : [ "certum level iv ca" ], - "sOn" : "Yandex LLC", - "sCn" : [ + "issuerON" : "Unizeto Technologies S.A.", + "notAfter" : 1451573263000, + "notBefore" : 1415717263000, + "serial" : "2e8f3b9cbc17eacb6459a3b8c0b598fb", + "subjectCN" : [ "mail.yandex.ru" ], - "sn" : "2e8f3b9cbc17eacb6459a3b8c0b598fb", - "diffDays" : 415, - "notBefore" : 1415717263, - "notAfter" : 1451573263, - "hash" : "59:4d:51:83:75:d0:9c:29:82:26:58:5f:ec:83:7b:60:ce:33:ba:d8", - "altcnt" : 12 + "subjectON" : "Yandex LLC", + "validDays" : 415 }, { - "notBefore" : 1236084865, - "notAfter" : 1709470465, "hash" : "12:e1:89:f7:dc:2f:a2:82:38:0b:48:77:31:9f:5f:1d:fe:af:6d:69", - "sOn" : "Unizeto Technologies S.A.", - "sCn" : [ + "issuerCN" : [ + "certum ca" + ], + "issuerON" : "Unizeto Sp. z o.o.", + "notAfter" : 1709470465000, + "notBefore" : 1236084865000, + "serial" : "4ca5fec6617c48b056382a8280e0508c", + "subjectCN" : [ "certum level iv ca" ], - "sn" : "4ca5fec6617c48b056382a8280e0508c", - "diffDays" : 5479, - "iOn" : "Unizeto Sp. z o.o.", - "iCn" : [ - "certum ca" - ] + "subjectON" : "Unizeto Technologies S.A.", + "validDays" : 5479 } ], - "mac2-term-cnt" : 2, - "psl" : [ + "certCnt" : 2, + "dstASN" : "AS0001 Cool Beans!", + "dstBytes" : 5216, + "dstDataBytes" : 5036, + "dstGEO" : "CA", + "dstIp" : "10.0.0.2", + "dstMac" : [ + "00:00:5e:00:01:02", + "00:1d:b5:ce:ef:c0" + ], + "dstMacCnt" : 2, + "dstOui" : [ + "ICANN, IANA Department", + "Juniper Networks" + ], + "dstOuiCnt" : 2, + "dstPackets" : 3, + "dstPayload8" : "160303005b020000", + "dstPort" : 443, + "dstRIR" : "TEST", + "fileId" : [], + "firstPacket" : 1421771550035, + "http" : { + "host" : [ + "mail.yandex.com" + ], + "hostCnt" : 1 + }, + "ipProtocol" : 6, + "lastPacket" : 1421771550224, + "length" : 188, + "node" : "test", + "packetLen" : [ 82, 82, 76, @@ -139,12 +89,7 @@ 76, 5106 ], - "ta" : [ - "dstip", - "srcip" - ], - "pa1" : 3, - "ps" : [ + "packetPos" : [ 24, 106, 188, @@ -152,33 +97,91 @@ 851, 927 ], - "g1" : "RUS", - "hocnt" : 1, - "ipSrc" : "10.0.0.1", - "a2" : "10.0.0.2", - "lpd" : 1421771550224, - "db1" : 517, - "by" : 5913, - "pr" : 6, - "tlscnt" : 2, - "rir2" : "TEST", - "pa" : 6, + "protocol" : [ + "tlsrulestest", + "tls", + "tcp" + ], + "protocolCnt" : 3, + "segmentCnt" : 1, + "srcASN" : "AS0000 This is neat", + "srcBytes" : 697, + "srcDataBytes" : 517, + "srcGEO" : "RU", + "srcIp" : "10.0.0.1", + "srcMac" : [ + "00:0f:f7:76:82:80" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Cisco Systems, Inc" + ], + "srcOuiCnt" : 1, + "srcPackets" : 3, + "srcPayload8" : "1603010200010001", + "srcPort" : 3627, + "tags" : [ + "dstip", + "srcip" + ], + "tagsCnt" : 2, "tcpflags" : { - "syn" : 1, - "fin" : 0, - "rst" : 0, "ack" : 2, + "dstZero" : 0, + "fin" : 0, "psh" : 2, - "urg" : 0, - "syn-ack" : 1 + "rst" : 0, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, + "test" : { + "ASN" : [ + "AS0000 This is neat" + ], + "GEO" : [ + "RU" + ], + "RIR" : [ + "" + ], + "ip" : [ + "10.0.0.1" + ], + "number" : [ + 33554442 + ], + "string.snow" : [ + "16777226:3627,33554442:443" + ] + }, + "timestamp" : "SET", + "tls" : { + "cipher" : [ + "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256" + ], + "cipherCnt" : 1, + "ja3" : [ + "8ca01c116f7fa9e77c6e1800eac1bec2" + ], + "ja3Cnt" : 1, + "srcSessionId" : [ + "51273b31b1b975c9276907197fbf6b855bc4387e341cea1d5cf727c086c0774b" + ], + "version" : [ + "TLSv1.2" + ], + "versionCnt" : 1 }, - "ss" : 1, - "a1" : "10.0.0.1" + "totBytes" : 5913, + "totDataBytes" : 5553, + "totPackets" : 6 }, "header" : { "index" : { - "_type" : "session", - "_index" : "tests_sessions-150120" + "_index" : "tests_sessions2-150120", + "_type" : "session" } } } diff --git a/tests/pcap/https2-301-get.test b/tests/pcap/https2-301-get.test index 14dc880d63..aafdd3e19a 100644 --- a/tests/pcap/https2-301-get.test +++ b/tests/pcap/https2-301-get.test @@ -1,119 +1,71 @@ { - "sessions" : [ + "sessions2" : [ { - "header" : { - "index" : { - "_index" : "tests_sessions-131125", - "_type" : "session" - } - }, "body" : { - "by1" : 1513, - "db2" : 4204, - "tls" : [ + "cert" : [ { - "sCn" : [ - "github.com" + "alt" : [ + "github.com", + "www.github.com" ], - "notBefore" : 1370822400, + "altCnt" : 2, "hash" : "d7:12:e9:69:65:dc:f2:36:c8:74:c7:03:7d:c0:b2:24:a9:3b:d2:33", - "notAfter" : 1441195200, - "diffDays" : 814, - "iCn" : [ + "issuerCN" : [ "digicert high assurance ev ca-1" ], - "iOn" : "DigiCert Inc", - "sn" : "047fbe2e4bde0084d2caf8e3ecfe7058", - "alt" : [ - "github.com", - "www.github.com" + "issuerON" : "DigiCert Inc", + "notAfter" : 1441195200000, + "notBefore" : 1370822400000, + "serial" : "047fbe2e4bde0084d2caf8e3ecfe7058", + "subjectCN" : [ + "github.com" ], - "sOn" : "GitHub, Inc.", - "altcnt" : 2 + "subjectON" : "GitHub, Inc.", + "validDays" : 814 }, { - "notAfter" : 1636502400, - "notBefore" : 1194609600, - "sCn" : [ - "digicert high assurance ev ca-1" - ], "hash" : "db:c7:e9:0b:0d:a5:d8:8a:55:35:43:0e:eb:66:5d:07:78:59:e8:e8", - "iOn" : "DigiCert Inc", - "sn" : "0337b928347c60a6aec5adb1217f3860", - "sOn" : "DigiCert Inc", - "diffDays" : 5114, - "iCn" : [ + "issuerCN" : [ "digicert high assurance ev root ca" - ] + ], + "issuerON" : "DigiCert Inc", + "notAfter" : 1636502400000, + "notBefore" : 1194609600000, + "serial" : "0337b928347c60a6aec5adb1217f3860", + "subjectCN" : [ + "digicert high assurance ev ca-1" + ], + "subjectON" : "DigiCert Inc", + "validDays" : 5114 } ], - "prot-term" : [ - "tls", - "tcp" - ], - "ss" : 1, - "lastPacket" : 1385410274128, - "g2" : "USA", - "lp" : 1385410274, - "fb1" : "8080010301005700", - "portDst" : 443, - "db" : 4817, - "p1" : 50754, - "ps" : [ - 24, - 118, - 208, - 290, - 502, - 2008, - 3514, - 4485, - 4567, - 4649, - 5045, - 5174, - 5256, - 5480, - 5955, - 6037, - 6146, - 6228, - 6337, - 6419, - 6501, - 6571, - 6641 - ], - "mac1-term" : [ - "00:1f:5b:ff:51:cb" - ], - "mac2-term-cnt" : 2, - "fs" : [], - "firstPacket" : 1385410273592, - "tlsver-termcnt" : 1, - "tlsdstid-term" : [ - "4fc128aa12f6c10f1b6f72c0d4447366fe600b41efff60c865e496ed7f838ba6" - ], - "ipSrc" : "10.180.156.141", - "tlscipher-termcnt" : 1, - "pa1" : 14, - "g1" : "USA", - "lpd" : 1385410274128, - "fpd" : 1385410273592, - "portSrc" : 50754, - "prot-term-cnt" : 2, - "db1" : 613, - "rir2" : "ARIN", - "fb2" : "1603010051020000", - "no" : "test", - "mac2-term" : [ + "certCnt" : 2, + "dstASN" : "AS36459 GitHub, Inc.", + "dstBytes" : 4806, + "dstDataBytes" : 4204, + "dstGEO" : "US", + "dstIp" : "192.30.252.131", + "dstMac" : [ "00:00:0c:07:ac:01", "00:d0:2b:d1:76:00" ], - "as2" : "AS36459 GitHub, Inc.", - "pa" : 23, - "p2" : 443, - "psl" : [ + "dstMacCnt" : 2, + "dstOui" : [ + "Cisco", + "Jetcell, Inc." + ], + "dstOuiCnt" : 2, + "dstPackets" : 9, + "dstPayload8" : "1603010051020000", + "dstPort" : 443, + "dstRIR" : "ARIN", + "fileId" : [], + "firstPacket" : 1385410273592, + "ipProtocol" : 6, + "lastPacket" : 1385410274128, + "length" : 536, + "node" : "test", + "packetLen" : [ 94, 90, 82, @@ -138,32 +90,85 @@ 70, 70 ], - "pa2" : 9, - "tlsver-term" : [ - "TLSv1" + "packetPos" : [ + 24, + 118, + 208, + 290, + 502, + 2008, + 3514, + 4485, + 4567, + 4649, + 5045, + 5174, + 5256, + 5480, + 5955, + 6037, + 6146, + 6228, + 6337, + 6419, + 6501, + 6571, + 6641 ], - "a2" : "192.30.252.131", - "by2" : 4806, - "a1" : "10.180.156.141", - "fp" : 1385410273, - "pr" : 6, - "sl" : 536, - "ipDst" : "192.30.252.131", - "mac1-term-cnt" : 1, - "timestamp" : "SET", - "tlscnt" : 2, - "tlscipher-term" : [ - "TLS_RSA_WITH_RC4_128_SHA" + "protocol" : [ + "tls", + "tcp" + ], + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 1513, + "srcDataBytes" : 613, + "srcGEO" : "US", + "srcIp" : "10.180.156.141", + "srcMac" : [ + "00:1f:5b:ff:51:cb" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Apple, Inc." ], - "by" : 6319, + "srcOuiCnt" : 1, + "srcPackets" : 14, + "srcPayload8" : "8080010301005700", + "srcPort" : 50754, "tcpflags" : { - "psh" : 8, "ack" : 8, + "dstZero" : 0, + "fin" : 2, + "psh" : 8, + "rst" : 3, + "srcZero" : 0, "syn" : 1, "syn-ack" : 1, - "fin" : 2, - "urg" : 0, - "rst" : 3 + "urg" : 0 + }, + "timestamp" : "SET", + "tls" : { + "cipher" : [ + "TLS_RSA_WITH_RC4_128_SHA" + ], + "cipherCnt" : 1, + "dstSessionId" : [ + "4fc128aa12f6c10f1b6f72c0d4447366fe600b41efff60c865e496ed7f838ba6" + ], + "version" : [ + "TLSv1" + ], + "versionCnt" : 1 + }, + "totBytes" : 6319, + "totDataBytes" : 4817, + "totPackets" : 23 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-131125", + "_type" : "session" } } } diff --git a/tests/pcap/https3-301-get.test b/tests/pcap/https3-301-get.test index c975ac5dc9..73637ad933 100644 --- a/tests/pcap/https3-301-get.test +++ b/tests/pcap/https3-301-get.test @@ -1,153 +1,77 @@ { - "sessions" : [ + "sessions2" : [ { - "header" : { - "index" : { - "_index" : "tests_sessions-131125", - "_type" : "session" - } - }, "body" : { - "ho" : [ - "www.github.com" - ], - "pa" : 23, - "by1" : 1530, - "fp" : 1385396821, - "db2" : 4208, - "prot-term-cnt" : 2, - "prot-term" : [ - "tls", - "tcp" - ], - "pa2" : 9, - "by2" : 4810, - "lastPacket" : 1385396821236, - "lp" : 1385396821, - "mac2-term-cnt" : 2, - "mac1-term" : [ - "00:1f:5b:ff:51:cb" - ], - "fb2" : "1603010055020000", - "tlscnt" : 2, - "firstPacket" : 1385396821013, - "ipDst" : "192.30.252.130", - "portDst" : 443, - "tlscipher-term" : [ - "TLS_RSA_WITH_RC4_128_SHA" - ], - "mac1-term-cnt" : 1, - "a2" : "192.30.252.130", - "hocnt" : 1, - "ss" : 1, - "sl" : 223, - "tcpflags" : { - "ack" : 8, - "psh" : 8, - "syn" : 1, - "syn-ack" : 1, - "rst" : 3, - "urg" : 0, - "fin" : 2 - }, - "tlsver-term" : [ - "TLSv1" - ], - "tlsja3-termcnt" : 1, - "tls" : [ + "cert" : [ { - "diffDays" : 814, "alt" : [ "github.com", "www.github.com" ], - "sCn" : [ - "github.com" - ], - "iCn" : [ + "altCnt" : 2, + "hash" : "d7:12:e9:69:65:dc:f2:36:c8:74:c7:03:7d:c0:b2:24:a9:3b:d2:33", + "issuerCN" : [ "digicert high assurance ev ca-1" ], - "sOn" : "GitHub, Inc.", - "altcnt" : 2, - "notBefore" : 1370822400, - "iOn" : "DigiCert Inc", - "hash" : "d7:12:e9:69:65:dc:f2:36:c8:74:c7:03:7d:c0:b2:24:a9:3b:d2:33", - "sn" : "047fbe2e4bde0084d2caf8e3ecfe7058", - "notAfter" : 1441195200 + "issuerON" : "DigiCert Inc", + "notAfter" : 1441195200000, + "notBefore" : 1370822400000, + "serial" : "047fbe2e4bde0084d2caf8e3ecfe7058", + "subjectCN" : [ + "github.com" + ], + "subjectON" : "GitHub, Inc.", + "validDays" : 814 }, { - "iOn" : "DigiCert Inc", - "notBefore" : 1194609600, "hash" : "db:c7:e9:0b:0d:a5:d8:8a:55:35:43:0e:eb:66:5d:07:78:59:e8:e8", - "sn" : "0337b928347c60a6aec5adb1217f3860", - "notAfter" : 1636502400, - "diffDays" : 5114, - "sCn" : [ + "issuerCN" : [ + "digicert high assurance ev root ca" + ], + "issuerON" : "DigiCert Inc", + "notAfter" : 1636502400000, + "notBefore" : 1194609600000, + "serial" : "0337b928347c60a6aec5adb1217f3860", + "subjectCN" : [ "digicert high assurance ev ca-1" ], - "sOn" : "DigiCert Inc", - "iCn" : [ - "digicert high assurance ev root ca" - ] + "subjectON" : "DigiCert Inc", + "validDays" : 5114 } ], - "fpd" : 1385396821013, - "tlscipher-termcnt" : 1, - "fs" : [], - "p1" : 62599, - "p2" : 443, - "no" : "test", - "portSrc" : 62599, - "ipSrc" : "10.180.156.141", - "tlsja3-term" : [ - "06a92bf69b367389d2feb0d70501ddfe" - ], - "fb1" : "1603010072010000", - "tlsdstid-term" : [ - "dddbfd5162689bb01e3f0ba158a586de9097b26cfc83599391cbb5e128d34b97" - ], - "pa1" : 14, - "g2" : "USA", - "lpd" : 1385396821236, - "as2" : "AS36459 GitHub, Inc.", - "a1" : "10.180.156.141", - "g1" : "USA", - "db1" : 630, - "mac2-term" : [ + "certCnt" : 2, + "dstASN" : "AS36459 GitHub, Inc.", + "dstBytes" : 4810, + "dstDataBytes" : 4208, + "dstGEO" : "US", + "dstIp" : "192.30.252.130", + "dstMac" : [ "00:00:0c:07:ac:01", "00:0e:d6:0b:98:80" ], - "tlsver-termcnt" : 1, - "rir2" : "ARIN", - "by" : 6340, - "timestamp" : "SET", - "pr" : 6, - "ps" : [ - 24, - 118, - 208, - 290, - 491, - 1997, - 3503, - 4478, - 4560, - 4642, - 5038, - 5167, - 5249, - 5501, - 5976, - 6058, - 6167, - 6249, - 6358, - 6440, - 6510, - 6580, - 6662 + "dstMacCnt" : 2, + "dstOui" : [ + "Cisco", + "Cisco Systems, Inc" ], - "psl" : [ + "dstOuiCnt" : 2, + "dstPackets" : 9, + "dstPayload8" : "1603010055020000", + "dstPort" : 443, + "dstRIR" : "ARIN", + "fileId" : [], + "firstPacket" : 1385396821013, + "http" : { + "host" : [ + "www.github.com" + ], + "hostCnt" : 1 + }, + "ipProtocol" : 6, + "lastPacket" : 1385396821236, + "length" : 223, + "node" : "test", + "packetLen" : [ 94, 90, 82, @@ -172,7 +96,90 @@ 82, 70 ], - "db" : 4838 + "packetPos" : [ + 24, + 118, + 208, + 290, + 491, + 1997, + 3503, + 4478, + 4560, + 4642, + 5038, + 5167, + 5249, + 5501, + 5976, + 6058, + 6167, + 6249, + 6358, + 6440, + 6510, + 6580, + 6662 + ], + "protocol" : [ + "tls", + "tcp" + ], + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 1530, + "srcDataBytes" : 630, + "srcGEO" : "US", + "srcIp" : "10.180.156.141", + "srcMac" : [ + "00:1f:5b:ff:51:cb" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Apple, Inc." + ], + "srcOuiCnt" : 1, + "srcPackets" : 14, + "srcPayload8" : "1603010072010000", + "srcPort" : 62599, + "tcpflags" : { + "ack" : 8, + "dstZero" : 0, + "fin" : 2, + "psh" : 8, + "rst" : 3, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, + "timestamp" : "SET", + "tls" : { + "cipher" : [ + "TLS_RSA_WITH_RC4_128_SHA" + ], + "cipherCnt" : 1, + "dstSessionId" : [ + "dddbfd5162689bb01e3f0ba158a586de9097b26cfc83599391cbb5e128d34b97" + ], + "ja3" : [ + "06a92bf69b367389d2feb0d70501ddfe" + ], + "ja3Cnt" : 1, + "version" : [ + "TLSv1" + ], + "versionCnt" : 1 + }, + "totBytes" : 6340, + "totDataBytes" : 4838, + "totPackets" : 23 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-131125", + "_type" : "session" + } } } ] diff --git a/tests/pcap/imap-tag.test b/tests/pcap/imap-tag.test index 2b7ece74a1..dabc581d59 100644 --- a/tests/pcap/imap-tag.test +++ b/tests/pcap/imap-tag.test @@ -1,110 +1,109 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "a1" : "10.0.0.1", - "ps" : [ + "dstASN" : "AS0001 Cool Beans!", + "dstBytes" : 134, + "dstDataBytes" : 18, + "dstGEO" : "CA", + "dstIp" : "10.0.0.2", + "dstMac" : [ + "00:00:5e:00:01:01", + "00:1d:b5:ce:ef:c0" + ], + "dstMacCnt" : 2, + "dstOui" : [ + "ICANN, IANA Department", + "Juniper Networks" + ], + "dstOuiCnt" : 2, + "dstPackets" : 2, + "dstPayload8" : "2a204f4b20494d41", + "dstPort" : 143, + "dstRIR" : "TEST", + "fileId" : [], + "firstPacket" : 1387759542002, + "ipProtocol" : 6, + "lastPacket" : 1387759542236, + "length" : 233, + "node" : "test", + "packetLen" : [ + 78, + 78, + 76, + 88 + ], + "packetPos" : [ 24, 102, 180, 256 ], - "g1" : "RUS", - "mac1-term-cnt" : 1, - "ta" : [ + "protocol" : [ + "imap", + "tcp" + ], + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcASN" : "AS0000 This is neat", + "srcBytes" : 122, + "srcDataBytes" : 0, + "srcGEO" : "RU", + "srcIp" : "10.0.0.1", + "srcMac" : [ + "00:0f:f7:76:82:80" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Cisco Systems, Inc" + ], + "srcOuiCnt" : 1, + "srcPackets" : 2, + "srcPort" : 4643, + "tags" : [ "dstip", "srcip" ], - "fpd" : 1387759542002, - "by1" : 122, - "a2" : "10.0.0.2", - "by" : 256, + "tagsCnt" : 2, "tcpflags" : { - "urg" : 0, + "ack" : 1, + "dstZero" : 0, "fin" : 0, "psh" : 1, - "syn-ack" : 1, "rst" : 0, + "srcZero" : 0, "syn" : 1, - "ack" : 1 + "syn-ack" : 1, + "urg" : 0 }, - "g2" : "CAN", - "p1" : 4643, - "ipSrc" : "10.0.0.1", - "sl" : 233, - "portSrc" : 4643, - "rir2" : "TEST", - "prot-term-cnt" : 2, - "mac2-term" : [ - "00:00:5e:00:01:01", - "00:1d:b5:ce:ef:c0" - ], - "fs" : [], - "lastPacket" : 1387759542236, - "tacnt" : 2, - "db2" : 18, - "as1" : "AS0000 This is neat", - "by2" : 134, - "mac2-term-cnt" : 2, - "lpd" : 1387759542236, - "psl" : [ - 78, - 78, - 76, - 88 - ], - "p2" : 143, - "pa2" : 2, - "mac1-term" : [ - "00:0f:f7:76:82:80" - ], - "tags-term" : [ - "srcip", - "dstip" - ], - "ss" : 1, - "as2" : "AS0001 Cool Beans!", - "fb2" : "2a204f4b20494d41", - "no" : "test", - "ipDst" : "10.0.0.2", - "firstPacket" : 1387759542002, - "portDst" : 143, - "prot-term" : [ - "imap", - "tcp" - ], "test" : { - "string" : [ - "16777226:4643,33554442:143" + "ASN" : [ + "AS0000 This is neat" ], - "number" : [ - 33554442 + "GEO" : [ + "RU" ], - "ip" : [ - 167772161 + "RIR" : [ + "" ], - "ip-asn" : [ - "AS0000 This is neat" + "ip" : [ + "10.0.0.1" ], - "ip-geo" : [ - "RUS" + "number" : [ + 33554442 ], - "ip-rir" : [ - "" + "string.snow" : [ + "16777226:4643,33554442:143" ] }, - "fp" : 1387759542, - "pr" : 6, - "db" : 18, - "pa" : 4, "timestamp" : "SET", - "pa1" : 2, - "lp" : 1387759542, - "db1" : 0 + "totBytes" : 256, + "totDataBytes" : 18, + "totPackets" : 4 }, "header" : { "index" : { - "_index" : "tests_sessions-131223", + "_index" : "tests_sessions2-131223", "_type" : "session" } } diff --git a/tests/pcap/ip-boundaries.test b/tests/pcap/ip-boundaries.test index f0edc537ca..41445a451a 100644 --- a/tests/pcap/ip-boundaries.test +++ b/tests/pcap/ip-boundaries.test @@ -1,61 +1,61 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "psl" : [ - 161 + "dstBytes" : 0, + "dstDataBytes" : 0, + "dstIp" : "0.0.0.0", + "dstMac" : [ + "00:10:db:ff:26:00" ], - "p1" : 50759, - "db2" : 0, - "pa2" : 0, - "a1" : "255.255.255.255", - "ps" : [ - 103 + "dstMacCnt" : 1, + "dstOui" : [ + "Juniper Networks" ], - "pa1" : 1, - "lpd" : 1387253713030, - "portSrc" : 50759, - "p2" : 3207, - "fb1" : "64313a6164323a69", - "ipSrc" : "255.255.255.255", + "dstOuiCnt" : 1, + "dstPackets" : 0, + "dstPort" : 3207, + "fileId" : [], "firstPacket" : 1387253713030, - "portDst" : 3207, - "prot-term-cnt" : 2, - "timestamp" : "SET", - "by2" : 0, - "fpd" : 1387253713030, - "mac2-term" : [ - "00:10:db:ff:26:00" - ], - "pr" : 17, + "ipProtocol" : 17, "lastPacket" : 1387253713030, - "sl" : 0, - "ss" : 1, - "mac2-term-cnt" : 1, - "pa" : 1, - "db1" : 137, - "by" : 145, - "by1" : 145, - "lp" : 1387253713, - "db" : 137, - "prot-term" : [ + "length" : 0, + "node" : "test", + "packetLen" : [ + 161 + ], + "packetPos" : [ + 103 + ], + "protocol" : [ "udp", "bittorrent" ], - "no" : "test", - "a2" : "0.0.0.0", - "mac1-term-cnt" : 1, - "ipDst" : "0.0.0.0", - "fs" : [], - "mac1-term" : [ + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 145, + "srcDataBytes" : 137, + "srcIp" : "255.255.255.255", + "srcMac" : [ "78:fe:3d:11:21:f2" ], - "fp" : 1387253713 + "srcMacCnt" : 1, + "srcOui" : [ + "Juniper Networks" + ], + "srcOuiCnt" : 1, + "srcPackets" : 1, + "srcPayload8" : "64313a6164323a69", + "srcPort" : 50759, + "timestamp" : "SET", + "totBytes" : 145, + "totDataBytes" : 137, + "totPackets" : 1 }, "header" : { "index" : { - "_type" : "session", - "_index" : "tests_sessions-131217" + "_index" : "tests_sessions2-131217", + "_type" : "session" } } } diff --git a/tests/pcap/irc-cap-req.test b/tests/pcap/irc-cap-req.test index ed8450e714..0156240749 100644 --- a/tests/pcap/irc-cap-req.test +++ b/tests/pcap/irc-cap-req.test @@ -1,9 +1,66 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "by1" : 1110, - "ps" : [ + "dstBytes" : 2067, + "dstDataBytes" : 1219, + "dstIp" : "10.11.11.11", + "dstMac" : [ + "00:1b:17:00:01:24" + ], + "dstMacCnt" : 1, + "dstOui" : [ + "Palo Alto Networks" + ], + "dstOuiCnt" : 1, + "dstPackets" : 12, + "dstPayload8" : "3a746d692e747769", + "dstPort" : 80, + "fileId" : [], + "firstPacket" : 1483561576585, + "ipProtocol" : 6, + "irc" : { + "channel" : [ + "#THE-USER" + ], + "channelCnt" : 1, + "nick" : [ + "THE-USER" + ], + "nickCnt" : 1 + }, + "lastPacket" : 1483561951602, + "length" : 375017, + "node" : "test", + "packetLen" : [ + 98, + 94, + 86, + 130, + 86, + 147, + 86, + 129, + 86, + 101, + 86, + 419, + 266, + 86, + 86, + 102, + 86, + 144, + 652, + 86, + 86, + 107, + 86, + 86, + 86, + 86 + ], + "packetPos" : [ 24, 122, 216, @@ -31,103 +88,50 @@ 3445, 3531 ], - "p2" : 80, - "db1" : 118, - "sl" : 375017, - "pa" : 26, - "mac1-term-cnt" : 1, - "fb2" : "3a746d692e747769", - "lp" : 1483561951, - "pr" : 6, - "portSrc" : 59604, - "pa2" : 12, - "vlan-cnt" : 1, - "mac2-term-cnt" : 1, - "ss" : 1, - "mac1-term" : [ - "00:22:83:3f:17:c5" + "protocol" : [ + "irc", + "tcp" ], - "portDst" : 80, - "vlan" : [ - 100 + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 1110, + "srcDataBytes" : 118, + "srcIp" : "10.10.10.10", + "srcMac" : [ + "00:22:83:3f:17:c5" ], - "lastPacket" : 1483561951602, - "db" : 1337, - "timestamp" : "SET", - "by" : 3177, - "ircnckcnt" : 1, - "pa1" : 14, - "prot-term-cnt" : 2, - "ircnck" : [ - "THE-USER" + "srcMacCnt" : 1, + "srcOui" : [ + "Juniper Networks" ], - "firstPacket" : 1483561576585, + "srcOuiCnt" : 1, + "srcPackets" : 14, + "srcPayload8" : "4341502052455120", + "srcPort" : 59604, "tcpflags" : { - "syn" : 1, - "syn-ack" : 1, - "fin" : 2, "ack" : 12, + "dstZero" : 0, + "fin" : 2, "psh" : 10, "rst" : 0, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, "urg" : 0 }, - "a1" : "10.10.10.10", - "psl" : [ - 98, - 94, - 86, - 130, - 86, - 147, - 86, - 129, - 86, - 101, - 86, - 419, - 266, - 86, - 86, - 102, - 86, - 144, - 652, - 86, - 86, - 107, - 86, - 86, - 86, - 86 - ], - "p1" : 59604, - "lpd" : 1483561951602, - "fb1" : "4341502052455120", - "db2" : 1219, - "prot-term" : [ - "irc", - "tcp" - ], - "ircchcnt" : 1, - "ipSrc" : "10.10.10.10", - "fpd" : 1483561576585, - "ipDst" : "10.11.11.11", - "fs" : [], - "fp" : 1483561576, - "ircch" : [ - "#THE-USER" - ], - "by2" : 2067, - "a2" : "10.11.11.11", - "mac2-term" : [ - "00:1b:17:00:01:24" + "timestamp" : "SET", + "totBytes" : 3177, + "totDataBytes" : 1337, + "totPackets" : 26, + "vlan" : [ + 100 ], - "no" : "test" + "vlanCnt" : 1 }, "header" : { "index" : { - "_type" : "session", - "_index" : "tests_sessions-170104" + "_index" : "tests_sessions2-170104", + "_type" : "session" } } } diff --git a/tests/pcap/irc.test b/tests/pcap/irc.test index a8064b7e15..7671b1a7ae 100644 --- a/tests/pcap/irc.test +++ b/tests/pcap/irc.test @@ -1,50 +1,43 @@ { - "sessions" : [ + "sessions2" : [ { - "header" : { - "index" : { - "_type" : "session", - "_index" : "tests_sessions-131220" - } - }, "body" : { - "fb2" : "3a636172642e6672", - "p1" : 45921, - "lpd" : 1387554256201, - "as2" : "AS23028 Team Cymru Inc.", - "fb1" : "5553455220787878", - "a1" : "10.180.156.249", - "ss" : 1, - "ircnck" : [ - "molochtest" - ], - "db1" : 114, - "ircch" : [ - "#moloch-fpc" - ], - "lastPacket" : 1387554256201, - "by" : 8945, - "no" : "test", - "mac2-term-cnt" : 2, - "fs" : [], - "ipDst" : "38.229.70.20", - "portDst" : 8000, - "sl" : 14568, - "prot-term" : [ - "irc", - "tcp" - ], - "a2" : "38.229.70.20", - "mac2-term" : [ + "dstASN" : "AS23028 Team Cymru Inc.", + "dstBytes" : 7899, + "dstDataBytes" : 6901, + "dstGEO" : "US", + "dstIp" : "38.229.70.20", + "dstMac" : [ "00:00:0c:07:ac:01", "00:d0:2b:d1:76:00" ], + "dstMacCnt" : 2, + "dstOui" : [ + "Cisco", + "Jetcell, Inc." + ], + "dstOuiCnt" : 2, + "dstPackets" : 15, + "dstPayload8" : "3a636172642e6672", + "dstPort" : 8000, + "dstRIR" : "ARIN", + "fileId" : [], "firstPacket" : 1387554241634, - "g1" : "USA", - "pa1" : 14, - "lp" : 1387554256, - "g2" : "USA", - "psl" : [ + "ipProtocol" : 6, + "irc" : { + "channel" : [ + "#moloch-fpc" + ], + "channelCnt" : 1, + "nick" : [ + "molochtest" + ], + "nickCnt" : 1 + }, + "lastPacket" : 1387554256201, + "length" : 14568, + "node" : "test", + "packetLen" : [ 90, 90, 82, @@ -75,35 +68,7 @@ 82, 638 ], - "pa2" : 15, - "p2" : 8000, - "prot-term-cnt" : 2, - "db" : 7015, - "ircnckcnt" : 1, - "pr" : 6, - "mac1-term-cnt" : 1, - "mac1-term" : [ - "00:13:72:c4:f1:e1" - ], - "timestamp" : "SET", - "db2" : 6901, - "pa" : 29, - "fp" : 1387554241, - "by2" : 7899, - "portSrc" : 45921, - "by1" : 1046, - "tcpflags" : { - "psh" : 11, - "fin" : 0, - "urg" : 0, - "syn" : 1, - "rst" : 0, - "ack" : 16, - "syn-ack" : 1 - }, - "ircchcnt" : 1, - "rir2" : "ARIN", - "ps" : [ + "packetPos" : [ 24, 114, 204, @@ -134,8 +99,48 @@ 8713, 8795 ], - "fpd" : 1387554241634, - "ipSrc" : "10.180.156.249" + "protocol" : [ + "irc", + "tcp" + ], + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 1046, + "srcDataBytes" : 114, + "srcGEO" : "US", + "srcIp" : "10.180.156.249", + "srcMac" : [ + "00:13:72:c4:f1:e1" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Dell Inc." + ], + "srcOuiCnt" : 1, + "srcPackets" : 14, + "srcPayload8" : "5553455220787878", + "srcPort" : 45921, + "tcpflags" : { + "ack" : 16, + "dstZero" : 0, + "fin" : 0, + "psh" : 11, + "rst" : 0, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, + "timestamp" : "SET", + "totBytes" : 8945, + "totDataBytes" : 7015, + "totPackets" : 29 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-131220", + "_type" : "session" + } } } ] diff --git a/tests/pcap/kafka.test b/tests/pcap/kafka.test index dab6f16216..3de801a123 100644 --- a/tests/pcap/kafka.test +++ b/tests/pcap/kafka.test @@ -1,90 +1,90 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "firstPacket" : 1483521857249, - "db2" : 0, - "timestamp" : "SET", - "p1" : 65056, - "a1" : "10.10.10.10", - "pr" : 6, - "ps" : [ - 24, - 122, - 216, - 302, - 467 + "dstBytes" : 78, + "dstDataBytes" : 0, + "dstIp" : "10.176.192.13", + "dstMac" : [ + "00:00:5e:00:01:01", + "3c:8a:b0:6e:77:c5" ], - "fpd" : 1483521857249, - "ss" : 1, - "db1" : 79, - "lp" : 1483521857, - "mac1-term-cnt" : 1, - "lpd" : 1483521857532, - "portDst" : 9092, - "vlan" : [ - 300 + "dstMacCnt" : 2, + "dstOui" : [ + "ICANN, IANA Department", + "Juniper Networks" ], - "no" : "test", - "sl" : 283, - "psl" : [ + "dstOuiCnt" : 2, + "dstPackets" : 1, + "dstPort" : 9092, + "fileId" : [], + "firstPacket" : 1483521857249, + "ipProtocol" : 6, + "lastPacket" : 1483521857532, + "length" : 283, + "node" : "test", + "packetLen" : [ 98, 94, 86, 165, 86 ], - "mac2-term" : [ - "00:00:5e:00:01:01", - "3c:8a:b0:6e:77:c5" + "packetPos" : [ + 24, + 122, + 216, + 302, + 467 ], - "prot-term" : [ + "protocol" : [ "kafka", "tcp" ], - "tags-term" : [ - "acked-unseen-segment-src" - ], - "lastPacket" : 1483521857532, - "vlan-cnt" : 1, - "a2" : "10.176.192.13", - "fs" : [], - "portSrc" : 65056, - "mac2-term-cnt" : 2, - "prot-term-cnt" : 2, - "p2" : 9092, - "tacnt" : 1, - "pa2" : 1, - "fb1" : "0000004b00020000", - "mac1-term" : [ + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 371, + "srcDataBytes" : 79, + "srcIp" : "10.10.10.10", + "srcMac" : [ "00:1b:17:00:02:30" ], - "by2" : 78, - "ipSrc" : "10.10.10.10", + "srcMacCnt" : 1, + "srcOui" : [ + "Palo Alto Networks" + ], + "srcOuiCnt" : 1, + "srcPackets" : 4, + "srcPayload8" : "0000004b00020000", + "srcPort" : 65056, + "tags" : [ + "acked-unseen-segment-src" + ], + "tagsCnt" : 1, "tcpflags" : { + "ack" : 1, + "dstZero" : 0, + "fin" : 1, "psh" : 1, "rst" : 0, + "srcZero" : 0, "syn" : 1, - "urg" : 0, "syn-ack" : 1, - "fin" : 1, - "ack" : 1 + "urg" : 0 }, - "pa" : 5, - "ta" : [ - "acked-unseen-segment-src" + "timestamp" : "SET", + "totBytes" : 449, + "totDataBytes" : 79, + "totPackets" : 5, + "vlan" : [ + 300 ], - "by1" : 371, - "ipDst" : "10.176.192.13", - "fp" : 1483521857, - "pa1" : 4, - "db" : 79, - "by" : 449 + "vlanCnt" : 1 }, "header" : { "index" : { - "_type" : "session", - "_index" : "tests_sessions-170104" + "_index" : "tests_sessions2-170104", + "_type" : "session" } } } diff --git a/tests/pcap/krb5-tcp.test b/tests/pcap/krb5-tcp.test index 4b3c282bd6..89ecc88c62 100644 --- a/tests/pcap/krb5-tcp.test +++ b/tests/pcap/krb5-tcp.test @@ -1,95 +1,97 @@ { - "sessions" : [ + "sessions2" : [ { - "header" : { - "index" : { - "_type" : "session", - "_index" : "tests_sessions-160618" - } - }, "body" : { - "sl" : 2, - "by1" : 461, - "db" : 273, + "dstBytes" : 70, + "dstDataBytes" : 0, + "dstIp" : "10.11.11.11", + "dstMac" : [ + "00:22:83:3f:17:cc" + ], + "dstMacCnt" : 1, + "dstOui" : [ + "Juniper Networks" + ], + "dstOuiCnt" : 1, + "dstPackets" : 1, + "dstPort" : 88, + "fileId" : [], + "firstPacket" : 1466265698248, + "ipProtocol" : 6, "krb5" : { - "realm-term" : [ - "xxxxxxxxxxxxxx.xxxxxx.xxx.COM" - ], - "cname-term" : [ + "cname" : [ "xxxxxxxxxxx$" ], - "realm-termcnt" : 1, - "sname-term" : [ + "cnameCnt" : 1, + "realm" : [ + "xxxxxxxxxxxxxx.xxxxxx.xxx.COM" + ], + "realmCnt" : 1, + "sname" : [ "krbtgt/xxxxxxxxxxxxxx.xxxxxx.xxx.COM" ], - "sname-termcnt" : 1, - "cname-termcnt" : 1 + "snameCnt" : 1 }, - "pa1" : 3, - "a2" : "10.11.11.11", - "portDst" : 88, - "ss" : 1, - "psl" : [ + "lastPacket" : 1466265698251, + "length" : 2, + "node" : "test", + "packetLen" : [ 86, 86, 76, 347 ], - "pa" : 4, - "mac2-term" : [ - "00:22:83:3f:17:cc" + "packetPos" : [ + 24, + 110, + 196, + 272 ], - "prot-term" : [ + "protocol" : [ "tcp", "krb5" ], - "timestamp" : "SET", - "no" : "test", + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 461, + "srcDataBytes" : 273, + "srcIp" : "10.10.10.10", + "srcMac" : [ + "00:22:83:3f:17:c5" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Juniper Networks" + ], + "srcOuiCnt" : 1, + "srcPackets" : 3, + "srcPayload8" : "0000010d6a820109", + "srcPort" : 57167, "tcpflags" : { - "syn" : 1, + "ack" : 1, + "dstZero" : 0, "fin" : 0, - "syn-ack" : 1, "psh" : 1, "rst" : 0, - "urg" : 0, - "ack" : 1 + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 }, - "db2" : 0, - "fpd" : 1466265698248, - "p2" : 88, - "by" : 531, - "fb1" : "0000010d6a820109", - "ipSrc" : "10.10.10.10", - "pa2" : 1, + "timestamp" : "SET", + "totBytes" : 531, + "totDataBytes" : 273, + "totPackets" : 4, "vlan" : [ 50 ], - "prot-term-cnt" : 2, - "mac1-term-cnt" : 1, - "a1" : "10.10.10.10", - "portSrc" : 57167, - "mac2-term-cnt" : 1, - "mac1-term" : [ - "00:22:83:3f:17:c5" - ], - "pr" : 6, - "lpd" : 1466265698251, - "firstPacket" : 1466265698248, - "by2" : 70, - "p1" : 57167, - "ps" : [ - 24, - 110, - 196, - 272 - ], - "fp" : 1466265698, - "fs" : [], - "vlan-cnt" : 1, - "ipDst" : "10.11.11.11", - "db1" : 273, - "lastPacket" : 1466265698251, - "lp" : 1466265698 + "vlanCnt" : 1 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-160618", + "_type" : "session" + } } } ] diff --git a/tests/pcap/krb5-udp.test b/tests/pcap/krb5-udp.test index 707540f4f8..3aee1fd227 100644 --- a/tests/pcap/krb5-udp.test +++ b/tests/pcap/krb5-udp.test @@ -1,78 +1,78 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "a2" : "10.150.10.150", - "fp" : 1466252055, - "p2" : 88, - "by" : 297, - "ps" : [ - 24 - ], - "vlan-cnt" : 1, - "lpd" : 1466252055201, - "firstPacket" : 1466252055201, - "a1" : "10.172.10.172", - "fpd" : 1466252055201, - "ss" : 1, - "p1" : 58412, - "pr" : 17, - "prot-term-cnt" : 2, - "prot-term" : [ - "udp", - "krb5" - ], - "fb1" : "6a81f83081f5a103", - "mac2-term" : [ + "dstBytes" : 0, + "dstDataBytes" : 0, + "dstIp" : "10.150.10.150", + "dstMac" : [ "00:00:5e:00:01:01" ], - "mac1-term" : [ - "00:1b:17:00:02:30" + "dstMacCnt" : 1, + "dstOui" : [ + "ICANN, IANA Department" ], - "by1" : 297, + "dstOuiCnt" : 1, + "dstPackets" : 0, + "dstPort" : 88, + "fileId" : [], + "firstPacket" : 1466252055201, + "ipProtocol" : 17, "krb5" : { - "sname-term" : [ - "krbtgt/xx.xxx.xxxxx.NET" - ], - "cname-termcnt" : 1, - "realm-termcnt" : 1, - "cname-term" : [ + "cname" : [ "xxxxxxxx$" ], - "sname-termcnt" : 1, - "realm-term" : [ + "cnameCnt" : 1, + "realm" : [ "xx.xxx.xxxxx.NET" - ] + ], + "realmCnt" : 1, + "sname" : [ + "krbtgt/xx.xxx.xxxxx.NET" + ], + "snameCnt" : 1 }, - "psl" : [ + "lastPacket" : 1466252055201, + "length" : 0, + "node" : "test", + "packetLen" : [ 313 ], - "mac1-term-cnt" : 1, - "lastPacket" : 1466252055201, - "by2" : 0, - "portDst" : 88, - "mac2-term-cnt" : 1, - "pa" : 1, + "packetPos" : [ + 24 + ], + "protocol" : [ + "udp", + "krb5" + ], + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 297, + "srcDataBytes" : 289, + "srcIp" : "10.172.10.172", + "srcMac" : [ + "00:1b:17:00:02:30" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Palo Alto Networks" + ], + "srcOuiCnt" : 1, + "srcPackets" : 1, + "srcPayload8" : "6a81f83081f5a103", + "srcPort" : 58412, "timestamp" : "SET", - "pa1" : 1, - "sl" : 0, - "db1" : 289, - "pa2" : 0, - "ipDst" : "10.150.10.150", - "portSrc" : 58412, + "totBytes" : 297, + "totDataBytes" : 289, + "totPackets" : 1, "vlan" : [ 300 ], - "no" : "test", - "db2" : 0, - "fs" : [], - "lp" : 1466252055, - "db" : 289, - "ipSrc" : "10.172.10.172" + "vlanCnt" : 1 }, "header" : { "index" : { - "_index" : "tests_sessions-160618", + "_index" : "tests_sessions2-160618", "_type" : "session" } } diff --git a/tests/pcap/ldap-and-search.test b/tests/pcap/ldap-and-search.test index 24421d77af..6be79de463 100644 --- a/tests/pcap/ldap-and-search.test +++ b/tests/pcap/ldap-and-search.test @@ -1,21 +1,53 @@ { - "sessions" : [ + "sessions2" : [ { - "header" : { - "index" : { - "_type" : "session", - "_index" : "tests_sessions-051115" - } - }, "body" : { - "mac1-term" : [ + "dstBytes" : 366, + "dstDataBytes" : 28, + "dstIp" : "127.0.0.1", + "dstMac" : [ "00:00:00:00:00:00" ], - "by1" : 610, - "db" : 102, - "ss" : 1, - "ipSrc" : "127.0.0.1", - "ps" : [ + "dstMacCnt" : 1, + "dstOui" : [ + "Officially Xerox, but 0:0:0:0:0:0 is more common" + ], + "dstOuiCnt" : 1, + "dstPackets" : 5, + "dstPayload8" : "300c02010161070a", + "dstPort" : 389, + "fileId" : [], + "firstPacket" : 1132019262677, + "ipProtocol" : 6, + "lastPacket" : 1132019262678, + "ldap" : { + "authtype" : [ + "none" + ], + "authtypeCnt" : 1, + "bindname" : [ + "" + ], + "bindnameCnt" : 1 + }, + "length" : 0, + "node" : "test", + "packetLen" : [ + 90, + 90, + 82, + 96, + 82, + 96, + 82, + 135, + 96, + 89, + 82, + 82, + 82 + ], + "packetPos" : [ 24, 114, 204, @@ -30,76 +62,46 @@ 1044, 1126 ], - "lastPacket" : 1132019262678, - "p2" : 389, - "pa2" : 5, - "prot-term" : [ + "protocol" : [ "ldap", "tcp" ], - "mac1-term-cnt" : 1, - "pa" : 13, - "a1" : "127.0.0.1", - "mac2-term-cnt" : 1, - "fp" : 1132019262, - "fs" : [], - "db1" : 74, - "by2" : 366, + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 610, + "srcDataBytes" : 74, + "srcIp" : "127.0.0.1", + "srcMac" : [ + "00:00:00:00:00:00" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Officially Xerox, but 0:0:0:0:0:0 is more common" + ], + "srcOuiCnt" : 1, + "srcPackets" : 8, + "srcPayload8" : "300c020101600702", + "srcPort" : 40848, "tcpflags" : { - "psh" : 5, - "syn-ack" : 1, - "rst" : 0, "ack" : 4, + "dstZero" : 0, "fin" : 2, + "psh" : 5, + "rst" : 0, + "srcZero" : 0, "syn" : 1, + "syn-ack" : 1, "urg" : 0 }, - "no" : "test", - "pr" : 6, - "sl" : 0, - "fb2" : "300c02010161070a", - "p1" : 40848, - "firstPacket" : 1132019262677, - "by" : 976, "timestamp" : "SET", - "fpd" : 1132019262677, - "fb1" : "300c020101600702", - "psl" : [ - 90, - 90, - 82, - 96, - 82, - 96, - 82, - 135, - 96, - 89, - 82, - 82, - 82 - ], - "lp" : 1132019262, - "lpd" : 1132019262678, - "ipDst" : "127.0.0.1", - "mac2-term" : [ - "00:00:00:00:00:00" - ], - "a2" : "127.0.0.1", - "portDst" : 389, - "pa1" : 8, - "portSrc" : 40848, - "db2" : 28, - "prot-term-cnt" : 2, - "ldap" : { - "authtype-term-cnt" : 1, - "bindname-term-cnt" : 1, - "bindname-term" : [ - "" - ], - "authtype-term" : [ - "none" - ] + "totBytes" : 976, + "totDataBytes" : 102, + "totPackets" : 13 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-051115", + "_type" : "session" } } } diff --git a/tests/pcap/ldap-simpleauth.test b/tests/pcap/ldap-simpleauth.test index d473ea1cd4..344807b165 100644 --- a/tests/pcap/ldap-simpleauth.test +++ b/tests/pcap/ldap-simpleauth.test @@ -1,36 +1,57 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "ss" : 1, - "portDst" : 3268, - "fb2" : "3084000000100201", - "db2" : 188, - "tacnt" : 2, - "as1" : "AS0000 This is neat", - "db1" : 258, - "pa2" : 4, - "lp" : 1463256637, - "fp" : 1463256456, - "portSrc" : 25936, - "fs" : [], - "by1" : 742, - "by" : 1174, - "g1" : "RUS", - "mac1-term" : [ - "00:22:83:3f:17:c5", - "2c:6b:f5:d6:17:c5" + "dstASN" : "AS0001 Cool Beans!", + "dstBytes" : 432, + "dstDataBytes" : 188, + "dstGEO" : "CA", + "dstIp" : "10.0.0.2", + "dstMac" : [ + "00:22:83:3f:17:cc", + "2c:6b:f5:d6:17:cc" ], - "pa1" : 8, - "db" : 446, - "pr" : 6, - "sl" : 181520, - "prot-term" : [ - "ldap", - "tcp" + "dstMacCnt" : 2, + "dstOui" : [ + "Juniper Networks" ], - "ipSrc" : "10.0.0.1", - "ps" : [ + "dstOuiCnt" : 1, + "dstPackets" : 4, + "dstPayload8" : "3084000000100201", + "dstPort" : 3268, + "dstRIR" : "TEST", + "fileId" : [], + "firstPacket" : 1463256456040, + "ipProtocol" : 6, + "lastPacket" : 1463256637560, + "ldap" : { + "authtype" : [ + "simple" + ], + "authtypeCnt" : 1, + "bindname" : [ + "CN=xxxxxxxx,OU=Users,OU=Accounts,DC=xx,DC=xxx,DC=xxxxx,DC=net", + "xxxxxxxxxxx@xx.xxx.xxxxx.net" + ], + "bindnameCnt" : 2 + }, + "length" : 181520, + "node" : "test", + "packetLen" : [ + 86, + 86, + 76, + 126, + 96, + 76, + 187, + 218, + 167, + 96, + 76, + 76 + ], + "packetPos" : [ 24, 110, 196, @@ -44,101 +65,78 @@ 1238, 1314 ], - "mac2-term" : [ - "00:22:83:3f:17:cc", - "2c:6b:f5:d6:17:cc" + "protocol" : [ + "ldap", + "tcp" ], - "mac2-term-cnt" : 2, - "no" : "test", - "mac1-term-cnt" : 2, - "ta" : [ + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcASN" : "AS0000 This is neat", + "srcBytes" : 742, + "srcDataBytes" : 258, + "srcGEO" : "RU", + "srcIp" : "10.0.0.1", + "srcMac" : [ + "00:22:83:3f:17:c5", + "2c:6b:f5:d6:17:c5" + ], + "srcMacCnt" : 2, + "srcOui" : [ + "Juniper Networks" + ], + "srcOuiCnt" : 1, + "srcPackets" : 8, + "srcPayload8" : "3032020101602d02", + "srcPort" : 25936, + "tags" : [ "dstip", "srcip" ], - "vlan-cnt" : 1, + "tagsCnt" : 2, "tcpflags" : { + "ack" : 3, + "dstZero" : 0, "fin" : 0, - "rst" : 1, "psh" : 6, + "rst" : 1, + "srcZero" : 0, "syn" : 1, - "urg" : 0, - "ack" : 3, - "syn-ack" : 1 + "syn-ack" : 1, + "urg" : 0 }, - "a1" : "10.0.0.1", - "rir2" : "TEST", "test" : { - "string" : [ - "16777226:25936,33554442:3268" + "ASN" : [ + "AS0000 This is neat" ], - "ip-rir" : [ - "" + "GEO" : [ + "RU" ], - "number" : [ - 33554442 + "RIR" : [ + "" ], "ip" : [ - 167772161 + "10.0.0.1" ], - "ip-asn" : [ - "AS0000 This is neat" + "number" : [ + 33554442 ], - "ip-geo" : [ - "RUS" + "string.snow" : [ + "16777226:25936,33554442:3268" ] }, - "prot-term-cnt" : 2, - "a2" : "10.0.0.2", - "ipDst" : "10.0.0.2", - "p1" : 25936, "timestamp" : "SET", - "fpd" : 1463256456040, - "by2" : 432, - "as2" : "AS0001 Cool Beans!", - "psl" : [ - 86, - 86, - 76, - 126, - 96, - 76, - 187, - 218, - 167, - 96, - 76, - 76 - ], - "g2" : "CAN", - "tags-term" : [ - "srcip", - "dstip" - ], - "firstPacket" : 1463256456040, - "fb1" : "3032020101602d02", - "lastPacket" : 1463256637560, - "lpd" : 1463256637560, - "p2" : 3268, - "ldap" : { - "bindname-term-cnt" : 2, - "bindname-term" : [ - "CN=xxxxxxxx,OU=Users,OU=Accounts,DC=xx,DC=xxx,DC=xxxxx,DC=net", - "xxxxxxxxxxx@xx.xxx.xxxxx.net" - ], - "authtype-term" : [ - "simple" - ], - "authtype-term-cnt" : 1 - }, - "pa" : 12, + "totBytes" : 1174, + "totDataBytes" : 446, + "totPackets" : 12, "vlan" : [ 50 - ] + ], + "vlanCnt" : 1 }, "header" : { "index" : { - "_type" : "session", - "_index" : "tests_sessions-160514" + "_index" : "tests_sessions2-160514", + "_type" : "session" } } } diff --git a/tests/pcap/ldap-ssl.test b/tests/pcap/ldap-ssl.test index 1f1767a144..5d3c4732b5 100644 --- a/tests/pcap/ldap-ssl.test +++ b/tests/pcap/ldap-ssl.test @@ -1,71 +1,82 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "portSrc" : 37386, - "tls" : [ + "cert" : [ { - "sn" : "008a07e08d4ab50a7b", - "notBefore" : 1422670709, - "notAfter" : 1738030709, - "iCn" : [ + "hash" : "2b:c2:6a:13:99:8b:2c:a0:1e:a4:65:dd:22:fb:fc:92:a0:99:fb:02", + "issuerCN" : [ "ldap ssl test" ], - "hash" : "2b:c2:6a:13:99:8b:2c:a0:1e:a4:65:dd:22:fb:fc:92:a0:99:fb:02", - "diffDays" : 3650, - "sCn" : [ + "notAfter" : 1738030709000, + "notBefore" : 1422670709000, + "serial" : "008a07e08d4ab50a7b", + "subjectCN" : [ "ldap ssl test" - ] + ], + "validDays" : 3650 } ], - "fp" : 1422673532, - "lpd" : 1422673558484, - "fb2" : "300c02010178070a", - "portDst" : 389, - "p2" : 389, - "tlsdstid-term" : [ - "3bacce112097291bccb0e59d56f92396277a9ae4a1b59a9610db211f878bd0fd" + "certCnt" : 1, + "dstBytes" : 3422, + "dstDataBytes" : 2050, + "dstIp" : "0000:0000:0000:0000:0000:0000:0000:0001", + "dstMac" : [ + "00:00:00:00:00:00" ], - "ipSrc" : "0.0.0.0", - "a2" : "0.0.0.1", - "no" : "test", - "tlsver-termcnt" : 1, - "tcpflags" : { - "psh" : 21, - "fin" : 1, - "urg" : 0, - "syn" : 1, - "rst" : 1, - "ack" : 11, - "syn-ack" : 1 - }, - "pa1" : 20, - "fb1" : "301d020101771880", - "db2" : 2050, - "tlscipher-termcnt" : 1, - "firstPacket" : 1422673532664, - "by" : 6801, - "tlscnt" : 1, - "lp" : 1422673558, - "timestamp" : "SET", - "db" : 3701, - "p1" : 37386, - "pr" : 6, - "mac1-term-cnt" : 1, - "tlsja3-term" : [ - "3f8b7a5c41878630f67e660533be80fc" + "dstMacCnt" : 1, + "dstOui" : [ + "Officially Xerox, but 0:0:0:0:0:0 is more common" ], + "dstOuiCnt" : 1, + "dstPackets" : 16, + "dstPayload8" : "300c02010178070a", + "dstPort" : 389, + "fileId" : [], + "firstPacket" : 1422673532664, + "ipProtocol" : 6, "lastPacket" : 1422673558484, - "prot-term-cnt" : 3, - "pa" : 36, - "tlsja3-termcnt" : 1, - "tlscipher-term" : [ - "TLS_RSA_WITH_AES_128_CBC_SHA" + "length" : 25820, + "node" : "test", + "packetLen" : [ + 110, + 110, + 102, + 133, + 102, + 116, + 102, + 219, + 188, + 102, + 631, + 102, + 111, + 102, + 241, + 108, + 235, + 102, + 108, + 283, + 102, + 443, + 331, + 251, + 102, + 171, + 587, + 102, + 619, + 267, + 102, + 171, + 299, + 102, + 331, + 90 ], - "tipv62-term" : "00000000000000000000000000000001", - "ss" : 1, - "sl" : 25820, - "ps" : [ + "packetPos" : [ 24, 134, 244, @@ -103,72 +114,67 @@ 6980, 7311 ], - "pa2" : 16, - "mac2-term" : [ - "00:00:00:00:00:00" - ], - "by2" : 3422, - "by1" : 3379, - "psl" : [ - 110, - 110, - 102, - 133, - 102, - 116, - 102, - 219, - 188, - 102, - 631, - 102, - 111, - 102, - 241, - 108, - 235, - 102, - 108, - 283, - 102, - 443, - 331, - 251, - 102, - 171, - 587, - 102, - 619, - 267, - 102, - 171, - 299, - 102, - 331, - 90 + "protocol" : [ + "tls", + "ldap", + "tcp" ], - "fs" : [], - "mac1-term" : [ + "protocolCnt" : 3, + "segmentCnt" : 1, + "srcBytes" : 3379, + "srcDataBytes" : 1651, + "srcIp" : "0000:0000:0000:0000:0000:0000:0000:0001", + "srcMac" : [ "00:00:00:00:00:00" ], - "a1" : "0.0.0.1", - "tlsver-term" : [ - "TLSv1.2" + "srcMacCnt" : 1, + "srcOui" : [ + "Officially Xerox, but 0:0:0:0:0:0 is more common" ], - "db1" : 1651, - "fpd" : 1422673532664, - "tipv61-term" : "00000000000000000000000000000001", - "ipDst" : "0.0.0.0", - "mac2-term-cnt" : 1, - "prot-term" : [ - "tls", - "ldap", - "tcp" - ] + "srcOuiCnt" : 1, + "srcPackets" : 20, + "srcPayload8" : "301d020101771880", + "srcPort" : 37386, + "tags" : [ + "cert:self-signed" + ], + "tagsCnt" : 1, + "tcpflags" : { + "ack" : 11, + "dstZero" : 0, + "fin" : 1, + "psh" : 21, + "rst" : 1, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, + "timestamp" : "SET", + "tls" : { + "cipher" : [ + "TLS_RSA_WITH_AES_128_CBC_SHA" + ], + "cipherCnt" : 1, + "dstSessionId" : [ + "3bacce112097291bccb0e59d56f92396277a9ae4a1b59a9610db211f878bd0fd" + ], + "ja3" : [ + "3f8b7a5c41878630f67e660533be80fc" + ], + "ja3Cnt" : 1, + "version" : [ + "TLSv1.2" + ], + "versionCnt" : 1 + }, + "totBytes" : 6801, + "totDataBytes" : 3701, + "totPackets" : 36 }, "header" : { "index" : { - "_index" : "tests_sessions-150131", + "_index" : "tests_sessions2-150131", "_type" : "session" } } diff --git a/tests/pcap/long-session.test b/tests/pcap/long-session.test index 856da35d9b..7dcc80d9e7 100644 --- a/tests/pcap/long-session.test +++ b/tests/pcap/long-session.test @@ -1,217 +1,215 @@ { - "sessions" : [ + "sessions2" : [ { - "header" : { - "index" : { - "_type" : "session", - "_index" : "tests_sessions-140529" - } - }, "body" : { - "ta" : [ - "dstip", - "srcip" + "dstASN" : "AS0001 Cool Beans!", + "dstBytes" : 66, + "dstDataBytes" : 0, + "dstGEO" : "CA", + "dstIp" : "10.0.0.2", + "dstMac" : [ + "00:00:0c:07:ac:01", + "00:0e:d6:0b:98:80" ], - "test" : { - "ip-rir" : [ - "" - ], - "ip-geo" : [ - "RUS" - ], - "number" : [ - 33554442 - ], - "ip" : [ - 167772161 - ], - "string" : [ - "16777226:54869,33554442:80" - ], - "ip-asn" : [ - "AS0000 This is neat" - ] - }, - "mac1-term" : [ - "00:13:72:c4:f1:e1" + "dstMacCnt" : 2, + "dstOui" : [ + "Cisco", + "Cisco Systems, Inc" ], - "pa2" : 1, - "pa" : 4, - "rir2" : "TEST", - "sl" : 908493, - "psl" : [ + "dstOuiCnt" : 2, + "dstPackets" : 1, + "dstPort" : 80, + "dstRIR" : "TEST", + "fileId" : [], + "firstPacket" : 1401385380102, + "ipProtocol" : 6, + "lastPacket" : 1401386288595, + "length" : 908493, + "node" : "test", + "packetLen" : [ 90, 82, 70, 75 ], - "firstPacket" : 1401385380102, - "g2" : "CAN", - "timestamp" : "SET", - "mac2-term-cnt" : 2, - "fs" : [], - "mac1-term-cnt" : 1, - "fpd" : 1401385380102, - "db" : 5, - "tags-term" : [ - "srcip", - "dstip" - ], - "db1" : 5, - "fb1" : "4745540d0a", - "no" : "test", - "ipDst" : "10.0.0.2", - "by1" : 187, - "lpd" : 1401386288595, - "by" : 253, - "as1" : "AS0000 This is neat", - "a2" : "10.0.0.2", - "prot-term" : [ + "packetPos" : [ + 24, + 114, + 196, + 266 + ], + "protocol" : [ "http", "tcp" ], - "g1" : "RUS", - "fp" : 1401385380, - "ro" : "SET", - "tacnt" : 2, - "a1" : "10.0.0.1", - "pa1" : 3, - "db2" : 0, - "portSrc" : 54869, + "protocolCnt" : 2, + "rootId" : "SET", + "segmentCnt" : 1, + "srcASN" : "AS0000 This is neat", + "srcBytes" : 187, + "srcDataBytes" : 5, + "srcGEO" : "RU", + "srcIp" : "10.0.0.1", + "srcMac" : [ + "00:13:72:c4:f1:e1" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Dell Inc." + ], + "srcOuiCnt" : 1, + "srcPackets" : 3, + "srcPayload8" : "4745540d0a", + "srcPort" : 54869, + "tags" : [ + "dstip", + "srcip" + ], + "tagsCnt" : 2, "tcpflags" : { + "ack" : 1, + "dstZero" : 0, + "fin" : 0, "psh" : 1, "rst" : 0, - "urg" : 0, - "ack" : 1, + "srcZero" : 0, "syn" : 1, - "fin" : 0, - "syn-ack" : 1 + "syn-ack" : 1, + "urg" : 0 }, - "p1" : 54869, - "p2" : 80, - "pr" : 6, - "as2" : "AS0001 Cool Beans!", - "by2" : 66, - "lastPacket" : 1401386288595, - "prot-term-cnt" : 2, - "lp" : 1401386288, - "ps" : [ - 24, - 114, - 196, - 266 - ], - "portDst" : 80, - "ipSrc" : "10.0.0.1", - "mac2-term" : [ - "00:00:0c:07:ac:01", - "00:0e:d6:0b:98:80" - ], - "ss" : 1 - } - }, - { + "test" : { + "ASN" : [ + "AS0000 This is neat" + ], + "GEO" : [ + "RU" + ], + "RIR" : [ + "" + ], + "ip" : [ + "10.0.0.1" + ], + "number" : [ + 33554442 + ], + "string.snow" : [ + "16777226:54869,33554442:80" + ] + }, + "timestamp" : "SET", + "totBytes" : 253, + "totDataBytes" : 5, + "totPackets" : 4 + }, "header" : { "index" : { - "_type" : "session", - "_index" : "tests_sessions-140529" + "_index" : "tests_sessions2-140529", + "_type" : "session" } - }, + } + }, + { "body" : { - "ps" : [ - 341 - ], - "prot-term-cnt" : 2, - "lp" : 1401386288, - "ss" : 2, - "mac2-term" : [ + "dstASN" : "AS0001 Cool Beans!", + "dstBytes" : 60, + "dstDataBytes" : 0, + "dstGEO" : "CA", + "dstIp" : "10.0.0.2", + "dstMac" : [ "00:00:0c:07:ac:01", "00:0e:d6:0b:98:80" ], - "portDst" : 80, - "ipSrc" : "10.0.0.1", - "tcpflags" : { - "urg" : 0, - "ack" : 0, - "psh" : 0, - "rst" : 1, - "syn-ack" : 0, - "fin" : 0, - "syn" : 0 - }, - "pa1" : 0, - "portSrc" : 54869, - "db2" : 0, + "dstMacCnt" : 2, + "dstOui" : [ + "Cisco", + "Cisco Systems, Inc" + ], + "dstOuiCnt" : 2, + "dstPackets" : 1, + "dstPort" : 80, + "dstRIR" : "TEST", + "fileId" : [], + "firstPacket" : 1401385380102, + "ipProtocol" : 6, "lastPacket" : 1401386288597, - "p1" : 54869, - "p2" : 80, - "as2" : "AS0001 Cool Beans!", - "pr" : 6, - "by2" : 60, - "prot-term" : [ + "length" : 908494, + "node" : "test", + "packetLen" : [ + 76 + ], + "packetPos" : [ + 341 + ], + "protocol" : [ "http", "tcp" ], - "a2" : "10.0.0.2", - "fp" : 1401385380, - "a1" : "10.0.0.1", - "tacnt" : 2, - "ro" : "SET", - "g1" : "RUS", - "fb1" : "4745540d0a", - "db1" : 0, - "tags-term" : [ - "srcip", - "dstip" - ], - "as1" : "AS0000 This is neat", - "lpd" : 1401386288597, - "by" : 60, - "no" : "test", - "ipDst" : "10.0.0.2", - "by1" : 0, - "mac2-term-cnt" : 2, - "fs" : [], - "mac1-term-cnt" : 1, - "db" : 0, - "fpd" : 1401385380102, - "g2" : "CAN", - "timestamp" : "SET", - "firstPacket" : 1401385380102, - "rir2" : "TEST", - "sl" : 908494, - "psl" : [ - 76 + "protocolCnt" : 2, + "rootId" : "SET", + "segmentCnt" : 2, + "srcASN" : "AS0000 This is neat", + "srcBytes" : 0, + "srcDataBytes" : 0, + "srcGEO" : "RU", + "srcIp" : "10.0.0.1", + "srcMac" : [ + "00:13:72:c4:f1:e1" ], - "ta" : [ + "srcMacCnt" : 1, + "srcOui" : [ + "Dell Inc." + ], + "srcOuiCnt" : 1, + "srcPackets" : 0, + "srcPayload8" : "4745540d0a", + "srcPort" : 54869, + "tags" : [ "dstip", "srcip" ], - "mac1-term" : [ - "00:13:72:c4:f1:e1" - ], + "tagsCnt" : 2, + "tcpflags" : { + "ack" : 0, + "dstZero" : 0, + "fin" : 0, + "psh" : 0, + "rst" : 1, + "srcZero" : 0, + "syn" : 0, + "syn-ack" : 0, + "urg" : 0 + }, "test" : { - "ip-rir" : [ - "" + "ASN" : [ + "AS0000 This is neat" ], - "number" : [ - 33554442 + "GEO" : [ + "RU" ], - "ip-geo" : [ - "RUS" + "RIR" : [ + "" ], "ip" : [ - 167772161 + "10.0.0.1" ], - "ip-asn" : [ - "AS0000 This is neat" + "number" : [ + 33554442 ], - "string" : [ + "string.snow" : [ "16777226:54869,33554442:80" ] }, - "pa" : 1, - "pa2" : 1 + "timestamp" : "SET", + "totBytes" : 60, + "totDataBytes" : 0, + "totPackets" : 1 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-140529", + "_type" : "session" + } } } ] diff --git a/tests/pcap/mongo.test b/tests/pcap/mongo.test index d3a5fcac2a..6d25f650ac 100644 --- a/tests/pcap/mongo.test +++ b/tests/pcap/mongo.test @@ -1,75 +1,28 @@ { - "sessions" : [ + "sessions2" : [ { - "header" : { - "index" : { - "_index" : "tests_sessions-170103", - "_type" : "session" - } - }, "body" : { - "ipSrc" : "10.10.10.10", - "ipDst" : "10.10.10.11", - "mac1-term-cnt" : 2, - "fb1" : "f700000098366e4d", - "lp" : 1483459979, - "ss" : 1, - "db2" : 0, - "vlan" : [ - 50, - 300 - ], - "db1" : 247, - "pa2" : 1, - "sl" : 343, - "fpd" : 1483459978959, - "mac2-term" : [ + "dstBytes" : 78, + "dstDataBytes" : 0, + "dstIp" : "10.10.10.11", + "dstMac" : [ "00:22:83:3f:17:c5", "2c:6b:f5:d6:17:c5" ], - "by2" : 78, - "fs" : [], - "pa1" : 5, - "vlan-cnt" : 2, - "fp" : 1483459978, - "by" : 699, - "prot-term-cnt" : 2, - "tcpflags" : { - "urg" : 0, - "syn-ack" : 1, - "ack" : 2, - "rst" : 0, - "psh" : 1, - "fin" : 0, - "syn" : 2 - }, - "p2" : 27017, - "mac1-term" : [ - "00:1b:17:00:02:30", - "2c:6b:f5:d6:17:cc" + "dstMacCnt" : 2, + "dstOui" : [ + "Juniper Networks" ], - "portDst" : 27017, - "lastPacket" : 1483459979301, - "lpd" : 1483459979301, - "pr" : 6, - "ps" : [ - 24, - 122, - 220, - 314, - 400, - 486 - ], - "db" : 247, - "timestamp" : "SET", - "a2" : "10.10.10.11", - "no" : "test", - "a1" : "10.10.10.10", - "p1" : 51822, - "by1" : 621, + "dstOuiCnt" : 1, + "dstPackets" : 1, + "dstPort" : 27017, + "fileId" : [], "firstPacket" : 1483459978959, - "mac2-term-cnt" : 2, - "psl" : [ + "ipProtocol" : 6, + "lastPacket" : 1483459979301, + "length" : 343, + "node" : "test", + "packetLen" : [ 98, 98, 94, @@ -77,66 +30,89 @@ 86, 333 ], - "prot-term" : [ + "packetPos" : [ + 24, + 122, + 220, + 314, + 400, + 486 + ], + "protocol" : [ "mongo", "tcp" ], - "portSrc" : 51822, - "pa" : 6 - } - }, - { - "body" : { + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 621, + "srcDataBytes" : 247, + "srcIp" : "10.10.10.10", + "srcMac" : [ + "00:1b:17:00:02:30", + "2c:6b:f5:d6:17:cc" + ], + "srcMacCnt" : 2, + "srcOui" : [ + "Juniper Networks", + "Palo Alto Networks" + ], + "srcOuiCnt" : 2, + "srcPackets" : 5, + "srcPayload8" : "f700000098366e4d", + "srcPort" : 51822, "tcpflags" : { "ack" : 2, - "urg" : 0, - "syn-ack" : 1, - "rst" : 0, + "dstZero" : 0, "fin" : 0, "psh" : 1, - "syn" : 2 + "rst" : 0, + "srcZero" : 0, + "syn" : 2, + "syn-ack" : 1, + "urg" : 0 }, - "mac1-term" : [ - "3c:8a:b0:6f:27:cc", - "00:1b:17:00:02:30" - ], - "p2" : 27017, - "prot-term-cnt" : 2, - "by" : 511, - "vlan-cnt" : 2, - "by2" : 78, - "fs" : [], - "pa1" : 5, - "fp" : 1483558834, - "fpd" : 1483558834969, - "db1" : 59, - "pa2" : 1, - "sl" : 163, - "mac2-term" : [ - "3c:8a:b0:6f:27:c5", - "00:00:5e:00:01:01", - "3c:8a:b0:6e:77:c5" - ], - "ss" : 1, + "timestamp" : "SET", + "totBytes" : 699, + "totDataBytes" : 247, + "totPackets" : 6, "vlan" : [ 50, 300 ], - "db2" : 0, - "lp" : 1483558835, - "mac1-term-cnt" : 2, - "fb1" : "3b00000000000000", - "ipDst" : "10.10.10.13", - "ipSrc" : "10.10.10.12", - "pa" : 6, - "prot-term" : [ - "mongo", - "tcp" + "vlanCnt" : 2 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-170103", + "_type" : "session" + } + } + }, + { + "body" : { + "dstBytes" : 78, + "dstDataBytes" : 0, + "dstIp" : "10.10.10.13", + "dstMac" : [ + "00:00:5e:00:01:01", + "3c:8a:b0:6e:77:c5", + "3c:8a:b0:6f:27:c5" ], - "portSrc" : 55582, + "dstMacCnt" : 3, + "dstOui" : [ + "ICANN, IANA Department", + "Juniper Networks" + ], + "dstOuiCnt" : 2, + "dstPackets" : 1, + "dstPort" : 27017, + "fileId" : [], "firstPacket" : 1483558834969, - "mac2-term-cnt" : 3, - "psl" : [ + "ipProtocol" : 6, + "lastPacket" : 1483558835131, + "length" : 163, + "node" : "test", + "packetLen" : [ 98, 98, 94, @@ -144,16 +120,7 @@ 86, 145 ], - "p1" : 55582, - "no" : "test", - "a1" : "10.10.10.12", - "by1" : 433, - "timestamp" : "SET", - "a2" : "10.10.10.13", - "db" : 59, - "pr" : 6, - "lpd" : 1483558835131, - "ps" : [ + "packetPos" : [ 819, 917, 1015, @@ -161,232 +128,239 @@ 1195, 1281 ], - "portDst" : 27017, - "lastPacket" : 1483558835131 + "protocol" : [ + "mongo", + "tcp" + ], + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 433, + "srcDataBytes" : 59, + "srcIp" : "10.10.10.12", + "srcMac" : [ + "00:1b:17:00:02:30", + "3c:8a:b0:6f:27:cc" + ], + "srcMacCnt" : 2, + "srcOui" : [ + "Juniper Networks", + "Palo Alto Networks" + ], + "srcOuiCnt" : 2, + "srcPackets" : 5, + "srcPayload8" : "3b00000000000000", + "srcPort" : 55582, + "tcpflags" : { + "ack" : 2, + "dstZero" : 0, + "fin" : 0, + "psh" : 1, + "rst" : 0, + "srcZero" : 0, + "syn" : 2, + "syn-ack" : 1, + "urg" : 0 + }, + "timestamp" : "SET", + "totBytes" : 511, + "totDataBytes" : 59, + "totPackets" : 6, + "vlan" : [ + 50, + 300 + ], + "vlanCnt" : 2 }, "header" : { "index" : { - "_type" : "session", - "_index" : "tests_sessions-170104" + "_index" : "tests_sessions2-170104", + "_type" : "session" } } }, { - "header" : { - "index" : { - "_type" : "session", - "_index" : "tests_sessions-170106" - } - }, "body" : { - "prot-term" : [ - "mongo", - "tcp" + "dstBytes" : 78, + "dstDataBytes" : 0, + "dstIp" : "10.10.10.15", + "dstMac" : [ + "00:1b:17:00:01:24" ], - "portSrc" : 61503, - "pa" : 4, - "a1" : "10.10.10.14", - "p1" : 61503, - "no" : "test", - "by1" : 280, + "dstMacCnt" : 1, + "dstOui" : [ + "Palo Alto Networks" + ], + "dstOuiCnt" : 1, + "dstPackets" : 1, + "dstPort" : 27017, + "fileId" : [], "firstPacket" : 1483726705497, - "mac2-term-cnt" : 1, - "psl" : [ + "ipProtocol" : 6, + "lastPacket" : 1483726705503, + "length" : 6, + "node" : "test", + "packetLen" : [ 98, 94, 86, 144 ], - "db" : 58, - "timestamp" : "SET", - "a2" : "10.10.10.15", - "portDst" : 27017, - "lastPacket" : 1483726705503, - "pr" : 6, - "ps" : [ + "packetPos" : [ 1426, 1524, 1618, 1704 ], - "lpd" : 1483726705503, - "prot-term-cnt" : 2, - "by" : 358, + "protocol" : [ + "mongo", + "tcp" + ], + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 280, + "srcDataBytes" : 58, + "srcIp" : "10.10.10.14", + "srcMac" : [ + "00:22:83:3f:17:c5" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Juniper Networks" + ], + "srcOuiCnt" : 1, + "srcPackets" : 3, + "srcPayload8" : "3a0000004c040000", + "srcPort" : 61503, "tcpflags" : { - "rst" : 0, "ack" : 1, - "syn-ack" : 1, - "urg" : 0, - "syn" : 1, + "dstZero" : 0, "fin" : 0, - "psh" : 1 + "psh" : 1, + "rst" : 0, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 }, - "mac1-term" : [ - "00:22:83:3f:17:c5" - ], - "p2" : 27017, - "fpd" : 1483726705497, - "db1" : 58, - "pa2" : 1, - "sl" : 6, - "mac2-term" : [ - "00:1b:17:00:01:24" - ], - "vlan-cnt" : 1, - "by2" : 78, - "fs" : [], - "pa1" : 3, - "fp" : 1483726705, - "lp" : 1483726705, - "ss" : 1, + "timestamp" : "SET", + "totBytes" : 358, + "totDataBytes" : 58, + "totPackets" : 4, "vlan" : [ 100 ], - "db2" : 0, - "ipDst" : "10.10.10.15", - "ipSrc" : "10.10.10.14", - "mac1-term-cnt" : 1, - "fb1" : "3a0000004c040000" - } - }, - { + "vlanCnt" : 1 + }, "header" : { "index" : { - "_index" : "tests_sessions-170106", + "_index" : "tests_sessions2-170106", "_type" : "session" } - }, + } + }, + { "body" : { - "by" : 569, - "prot-term-cnt" : 2, - "tcpflags" : { - "psh" : 1, - "fin" : 0, - "syn" : 1, - "syn-ack" : 1, - "urg" : 0, - "ack" : 1, - "rst" : 0 - }, - "p2" : 27017, - "mac1-term" : [ - "00:22:83:3f:17:c5", - "2c:6b:f5:d6:17:c5" - ], - "db1" : 269, - "pa2" : 1, - "sl" : 5, - "fpd" : 1483737232974, - "mac2-term" : [ + "dstBytes" : 78, + "dstDataBytes" : 0, + "dstIp" : "10.10.10.17", + "dstMac" : [ "00:1b:17:00:01:24" ], - "by2" : 78, - "fs" : [], - "pa1" : 3, - "vlan-cnt" : 1, - "fp" : 1483737232, - "lp" : 1483737232, - "ss" : 1, - "db2" : 0, - "vlan" : [ - 100 + "dstMacCnt" : 1, + "dstOui" : [ + "Palo Alto Networks" ], - "ipSrc" : "10.10.10.16", - "ipDst" : "10.10.10.17", - "mac1-term-cnt" : 2, - "fb1" : "0d01000000000000", - "prot-term" : [ - "mongo", - "tcp" - ], - "portSrc" : 51358, - "pa" : 4, - "a1" : "10.10.10.16", - "p1" : 51358, - "no" : "test", - "by1" : 491, + "dstOuiCnt" : 1, + "dstPackets" : 1, + "dstPort" : 27017, + "fileId" : [], "firstPacket" : 1483737232974, - "mac2-term-cnt" : 1, - "psl" : [ + "ipProtocol" : 6, + "lastPacket" : 1483737232979, + "length" : 5, + "node" : "test", + "packetLen" : [ 98, 94, 86, 355 ], - "db" : 269, - "timestamp" : "SET", - "a2" : "10.10.10.17", - "portDst" : 27017, - "lastPacket" : 1483737232979, - "ps" : [ + "packetPos" : [ 1848, 1946, 2040, 2126 ], - "lpd" : 1483737232979, - "pr" : 6 - } - }, - { - "body" : { - "mac1-term-cnt" : 2, - "fb1" : "4900000087910000", - "ipDst" : "10.10.10.19", - "ipSrc" : "10.10.10.18", - "ss" : 1, - "vlan" : [ - 50, - 300 + "protocol" : [ + "mongo", + "tcp" ], - "db2" : 0, - "lp" : 1483814916, - "vlan-cnt" : 2, - "fs" : [], - "by2" : 156, - "pa1" : 5, - "fp" : 1483814916, - "fpd" : 1483814916005, - "db1" : 73, - "pa2" : 2, - "sl" : 103, - "mac2-term" : [ + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 491, + "srcDataBytes" : 269, + "srcIp" : "10.10.10.16", + "srcMac" : [ + "00:22:83:3f:17:c5", "2c:6b:f5:d6:17:c5" ], + "srcMacCnt" : 2, + "srcOui" : [ + "Juniper Networks" + ], + "srcOuiCnt" : 1, + "srcPackets" : 3, + "srcPayload8" : "0d01000000000000", + "srcPort" : 51358, "tcpflags" : { + "ack" : 1, + "dstZero" : 0, "fin" : 0, "psh" : 1, - "syn" : 2, - "ack" : 2, - "syn-ack" : 2, - "urg" : 0, - "rst" : 0 + "rst" : 0, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 }, - "mac1-term" : [ - "00:1b:17:00:02:30", - "2c:6b:f5:d6:17:cc" + "timestamp" : "SET", + "totBytes" : 569, + "totDataBytes" : 269, + "totPackets" : 4, + "vlan" : [ + 100 ], - "p2" : 30000, - "prot-term-cnt" : 2, - "by" : 603, - "lpd" : 1483814916108, - "pr" : 6, - "ps" : [ - 2481, - 2579, - 2677, - 2771, - 2865, - 2951, - 3037 + "vlanCnt" : 1 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-170106", + "_type" : "session" + } + } + }, + { + "body" : { + "dstBytes" : 156, + "dstDataBytes" : 0, + "dstIp" : "10.10.10.19", + "dstMac" : [ + "2c:6b:f5:d6:17:c5" ], - "portDst" : 30000, - "lastPacket" : 1483814916108, - "timestamp" : "SET", - "a2" : "10.10.10.19", - "db" : 73, + "dstMacCnt" : 1, + "dstOui" : [ + "Juniper Networks" + ], + "dstOuiCnt" : 1, + "dstPackets" : 2, + "dstPort" : 30000, + "fileId" : [], "firstPacket" : 1483814916005, - "psl" : [ + "ipProtocol" : 6, + "lastPacket" : 1483814916108, + "length" : 103, + "node" : "test", + "packetLen" : [ 98, 98, 94, @@ -395,21 +369,61 @@ 86, 159 ], - "mac2-term-cnt" : 1, - "no" : "test", - "p1" : 64566, - "a1" : "10.10.10.18", - "by1" : 447, - "pa" : 7, - "prot-term" : [ + "packetPos" : [ + 2481, + 2579, + 2677, + 2771, + 2865, + 2951, + 3037 + ], + "protocol" : [ "mongo", "tcp" ], - "portSrc" : 64566 + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 447, + "srcDataBytes" : 73, + "srcIp" : "10.10.10.18", + "srcMac" : [ + "00:1b:17:00:02:30", + "2c:6b:f5:d6:17:cc" + ], + "srcMacCnt" : 2, + "srcOui" : [ + "Juniper Networks", + "Palo Alto Networks" + ], + "srcOuiCnt" : 2, + "srcPackets" : 5, + "srcPayload8" : "4900000087910000", + "srcPort" : 64566, + "tcpflags" : { + "ack" : 2, + "dstZero" : 0, + "fin" : 0, + "psh" : 1, + "rst" : 0, + "srcZero" : 0, + "syn" : 2, + "syn-ack" : 2, + "urg" : 0 + }, + "timestamp" : "SET", + "totBytes" : 603, + "totDataBytes" : 73, + "totPackets" : 7, + "vlan" : [ + 50, + 300 + ], + "vlanCnt" : 2 }, "header" : { "index" : { - "_index" : "tests_sessions-170107", + "_index" : "tests_sessions2-170107", "_type" : "session" } } diff --git a/tests/pcap/mpls-basic.test b/tests/pcap/mpls-basic.test index c231587301..8b80d07747 100644 --- a/tests/pcap/mpls-basic.test +++ b/tests/pcap/mpls-basic.test @@ -1,42 +1,31 @@ { - "sessions" : [ + "sessions2" : [ { - "header" : { - "index" : { - "_type" : "session", - "_index" : "tests_sessions-000303" - } - }, "body" : { - "prot-term" : [ - "udp" - ], - "mac1-term" : [ - "00:30:96:05:28:38" + "dstBytes" : 0, + "dstDataBytes" : 0, + "dstIp" : "255.255.255.255", + "dstMac" : [ + "ff:ff:ff:ff:ff:ff" ], - "pa" : 6, - "db" : 324, - "lpd" : 952109351633, - "db1" : 324, - "ss" : 1, - "mac2-term-cnt" : 1, - "ipSrc" : "10.1.2.1", + "dstMacCnt" : 1, + "dstPackets" : 0, + "dstPort" : 711, + "fileId" : [], "firstPacket" : 952109329022, - "fpd" : 952109329022, - "db2" : 0, - "ipDst" : "255.255.255.255", - "by" : 372, - "portDst" : 711, + "ipProtocol" : 17, "lastPacket" : 952109351633, - "by2" : 0, - "timestamp" : "SET", - "prot-term-cnt" : 1, - "portSrc" : 711, - "p2" : 711, - "fb1" : "000100100a010001", - "p1" : 711, - "fs" : [], - "ps" : [ + "length" : 22610, + "node" : "test", + "packetLen" : [ + 78, + 78, + 78, + 78, + 78, + 78 + ], + "packetPos" : [ 24, 436, 2002, @@ -44,64 +33,63 @@ 3181, 5412 ], - "pa1" : 6, - "mac2-term" : [ - "ff:ff:ff:ff:ff:ff" + "protocol" : [ + "udp" ], - "psl" : [ - 78, - 78, - 78, - 78, - 78, - 78 + "protocolCnt" : 1, + "segmentCnt" : 1, + "srcBytes" : 372, + "srcDataBytes" : 324, + "srcIp" : "10.1.2.1", + "srcMac" : [ + "00:30:96:05:28:38" ], - "pr" : 17, - "a2" : "255.255.255.255", - "mac1-term-cnt" : 1, - "by1" : 372, - "lp" : 952109351, - "pa2" : 0, - "sl" : 22610, - "no" : "test", - "a1" : "10.1.2.1", - "fp" : 952109329 - } - }, - { + "srcMacCnt" : 1, + "srcOui" : [ + "Cisco Systems, Inc" + ], + "srcOuiCnt" : 1, + "srcPackets" : 6, + "srcPayload8" : "000100100a010001", + "srcPort" : 711, + "timestamp" : "SET", + "totBytes" : 372, + "totDataBytes" : 324, + "totPackets" : 6 + }, "header" : { "index" : { - "_type" : "session", - "_index" : "tests_sessions-000303" + "_index" : "tests_sessions2-000303", + "_type" : "session" } - }, + } + }, + { "body" : { - "mac1-term" : [ - "00:30:96:e6:fc:39" - ], - "pa" : 6, - "db" : 324, - "db1" : 324, - "lpd" : 952109353515, - "prot-term" : [ - "udp" + "dstBytes" : 0, + "dstDataBytes" : 0, + "dstIp" : "255.255.255.255", + "dstMac" : [ + "ff:ff:ff:ff:ff:ff" ], - "mac2-term-cnt" : 1, - "ss" : 1, + "dstMacCnt" : 1, + "dstPackets" : 0, + "dstPort" : 711, + "fileId" : [], "firstPacket" : 952109332196, - "fpd" : 952109332196, - "ipSrc" : "10.1.2.2", - "ipDst" : "255.255.255.255", - "db2" : 0, - "timestamp" : "SET", - "by2" : 0, - "prot-term-cnt" : 1, - "portSrc" : 711, - "by" : 372, - "portDst" : 711, + "ipProtocol" : 17, "lastPacket" : 952109353515, - "fs" : [], - "ps" : [ + "length" : 21318, + "node" : "test", + "packetLen" : [ + 78, + 78, + 78, + 78, + 78, + 78 + ], + "packetPos" : [ 102, 514, 2601, @@ -109,78 +97,80 @@ 5154, 5566 ], - "pa1" : 6, - "p2" : 711, - "fb1" : "000100100a200001", - "p1" : 711, - "psl" : [ - 78, - 78, - 78, - 78, - 78, - 78 + "protocol" : [ + "udp" ], - "pr" : 17, - "a2" : "255.255.255.255", - "mac2-term" : [ - "ff:ff:ff:ff:ff:ff" + "protocolCnt" : 1, + "segmentCnt" : 1, + "srcBytes" : 372, + "srcDataBytes" : 324, + "srcIp" : "10.1.2.2", + "srcMac" : [ + "00:30:96:e6:fc:39" ], - "pa2" : 0, - "sl" : 21318, - "no" : "test", - "a1" : "10.1.2.2", - "fp" : 952109332, - "mac1-term-cnt" : 1, - "lp" : 952109353, - "by1" : 372 - } - }, - { + "srcMacCnt" : 1, + "srcOui" : [ + "Cisco Systems, Inc" + ], + "srcOuiCnt" : 1, + "srcPackets" : 6, + "srcPayload8" : "000100100a200001", + "srcPort" : 711, + "timestamp" : "SET", + "totBytes" : 372, + "totDataBytes" : 324, + "totPackets" : 6 + }, "header" : { "index" : { - "_index" : "tests_sessions-000303", + "_index" : "tests_sessions2-000303", "_type" : "session" } - }, + } + }, + { "body" : { - "prot-term" : [ - "icmp", - "mpls" - ], - "db" : 0, - "lpd" : 952109337204, - "db1" : 0, - "mac1-term" : [ - "00:30:96:05:28:38" + "dstBytes" : 570, + "dstDataBytes" : 0, + "dstIp" : "10.34.0.1", + "dstMac" : [ + "00:30:96:e6:fc:39" ], - "pa" : 10, - "ss" : 1, - "mac2-term-cnt" : 1, - "ipSrc" : "10.1.2.1", - "icmpCode" : [ - 0 + "dstMacCnt" : 1, + "dstOui" : [ + "Cisco Systems, Inc" ], - "fpd" : 952109337199, + "dstOuiCnt" : 1, + "dstPackets" : 5, + "dstPort" : 0, + "fileId" : [], "firstPacket" : 952109337199, - "db2" : 0, - "ipDst" : "10.34.0.1", - "portDst" : 0, + "icmp" : { + "code" : [ + 0 + ], + "type" : [ + 8, + 0 + ] + }, + "ipProtocol" : 1, "lastPacket" : 952109337204, - "by" : 1160, - "portSrc" : 0, - "timestamp" : "SET", - "by2" : 570, - "prot-term-cnt" : 2, - "icmpType" : [ - 8, - 0 + "length" : 4, + "node" : "test", + "packetLen" : [ + 134, + 130, + 134, + 130, + 134, + 130, + 134, + 130, + 134, + 130 ], - "p1" : 0, - "p2" : 0, - "pa1" : 5, - "fs" : [], - "ps" : [ + "packetPos" : [ 682, 816, 946, @@ -192,112 +182,60 @@ 1738, 1872 ], - "mac2-term" : [ - "00:30:96:e6:fc:39" + "protocol" : [ + "icmp", + "mpls" ], - "a2" : "10.34.0.1", - "pr" : 1, - "psl" : [ - 134, - 130, - 134, - 130, - 134, - 130, - 134, - 130, - 134, - 130 + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 590, + "srcDataBytes" : 0, + "srcIp" : "10.1.2.1", + "srcMac" : [ + "00:30:96:05:28:38" ], - "mac1-term-cnt" : 1, - "by1" : 590, - "lp" : 952109337, - "no" : "test", - "a1" : "10.1.2.1", - "fp" : 952109337, - "pa2" : 5, - "sl" : 4 - } - }, - { + "srcMacCnt" : 1, + "srcOui" : [ + "Cisco Systems, Inc" + ], + "srcOuiCnt" : 1, + "srcPackets" : 5, + "srcPort" : 0, + "timestamp" : "SET", + "totBytes" : 1160, + "totDataBytes" : 0, + "totPackets" : 10 + }, "header" : { "index" : { - "_type" : "session", - "_index" : "tests_sessions-000303" + "_index" : "tests_sessions2-000303", + "_type" : "session" } - }, + } + }, + { "body" : { - "mac2-term-cnt" : 1, - "ss" : 1, - "pa" : 19, - "mac1-term" : [ - "00:30:96:05:28:38" + "dstBytes" : 517, + "dstDataBytes" : 49, + "dstIp" : "10.34.0.1", + "dstMac" : [ + "00:30:96:e6:fc:39" ], - "db" : 75, - "lpd" : 952109348977, - "db1" : 26, - "tcpflags" : { - "psh" : 9, - "rst" : 3, - "urg" : 0, - "syn-ack" : 1, - "fin" : 2, - "syn" : 1, - "ack" : 5 - }, - "prot-term" : [ - "tcp", - "mpls" + "dstMacCnt" : 1, + "dstOui" : [ + "Cisco Systems, Inc" ], - "ipDst" : "10.34.0.1", - "db2" : 49, + "dstOuiCnt" : 1, + "dstPackets" : 8, + "dstPayload8" : "fffb01fffb03fffd", + "dstPort" : 23, + "fileId" : [], "firstPacket" : 952109346874, - "fpd" : 952109346874, - "ipSrc" : "10.1.2.1", - "fb2" : "fffb01fffb03fffd", - "fs" : [], - "ps" : [ - 3349, - 3427, - 3503, - 3579, - 3662, - 3738, - 3820, - 3897, - 3974, - 4057, - 4164, - 4240, - 4622, - 4698, - 4774, - 4850, - 4926, - 5002, - 5078 - ], - "pa1" : 11, - "p2" : 23, - "fb1" : "fffd03fffb1ffffb", - "p1" : 11001, - "by2" : 517, - "timestamp" : "SET", - "prot-term-cnt" : 2, - "portSrc" : 11001, - "by" : 1195, - "portDst" : 23, + "ipProtocol" : 6, "lastPacket" : 952109348977, - "sl" : 2103, - "pa2" : 8, - "no" : "test", - "a1" : "10.1.2.1", - "fp" : 952109346, - "by1" : 678, - "mac1-term-cnt" : 1, - "lp" : 952109348, - "pr" : 6, - "psl" : [ + "length" : 2103, + "node" : "test", + "packetLen" : [ 78, 76, 76, @@ -318,10 +256,68 @@ 76, 76 ], - "a2" : "10.34.0.1", - "mac2-term" : [ - "00:30:96:e6:fc:39" - ] + "packetPos" : [ + 3349, + 3427, + 3503, + 3579, + 3662, + 3738, + 3820, + 3897, + 3974, + 4057, + 4164, + 4240, + 4622, + 4698, + 4774, + 4850, + 4926, + 5002, + 5078 + ], + "protocol" : [ + "tcp", + "mpls" + ], + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 678, + "srcDataBytes" : 26, + "srcIp" : "10.1.2.1", + "srcMac" : [ + "00:30:96:05:28:38" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Cisco Systems, Inc" + ], + "srcOuiCnt" : 1, + "srcPackets" : 11, + "srcPayload8" : "fffd03fffb1ffffb", + "srcPort" : 11001, + "tcpflags" : { + "ack" : 5, + "dstZero" : 0, + "fin" : 2, + "psh" : 9, + "rst" : 3, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, + "timestamp" : "SET", + "totBytes" : 1195, + "totDataBytes" : 75, + "totPackets" : 19 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-000303", + "_type" : "session" + } } } ] diff --git a/tests/pcap/mysql-allow.test b/tests/pcap/mysql-allow.test index 6dbfa912f1..2729a60569 100644 --- a/tests/pcap/mysql-allow.test +++ b/tests/pcap/mysql-allow.test @@ -1,49 +1,46 @@ { - "sessions" : [ + "sessions2" : [ { - "header" : { - "index" : { - "_index" : "tests_sessions-140423", - "_type" : "session" - } - }, "body" : { - "a1" : "192.168.1.3", - "p1" : 42803, - "a2" : "192.168.1.3", + "dstBytes" : 523, + "dstDataBytes" : 185, + "dstIp" : "192.168.1.3", + "dstMac" : [ + "00:00:00:00:00:00" + ], + "dstMacCnt" : 1, + "dstOui" : [ + "Officially Xerox, but 0:0:0:0:0:0 is more common" + ], + "dstOuiCnt" : 1, + "dstPackets" : 5, + "dstPayload8" : "5b0000000a352e35", + "dstPort" : 3306, + "dstRIR" : "ARIN", + "fileId" : [], + "firstPacket" : 1398217358961, + "ipProtocol" : 6, + "lastPacket" : 1398217359003, + "length" : 43, "mysql" : { - "user-term" : "user10", - "ver-term" : "5.5.35-0ubuntu0.12.04.2" + "user" : "user10", + "version" : "5.5.35-0ubuntu0.12.04.2" }, - "fb2" : "5b0000000a352e35", - "sl" : 43, - "fs" : [], - "prot-term" : [ - "mysql", - "tcp" + "node" : "test", + "packetLen" : [ + 90, + 90, + 82, + 177, + 82, + 168, + 82, + 93, + 119, + 161, + 82 ], - "db1" : 123, - "fpd" : 1398217358961, - "lpd" : 1398217359003, - "by1" : 527, - "mac1-term-cnt" : 1, - "by" : 1050, - "by2" : 523, - "p2" : 3306, - "portSrc" : 42803, - "fp" : 1398217358, - "fb1" : "5200000105a60f00", - "pa" : 11, - "db2" : 185, - "lp" : 1398217359, - "pr" : 6, - "db" : 308, - "firstPacket" : 1398217358961, - "ipDst" : "192.168.1.3", - "ipSrc" : "192.168.1.3", - "timestamp" : "SET", - "mac2-term-cnt" : 1, - "ps" : [ + "packetPos" : [ 24, 114, 204, @@ -56,42 +53,47 @@ 1007, 1168 ], - "rir2" : "ARIN", - "pa2" : 5, - "portDst" : 3306, - "pa1" : 6, - "rir1" : "ARIN", - "lastPacket" : 1398217359003, - "mac2-term" : [ - "00:00:00:00:00:00" + "protocol" : [ + "mysql", + "tcp" ], - "mac1-term" : [ + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 527, + "srcDataBytes" : 123, + "srcIp" : "192.168.1.3", + "srcMac" : [ "00:00:00:00:00:00" ], - "ss" : 1, - "no" : "test", - "prot-term-cnt" : 2, - "psl" : [ - 90, - 90, - 82, - 177, - 82, - 168, - 82, - 93, - 119, - 161, - 82 + "srcMacCnt" : 1, + "srcOui" : [ + "Officially Xerox, but 0:0:0:0:0:0 is more common" ], + "srcOuiCnt" : 1, + "srcPackets" : 6, + "srcPayload8" : "5200000105a60f00", + "srcPort" : 42803, + "srcRIR" : "ARIN", "tcpflags" : { - "rst" : 0, - "syn-ack" : 1, - "urg" : 0, - "syn" : 1, "ack" : 4, + "dstZero" : 0, + "fin" : 0, "psh" : 5, - "fin" : 0 + "rst" : 0, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, + "timestamp" : "SET", + "totBytes" : 1050, + "totDataBytes" : 308, + "totPackets" : 11 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-140423", + "_type" : "session" } } } diff --git a/tests/pcap/mysql-deny.test b/tests/pcap/mysql-deny.test index 5dacb49786..6df7a6d1b9 100644 --- a/tests/pcap/mysql-deny.test +++ b/tests/pcap/mysql-deny.test @@ -1,30 +1,33 @@ { - "sessions" : [ + "sessions2" : [ { - "header" : { - "index" : { - "_type" : "session", - "_index" : "tests_sessions-140422" - } - }, "body" : { - "a1" : "192.168.1.3", - "mysql" : { - "ver-term" : "5.5.35-0FUNntu0.12.04.2", - "user-term" : "user0" - }, - "rir2" : "ARIN", - "pa2" : 5, - "mac2-term-cnt" : 1, - "mac1-term-cnt" : 1, - "mac1-term" : [ + "dstBytes" : 513, + "dstDataBytes" : 175, + "dstIp" : "192.168.1.3", + "dstMac" : [ "00:00:00:00:00:00" ], - "fp" : 1398195861, - "a2" : "192.168.1.3", + "dstMacCnt" : 1, + "dstOui" : [ + "Officially Xerox, but 0:0:0:0:0:0 is more common" + ], + "dstOuiCnt" : 1, + "dstPackets" : 5, + "dstPayload8" : "5b0000000a352e35", + "dstPort" : 3306, + "dstRIR" : "ARIN", + "fileId" : [], + "firstPacket" : 1398195861493, + "ipProtocol" : 6, "lastPacket" : 1398195861496, - "pa" : 10, - "psl" : [ + "length" : 2, + "mysql" : { + "user" : "user0", + "version" : "5.5.35-0FUNntu0.12.04.2" + }, + "node" : "test", + "packetLen" : [ 90, 90, 82, @@ -36,31 +39,7 @@ 82, 82 ], - "timestamp" : "SET", - "portDst" : 3306, - "prot-term-cnt" : 2, - "rir1" : "ARIN", - "by" : 916, - "ss" : 1, - "sl" : 2, - "pr" : 6, - "ipDst" : "192.168.1.3", - "p2" : 3306, - "lp" : 1398195861, - "p1" : 42418, - "fs" : [], - "by2" : 513, - "portSrc" : 42418, - "tcpflags" : { - "rst" : 0, - "ack" : 3, - "urg" : 0, - "syn" : 1, - "syn-ack" : 1, - "psh" : 3, - "fin" : 2 - }, - "ps" : [ + "packetPos" : [ 24, 114, 204, @@ -72,25 +51,48 @@ 936, 1018 ], - "fpd" : 1398195861493, - "prot-term" : [ + "protocol" : [ "mysql", "tcp" ], - "mac2-term" : [ + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 403, + "srcDataBytes" : 65, + "srcIp" : "192.168.1.3", + "srcMac" : [ "00:00:00:00:00:00" ], - "fb1" : "3d00000105a60f00", - "no" : "test", - "by1" : 403, - "firstPacket" : 1398195861493, - "db1" : 65, - "lpd" : 1398195861496, - "fb2" : "5b0000000a352e35", - "db" : 240, - "db2" : 175, - "ipSrc" : "192.168.1.3", - "pa1" : 5 + "srcMacCnt" : 1, + "srcOui" : [ + "Officially Xerox, but 0:0:0:0:0:0 is more common" + ], + "srcOuiCnt" : 1, + "srcPackets" : 5, + "srcPayload8" : "3d00000105a60f00", + "srcPort" : 42418, + "srcRIR" : "ARIN", + "tcpflags" : { + "ack" : 3, + "dstZero" : 0, + "fin" : 2, + "psh" : 3, + "rst" : 0, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, + "timestamp" : "SET", + "totBytes" : 916, + "totDataBytes" : 240, + "totPackets" : 10 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-140422", + "_type" : "session" + } } } ] diff --git a/tests/pcap/mysql-tls.test b/tests/pcap/mysql-tls.test index d2a88650e2..2a4d640630 100644 --- a/tests/pcap/mysql-tls.test +++ b/tests/pcap/mysql-tls.test @@ -1,19 +1,48 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "lpd" : 1482552218503, - "vlan-cnt" : 1, - "a1" : "10.10.10.10", - "sl" : 4, - "portSrc" : 41324, - "mac1-term-cnt" : 2, - "tlscipher-termcnt" : 1, - "mac1-term" : [ - "00:22:83:3f:17:c5", - "2c:6b:f5:d6:17:c5" + "cert" : [ + { + "hash" : "0f:74:24:1f:1a:ad:c5:74:02:b5:97:16:7c:17:7d:f9:d9:d3:45:d8", + "issuerCN" : [ + "mysql_server_5.7.12_auto_generated_ca_certificate" + ], + "notAfter" : 1779463436000, + "notBefore" : 1464103436000, + "serial" : "02", + "subjectCN" : [ + "mysql_server_5.7.12_auto_generated_server_certificate" + ], + "validDays" : 3650 + } + ], + "certCnt" : 1, + "dstBytes" : 1309, + "dstDataBytes" : 951, + "dstIp" : "10.10.10.11", + "dstMac" : [ + "00:22:83:3f:17:cc", + "2c:6b:f5:d6:17:cc" ], - "psl" : [ + "dstMacCnt" : 2, + "dstOui" : [ + "Juniper Networks" + ], + "dstOuiCnt" : 1, + "dstPackets" : 5, + "dstPayload8" : "4a0000000a352e37", + "dstPort" : 3306, + "fileId" : [], + "firstPacket" : 1482552218498, + "ipProtocol" : 6, + "lastPacket" : 1482552218503, + "length" : 4, + "mysql" : { + "version" : "5.7.16" + }, + "node" : "test", + "packetLen" : [ 94, 94, 86, @@ -25,17 +54,7 @@ 86, 959 ], - "fb2" : "4a0000000a352e37", - "fs" : [], - "fpd" : 1482552218498, - "p1" : 41324, - "mac2-term" : [ - "00:22:83:3f:17:cc", - "2c:6b:f5:d6:17:cc" - ], - "tlsja3-termcnt" : 1, - "fb1" : "200000018faa3a00", - "ps" : [ + "packetPos" : [ 24, 118, 212, @@ -47,83 +66,68 @@ 998, 1084 ], - "prot-term-cnt" : 3, - "mysql" : { - "ver-term" : "5.7.16" - }, - "timestamp" : "SET", - "pa" : 10, - "portDst" : 3306, - "vlan" : [ - 50 - ], - "by" : 1859, - "prot-term" : [ + "protocol" : [ "tls", "mysql", "tcp" ], - "ss" : 1, - "tls" : [ - { - "diffDays" : 3650, - "iCn" : [ - "mysql_server_5.7.12_auto_generated_ca_certificate" - ], - "hash" : "0f:74:24:1f:1a:ad:c5:74:02:b5:97:16:7c:17:7d:f9:d9:d3:45:d8", - "sCn" : [ - "mysql_server_5.7.12_auto_generated_server_certificate" - ], - "notAfter" : 1779463436, - "sn" : "02", - "notBefore" : 1464103436 - } + "protocolCnt" : 3, + "segmentCnt" : 1, + "srcBytes" : 550, + "srcDataBytes" : 192, + "srcIp" : "10.10.10.10", + "srcMac" : [ + "00:22:83:3f:17:c5", + "2c:6b:f5:d6:17:c5" ], - "pr" : 6, - "tlscipher-term" : [ - "TLS_RSA_WITH_AES_256_CBC_SHA" + "srcMacCnt" : 2, + "srcOui" : [ + "Juniper Networks" ], + "srcOuiCnt" : 1, + "srcPackets" : 5, + "srcPayload8" : "200000018faa3a00", + "srcPort" : 41324, "tcpflags" : { + "ack" : 4, + "dstZero" : 0, "fin" : 0, "psh" : 4, - "syn" : 1, - "urg" : 0, "rst" : 0, + "srcZero" : 0, + "syn" : 1, "syn-ack" : 1, - "ack" : 4 + "urg" : 0 }, - "db1" : 192, - "ipDst" : "10.10.10.11", - "fp" : 1482552218, - "firstPacket" : 1482552218498, - "p2" : 3306, - "by2" : 1309, - "lp" : 1482552218, - "a2" : "10.10.10.11", - "pa1" : 5, - "tlscnt" : 1, - "mac2-term-cnt" : 2, - "by1" : 550, - "tlsja3-term" : [ - "3509fd9d38bde48d8abc69cd321c0ba0" - ], - "tlsver-term" : [ - "TLSv1.1" - ], - "lastPacket" : 1482552218503, - "db" : 1143, - "no" : "test", - "ipSrc" : "10.10.10.10", - "tlsver-termcnt" : 1, - "pa2" : 5, - "tlsdstid-term" : [ - "6ecd687d8a5b4d46007ae3294abff81da76c864b02e63e5cf78d70012df8629c" + "timestamp" : "SET", + "tls" : { + "cipher" : [ + "TLS_RSA_WITH_AES_256_CBC_SHA" + ], + "cipherCnt" : 1, + "dstSessionId" : [ + "6ecd687d8a5b4d46007ae3294abff81da76c864b02e63e5cf78d70012df8629c" + ], + "ja3" : [ + "3509fd9d38bde48d8abc69cd321c0ba0" + ], + "ja3Cnt" : 1, + "version" : [ + "TLSv1.1" + ], + "versionCnt" : 1 + }, + "totBytes" : 1859, + "totDataBytes" : 1143, + "totPackets" : 10, + "vlan" : [ + 50 ], - "db2" : 951 + "vlanCnt" : 1 }, "header" : { "index" : { - "_index" : "tests_sessions-161224", + "_index" : "tests_sessions2-161224", "_type" : "session" } } diff --git a/tests/pcap/nflog.test b/tests/pcap/nflog.test index 114dcaf0d3..64d533a1aa 100644 --- a/tests/pcap/nflog.test +++ b/tests/pcap/nflog.test @@ -1,88 +1,90 @@ { - "sessions" : [ + "sessions2" : [ { - "header" : { - "index" : { - "_type" : "session", - "_index" : "tests_sessions-170315" - } - }, "body" : { - "timestamp" : "SET", - "rir2" : "ARIN", - "no" : "test", - "ss" : 1, - "hsvercnt" : 1, - "db" : 1339, - "hh1cnt" : 3, - "us" : [ - "//www.aol.com/" - ], - "by1" : 764, - "hh2" : [ - "http:header:cache-control", - "http:header:connection", - "http:header:content-length", - "http:header:content-type", - "http:header:date", - "http:header:location", - "http:header:request-id", - "http:header:server", - "http:header:set-cookie", - "http:header:vary", - "http:header:x-aol-hn", - "http:header:x-content-type-options", - "http:header:x-xss-protection" - ], - "lp" : 1489603870, - "ipSrc" : "10.10.10.10", - "fb2" : "485454502f312e31", - "hh2cnt" : 13, - "hdvercnt" : 1, - "ps" : [ - 216, - 356, - 552, - 684, - 988, - 1176, - 2532, - 2664, - 2796, - 2984 - ], - "pr" : 6, - "prot-term-cnt" : 2, - "a2" : "52.43.228.156", - "ho" : [ - "www.aol.com" - ], - "hmd5cnt" : 1, - "pa" : 10, - "ipDst" : "52.43.228.156", - "p2" : 80, - "db2" : 1167, - "uacnt" : 1, - "fp" : 1489603870, - "prot-term" : [ - "http", - "tcp" - ], - "hsver" : [ - "1.1" - ], - "fs" : [], + "dstASN" : "AS16509 Amazon.com, Inc.", + "dstBytes" : 1800, + "dstDataBytes" : 1167, + "dstGEO" : "US", + "dstIp" : "52.43.228.156", + "dstPackets" : 4, + "dstPayload8" : "485454502f312e31", + "dstPort" : 80, + "dstRIR" : "ARIN", + "fileId" : [], + "firstPacket" : 1489603870937, "http" : { - "method-term" : [ + "clientVersion" : [ + "1.1" + ], + "clientVersionCnt" : 1, + "host" : [ + "www.aol.com" + ], + "hostCnt" : 1, + "md5" : [ + "cbd66d3fd40db184cf2af858b4184f87" + ], + "md5Cnt" : 1, + "method" : [ "GET" ], + "methodCnt" : 1, + "path" : [ + "/" + ], + "pathCnt" : 1, + "requestHeader" : [ + "accept", + "user-agent", + "host" + ], + "requestHeaderCnt" : 3, + "response-location" : [ + "https://www.aol.com/" + ], + "responseHeader" : [ + "content-type", + "x-aol-hn", + "vary", + "cache-control", + "x-xss-protection", + "request-id", + "content-length", + "date", + "connection", + "server", + "set-cookie", + "x-content-type-options", + "location" + ], + "responseHeaderCnt" : 13, + "serverVersion" : [ + "1.1" + ], + "serverVersionCnt" : 1, + "sha256" : [ + "2ddd74d069d1623fa32ba5dce66e7aaf18e1e8ed09bfab55fd8d96bef61a30f7" + ], + "sha256Cnt" : 1, "statuscode" : [ 301 ], - "statuscode-cnt" : 1, - "method-term-cnt" : 1 + "statuscodeCnt" : 1, + "uri" : [ + "www.aol.com/" + ], + "uriCnt" : 1, + "useragent" : [ + "curl/7.19.7 (x86_64-redhat-linux-gnu) libcurl/7.19.7 NSS/3.21 Basic ECC zlib/1.2.3 libidn/1.18 libssh2/1.4.2" + ], + "useragentCnt" : 1 }, - "psl" : [ + "ipProtocol" : 6, + "lastPacket" : 1489603870937, + "length" : 0, + "node" : "test", + "packetLen" : [ 124, 180, 116, @@ -94,57 +96,51 @@ 172, 100 ], + "packetPos" : [ + 216, + 356, + 552, + 684, + 988, + 1176, + 2532, + 2664, + 2796, + 2984 + ], + "protocol" : [ + "http", + "tcp" + ], + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 764, + "srcDataBytes" : 172, + "srcIp" : "10.10.10.10", + "srcPackets" : 6, + "srcPayload8" : "474554202f204854", + "srcPort" : 54064, "tcpflags" : { - "urg" : 0, + "ack" : 4, + "dstZero" : 0, "fin" : 2, "psh" : 2, - "ack" : 4, + "rst" : 0, + "srcZero" : 0, "syn" : 1, "syn-ack" : 1, - "rst" : 0 - }, - "p1" : 54064, - "portDst" : 80, - "sl" : 0, - "fb1" : "474554202f204854", - "pa2" : 4, - "hpathcnt" : 1, - "hmd5" : [ - "cbd66d3fd40db184cf2af858b4184f87" - ], - "g2" : "USA", - "db1" : 172, - "hh1" : [ - "http:header:accept", - "http:header:host", - "http:header:user-agent" - ], - "ua" : [ - "curl/7.19.7 (x86_64-redhat-linux-gnu) libcurl/7.19.7 NSS/3.21 Basic ECC zlib/1.2.3 libidn/1.18 libssh2/1.4.2" - ], - "portSrc" : 54064, - "hdver" : [ - "1.1" - ], - "lastPacket" : 1489603870937, - "by2" : 1800, - "by" : 2564, - "lpd" : 1489603870937, - "hpath" : [ - "/" - ], - "firstPacket" : 1489603870937, - "hdrs" : { - "hres-location" : [ - "https://www.aol.com/" - ] + "urg" : 0 }, - "as2" : "AS16509 Amazon.com, Inc.", - "pa1" : 6, - "uscnt" : 1, - "fpd" : 1489603870937, - "a1" : "10.10.10.10", - "hocnt" : 1 + "timestamp" : "SET", + "totBytes" : 2564, + "totDataBytes" : 1339, + "totPackets" : 10 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-170315", + "_type" : "session" + } } } ] diff --git a/tests/pcap/no-syn-ack.test b/tests/pcap/no-syn-ack.test index bfcc3c5ab1..cd372c6dba 100644 --- a/tests/pcap/no-syn-ack.test +++ b/tests/pcap/no-syn-ack.test @@ -1,241 +1,238 @@ { - "sessions" : [ + "sessions2" : [ { - "header" : { - "index" : { - "_index" : "tests_sessions-161222", - "_type" : "session" - } - }, "body" : { - "vlan" : [ - 50 + "dstBytes" : 152, + "dstDataBytes" : 82, + "dstIp" : "10.176.176.11", + "dstMac" : [ + "00:22:83:3f:17:c5" ], - "ta" : [ - "acked-unseen-segment-src", - "no-syn-ack" + "dstMacCnt" : 1, + "dstOui" : [ + "Juniper Networks" ], + "dstOuiCnt" : 1, + "dstPackets" : 1, + "dstPayload8" : "4e0000000a352e36", + "dstPort" : 3306, + "fileId" : [], + "firstPacket" : 1482381409512, + "ipProtocol" : 6, + "lastPacket" : 1482381409513, + "length" : 0, "mysql" : { - "ver-term" : "5.6.12-log", - "user-term" : "xxxxxxxxxxxxx" - }, - "tags-term" : [ - "no-syn-ack", - "acked-unseen-segment-src" - ], - "vlan-cnt" : 1, - "ipSrc" : "10.10.10.10", - "mac2-term-cnt" : 1, - "mac1-term-cnt" : 1, - "pa2" : 1, - "tcpflags" : { - "syn" : 1, - "psh" : 2, - "rst" : 0, - "fin" : 0, - "syn-ack" : 0, - "urg" : 0, - "ack" : 2 + "user" : "xxxxxxxxxxxxx", + "version" : "5.6.12-log" }, - "pa" : 5, - "p2" : 3306, - "by2" : 152, - "fb2" : "4e0000000a352e36", - "tacnt" : 2, - "sl" : 0, - "fs" : [], - "a2" : "10.176.176.11", - "portDst" : 3306, - "prot-term" : [ - "mysql", - "tcp" - ], - "db" : 164, - "pa1" : 4, - "pr" : 6, - "mac1-term" : [ - "00:22:83:3f:17:cc" + "node" : "test", + "packetLen" : [ + 94, + 86, + 168, + 86, + 168 ], - "timestamp" : "SET", - "fpd" : 1482381409512, - "no" : "test", - "fb1" : "4e0000018da20200", - "prot-term-cnt" : 2, - "by1" : 370, - "p1" : 40041, - "a1" : "10.10.10.10", - "db2" : 82, - "ps" : [ + "packetPos" : [ 24, 118, 204, 372, 458 ], - "ss" : 1, - "psl" : [ - 94, - 86, - 168, - 86, - 168 + "protocol" : [ + "mysql", + "tcp" ], - "firstPacket" : 1482381409512, - "lpd" : 1482381409513, - "lp" : 1482381409, - "portSrc" : 40041, - "db1" : 82, - "fp" : 1482381409, - "ipDst" : "10.176.176.11", - "by" : 522, - "mac2-term" : [ - "00:22:83:3f:17:c5" + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 370, + "srcDataBytes" : 82, + "srcIp" : "10.10.10.10", + "srcMac" : [ + "00:22:83:3f:17:cc" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Juniper Networks" ], - "lastPacket" : 1482381409513 + "srcOuiCnt" : 1, + "srcPackets" : 4, + "srcPayload8" : "4e0000018da20200", + "srcPort" : 40041, + "tags" : [ + "acked-unseen-segment-src", + "no-syn-ack" + ], + "tagsCnt" : 2, + "tcpflags" : { + "ack" : 2, + "dstZero" : 0, + "fin" : 0, + "psh" : 2, + "rst" : 0, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 0, + "urg" : 0 + }, + "timestamp" : "SET", + "totBytes" : 522, + "totDataBytes" : 164, + "totPackets" : 5, + "vlan" : [ + 50 + ], + "vlanCnt" : 1 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-161222", + "_type" : "session" + } } }, { "body" : { - "db1" : 413, - "hsver" : [ - "1.1" - ], - "portSrc" : 61181, - "lp" : 1483544164, - "hsvercnt" : 1, - "fp" : 1483544164, - "firstPacket" : 1483544164032, - "uscnt" : 1, - "lpd" : 1483544164045, - "hh1cnt" : 7, - "hh2" : [ - "http:header:accept-ranges", - "http:header:access-control-allow-origin", - "http:header:connection", - "http:header:content-length", - "http:header:content-type", - "http:header:date", - "http:header:etag", - "http:header:last-modified", - "http:header:server" - ], - "mac2-term" : [ + "dstBytes" : 1582, + "dstDataBytes" : 1460, + "dstIp" : "10.13.13.13", + "dstMac" : [ "00:1b:17:00:01:24" ], - "lastPacket" : 1483544164045, - "ipDst" : "10.13.13.13", - "hh2cnt" : 9, - "by" : 2183, - "us" : [ - "//xxxx.xxxxxxxx.com/xxxxxx/xxxxxxxxx/xxxxxxxxxx/xxx/xx/xxxxxxx/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.ts" - ], - "no" : "test", - "ua" : [ - "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36" - ], - "prot-term-cnt" : 2, - "fb1" : "474554202f787878", - "pa1" : 3, - "pr" : 6, - "timestamp" : "SET", - "fpd" : 1483544164032, - "mac1-term" : [ - "00:22:83:3f:17:c5", - "2c:6b:f5:d6:17:c5" + "dstMacCnt" : 1, + "dstOui" : [ + "Palo Alto Networks" ], - "a1" : "10.12.12.12", - "db2" : 1460, - "psl" : [ + "dstOuiCnt" : 1, + "dstPackets" : 2, + "dstPayload8" : "485454502f312e31", + "dstPort" : 80, + "fileId" : [], + "firstPacket" : 1483544164032, + "http" : { + "clientVersion" : [ + "1.1" + ], + "clientVersionCnt" : 1, + "host" : [ + "xxxx.xxxxxxxx.com" + ], + "hostCnt" : 1, + "method" : [ + "GET" + ], + "methodCnt" : 1, + "path" : [ + "/xxxxxx/xxxxxxxxx/xxxxxxxxxx/xxx/xx/xxxxxxx/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.ts" + ], + "pathCnt" : 1, + "requestHeader" : [ + "x-requested-with", + "accept", + "user-agent", + "accept-encoding", + "connection", + "host", + "accept-language" + ], + "requestHeaderCnt" : 7, + "responseHeader" : [ + "content-type", + "access-control-allow-origin", + "accept-ranges", + "content-length", + "etag", + "date", + "last-modified", + "connection", + "server" + ], + "responseHeaderCnt" : 9, + "serverVersion" : [ + "1.1" + ], + "serverVersionCnt" : 1, + "statuscode" : [ + 200 + ], + "statuscodeCnt" : 1, + "uri" : [ + "xxxx.xxxxxxxx.com/xxxxxx/xxxxxxxxx/xxxxxxxxxx/xxx/xx/xxxxxxx/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.ts" + ], + "uriCnt" : 1, + "useragent" : [ + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36" + ], + "useragentCnt" : 1 + }, + "ipProtocol" : 6, + "lastPacket" : 1483544164045, + "length" : 13, + "node" : "test", + "packetLen" : [ 86, 76, 487, 80, 1534 ], - "ps" : [ + "packetPos" : [ 626, 712, 788, 1275, 1355 ], - "ss" : 1, - "hpathcnt" : 1, - "hdvercnt" : 1, - "by1" : 601, - "p1" : 61181, - "pa" : 5, - "by2" : 1582, - "http" : { - "method-term" : [ - "GET" - ], - "statuscode-cnt" : 1, - "method-term-cnt" : 1, - "statuscode" : [ - 200 - ] - }, - "p2" : 80, - "uacnt" : 1, - "a2" : "10.13.13.13", - "fs" : [], - "portDst" : 80, - "sl" : 13, - "tacnt" : 1, - "ho" : [ - "xxxx.xxxxxxxx.com" - ], - "db" : 1873, - "prot-term" : [ + "protocol" : [ "http", "tcp" ], - "fb2" : "485454502f312e31", - "hpath" : [ - "/xxxxxx/xxxxxxxxx/xxxxxxxxxx/xxx/xx/xxxxxxx/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.ts" - ], - "tags-term" : [ - "no-syn-ack" + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 601, + "srcDataBytes" : 413, + "srcIp" : "10.12.12.12", + "srcMac" : [ + "00:22:83:3f:17:c5", + "2c:6b:f5:d6:17:c5" ], - "vlan" : [ - 100 + "srcMacCnt" : 2, + "srcOui" : [ + "Juniper Networks" ], - "ta" : [ + "srcOuiCnt" : 1, + "srcPackets" : 3, + "srcPayload8" : "474554202f787878", + "srcPort" : 61181, + "tags" : [ "no-syn-ack" ], - "hh1" : [ - "http:header:accept", - "http:header:accept-encoding", - "http:header:accept-language", - "http:header:connection", - "http:header:host", - "http:header:user-agent", - "http:header:x-requested-with" - ], - "pa2" : 2, - "hocnt" : 1, + "tagsCnt" : 1, "tcpflags" : { - "urg" : 0, "ack" : 3, + "dstZero" : 0, "fin" : 0, - "syn-ack" : 0, - "rst" : 0, "psh" : 1, - "syn" : 1 + "rst" : 0, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 0, + "urg" : 0 }, - "mac1-term-cnt" : 2, - "ipSrc" : "10.12.12.12", - "hdver" : [ - "1.1" + "timestamp" : "SET", + "totBytes" : 2183, + "totDataBytes" : 1873, + "totPackets" : 5, + "vlan" : [ + 100 ], - "vlan-cnt" : 1, - "mac2-term-cnt" : 1 + "vlanCnt" : 1 }, "header" : { "index" : { - "_type" : "session", - "_index" : "tests_sessions-170104" + "_index" : "tests_sessions2-170104", + "_type" : "session" } } } diff --git a/tests/pcap/openssl-ssl3.test b/tests/pcap/openssl-ssl3.test index 2371952905..4ea24bad1e 100644 --- a/tests/pcap/openssl-ssl3.test +++ b/tests/pcap/openssl-ssl3.test @@ -1,129 +1,36 @@ { - "sessions" : [ + "sessions2" : [ { - "header" : { - "index" : { - "_type" : "session", - "_index" : "tests_sessions-141015" - } - }, "body" : { - "tlsja3-termcnt" : 1, - "fpd" : 1413337769995, - "firstPacket" : 1413337769995, - "fb2" : "1603000051020000", - "pa" : 20, - "as2" : "AS15169 Google LLC", - "db1" : 313, - "pa2" : 8, - "by" : 5986, - "ipDst" : "74.125.228.238", - "tlsver-termcnt" : 1, - "p1" : 45680, - "mac1-term-cnt" : 1, - "psl" : [ - 90, - 90, - 82, - 222, - 82, - 1500, - 82, - 1500, - 1342, - 252, - 82, - 82, - 82, - 228, - 153, - 82, - 109, - 82, - 82, - 82 - ], - "tlscipher-termcnt" : 1, - "ss" : 1, - "g2" : "USA", - "no" : "test", - "g1" : "RUS", - "by2" : 4873, - "lpd" : 1413337770679, - "timestamp" : "SET", - "ipSrc" : "10.0.0.1", - "test" : { - "string" : [ - "16777226:45680,-287015606:443" - ], - "number" : [ - 4007951690 - ], - "ip-rir" : [ - "" - ], - "ip-geo" : [ - "RUS" - ], - "ip" : [ - 167772161 - ], - "ip-asn" : [ - "AS0000 This is neat" - ] - }, - "tacnt" : 1, - "fs" : [], - "lp" : 1413337770, - "ta" : [ - "srcip" - ], - "as1" : "AS0000 This is neat", - "a1" : "10.0.0.1", - "sl" : 685, - "tls" : [ + "cert" : [ { - "sn" : "12bbe6", - "diffDays" : 5936, "hash" : "73:59:75:5c:6d:f9:a0:ab:c3:06:0b:ce:36:95:64:c8:ec:45:42:a3", - "iOn" : "Equifax", - "notAfter" : 1534824000, - "notBefore" : 1021953600, - "sOn" : "GeoTrust Inc.", - "sCn" : [ + "issuerON" : "Equifax", + "notAfter" : 1534824000000, + "notBefore" : 1021953600000, + "serial" : "12bbe6", + "subjectCN" : [ "geotrust global ca" - ] + ], + "subjectON" : "GeoTrust Inc.", + "validDays" : 5936 }, { - "sn" : "023a76", "hash" : "bb:dc:e1:3e:9d:53:7a:52:29:91:5c:b1:23:c7:aa:b0:a8:55:e7:98", - "diffDays" : 1366, - "iOn" : "GeoTrust Inc.", - "notAfter" : 1483228799, - "iCn" : [ + "issuerCN" : [ "geotrust global ca" ], - "sOn" : "Google Inc", - "notBefore" : 1365174955, - "sCn" : [ + "issuerON" : "GeoTrust Inc.", + "notAfter" : 1483228799000, + "notBefore" : 1365174955000, + "serial" : "023a76", + "subjectCN" : [ "google internet authority g2" - ] + ], + "subjectON" : "Google Inc", + "validDays" : 1366 }, { - "notAfter" : 1419292800, - "iOn" : "Google Inc", - "iCn" : [ - "google internet authority g2" - ], - "notBefore" : 1411553285, - "sOn" : "Google Inc", - "sCn" : [ - "*.google.com" - ], - "sn" : "7a5b0bd895632f87", - "altcnt" : 49, - "diffDays" : 89, - "hash" : "0e:a3:27:7c:eb:7f:b2:8c:2b:5d:7d:d7:6b:e9:ba:1a:ec:0d:ff:91", "alt" : [ "*.google.com", "*.android.com", @@ -174,19 +81,72 @@ "youtu.be", "youtube.com", "youtubeeducation.com" - ] + ], + "altCnt" : 49, + "hash" : "0e:a3:27:7c:eb:7f:b2:8c:2b:5d:7d:d7:6b:e9:ba:1a:ec:0d:ff:91", + "issuerCN" : [ + "google internet authority g2" + ], + "issuerON" : "Google Inc", + "notAfter" : 1419292800000, + "notBefore" : 1411553285000, + "serial" : "7a5b0bd895632f87", + "subjectCN" : [ + "*.google.com" + ], + "subjectON" : "Google Inc", + "validDays" : 89 } ], - "pr" : 6, - "pa1" : 12, - "tlsja3-term" : [ - "f917ae2520ef9784557b0a3c02489c2c" + "certCnt" : 3, + "dstASN" : "AS15169 Google LLC", + "dstBytes" : 4873, + "dstDataBytes" : 4337, + "dstGEO" : "US", + "dstIp" : "74.125.228.238", + "dstMac" : [ + "00:00:0c:07:ac:01", + "00:0e:d6:0b:98:80" ], - "rir2" : "ARIN", - "portDst" : 443, - "fb1" : "1603000087010000", - "p2" : 443, - "ps" : [ + "dstMacCnt" : 2, + "dstOui" : [ + "Cisco", + "Cisco Systems, Inc" + ], + "dstOuiCnt" : 2, + "dstPackets" : 8, + "dstPayload8" : "1603000051020000", + "dstPort" : 443, + "dstRIR" : "ARIN", + "fileId" : [], + "firstPacket" : 1413337769995, + "ipProtocol" : 6, + "lastPacket" : 1413337770679, + "length" : 685, + "node" : "test", + "packetLen" : [ + 90, + 90, + 82, + 222, + 82, + 1500, + 82, + 1500, + 1342, + 252, + 82, + 82, + 82, + 228, + 153, + 82, + 109, + 82, + 82, + 82 + ], + "packetPos" : [ 24, 114, 204, @@ -208,48 +168,90 @@ 6166, 6248 ], - "by1" : 1113, - "db2" : 4337, - "portSrc" : 45680, - "prot-term-cnt" : 2, - "fp" : 1413337769, - "a2" : "74.125.228.238", - "mac1-term" : [ - "00:13:72:c4:f1:e1" - ], - "tcpflags" : { - "psh" : 6, - "rst" : 0, - "fin" : 2, - "syn-ack" : 1, - "urg" : 0, - "syn" : 1, - "ack" : 10 - }, - "mac2-term-cnt" : 2, - "tlscnt" : 3, - "prot-term" : [ + "protocol" : [ "tls", "tcp" ], - "tlsver-term" : [ - "SSLv3" + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcASN" : "AS0000 This is neat", + "srcBytes" : 1113, + "srcDataBytes" : 313, + "srcGEO" : "RU", + "srcIp" : "10.0.0.1", + "srcMac" : [ + "00:13:72:c4:f1:e1" ], - "lastPacket" : 1413337770679, - "mac2-term" : [ - "00:00:0c:07:ac:01", - "00:0e:d6:0b:98:80" + "srcMacCnt" : 1, + "srcOui" : [ + "Dell Inc." ], - "tags-term" : [ + "srcOuiCnt" : 1, + "srcPackets" : 12, + "srcPayload8" : "1603000087010000", + "srcPort" : 45680, + "tags" : [ "srcip" ], - "tlsdstid-term" : [ - "95c3c5ac6f647c1b7214c7bcf4387b6f34819f9f17095c690caafbdbcf6a36ec" - ], - "db" : 4650, - "tlscipher-term" : [ - "TLS_ECDHE_RSA_WITH_RC4_128_SHA" - ] + "tagsCnt" : 1, + "tcpflags" : { + "ack" : 10, + "dstZero" : 0, + "fin" : 2, + "psh" : 6, + "rst" : 0, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, + "test" : { + "ASN" : [ + "AS0000 This is neat" + ], + "GEO" : [ + "RU" + ], + "RIR" : [ + "" + ], + "ip" : [ + "10.0.0.1" + ], + "number" : [ + 4007951690 + ], + "string.snow" : [ + "16777226:45680,-287015606:443" + ] + }, + "timestamp" : "SET", + "tls" : { + "cipher" : [ + "TLS_ECDHE_RSA_WITH_RC4_128_SHA" + ], + "cipherCnt" : 1, + "dstSessionId" : [ + "95c3c5ac6f647c1b7214c7bcf4387b6f34819f9f17095c690caafbdbcf6a36ec" + ], + "ja3" : [ + "f917ae2520ef9784557b0a3c02489c2c" + ], + "ja3Cnt" : 1, + "version" : [ + "SSLv3" + ], + "versionCnt" : 1 + }, + "totBytes" : 5986, + "totDataBytes" : 4650, + "totPackets" : 20 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-141015", + "_type" : "session" + } } } ] diff --git a/tests/pcap/openssl-tls1-tls1_2.test b/tests/pcap/openssl-tls1-tls1_2.test index 45fbb975f1..9a4fdce9ff 100644 --- a/tests/pcap/openssl-tls1-tls1_2.test +++ b/tests/pcap/openssl-tls1-tls1_2.test @@ -1,160 +1,36 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "lastPacket" : 1413338208189, - "g2" : "USA", - "a2" : "74.125.228.103", - "prot-term" : [ - "tls", - "tcp" - ], - "tlsja3-termcnt" : 1, - "a1" : "10.0.0.1", - "pa" : 20, - "tlsver-termcnt" : 1, - "fpd" : 1413338207695, - "mac1-term" : [ - "00:13:72:c4:f1:e1" - ], - "lp" : 1413338208, - "ta" : [ - "srcip" - ], - "test" : { - "ip-rir" : [ - "" - ], - "number" : [ - 1743027530 - ], - "string" : [ - "16777226:42431,1743027530:443" - ], - "ip-asn" : [ - "AS0000 This is neat" - ], - "ip" : [ - 167772161 - ], - "ip-geo" : [ - "RUS" - ] - }, - "firstPacket" : 1413338207695, - "db" : 4898, - "pr" : 6, - "portSrc" : 42431, - "fs" : [], - "pa2" : 8, - "prot-term-cnt" : 2, - "portDst" : 443, - "tlsver-term" : [ - "TLSv1.2" - ], - "timestamp" : "SET", - "no" : "test", - "by1" : 1220, - "p1" : 42431, - "mac1-term-cnt" : 1, - "pa1" : 12, - "tlscipher-term" : [ - "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256" - ], - "tlsja3-term" : [ - "609a9998ac9d232d213aee990ec5162f" - ], - "psl" : [ - 90, - 90, - 82, - 345, - 82, - 1500, - 82, - 1500, - 1342, - 234, - 82, - 82, - 82, - 208, - 312, - 82, - 113, - 82, - 82, - 82 - ], - "ipDst" : "74.125.228.103", - "tags-term" : [ - "srcip" - ], - "rir2" : "ARIN", - "db1" : 420, - "g1" : "RUS", - "ps" : [ - 24, - 114, - 204, - 286, - 631, - 713, - 2213, - 2295, - 3795, - 5137, - 5371, - 5453, - 5535, - 5617, - 5825, - 6137, - 6219, - 6332, - 6414, - 6496 - ], - "by2" : 5014, - "as1" : "AS0000 This is neat", - "sl" : 494, - "tls" : [ + "cert" : [ { - "sCn" : [ + "hash" : "73:59:75:5c:6d:f9:a0:ab:c3:06:0b:ce:36:95:64:c8:ec:45:42:a3", + "issuerON" : "Equifax", + "notAfter" : 1534824000000, + "notBefore" : 1021953600000, + "serial" : "12bbe6", + "subjectCN" : [ "geotrust global ca" ], - "notAfter" : 1534824000, - "iOn" : "Equifax", - "hash" : "73:59:75:5c:6d:f9:a0:ab:c3:06:0b:ce:36:95:64:c8:ec:45:42:a3", - "sn" : "12bbe6", - "diffDays" : 5936, - "notBefore" : 1021953600, - "sOn" : "GeoTrust Inc." + "subjectON" : "GeoTrust Inc.", + "validDays" : 5936 }, { - "sCn" : [ - "google internet authority g2" - ], - "iOn" : "GeoTrust Inc.", - "notAfter" : 1483228799, "hash" : "bb:dc:e1:3e:9d:53:7a:52:29:91:5c:b1:23:c7:aa:b0:a8:55:e7:98", - "sn" : "023a76", - "diffDays" : 1366, - "iCn" : [ + "issuerCN" : [ "geotrust global ca" ], - "notBefore" : 1365174955, - "sOn" : "Google Inc" - }, - { - "sOn" : "Google Inc", - "altcnt" : 49, - "notBefore" : 1411553285, - "diffDays" : 89, - "iCn" : [ + "issuerON" : "GeoTrust Inc.", + "notAfter" : 1483228799000, + "notBefore" : 1365174955000, + "serial" : "023a76", + "subjectCN" : [ "google internet authority g2" ], - "sn" : "7a5b0bd895632f87", + "subjectON" : "Google Inc", + "validDays" : 1366 + }, + { "alt" : [ "*.google.com", "*.android.com", @@ -206,46 +82,172 @@ "youtube.com", "youtubeeducation.com" ], + "altCnt" : 49, "hash" : "0e:a3:27:7c:eb:7f:b2:8c:2b:5d:7d:d7:6b:e9:ba:1a:ec:0d:ff:91", - "iOn" : "Google Inc", - "notAfter" : 1419292800, - "sCn" : [ + "issuerCN" : [ + "google internet authority g2" + ], + "issuerON" : "Google Inc", + "notAfter" : 1419292800000, + "notBefore" : 1411553285000, + "serial" : "7a5b0bd895632f87", + "subjectCN" : [ "*.google.com" - ] + ], + "subjectON" : "Google Inc", + "validDays" : 89 } ], - "fb1" : "1603010102010000", - "mac2-term" : [ + "certCnt" : 3, + "dstASN" : "AS15169 Google LLC", + "dstBytes" : 5014, + "dstDataBytes" : 4478, + "dstGEO" : "US", + "dstIp" : "74.125.228.103", + "dstMac" : [ "00:00:0c:07:ac:01", "00:d0:2b:d1:76:00" ], - "as2" : "AS15169 Google LLC", - "tacnt" : 1, - "tlscnt" : 3, - "tlscipher-termcnt" : 1, - "fb2" : "160303003d020000", - "ipSrc" : "10.0.0.1", - "mac2-term-cnt" : 2, - "by" : 6234, - "fp" : 1413338207, - "ss" : 1, - "lpd" : 1413338208189, + "dstMacCnt" : 2, + "dstOui" : [ + "Cisco", + "Jetcell, Inc." + ], + "dstOuiCnt" : 2, + "dstPackets" : 8, + "dstPayload8" : "160303003d020000", + "dstPort" : 443, + "dstRIR" : "ARIN", + "fileId" : [], + "firstPacket" : 1413338207695, + "ipProtocol" : 6, + "lastPacket" : 1413338208189, + "length" : 494, + "node" : "test", + "packetLen" : [ + 90, + 90, + 82, + 345, + 82, + 1500, + 82, + 1500, + 1342, + 234, + 82, + 82, + 82, + 208, + 312, + 82, + 113, + 82, + 82, + 82 + ], + "packetPos" : [ + 24, + 114, + 204, + 286, + 631, + 713, + 2213, + 2295, + 3795, + 5137, + 5371, + 5453, + 5535, + 5617, + 5825, + 6137, + 6219, + 6332, + 6414, + 6496 + ], + "protocol" : [ + "tls", + "tcp" + ], + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcASN" : "AS0000 This is neat", + "srcBytes" : 1220, + "srcDataBytes" : 420, + "srcGEO" : "RU", + "srcIp" : "10.0.0.1", + "srcMac" : [ + "00:13:72:c4:f1:e1" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Dell Inc." + ], + "srcOuiCnt" : 1, + "srcPackets" : 12, + "srcPayload8" : "1603010102010000", + "srcPort" : 42431, + "tags" : [ + "srcip" + ], + "tagsCnt" : 1, "tcpflags" : { "ack" : 10, - "syn" : 1, - "rst" : 0, - "urg" : 0, + "dstZero" : 0, "fin" : 2, + "psh" : 6, + "rst" : 0, + "srcZero" : 0, + "syn" : 1, "syn-ack" : 1, - "psh" : 6 + "urg" : 0 + }, + "test" : { + "ASN" : [ + "AS0000 This is neat" + ], + "GEO" : [ + "RU" + ], + "RIR" : [ + "" + ], + "ip" : [ + "10.0.0.1" + ], + "number" : [ + 1743027530 + ], + "string.snow" : [ + "16777226:42431,1743027530:443" + ] + }, + "timestamp" : "SET", + "tls" : { + "cipher" : [ + "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256" + ], + "cipherCnt" : 1, + "ja3" : [ + "609a9998ac9d232d213aee990ec5162f" + ], + "ja3Cnt" : 1, + "version" : [ + "TLSv1.2" + ], + "versionCnt" : 1 }, - "db2" : 4478, - "p2" : 443 + "totBytes" : 6234, + "totDataBytes" : 4898, + "totPackets" : 20 }, "header" : { "index" : { - "_type" : "session", - "_index" : "tests_sessions-141015" + "_index" : "tests_sessions2-141015", + "_type" : "session" } } } diff --git a/tests/pcap/openssl-tls1.test b/tests/pcap/openssl-tls1.test index bd11f6e7c7..74ba09f180 100644 --- a/tests/pcap/openssl-tls1.test +++ b/tests/pcap/openssl-tls1.test @@ -1,131 +1,36 @@ { - "sessions" : [ + "sessions2" : [ { - "header" : { - "index" : { - "_type" : "session", - "_index" : "tests_sessions-141015" - } - }, "body" : { - "as1" : "AS0000 This is neat", - "portDst" : 443, - "mac1-term-cnt" : 1, - "mac2-term" : [ - "00:00:0c:07:ac:01", - "00:0e:d6:0b:98:80" - ], - "tlscnt" : 3, - "fp" : 1413337821, - "fs" : [], - "portSrc" : 40111, - "tcpflags" : { - "rst" : 0, - "psh" : 6, - "syn" : 1, - "urg" : 0, - "syn-ack" : 1, - "ack" : 10, - "fin" : 2 - }, - "tlsja3-termcnt" : 1, - "pa2" : 8, - "ss" : 1, - "ipSrc" : "10.0.0.1", - "mac2-term-cnt" : 2, - "mac1-term" : [ - "00:13:72:c4:f1:e1" - ], - "g2" : "USA", - "rir2" : "ARIN", - "tags-term" : [ - "srcip" - ], - "sl" : 1138, - "prot-term-cnt" : 2, - "tlscipher-term" : [ - "TLS_ECDHE_RSA_WITH_RC4_128_SHA" - ], - "by1" : 1118, - "pr" : 6, - "by2" : 5008, - "fb2" : "160301003d020000", - "ipDst" : "74.125.228.39", - "pa1" : 12, - "tlsver-termcnt" : 1, - "a2" : "74.125.228.39", - "ta" : [ - "srcip" - ], - "as2" : "AS15169 Google LLC", - "by" : 6126, - "lastPacket" : 1413337822763, - "p2" : 443, - "db" : 4790, - "tlsja3-term" : [ - "ec20ec16b7e120c02817c84f825ca65d" - ], - "no" : "test", - "psl" : [ - 90, - 90, - 82, - 251, - 82, - 1500, - 1500, - 82, - 82, - 1342, - 232, - 82, - 82, - 204, - 308, - 82, - 109, - 82, - 82, - 82 - ], - "tacnt" : 1, - "firstPacket" : 1413337821624, - "tls" : [ + "cert" : [ { - "iOn" : "Equifax", - "sOn" : "GeoTrust Inc.", - "diffDays" : 5936, - "sCn" : [ + "hash" : "73:59:75:5c:6d:f9:a0:ab:c3:06:0b:ce:36:95:64:c8:ec:45:42:a3", + "issuerON" : "Equifax", + "notAfter" : 1534824000000, + "notBefore" : 1021953600000, + "serial" : "12bbe6", + "subjectCN" : [ "geotrust global ca" ], - "sn" : "12bbe6", - "hash" : "73:59:75:5c:6d:f9:a0:ab:c3:06:0b:ce:36:95:64:c8:ec:45:42:a3", - "notAfter" : 1534824000, - "notBefore" : 1021953600 + "subjectON" : "GeoTrust Inc.", + "validDays" : 5936 }, { - "diffDays" : 1366, - "iOn" : "GeoTrust Inc.", - "sOn" : "Google Inc", - "sCn" : [ - "google internet authority g2" - ], - "sn" : "023a76", "hash" : "bb:dc:e1:3e:9d:53:7a:52:29:91:5c:b1:23:c7:aa:b0:a8:55:e7:98", - "notBefore" : 1365174955, - "iCn" : [ + "issuerCN" : [ "geotrust global ca" ], - "notAfter" : 1483228799 + "issuerON" : "GeoTrust Inc.", + "notAfter" : 1483228799000, + "notBefore" : 1365174955000, + "serial" : "023a76", + "subjectCN" : [ + "google internet authority g2" + ], + "subjectON" : "Google Inc", + "validDays" : 1366 }, { - "sn" : "7a5b0bd895632f87", - "sCn" : [ - "*.google.com" - ], - "iOn" : "Google Inc", - "sOn" : "Google Inc", - "diffDays" : 89, "alt" : [ "*.google.com", "*.android.com", @@ -177,16 +82,71 @@ "youtube.com", "youtubeeducation.com" ], - "notBefore" : 1411553285, - "iCn" : [ + "altCnt" : 49, + "hash" : "0e:a3:27:7c:eb:7f:b2:8c:2b:5d:7d:d7:6b:e9:ba:1a:ec:0d:ff:91", + "issuerCN" : [ "google internet authority g2" ], - "notAfter" : 1419292800, - "altcnt" : 49, - "hash" : "0e:a3:27:7c:eb:7f:b2:8c:2b:5d:7d:d7:6b:e9:ba:1a:ec:0d:ff:91" + "issuerON" : "Google Inc", + "notAfter" : 1419292800000, + "notBefore" : 1411553285000, + "serial" : "7a5b0bd895632f87", + "subjectCN" : [ + "*.google.com" + ], + "subjectON" : "Google Inc", + "validDays" : 89 } ], - "ps" : [ + "certCnt" : 3, + "dstASN" : "AS15169 Google LLC", + "dstBytes" : 5008, + "dstDataBytes" : 4472, + "dstGEO" : "US", + "dstIp" : "74.125.228.39", + "dstMac" : [ + "00:00:0c:07:ac:01", + "00:0e:d6:0b:98:80" + ], + "dstMacCnt" : 2, + "dstOui" : [ + "Cisco", + "Cisco Systems, Inc" + ], + "dstOuiCnt" : 2, + "dstPackets" : 8, + "dstPayload8" : "160301003d020000", + "dstPort" : 443, + "dstRIR" : "ARIN", + "fileId" : [], + "firstPacket" : 1413337821624, + "ipProtocol" : 6, + "lastPacket" : 1413337822763, + "length" : 1138, + "node" : "test", + "packetLen" : [ + 90, + 90, + 82, + 251, + 82, + 1500, + 1500, + 82, + 82, + 1342, + 232, + 82, + 82, + 204, + 308, + 82, + 109, + 82, + 82, + 82 + ], + "packetPos" : [ 24, 114, 204, @@ -208,45 +168,87 @@ 6306, 6388 ], - "db1" : 318, - "prot-term" : [ + "protocol" : [ "tls", "tcp" ], + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcASN" : "AS0000 This is neat", + "srcBytes" : 1118, + "srcDataBytes" : 318, + "srcGEO" : "RU", + "srcIp" : "10.0.0.1", + "srcMac" : [ + "00:13:72:c4:f1:e1" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Dell Inc." + ], + "srcOuiCnt" : 1, + "srcPackets" : 12, + "srcPayload8" : "16030100a4010000", + "srcPort" : 40111, + "tags" : [ + "srcip" + ], + "tagsCnt" : 1, + "tcpflags" : { + "ack" : 10, + "dstZero" : 0, + "fin" : 2, + "psh" : 6, + "rst" : 0, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, "test" : { - "ip-asn" : [ + "ASN" : [ "AS0000 This is neat" ], - "ip-geo" : [ - "RUS" + "GEO" : [ + "RU" ], - "string" : [ - "16777226:40111,669285706:443" + "RIR" : [ + "" ], "ip" : [ - 167772161 - ], - "ip-rir" : [ - "" + "10.0.0.1" ], "number" : [ 669285706 + ], + "string.snow" : [ + "16777226:40111,669285706:443" ] }, - "fpd" : 1413337821624, - "lp" : 1413337822, - "p1" : 40111, - "pa" : 20, - "lpd" : 1413337822763, - "tlsver-term" : [ - "TLSv1" - ], - "db2" : 4472, - "a1" : "10.0.0.1", - "tlscipher-termcnt" : 1, - "fb1" : "16030100a4010000", - "g1" : "RUS", - "timestamp" : "SET" + "timestamp" : "SET", + "tls" : { + "cipher" : [ + "TLS_ECDHE_RSA_WITH_RC4_128_SHA" + ], + "cipherCnt" : 1, + "ja3" : [ + "ec20ec16b7e120c02817c84f825ca65d" + ], + "ja3Cnt" : 1, + "version" : [ + "TLSv1" + ], + "versionCnt" : 1 + }, + "totBytes" : 6126, + "totDataBytes" : 4790, + "totPackets" : 20 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-141015", + "_type" : "session" + } } } ] diff --git a/tests/pcap/openssl-tls1_1.test b/tests/pcap/openssl-tls1_1.test index fa3fb46ac3..79c3ab5aab 100644 --- a/tests/pcap/openssl-tls1_1.test +++ b/tests/pcap/openssl-tls1_1.test @@ -1,51 +1,36 @@ { - "sessions" : [ + "sessions2" : [ { - "header" : { - "index" : { - "_index" : "tests_sessions-141015", - "_type" : "session" - } - }, "body" : { - "pa1" : 12, - "tls" : [ + "cert" : [ { - "sCn" : [ + "hash" : "73:59:75:5c:6d:f9:a0:ab:c3:06:0b:ce:36:95:64:c8:ec:45:42:a3", + "issuerON" : "Equifax", + "notAfter" : 1534824000000, + "notBefore" : 1021953600000, + "serial" : "12bbe6", + "subjectCN" : [ "geotrust global ca" ], - "notAfter" : 1534824000, - "iOn" : "Equifax", - "hash" : "73:59:75:5c:6d:f9:a0:ab:c3:06:0b:ce:36:95:64:c8:ec:45:42:a3", - "sOn" : "GeoTrust Inc.", - "diffDays" : 5936, - "sn" : "12bbe6", - "notBefore" : 1021953600 + "subjectON" : "GeoTrust Inc.", + "validDays" : 5936 }, { - "sn" : "023a76", - "notBefore" : 1365174955, - "sOn" : "Google Inc", - "diffDays" : 1366, - "notAfter" : 1483228799, - "iCn" : [ + "hash" : "bb:dc:e1:3e:9d:53:7a:52:29:91:5c:b1:23:c7:aa:b0:a8:55:e7:98", + "issuerCN" : [ "geotrust global ca" ], - "iOn" : "GeoTrust Inc.", - "hash" : "bb:dc:e1:3e:9d:53:7a:52:29:91:5c:b1:23:c7:aa:b0:a8:55:e7:98", - "sCn" : [ + "issuerON" : "GeoTrust Inc.", + "notAfter" : 1483228799000, + "notBefore" : 1365174955000, + "serial" : "023a76", + "subjectCN" : [ "google internet authority g2" - ] + ], + "subjectON" : "Google Inc", + "validDays" : 1366 }, { - "sCn" : [ - "*.google.com" - ], - "diffDays" : 89, - "sOn" : "Google Inc", - "hash" : "0e:a3:27:7c:eb:7f:b2:8c:2b:5d:7d:d7:6b:e9:ba:1a:ec:0d:ff:91", - "iOn" : "Google Inc", - "notAfter" : 1419292800, "alt" : [ "*.google.com", "*.android.com", @@ -97,39 +82,49 @@ "youtube.com", "youtubeeducation.com" ], - "iCn" : [ + "altCnt" : 49, + "hash" : "0e:a3:27:7c:eb:7f:b2:8c:2b:5d:7d:d7:6b:e9:ba:1a:ec:0d:ff:91", + "issuerCN" : [ "google internet authority g2" ], - "notBefore" : 1411553285, - "sn" : "7a5b0bd895632f87", - "altcnt" : 49 + "issuerON" : "Google Inc", + "notAfter" : 1419292800000, + "notBefore" : 1411553285000, + "serial" : "7a5b0bd895632f87", + "subjectCN" : [ + "*.google.com" + ], + "subjectON" : "Google Inc", + "validDays" : 89 } ], - "fpd" : 1413337837095, - "tlscipher-term" : [ - "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA" - ], - "mac2-term" : [ + "certCnt" : 3, + "dstASN" : "AS15169 Google LLC", + "dstBytes" : 5036, + "dstDataBytes" : 4500, + "dstGEO" : "US", + "dstIp" : "74.125.228.226", + "dstMac" : [ "00:00:0c:07:ac:01", "00:d0:2b:d1:76:00" ], - "ipSrc" : "10.0.0.1", - "lp" : 1413337838, - "db" : 4872, - "g2" : "USA", - "rir2" : "ARIN", - "tcpflags" : { - "urg" : 0, - "fin" : 2, - "syn" : 1, - "syn-ack" : 1, - "ack" : 10, - "psh" : 6, - "rst" : 0 - }, - "as1" : "AS0000 This is neat", - "by" : 6208, - "psl" : [ + "dstMacCnt" : 2, + "dstOui" : [ + "Cisco", + "Jetcell, Inc." + ], + "dstOuiCnt" : 2, + "dstPackets" : 8, + "dstPayload8" : "160302003d020000", + "dstPort" : 443, + "dstRIR" : "ARIN", + "fileId" : [], + "firstPacket" : 1413337837095, + "ipProtocol" : 6, + "lastPacket" : 1413337838083, + "length" : 988, + "node" : "test", + "packetLen" : [ 90, 90, 82, @@ -151,65 +146,7 @@ 82, 82 ], - "ipDst" : "74.125.228.226", - "test" : { - "string" : [ - "16777226:43868,-488342198:443" - ], - "ip-rir" : [ - "" - ], - "number" : [ - 3806625098 - ], - "ip" : [ - 167772161 - ], - "ip-geo" : [ - "RUS" - ], - "ip-asn" : [ - "AS0000 This is neat" - ] - }, - "lastPacket" : 1413337838083, - "tlscipher-termcnt" : 1, - "mac1-term" : [ - "00:13:72:c4:f1:e1" - ], - "ta" : [ - "srcip" - ], - "fb2" : "160302003d020000", - "timestamp" : "SET", - "p1" : 43868, - "tlscnt" : 3, - "a1" : "10.0.0.1", - "fp" : 1413337837, - "fs" : [], - "sl" : 988, - "a2" : "74.125.228.226", - "as2" : "AS15169 Google LLC", - "tags-term" : [ - "srcip" - ], - "g1" : "RUS", - "p2" : 443, - "prot-term" : [ - "tls", - "tcp" - ], - "tlsver-term" : [ - "TLSv1.1" - ], - "by2" : 5036, - "pr" : 6, - "pa" : 20, - "tlsja3-term" : [ - "0358fddc04b374f06c17e363499cf6fb" - ], - "tlsver-termcnt" : 1, - "ps" : [ + "packetPos" : [ 24, 114, 204, @@ -231,22 +168,87 @@ 6388, 6470 ], - "portSrc" : 43868, - "pa2" : 8, - "tlsja3-termcnt" : 1, - "db1" : 372, - "no" : "test", - "by1" : 1172, - "firstPacket" : 1413337837095, - "prot-term-cnt" : 2, - "mac1-term-cnt" : 1, - "portDst" : 443, - "tacnt" : 1, - "ss" : 1, - "lpd" : 1413337838083, - "mac2-term-cnt" : 2, - "fb1" : "16030100a4010000", - "db2" : 4500 + "protocol" : [ + "tls", + "tcp" + ], + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcASN" : "AS0000 This is neat", + "srcBytes" : 1172, + "srcDataBytes" : 372, + "srcGEO" : "RU", + "srcIp" : "10.0.0.1", + "srcMac" : [ + "00:13:72:c4:f1:e1" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Dell Inc." + ], + "srcOuiCnt" : 1, + "srcPackets" : 12, + "srcPayload8" : "16030100a4010000", + "srcPort" : 43868, + "tags" : [ + "srcip" + ], + "tagsCnt" : 1, + "tcpflags" : { + "ack" : 10, + "dstZero" : 0, + "fin" : 2, + "psh" : 6, + "rst" : 0, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, + "test" : { + "ASN" : [ + "AS0000 This is neat" + ], + "GEO" : [ + "RU" + ], + "RIR" : [ + "" + ], + "ip" : [ + "10.0.0.1" + ], + "number" : [ + 3806625098 + ], + "string.snow" : [ + "16777226:43868,-488342198:443" + ] + }, + "timestamp" : "SET", + "tls" : { + "cipher" : [ + "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA" + ], + "cipherCnt" : 1, + "ja3" : [ + "0358fddc04b374f06c17e363499cf6fb" + ], + "ja3Cnt" : 1, + "version" : [ + "TLSv1.1" + ], + "versionCnt" : 1 + }, + "totBytes" : 6208, + "totDataBytes" : 4872, + "totPackets" : 20 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-141015", + "_type" : "session" + } } } ] diff --git a/tests/pcap/openssl-tls1_2-tls1.test b/tests/pcap/openssl-tls1_2-tls1.test index 21ec24f9f6..4bca0459d7 100644 --- a/tests/pcap/openssl-tls1_2-tls1.test +++ b/tests/pcap/openssl-tls1_2-tls1.test @@ -1,125 +1,24 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "firstPacket" : 1413338074954, - "fs" : [], - "tlscnt" : 2, - "by1" : 1210, - "tlscipher-term" : [ - "TLS_RSA_WITH_RC4_128_MD5" - ], - "pa1" : 11, - "fb2" : "1603010f75020000", - "p2" : 443, - "tlsja3-term" : [ - "609a9998ac9d232d213aee990ec5162f" - ], - "g2" : "USA", - "ps" : [ - 24, - 114, - 196, - 266, - 599, - 2129, - 2199, - 3729, - 4841, - 4911, - 4981, - 5361, - 5437, - 5550, - 5620, - 5713, - 5783, - 5859, - 5935 - ], - "tacnt" : 1, - "by" : 5677, - "ss" : 1, - "ipDst" : "64.12.21.3", - "psl" : [ - 90, - 82, - 70, - 333, - 1530, - 70, - 1530, - 1112, - 70, - 70, - 380, - 76, - 113, - 70, - 93, - 70, - 76, - 76, - 70 - ], - "g1" : "RUS", - "mac2-term-cnt" : 2, - "lp" : 1413338075, - "db1" : 596, - "prot-term-cnt" : 2, - "tlsdstid-term" : [ - "9938c44043b6f99f355b9381131c10d511608be1435144a0b0339d8d68aaf38e" - ], - "prot-term" : [ - "tls", - "tcp" - ], - "mac1-term-cnt" : 1, - "db" : 4601, - "fb1" : "1603010102010000", - "tlscipher-termcnt" : 1, - "lpd" : 1413338075560, - "portDst" : 443, - "no" : "test", - "rir2" : "ARIN", - "tags-term" : [ - "srcip" - ], - "tlsver-term" : [ - "TLSv1" - ], - "as2" : "AS1668 AOL Transit Data Network", - "tls" : [ + "cert" : [ { - "notBefore" : 1321026040, - "notAfter" : 1636685477, - "sn" : "4c0e8c39", - "diffDays" : 3653, "hash" : "c5:3e:73:07:3f:93:ce:78:95:de:74:84:12:6b:c3:03:da:b9:e6:57", - "iCn" : [ + "issuerCN" : [ "entrust.net certification authority (2048)" ], - "sCn" : [ + "issuerON" : "Entrust.net", + "notAfter" : 1636685477000, + "notBefore" : 1321026040000, + "serial" : "4c0e8c39", + "subjectCN" : [ "entrust certification authority - l1c" ], - "iOn" : "Entrust.net", - "sOn" : "Entrust, Inc." + "subjectON" : "Entrust, Inc.", + "validDays" : 3653 }, { - "notAfter" : 1497242632, - "notBefore" : 1402510723, - "sn" : "4c235548", - "diffDays" : 1096, - "hash" : "2f:b6:7a:b6:34:99:c2:65:69:0c:cc:f6:8d:8a:73:ee:e2:0e:7a:8b", - "iCn" : [ - "entrust certification authority - l1c" - ], - "altcnt" : 62, - "iOn" : "Entrust, Inc.", - "sCn" : [ - "www.aol.com" - ], - "sOn" : "AOL Inc.", "alt" : [ "toshiba.aol.ca", "main-w.welcomescreen.aol.com", @@ -183,70 +82,173 @@ "welcomescreen.aol.com", "m.article.aol.com", "hp.aol.ca" - ] + ], + "altCnt" : 62, + "hash" : "2f:b6:7a:b6:34:99:c2:65:69:0c:cc:f6:8d:8a:73:ee:e2:0e:7a:8b", + "issuerCN" : [ + "entrust certification authority - l1c" + ], + "issuerON" : "Entrust, Inc.", + "notAfter" : 1497242632000, + "notBefore" : 1402510723000, + "serial" : "4c235548", + "subjectCN" : [ + "www.aol.com" + ], + "subjectON" : "AOL Inc.", + "validDays" : 1096 } ], - "pr" : 6, + "certCnt" : 2, + "dstASN" : "AS1668 AOL Transit Data Network", + "dstBytes" : 4467, + "dstDataBytes" : 4005, + "dstGEO" : "US", + "dstIp" : "64.12.21.3", + "dstMac" : [ + "00:00:0c:07:ac:01", + "00:0e:d6:0b:98:80" + ], + "dstMacCnt" : 2, + "dstOui" : [ + "Cisco", + "Cisco Systems, Inc" + ], + "dstOuiCnt" : 2, + "dstPackets" : 8, + "dstPayload8" : "1603010f75020000", + "dstPort" : 443, + "dstRIR" : "ARIN", + "fileId" : [], + "firstPacket" : 1413338074954, + "ipProtocol" : 6, + "lastPacket" : 1413338075560, + "length" : 606, + "node" : "test", + "packetLen" : [ + 90, + 82, + 70, + 333, + 1530, + 70, + 1530, + 1112, + 70, + 70, + 380, + 76, + 113, + 70, + 93, + 70, + 76, + 76, + 70 + ], + "packetPos" : [ + 24, + 114, + 196, + 266, + 599, + 2129, + 2199, + 3729, + 4841, + 4911, + 4981, + 5361, + 5437, + 5550, + 5620, + 5713, + 5783, + 5859, + 5935 + ], + "protocol" : [ + "tls", + "tcp" + ], + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcASN" : "AS0000 This is neat", + "srcBytes" : 1210, + "srcDataBytes" : 596, + "srcGEO" : "RU", + "srcIp" : "10.0.0.1", + "srcMac" : [ + "00:13:72:c4:f1:e1" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Dell Inc." + ], + "srcOuiCnt" : 1, + "srcPackets" : 11, + "srcPayload8" : "1603010102010000", + "srcPort" : 40291, + "tags" : [ + "srcip" + ], + "tagsCnt" : 1, + "tcpflags" : { + "ack" : 8, + "dstZero" : 0, + "fin" : 2, + "psh" : 7, + "rst" : 0, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, "test" : { - "ip-asn" : [ + "ASN" : [ "AS0000 This is neat" ], - "string" : [ - "16777226:40291,51711040:443" + "GEO" : [ + "RU" ], - "ip" : [ - 167772161 - ], - "ip-geo" : [ - "RUS" - ], - "ip-rir" : [ + "RIR" : [ "" ], + "ip" : [ + "10.0.0.1" + ], "number" : [ 51711040 + ], + "string.snow" : [ + "16777226:40291,51711040:443" ] }, - "portSrc" : 40291, - "tcpflags" : { - "urg" : 0, - "syn-ack" : 1, - "rst" : 0, - "fin" : 2, - "ack" : 8, - "syn" : 1, - "psh" : 7 - }, - "a2" : "64.12.21.3", - "ta" : [ - "srcip" - ], - "mac2-term" : [ - "00:00:0c:07:ac:01", - "00:0e:d6:0b:98:80" - ], - "lastPacket" : 1413338075560, - "a1" : "10.0.0.1", "timestamp" : "SET", - "fp" : 1413338074, - "db2" : 4005, - "pa2" : 8, - "ipSrc" : "10.0.0.1", - "tlsver-termcnt" : 1, - "tlsja3-termcnt" : 1, - "mac1-term" : [ - "00:13:72:c4:f1:e1" - ], - "p1" : 40291, - "as1" : "AS0000 This is neat", - "fpd" : 1413338074954, - "pa" : 19, - "by2" : 4467, - "sl" : 606 + "tls" : { + "cipher" : [ + "TLS_RSA_WITH_RC4_128_MD5" + ], + "cipherCnt" : 1, + "dstSessionId" : [ + "9938c44043b6f99f355b9381131c10d511608be1435144a0b0339d8d68aaf38e" + ], + "ja3" : [ + "609a9998ac9d232d213aee990ec5162f" + ], + "ja3Cnt" : 1, + "version" : [ + "TLSv1" + ], + "versionCnt" : 1 + }, + "totBytes" : 5677, + "totDataBytes" : 4601, + "totPackets" : 19 }, "header" : { "index" : { - "_index" : "tests_sessions-141015", + "_index" : "tests_sessions2-141015", "_type" : "session" } } diff --git a/tests/pcap/openssl-tls1_2.test b/tests/pcap/openssl-tls1_2.test index 0dc3874e0d..7eb07f0811 100644 --- a/tests/pcap/openssl-tls1_2.test +++ b/tests/pcap/openssl-tls1_2.test @@ -1,155 +1,36 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "as1" : "AS0000 This is neat", - "tlsja3-termcnt" : 1, - "lastPacket" : 1413337896584, - "psl" : [ - 90, - 90, - 82, - 345, - 82, - 1500, - 1500, - 82, - 82, - 1342, - 234, - 82, - 82, - 208, - 312, - 82, - 113, - 82, - 82, - 82 - ], - "mac1-term" : [ - "00:13:72:c4:f1:e1" - ], - "a1" : "10.0.0.1", - "db1" : 420, - "portSrc" : 57413, - "p1" : 57413, - "tlscnt" : 3, - "lp" : 1413337896, - "lpd" : 1413337896584, - "portDst" : 443, - "ta" : [ - "srcip" - ], - "prot-term" : [ - "tls", - "tcp" - ], - "firstPacket" : 1413337896106, - "fp" : 1413337896, - "fpd" : 1413337896106, - "timestamp" : "SET", - "pr" : 6, - "by" : 6234, - "tags-term" : [ - "srcip" - ], - "fs" : [], - "db2" : 4478, - "fb2" : "160303003d020000", - "tlsver-termcnt" : 1, - "prot-term-cnt" : 2, - "tlscipher-termcnt" : 1, - "ss" : 1, - "fb1" : "1603010102010000", - "mac2-term-cnt" : 2, - "as2" : "AS15169 Google LLC", - "pa" : 20, - "tlscipher-term" : [ - "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256" - ], - "tlsja3-term" : [ - "609a9998ac9d232d213aee990ec5162f" - ], - "no" : "test", - "rir2" : "ARIN", - "db" : 4898, - "pa1" : 12, - "tacnt" : 1, - "tcpflags" : { - "psh" : 6, - "fin" : 2, - "syn" : 1, - "syn-ack" : 1, - "urg" : 0, - "rst" : 0, - "ack" : 10 - }, - "test" : { - "ip" : [ - 167772161 - ], - "ip-geo" : [ - "RUS" - ], - "ip-rir" : [ - "" - ], - "string" : [ - "16777226:57413,635731274:443" - ], - "number" : [ - 635731274 - ], - "ip-asn" : [ - "AS0000 This is neat" - ] - }, - "p2" : 443, - "by1" : 1220, - "g2" : "USA", - "tls" : [ + "cert" : [ { - "sOn" : "GeoTrust Inc.", - "notAfter" : 1534824000, - "sn" : "12bbe6", - "iOn" : "Equifax", - "notBefore" : 1021953600, - "sCn" : [ + "hash" : "73:59:75:5c:6d:f9:a0:ab:c3:06:0b:ce:36:95:64:c8:ec:45:42:a3", + "issuerON" : "Equifax", + "notAfter" : 1534824000000, + "notBefore" : 1021953600000, + "serial" : "12bbe6", + "subjectCN" : [ "geotrust global ca" ], - "hash" : "73:59:75:5c:6d:f9:a0:ab:c3:06:0b:ce:36:95:64:c8:ec:45:42:a3", - "diffDays" : 5936 + "subjectON" : "GeoTrust Inc.", + "validDays" : 5936 }, { - "notBefore" : 1365174955, - "sCn" : [ - "google internet authority g2" - ], "hash" : "bb:dc:e1:3e:9d:53:7a:52:29:91:5c:b1:23:c7:aa:b0:a8:55:e7:98", - "diffDays" : 1366, - "iCn" : [ + "issuerCN" : [ "geotrust global ca" ], - "sn" : "023a76", - "iOn" : "GeoTrust Inc.", - "notAfter" : 1483228799, - "sOn" : "Google Inc" - }, - { - "notAfter" : 1419292800, - "sOn" : "Google Inc", - "altcnt" : 49, - "notBefore" : 1411553285, - "diffDays" : 89, - "hash" : "0e:a3:27:7c:eb:7f:b2:8c:2b:5d:7d:d7:6b:e9:ba:1a:ec:0d:ff:91", - "sCn" : [ - "*.google.com" - ], - "sn" : "7a5b0bd895632f87", - "iCn" : [ + "issuerON" : "GeoTrust Inc.", + "notAfter" : 1483228799000, + "notBefore" : 1365174955000, + "serial" : "023a76", + "subjectCN" : [ "google internet authority g2" ], + "subjectON" : "Google Inc", + "validDays" : 1366 + }, + { "alt" : [ "*.google.com", "*.android.com", @@ -201,19 +82,71 @@ "youtube.com", "youtubeeducation.com" ], - "iOn" : "Google Inc" + "altCnt" : 49, + "hash" : "0e:a3:27:7c:eb:7f:b2:8c:2b:5d:7d:d7:6b:e9:ba:1a:ec:0d:ff:91", + "issuerCN" : [ + "google internet authority g2" + ], + "issuerON" : "Google Inc", + "notAfter" : 1419292800000, + "notBefore" : 1411553285000, + "serial" : "7a5b0bd895632f87", + "subjectCN" : [ + "*.google.com" + ], + "subjectON" : "Google Inc", + "validDays" : 89 } ], - "g1" : "RUS", - "tlsver-term" : [ - "TLSv1.2" - ], - "ipDst" : "74.125.228.37", - "mac2-term" : [ + "certCnt" : 3, + "dstASN" : "AS15169 Google LLC", + "dstBytes" : 5014, + "dstDataBytes" : 4478, + "dstGEO" : "US", + "dstIp" : "74.125.228.37", + "dstMac" : [ "00:00:0c:07:ac:01", "00:d0:2b:d1:76:00" ], - "ps" : [ + "dstMacCnt" : 2, + "dstOui" : [ + "Cisco", + "Jetcell, Inc." + ], + "dstOuiCnt" : 2, + "dstPackets" : 8, + "dstPayload8" : "160303003d020000", + "dstPort" : 443, + "dstRIR" : "ARIN", + "fileId" : [], + "firstPacket" : 1413337896106, + "ipProtocol" : 6, + "lastPacket" : 1413337896584, + "length" : 477, + "node" : "test", + "packetLen" : [ + 90, + 90, + 82, + 345, + 82, + 1500, + 1500, + 82, + 82, + 1342, + 234, + 82, + 82, + 208, + 312, + 82, + 113, + 82, + 82, + 82 + ], + "packetPos" : [ 24, 114, 204, @@ -235,16 +168,85 @@ 6414, 6496 ], - "ipSrc" : "10.0.0.1", - "sl" : 477, - "pa2" : 8, - "mac1-term-cnt" : 1, - "by2" : 5014, - "a2" : "74.125.228.37" + "protocol" : [ + "tls", + "tcp" + ], + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcASN" : "AS0000 This is neat", + "srcBytes" : 1220, + "srcDataBytes" : 420, + "srcGEO" : "RU", + "srcIp" : "10.0.0.1", + "srcMac" : [ + "00:13:72:c4:f1:e1" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Dell Inc." + ], + "srcOuiCnt" : 1, + "srcPackets" : 12, + "srcPayload8" : "1603010102010000", + "srcPort" : 57413, + "tags" : [ + "srcip" + ], + "tagsCnt" : 1, + "tcpflags" : { + "ack" : 10, + "dstZero" : 0, + "fin" : 2, + "psh" : 6, + "rst" : 0, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, + "test" : { + "ASN" : [ + "AS0000 This is neat" + ], + "GEO" : [ + "RU" + ], + "RIR" : [ + "" + ], + "ip" : [ + "10.0.0.1" + ], + "number" : [ + 635731274 + ], + "string.snow" : [ + "16777226:57413,635731274:443" + ] + }, + "timestamp" : "SET", + "tls" : { + "cipher" : [ + "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256" + ], + "cipherCnt" : 1, + "ja3" : [ + "609a9998ac9d232d213aee990ec5162f" + ], + "ja3Cnt" : 1, + "version" : [ + "TLSv1.2" + ], + "versionCnt" : 1 + }, + "totBytes" : 6234, + "totDataBytes" : 4898, + "totPackets" : 20 }, "header" : { "index" : { - "_index" : "tests_sessions-141015", + "_index" : "tests_sessions2-141015", "_type" : "session" } } diff --git a/tests/pcap/oracle.test b/tests/pcap/oracle.test index 75e858593a..8071f6ef3c 100644 --- a/tests/pcap/oracle.test +++ b/tests/pcap/oracle.test @@ -1,8 +1,34 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "psl" : [ + "dstBytes" : 78, + "dstDataBytes" : 0, + "dstIp" : "10.0.0.17", + "dstMac" : [ + "00:00:5e:00:01:01", + "80:71:1f:84:ef:c5" + ], + "dstMacCnt" : 2, + "dstOui" : [ + "ICANN, IANA Department", + "Juniper Networks" + ], + "dstOuiCnt" : 2, + "dstPackets" : 1, + "dstPort" : 1522, + "fileId" : [], + "firstPacket" : 1476102172344, + "ipProtocol" : 6, + "lastPacket" : 1476102172528, + "length" : 183, + "node" : "test", + "oracle" : { + "host" : "10.000.000.11", + "service" : "xxxxx.xxx", + "user" : "user" + }, + "packetLen" : [ 98, 98, 94, @@ -10,12 +36,7 @@ 86, 310 ], - "ipDst" : "10.0.0.17", - "sl" : 183, - "lp" : 1476102172, - "mac1-term-cnt" : 2, - "a1" : "10.0.0.16", - "ps" : [ + "packetPos" : [ 24, 122, 220, @@ -23,69 +44,52 @@ 400, 486 ], - "a2" : "10.0.0.17", - "vlan" : [ - 50, - 300 + "protocol" : [ + "tcp", + "oracle" ], - "fpd" : 1476102172344, - "by" : 676, - "fs" : [], - "fb1" : "00e0000001000000", - "ipSrc" : "10.0.0.16", - "pa" : 6, - "timestamp" : "SET", - "db2" : 0, - "lpd" : 1476102172528, - "mac1-term" : [ + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 598, + "srcDataBytes" : 224, + "srcIp" : "10.0.0.16", + "srcMac" : [ "00:1b:17:00:14:30", "80:71:1f:84:ef:cc" ], - "mac2-term" : [ - "00:00:5e:00:01:01", - "80:71:1f:84:ef:c5" + "srcMacCnt" : 2, + "srcOui" : [ + "Juniper Networks", + "Palo Alto Networks" ], - "ss" : 1, - "pa2" : 1, - "pa1" : 5, - "fp" : 1476102172, - "portDst" : 1522, - "p2" : 1522, - "db1" : 224, - "firstPacket" : 1476102172344, - "portSrc" : 64084, - "no" : "test", - "db" : 224, - "by2" : 78, - "pr" : 6, - "lastPacket" : 1476102172528, - "oracle" : { - "service-term" : "xxxxx.xxx", - "host-term" : "10.000.000.11", - "user-term" : "user" - }, + "srcOuiCnt" : 2, + "srcPackets" : 5, + "srcPayload8" : "00e0000001000000", + "srcPort" : 64084, "tcpflags" : { + "ack" : 2, + "dstZero" : 0, + "fin" : 0, + "psh" : 1, "rst" : 0, + "srcZero" : 0, "syn" : 2, - "urg" : 0, "syn-ack" : 1, - "ack" : 2, - "psh" : 1, - "fin" : 0 + "urg" : 0 }, - "mac2-term-cnt" : 2, - "p1" : 64084, - "prot-term-cnt" : 2, - "vlan-cnt" : 2, - "by1" : 598, - "prot-term" : [ - "tcp", - "oracle" - ] + "timestamp" : "SET", + "totBytes" : 676, + "totDataBytes" : 224, + "totPackets" : 6, + "vlan" : [ + 50, + 300 + ], + "vlanCnt" : 2 }, "header" : { "index" : { - "_index" : "tests_sessions-161010", + "_index" : "tests_sessions2-161010", "_type" : "session" } } diff --git a/tests/pcap/pop3-tag.test b/tests/pcap/pop3-tag.test index 2197ab9132..62727d8c7e 100644 --- a/tests/pcap/pop3-tag.test +++ b/tests/pcap/pop3-tag.test @@ -1,116 +1,112 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "mac2-term" : [ + "dstASN" : "AS0001 Cool Beans!", + "dstBytes" : 132, + "dstDataBytes" : 16, + "dstGEO" : "CA", + "dstIp" : "10.0.0.2", + "dstMac" : [ "00:00:5e:00:01:02", "00:1d:b5:ce:ef:c1" ], - "p1" : 2464, - "ipDst" : "10.0.0.2", - "g2" : "CAN", - "tags-term" : [ - "yara:tag1", - "yara:tag2", - "yara:Pop3Yara", - "srcip", - "dstip" + "dstMacCnt" : 2, + "dstOui" : [ + "ICANN, IANA Department", + "Juniper Networks" ], - "ps" : [ - 24, - 102, - 180, - 256 - ], - "prot-term-cnt" : 2, - "db2" : 16, - "lpd" : 1387659690419, - "no" : "test", - "psl" : [ + "dstOuiCnt" : 2, + "dstPackets" : 2, + "dstPayload8" : "2b4f4b20504f5033", + "dstPort" : 110, + "dstRIR" : "TEST", + "fileId" : [], + "firstPacket" : 1387659689446, + "ipProtocol" : 6, + "lastPacket" : 1387659690419, + "length" : 973, + "node" : "test", + "packetLen" : [ 78, 78, 76, 86 ], - "a2" : "10.0.0.2", - "lastPacket" : 1387659690419, - "p2" : 110, - "lp" : 1387659690, - "by1" : 122, - "pa1" : 2, - "fs" : [], - "db1" : 0, - "mac1-term" : [ - "00:0f:f7:76:7d:40" + "packetPos" : [ + 24, + 102, + 180, + 256 ], - "pa2" : 2, - "mac2-term-cnt" : 2, - "by2" : 132, - "mac1-term-cnt" : 1, - "prot-term" : [ + "protocol" : [ "pop3", "tcp" ], - "by" : 254, - "pr" : 6, - "db" : 16, - "fp" : 1387659689, + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcASN" : "AS0000 This is neat", + "srcBytes" : 122, + "srcDataBytes" : 0, + "srcGEO" : "RU", + "srcIp" : "10.0.0.1", + "srcMac" : [ + "00:0f:f7:76:7d:40" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Cisco Systems, Inc" + ], + "srcOuiCnt" : 1, + "srcPackets" : 2, + "srcPort" : 2464, + "tags" : [ + "dstip", + "srcip", + "yara:Pop3Yara", + "yara:tag1", + "yara:tag2" + ], + "tagsCnt" : 5, "tcpflags" : { "ack" : 1, + "dstZero" : 0, "fin" : 0, "psh" : 1, - "urg" : 0, - "syn-ack" : 1, + "rst" : 0, + "srcZero" : 0, "syn" : 1, - "rst" : 0 + "syn-ack" : 1, + "urg" : 0 }, - "ss" : 1, - "a1" : "10.0.0.1", - "pa" : 4, - "rir2" : "TEST", - "ipSrc" : "10.0.0.1", "test" : { - "ip-geo" : [ - "RUS" + "ASN" : [ + "AS0000 This is neat" ], - "ip" : [ - 167772161 + "GEO" : [ + "RU" ], - "ip-rir" : [ + "RIR" : [ "" ], - "ip-asn" : [ - "AS0000 This is neat" - ], - "string" : [ - "16777226:2464,33554442:110" + "ip" : [ + "10.0.0.1" ], "number" : [ 33554442 + ], + "string.snow" : [ + "16777226:2464,33554442:110" ] }, - "g1" : "RUS", - "as2" : "AS0001 Cool Beans!", - "as1" : "AS0000 This is neat", "timestamp" : "SET", - "portSrc" : 2464, - "ta" : [ - "dstip", - "srcip", - "yara:Pop3Yara", - "yara:tag1", - "yara:tag2" - ], - "sl" : 973, - "tacnt" : 5, - "portDst" : 110, - "fb2" : "2b4f4b20504f5033", - "firstPacket" : 1387659689446, - "fpd" : 1387659689446 + "totBytes" : 254, + "totDataBytes" : 16, + "totPackets" : 4 }, "header" : { "index" : { - "_index" : "tests_sessions-131221", + "_index" : "tests_sessions2-131221", "_type" : "session" } } diff --git a/tests/pcap/postgres-badpass.test b/tests/pcap/postgres-badpass.test index a40be842c5..e37155971c 100644 --- a/tests/pcap/postgres-badpass.test +++ b/tests/pcap/postgres-badpass.test @@ -1,101 +1,31 @@ { - "sessions" : [ + "sessions2" : [ { - "header" : { - "index" : { - "_index" : "tests_sessions-140505", - "_type" : "session" - } - }, "body" : { - "ss" : 1, - "mac2-term-cnt" : 1, - "tcpflags" : { - "syn" : 1, - "rst" : 0, - "syn-ack" : 1, - "urg" : 0, - "ack" : 5, - "psh" : 4, - "fin" : 2 - }, - "no" : "test", - "tags-term" : [ - "srcip", - "dstip" - ], - "lp" : 1399312748, - "test" : { - "string" : [ - "16777226:53499,33554442:5432" - ], - "ip-rir" : [ - "" - ], - "ip-asn" : [ - "AS0000 This is neat" - ], - "ip-geo" : [ - "RUS" - ], - "number" : [ - 33554442 - ], - "ip" : [ - 167772161 - ] - }, - "mac2-term" : [ + "dstASN" : "AS0001 Cool Beans!", + "dstBytes" : 352, + "dstDataBytes" : 14, + "dstGEO" : "CA", + "dstIp" : "10.0.0.2", + "dstMac" : [ "00:13:72:c4:f1:e1" ], - "a2" : "10.0.0.2", - "firstPacket" : 1399312748531, - "fb2" : "4e520000000c0000", - "prot-term" : [ - "tcp", - "postgresql" - ], - "ta" : [ - "dstip", - "srcip" - ], - "a1" : "10.0.0.1", - "db1" : 82, - "mac1-term-cnt" : 2, - "ipSrc" : "10.0.0.1", - "mac1-term" : [ - "00:00:0c:07:ac:01", - "00:d0:2b:d1:76:00" + "dstMacCnt" : 1, + "dstOui" : [ + "Dell Inc." ], - "by" : 974, - "pa2" : 5, - "ipDst" : "10.0.0.2", - "portSrc" : 53499, - "g2" : "CAN", - "as1" : "AS0000 This is neat", - "portDst" : 5432, - "postgresql" : { - "app-term" : "psql", - "db-term" : "bar", - "user-term" : "foo" - }, - "as2" : "AS0001 Cool Beans!", - "p2" : 5432, - "fs" : [], - "tacnt" : 2, - "g1" : "RUS", - "pa" : 13, - "p1" : 53499, - "fp" : 1399312748, - "prot-term-cnt" : 2, - "sl" : 18, - "db" : 96, - "fb1" : "0000000804d2162f", - "pr" : 6, - "lpd" : 1399312748549, - "by2" : 352, - "fpd" : 1399312748531, - "psl" : [ + "dstOuiCnt" : 1, + "dstPackets" : 5, + "dstPayload8" : "4e520000000c0000", + "dstPort" : 5432, + "dstRIR" : "TEST", + "fileId" : [], + "firstPacket" : 1399312748531, + "ipProtocol" : 6, + "lastPacket" : 1399312748549, + "length" : 18, + "node" : "test", + "packetLen" : [ 94, 90, 82, @@ -110,11 +40,7 @@ 82, 82 ], - "timestamp" : "SET", - "rir2" : "TEST", - "pa1" : 8, - "by1" : 622, - "ps" : [ + "packetPos" : [ 24, 118, 208, @@ -129,8 +55,81 @@ 1042, 1124 ], - "lastPacket" : 1399312748549, - "db2" : 14 + "postgresql" : { + "app" : "psql", + "db" : "bar", + "user" : "foo" + }, + "protocol" : [ + "tcp", + "postgresql" + ], + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcASN" : "AS0000 This is neat", + "srcBytes" : 622, + "srcDataBytes" : 82, + "srcGEO" : "RU", + "srcIp" : "10.0.0.1", + "srcMac" : [ + "00:00:0c:07:ac:01", + "00:d0:2b:d1:76:00" + ], + "srcMacCnt" : 2, + "srcOui" : [ + "Cisco", + "Jetcell, Inc." + ], + "srcOuiCnt" : 2, + "srcPackets" : 8, + "srcPayload8" : "0000000804d2162f", + "srcPort" : 53499, + "tags" : [ + "dstip", + "srcip" + ], + "tagsCnt" : 2, + "tcpflags" : { + "ack" : 5, + "dstZero" : 0, + "fin" : 2, + "psh" : 4, + "rst" : 0, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, + "test" : { + "ASN" : [ + "AS0000 This is neat" + ], + "GEO" : [ + "RU" + ], + "RIR" : [ + "" + ], + "ip" : [ + "10.0.0.1" + ], + "number" : [ + 33554442 + ], + "string.snow" : [ + "16777226:53499,33554442:5432" + ] + }, + "timestamp" : "SET", + "totBytes" : 974, + "totDataBytes" : 96, + "totPackets" : 13 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-140505", + "_type" : "session" + } } } ] diff --git a/tests/pcap/postgres-good.test b/tests/pcap/postgres-good.test index d843cca76d..7eb669fd54 100644 --- a/tests/pcap/postgres-good.test +++ b/tests/pcap/postgres-good.test @@ -1,25 +1,28 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "ss" : 1, - "db1" : 129, - "portDst" : 5432, - "pa" : 11, - "db2" : 327, - "portSrc" : 36060, - "tacnt" : 1, - "by1" : 533, - "mac2-term-cnt" : 1, - "as1" : "AS0000 This is neat", - "g1" : "RUS", - "a1" : "10.0.0.1", + "dstBytes" : 665, + "dstDataBytes" : 327, + "dstIp" : "10.0.13.120", + "dstMac" : [ + "00:0c:29:18:7f:fe" + ], + "dstMacCnt" : 1, + "dstOui" : [ + "VMware, Inc." + ], + "dstOuiCnt" : 1, + "dstPackets" : 5, + "dstPayload8" : "4e520000000c0000", + "dstPort" : 5432, + "fileId" : [], + "firstPacket" : 1399300685023, + "ipProtocol" : 6, "lastPacket" : 1399300685032, - "fs" : [], - "pr" : 6, - "a2" : "10.0.13.120", - "by2" : 665, - "psl" : [ + "length" : 8, + "node" : "test", + "packetLen" : [ 90, 90, 82, @@ -32,22 +35,7 @@ 123, 395 ], - "sl" : 8, - "db" : 456, - "no" : "test", - "pa1" : 6, - "fb2" : "4e520000000c0000", - "by" : 1198, - "fpd" : 1399300685023, - "mac1-term" : [ - "90:e2:ba:52:f6:2a", - "00:00:5e:00:01:71" - ], - "ipDst" : "10.0.13.120", - "ta" : [ - "srcip" - ], - "ps" : [ + "packetPos" : [ 24, 114, 204, @@ -60,67 +48,79 @@ 880, 1003 ], - "fb1" : "0000000804d2162f", - "tags-term" : [ + "postgresql" : { + "app" : "pgAdmin III - Browser", + "db" : "dbdbdbdb", + "user" : "cooluser" + }, + "protocol" : [ + "tcp", + "postgresql" + ], + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcASN" : "AS0000 This is neat", + "srcBytes" : 533, + "srcDataBytes" : 129, + "srcGEO" : "RU", + "srcIp" : "10.0.0.1", + "srcMac" : [ + "00:00:5e:00:01:71", + "90:e2:ba:52:f6:2a" + ], + "srcMacCnt" : 2, + "srcOui" : [ + "ICANN, IANA Department", + "Intel Corporate" + ], + "srcOuiCnt" : 2, + "srcPackets" : 6, + "srcPayload8" : "0000000804d2162f", + "srcPort" : 36060, + "tags" : [ "srcip" ], - "mac1-term-cnt" : 2, - "timestamp" : "SET", - "lpd" : 1399300685032, + "tagsCnt" : 1, + "tcpflags" : { + "ack" : 3, + "dstZero" : 0, + "fin" : 0, + "psh" : 6, + "rst" : 0, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, "test" : { - "ip-rir" : [ - "" - ], - "ip-asn" : [ + "ASN" : [ "AS0000 This is neat" ], + "GEO" : [ + "RU" + ], + "RIR" : [ + "" + ], "ip" : [ - 167772161 + "10.0.0.1" ], "number" : [ 2014117898 ], - "ip-geo" : [ - "RUS" - ], - "string" : [ + "string.snow" : [ "16777226:36060,2014117898:5432" ] }, - "prot-term" : [ - "tcp", - "postgresql" - ], - "lp" : 1399300685, - "pa2" : 5, - "prot-term-cnt" : 2, - "postgresql" : { - "user-term" : "cooluser", - "app-term" : "pgAdmin III - Browser", - "db-term" : "dbdbdbdb" - }, - "mac2-term" : [ - "00:0c:29:18:7f:fe" - ], - "ipSrc" : "10.0.0.1", - "fp" : 1399300685, - "p2" : 5432, - "p1" : 36060, - "tcpflags" : { - "fin" : 0, - "psh" : 6, - "syn-ack" : 1, - "urg" : 0, - "rst" : 0, - "syn" : 1, - "ack" : 3 - }, - "firstPacket" : 1399300685023 + "timestamp" : "SET", + "totBytes" : 1198, + "totDataBytes" : 456, + "totPackets" : 11 }, "header" : { "index" : { - "_type" : "session", - "_index" : "tests_sessions-140505" + "_index" : "tests_sessions2-140505", + "_type" : "session" } } } diff --git a/tests/pcap/postgres-no-sslrequest.test b/tests/pcap/postgres-no-sslrequest.test index 2473dd4be7..91f272834f 100644 --- a/tests/pcap/postgres-no-sslrequest.test +++ b/tests/pcap/postgres-no-sslrequest.test @@ -1,46 +1,28 @@ { - "sessions" : [ + "sessions2" : [ { - "header" : { - "index" : { - "_index" : "tests_sessions-041219", - "_type" : "session" - } - }, "body" : { - "a1" : "127.0.0.1", - "fpd" : 1103485433560, - "db1" : 96, - "p1" : 57827, - "mac1-term-cnt" : 1, - "db" : 256, - "pa" : 11, - "lp" : 1103485433, - "fs" : [], - "postgresql" : { - "db-term" : "dbdb", - "user-term" : "user" - }, - "portSrc" : 57827, - "db2" : 160, - "by1" : 566, - "ipDst" : "127.0.0.1", - "pa1" : 7, - "pr" : 6, - "portDst" : 5432, - "prot-term-cnt" : 2, - "prot-term" : [ - "tcp", - "postgresql" - ], - "ipSrc" : "127.0.0.1", - "fb2" : "5200000008000000", - "fb1" : "0000005b00030000", - "no" : "test", - "mac1-term" : [ + "dstBytes" : 432, + "dstDataBytes" : 160, + "dstIp" : "127.0.0.1", + "dstMac" : [ "00:00:00:00:00:00" ], - "psl" : [ + "dstMacCnt" : 1, + "dstOui" : [ + "Officially Xerox, but 0:0:0:0:0:0 is more common" + ], + "dstOuiCnt" : 1, + "dstPackets" : 4, + "dstPayload8" : "5200000008000000", + "dstPort" : 5432, + "fileId" : [], + "firstPacket" : 1103485433560, + "ipProtocol" : 6, + "lastPacket" : 1103485433664, + "length" : 104, + "node" : "test", + "packetLen" : [ 90, 90, 82, @@ -53,25 +35,7 @@ 82, 82 ], - "lastPacket" : 1103485433664, - "fp" : 1103485433, - "a2" : "127.0.0.1", - "sl" : 104, - "mac2-term-cnt" : 1, - "ss" : 1, - "tcpflags" : { - "psh" : 3, - "fin" : 2, - "syn-ack" : 1, - "rst" : 0, - "ack" : 4, - "syn" : 1, - "urg" : 0 - }, - "lpd" : 1103485433664, - "pa2" : 4, - "p2" : 5432, - "ps" : [ + "packetPos" : [ 24, 114, 204, @@ -84,13 +48,51 @@ 1034, 1116 ], - "mac2-term" : [ + "postgresql" : { + "db" : "dbdb", + "user" : "user" + }, + "protocol" : [ + "tcp", + "postgresql" + ], + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 566, + "srcDataBytes" : 96, + "srcIp" : "127.0.0.1", + "srcMac" : [ "00:00:00:00:00:00" ], - "by2" : 432, + "srcMacCnt" : 1, + "srcOui" : [ + "Officially Xerox, but 0:0:0:0:0:0 is more common" + ], + "srcOuiCnt" : 1, + "srcPackets" : 7, + "srcPayload8" : "0000005b00030000", + "srcPort" : 57827, + "tcpflags" : { + "ack" : 4, + "dstZero" : 0, + "fin" : 2, + "psh" : 3, + "rst" : 0, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, "timestamp" : "SET", - "by" : 998, - "firstPacket" : 1103485433560 + "totBytes" : 998, + "totDataBytes" : 256, + "totPackets" : 11 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-041219", + "_type" : "session" + } } } ] diff --git a/tests/pcap/pppoe.test b/tests/pcap/pppoe.test index 1e13983f10..45e92d5aed 100644 --- a/tests/pcap/pppoe.test +++ b/tests/pcap/pppoe.test @@ -1,154 +1,158 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "ss" : 1, - "p2" : 443, - "by2" : 3596, - "mac2-term-cnt" : 1, - "prot-term" : [ - "pppoe", - "tls", - "tcp" - ], - "db" : 3564, - "fs" : [], - "ps" : [ - 24, - 110, - 192, - 270, - 426, - 1864, - 3302, - 3380, - 3953, - 4235 - ], - "by" : 4196, - "rir1" : "ARIN", - "firstPacket" : 1193676055856, - "no" : "test", - "rir2" : "ARIN", - "lpd" : 1193676056235, - "psl" : [ - 86, - 82, - 78, - 156, - 1438, - 1438, - 78, - 573, - 282, - 145 - ], - "pa" : 10, - "mac2-term" : [ - "00:90:1a:a0:3d:a4" - ], - "portSrc" : 1063, - "prot-term-cnt" : 3, - "pr" : 6, - "g2" : "USA", - "tlsver-termcnt" : 1, - "g1" : "GBR", - "tlsdstid-term" : [ - "dd709157b76b42f05802dfaa6501a55ec5a2a62c2c2ec856f1fcbc1cc8dc5717" - ], - "tlscipher-term" : [ - "TLS_RSA_WITH_RC4_128_MD5" - ], - "by1" : 600, - "as2" : "AS1668 AOL Transit Data Network", - "ipSrc" : "172.202.246.57", - "db2" : 3282, - "fpd" : 1193676055856, - "pa1" : 5, - "mac1-term-cnt" : 1, - "lastPacket" : 1193676056235, - "a2" : "64.12.189.217", - "as1" : "AS1668 AOL Transit Data Network", - "mac1-term" : [ - "00:11:f5:13:d7:a3" - ], - "fp" : 1193676055, - "lp" : 1193676056, - "fb2" : "160300004a020000", - "db1" : 282, - "portDst" : 443, - "a1" : "172.202.246.57", - "fb1" : "804c010300003300", - "sl" : 380, - "tls" : [ + "cert" : [ { "hash" : "cc:04:2e:48:2f:29:6a:3a:dc:a4:8b:fb:79:a6:cd:5f:67:8a:c1:e2", - "notBefore" : 1168837200, - "sn" : "0119", - "iCn" : [ + "issuerCN" : [ "aol member ca" ], - "notAfter" : 1231995540, - "diffDays" : 730, - "iOn" : "America Online Inc.", - "sOn" : "AOL LLC", - "sCn" : [ + "issuerON" : "America Online Inc.", + "notAfter" : 1231995540000, + "notBefore" : 1168837200000, + "serial" : "0119", + "subjectCN" : [ "kdc.uas.aol.com" - ] + ], + "subjectON" : "AOL LLC", + "validDays" : 730 }, { - "iCn" : [ + "hash" : "39:21:c1:15:c1:5d:0e:ca:5c:cb:5b:c4:f0:7d:21:d8:05:0b:56:6a", + "issuerCN" : [ "america online root certification authority 1" ], - "notBefore" : 1022565600, - "hash" : "39:21:c1:15:c1:5d:0e:ca:5c:cb:5b:c4:f0:7d:21:d8:05:0b:56:6a", - "sn" : "01", - "sCn" : [ + "issuerON" : "America Online Inc.", + "notAfter" : 2142276180000, + "notBefore" : 1022565600000, + "serial" : "01", + "subjectCN" : [ "america online root certification authority 1" ], - "sOn" : "America Online Inc.", - "diffDays" : 12959, - "iOn" : "America Online Inc.", - "notAfter" : 2142276180 + "subjectON" : "America Online Inc.", + "validDays" : 12959 }, { - "iCn" : [ + "hash" : "a1:44:6b:ce:0c:87:4d:f0:f2:c3:f6:1d:a5:c9:a2:bc:f9:da:b2:04", + "issuerCN" : [ "america online root certification authority 1" ], - "sn" : "07", - "hash" : "a1:44:6b:ce:0c:87:4d:f0:f2:c3:f6:1d:a5:c9:a2:bc:f9:da:b2:04", - "notBefore" : 1086369999, - "sOn" : "America Online Inc.", - "sCn" : [ + "issuerON" : "America Online Inc.", + "notAfter" : 1875288399000, + "notBefore" : 1086369999000, + "serial" : "07", + "subjectCN" : [ "aol member ca" ], - "iOn" : "America Online Inc.", - "diffDays" : 9131, - "notAfter" : 1875288399 + "subjectON" : "America Online Inc.", + "validDays" : 9131 } ], - "tlsver-term" : [ - "SSLv3" + "certCnt" : 3, + "dstASN" : "AS1668 AOL Transit Data Network", + "dstBytes" : 3596, + "dstDataBytes" : 3282, + "dstGEO" : "US", + "dstIp" : "64.12.189.217", + "dstMac" : [ + "00:90:1a:a0:3d:a4" ], - "tlscnt" : 3, - "pa2" : 5, - "tlscipher-termcnt" : 1, - "p1" : 1063, - "timestamp" : "SET", + "dstMacCnt" : 1, + "dstOui" : [ + "Unisphere Solutions" + ], + "dstOuiCnt" : 1, + "dstPackets" : 5, + "dstPayload8" : "160300004a020000", + "dstPort" : 443, + "dstRIR" : "ARIN", + "fileId" : [], + "firstPacket" : 1193676055856, + "ipProtocol" : 6, + "lastPacket" : 1193676056235, + "length" : 380, + "node" : "test", + "packetLen" : [ + 86, + 82, + 78, + 156, + 1438, + 1438, + 78, + 573, + 282, + 145 + ], + "packetPos" : [ + 24, + 110, + 192, + 270, + 426, + 1864, + 3302, + 3380, + 3953, + 4235 + ], + "protocol" : [ + "pppoe", + "tls", + "tcp" + ], + "protocolCnt" : 3, + "segmentCnt" : 1, + "srcASN" : "AS1668 AOL Transit Data Network", + "srcBytes" : 600, + "srcDataBytes" : 282, + "srcGEO" : "GB", + "srcIp" : "172.202.246.57", + "srcMac" : [ + "00:11:f5:13:d7:a3" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Askey Computer Corp" + ], + "srcOuiCnt" : 1, + "srcPackets" : 5, + "srcPayload8" : "804c010300003300", + "srcPort" : 1063, + "srcRIR" : "ARIN", "tcpflags" : { + "ack" : 4, + "dstZero" : 0, + "fin" : 0, "psh" : 4, "rst" : 0, - "ack" : 4, + "srcZero" : 0, "syn" : 1, - "fin" : 0, "syn-ack" : 1, "urg" : 0 }, - "ipDst" : "64.12.189.217" + "timestamp" : "SET", + "tls" : { + "cipher" : [ + "TLS_RSA_WITH_RC4_128_MD5" + ], + "cipherCnt" : 1, + "dstSessionId" : [ + "dd709157b76b42f05802dfaa6501a55ec5a2a62c2c2ec856f1fcbc1cc8dc5717" + ], + "version" : [ + "SSLv3" + ], + "versionCnt" : 1 + }, + "totBytes" : 4196, + "totDataBytes" : 3564, + "totPackets" : 10 }, "header" : { "index" : { - "_index" : "tests_sessions-071029", + "_index" : "tests_sessions2-071029", "_type" : "session" } } diff --git a/tests/pcap/quic24-wireshark.test b/tests/pcap/quic24-wireshark.test index c587487f9a..fc0c260502 100644 --- a/tests/pcap/quic24-wireshark.test +++ b/tests/pcap/quic24-wireshark.test @@ -1,86 +1,87 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "by1" : 2866, - "p2" : 443, - "pa2" : 2, - "timestamp" : "SET", - "pr" : 17, - "g2" : "USA", - "fp" : 1429686316, - "pa1" : 3, - "db" : 5610, - "ipDst" : "216.58.208.195", - "fb1" : "0d00db40daa3bbe2", - "db2" : 2768, - "portDst" : 443, - "quic" : { - "ua-term" : [ - "canary Chrome/44.0.2375.0" - ], - "version-termcnt" : 1, - "host-termcnt" : 1, - "host-term" : [ - "www.google.fr" - ], - "ua-termcnt" : 1, - "version-term" : [ - "Q024" - ] - }, + "dstASN" : "AS15169 Google LLC", + "dstBytes" : 2784, + "dstDataBytes" : 2768, + "dstGEO" : "US", + "dstIp" : "216.58.208.195", + "dstMac" : [ + "00:12:ef:c0:d7:a3" + ], + "dstMacCnt" : 1, + "dstOui" : [ + "OneAccess SA" + ], + "dstOuiCnt" : 1, + "dstPackets" : 2, + "dstPayload8" : "0c00db40daa3bbe2", + "dstPort" : 443, + "dstRIR" : "ARIN", + "fileId" : [], "firstPacket" : 1429686316293, - "ps" : [ + "ipProtocol" : 17, + "lastPacket" : 1429686316327, + "length" : 33, + "node" : "test", + "packetLen" : [ + 1408, + 1408, + 1408, + 98, + 1408 + ], + "packetPos" : [ 24, 1432, 2840, 4248, 4346 ], - "a1" : "10.44.100.22", - "prot-term" : [ + "protocol" : [ "udp", + "iprulztest", "quic" ], - "prot-term-cnt" : 2, - "lpd" : 1429686316327, - "mac2-term" : [ - "00:12:ef:c0:d7:a3" - ], - "psl" : [ - 1408, - 1408, - 1408, - 98, - 1408 - ], - "ss" : 1, - "lp" : 1429686316, - "fs" : [], - "as2" : "AS15169 Google LLC", - "sl" : 33, - "a2" : "216.58.208.195", - "by" : 5650, - "no" : "test", - "rir2" : "ARIN", - "db1" : 2842, - "by2" : 2784, - "ipSrc" : "10.44.100.22", - "lastPacket" : 1429686316327, - "mac2-term-cnt" : 1, - "mac1-term" : [ + "protocolCnt" : 3, + "quic" : { + "host" : [ + "www.google.fr" + ], + "hostCnt" : 1, + "useragent" : [ + "canary Chrome/44.0.2375.0" + ], + "useragentCnt" : 1, + "version" : [ + "Q024" + ], + "versionCnt" : 1 + }, + "segmentCnt" : 1, + "srcBytes" : 2866, + "srcDataBytes" : 2842, + "srcIp" : "10.44.100.22", + "srcMac" : [ "a0:88:b4:1e:16:0c" ], - "p1" : 50509, - "mac1-term-cnt" : 1, - "fpd" : 1429686316293, - "pa" : 5, - "fb2" : "0c00db40daa3bbe2", - "portSrc" : 50509 + "srcMacCnt" : 1, + "srcOui" : [ + "Intel Corporate" + ], + "srcOuiCnt" : 1, + "srcPackets" : 3, + "srcPayload8" : "0d00db40daa3bbe2", + "srcPort" : 50509, + "timestamp" : "SET", + "totBytes" : 5650, + "totDataBytes" : 5610, + "totPackets" : 5 }, "header" : { "index" : { - "_index" : "tests_sessions-150422", + "_index" : "tests_sessions2-150422", "_type" : "session" } } diff --git a/tests/pcap/quic33-wireshark.test b/tests/pcap/quic33-wireshark.test index f2aafd9a80..62addc0efd 100644 --- a/tests/pcap/quic33-wireshark.test +++ b/tests/pcap/quic33-wireshark.test @@ -1,80 +1,80 @@ { - "sessions" : [ + "sessions2" : [ { - "header" : { - "index" : { - "_type" : "session", - "_index" : "tests_sessions-160511" - } - }, "body" : { - "prot-term" : [ + "dstASN" : "AS36040 Google LLC", + "dstBytes" : 119, + "dstDataBytes" : 111, + "dstGEO" : "US", + "dstIp" : "64.15.116.182", + "dstMac" : [ + "52:54:00:12:35:02" + ], + "dstMacCnt" : 1, + "dstOui" : [ + "Realtek (UpTech? also reported)" + ], + "dstOuiCnt" : 1, + "dstPackets" : 1, + "dstPayload8" : "0a79f24eb64a9c07", + "dstPort" : 443, + "dstRIR" : "ARIN", + "fileId" : [], + "firstPacket" : 1462966377795, + "ipProtocol" : 17, + "lastPacket" : 1462966377990, + "length" : 195, + "node" : "test", + "packetLen" : [ + 1408, + 1408, + 135 + ], + "packetPos" : [ + 24, + 1432, + 2840 + ], + "protocol" : [ "udp", "quic" ], - "pr" : 17, - "a2" : "64.15.116.182", - "db1" : 2768, - "prot-term-cnt" : 2, - "fp" : 1462966377, + "protocolCnt" : 2, "quic" : { - "version-term" : [ - "Q033" - ], - "host-term" : [ + "host" : [ "www.google.com" ], - "host-termcnt" : 1, - "version-termcnt" : 1 + "hostCnt" : 1, + "version" : [ + "Q033" + ], + "versionCnt" : 1 }, - "portSrc" : 40482, - "portDst" : 443, - "mac1-term-cnt" : 1, - "pa" : 3, - "fpd" : 1462966377795, - "g2" : "USA", - "lp" : 1462966377, - "ipDst" : "64.15.116.182", - "db" : 2879, - "ss" : 1, - "pa1" : 2, - "ipSrc" : "10.0.2.15", - "psl" : [ - 1408, - 1408, - 135 - ], - "pa2" : 1, - "mac2-term-cnt" : 1, - "lastPacket" : 1462966377990, - "p1" : 40482, - "by2" : 119, - "by" : 2903, - "fs" : [], - "p2" : 443, - "fb1" : "0979f24eb64a9c07", - "fb2" : "0a79f24eb64a9c07", - "as2" : "AS36040 Google LLC", - "no" : "test", - "db2" : 111, - "a1" : "10.0.2.15", - "lpd" : 1462966377990, - "by1" : 2784, - "timestamp" : "SET", - "mac1-term" : [ + "segmentCnt" : 1, + "srcBytes" : 2784, + "srcDataBytes" : 2768, + "srcIp" : "10.0.2.15", + "srcMac" : [ "08:00:27:cc:18:37" ], - "ps" : [ - 24, - 1432, - 2840 + "srcMacCnt" : 1, + "srcOui" : [ + "PCS Computer Systems GmbH" ], - "firstPacket" : 1462966377795, - "sl" : 195, - "rir2" : "ARIN", - "mac2-term" : [ - "52:54:00:12:35:02" - ] + "srcOuiCnt" : 1, + "srcPackets" : 2, + "srcPayload8" : "0979f24eb64a9c07", + "srcPort" : 40482, + "timestamp" : "SET", + "totBytes" : 2903, + "totDataBytes" : 2879, + "totPackets" : 3 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-160511", + "_type" : "session" + } } } ] diff --git a/tests/pcap/quic34.test b/tests/pcap/quic34.test index 3f24eeb00e..1a8474e7a0 100644 --- a/tests/pcap/quic34.test +++ b/tests/pcap/quic34.test @@ -1,83 +1,83 @@ { - "sessions" : [ + "sessions2" : [ { - "header" : { - "index" : { - "_type" : "session", - "_index" : "tests_sessions-160629" - } - }, "body" : { + "dstASN" : "AS15169 Google LLC", + "dstBytes" : 0, + "dstDataBytes" : 0, + "dstGEO" : "US", + "dstIp" : "216.58.194.195", + "dstMac" : [ + "00:00:5e:00:01:01" + ], + "dstMacCnt" : 1, + "dstOui" : [ + "ICANN, IANA Department" + ], + "dstOuiCnt" : 1, + "dstPackets" : 0, + "dstPort" : 443, + "dstRIR" : "ARIN", + "fileId" : [], "firstPacket" : 1467241187446, - "by1" : 1396, - "ipSrc" : "10.0.0.4", - "vlan" : [ - 300 + "ipProtocol" : 17, + "lastPacket" : 1467241187446, + "length" : 0, + "node" : "test", + "packetLen" : [ + 1412 ], - "ps" : [ + "packetPos" : [ 24 ], - "by" : 1396, - "by2" : 0, - "rir2" : "ARIN", - "portDst" : 443, - "fpd" : 1467241187446, + "protocol" : [ + "udp", + "quic" + ], + "protocolCnt" : 2, "quic" : { - "ua-term" : [ - "canary Chrome/53.0.2782.0 Windows NT 6.1; Win64; x64" - ], - "host-term" : [ + "host" : [ "ssl.gstatic.com" ], - "version-term" : [ + "hostCnt" : 1, + "useragent" : [ + "canary Chrome/53.0.2782.0 Windows NT 6.1; Win64; x64" + ], + "useragentCnt" : 1, + "version" : [ "Q034" ], - "host-termcnt" : 1, - "ua-termcnt" : 1, - "version-termcnt" : 1 + "versionCnt" : 1 }, - "as2" : "AS15169 Google LLC", - "a1" : "10.0.0.4", - "ss" : 1, - "no" : "test", - "p1" : 54800, - "p2" : 443, - "sl" : 0, - "vlan-cnt" : 1, - "db2" : 0, - "mac1-term" : [ + "segmentCnt" : 1, + "srcBytes" : 1396, + "srcDataBytes" : 1388, + "srcIp" : "10.0.0.4", + "srcMac" : [ "00:1b:17:00:02:30" ], - "timestamp" : "SET", - "mac1-term-cnt" : 1, - "fb1" : "0d9300a653775210", - "pa" : 1, - "db" : 1388, - "pa1" : 1, - "db1" : 1388, - "lp" : 1467241187, - "lpd" : 1467241187446, - "a2" : "216.58.194.195", - "fs" : [], - "pr" : 17, - "portSrc" : 54800, - "g2" : "USA", - "prot-term-cnt" : 2, - "ipDst" : "216.58.194.195", - "mac2-term" : [ - "00:00:5e:00:01:01" - ], - "mac2-term-cnt" : 1, - "psl" : [ - 1412 + "srcMacCnt" : 1, + "srcOui" : [ + "Palo Alto Networks" ], - "lastPacket" : 1467241187446, - "fp" : 1467241187, - "prot-term" : [ - "udp", - "quic" + "srcOuiCnt" : 1, + "srcPackets" : 1, + "srcPayload8" : "0d9300a653775210", + "srcPort" : 54800, + "timestamp" : "SET", + "totBytes" : 1396, + "totDataBytes" : 1388, + "totPackets" : 1, + "vlan" : [ + 300 ], - "pa2" : 0 + "vlanCnt" : 1 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-160629", + "_type" : "session" + } } } ] diff --git a/tests/pcap/radius.test b/tests/pcap/radius.test index 226f1be76d..9c97163e70 100644 --- a/tests/pcap/radius.test +++ b/tests/pcap/radius.test @@ -1,125 +1,121 @@ { - "sessions" : [ + "sessions2" : [ { - "header" : { - "index" : { - "_type" : "session", - "_index" : "tests_sessions-160515" - } - }, "body" : { - "test" : { - "string" : [ - "16777226:45210,33554442:1813" - ], - "ip-asn" : [ - "AS0000 This is neat" - ], - "ip-geo" : [ - "RUS" - ], - "number" : [ - 33554442 - ], - "ip" : [ - 167772161 - ], - "ip-rir" : [ - "" - ] - }, - "db1" : 381, - "lp" : 1463339353, - "by" : 455, - "ta" : [ - "dstip", - "srcip" - ], - "lastPacket" : 1463339353461, - "a1" : "10.0.0.1", - "timestamp" : "SET", - "pa2" : 1, - "fb2" : "05e3001400000000", - "ipDst" : "10.0.0.2", - "db" : 439, - "pa" : 2, - "tags-term" : [ - "srcip", - "dstip" - ], - "portSrc" : 45210, - "db2" : 58, - "tacnt" : 2, - "ipSrc" : "10.0.0.1", - "as1" : "AS0000 This is neat", - "vlan" : [ - 50 - ], - "fpd" : 1463339353458, - "lpd" : 1463339353461, - "no" : "test", - "mac2-term" : [ + "dstASN" : "AS0001 Cool Beans!", + "dstBytes" : 66, + "dstDataBytes" : 58, + "dstGEO" : "CA", + "dstIp" : "10.0.0.2", + "dstMac" : [ "00:22:83:3f:17:c5" ], - "p1" : 45210, - "g1" : "RUS", - "prot-term" : [ - "udp", - "radius" - ], - "mac1-term-cnt" : 1, - "prot-term-cnt" : 2, - "pr" : 17, - "p2" : 1813, - "mac1-term" : [ - "00:22:83:3f:17:cc" + "dstMacCnt" : 1, + "dstOui" : [ + "Juniper Networks" ], - "by2" : 66, - "g2" : "CAN", - "portDst" : 1813, + "dstOuiCnt" : 1, + "dstPackets" : 1, + "dstPayload8" : "05e3001400000000", + "dstPort" : 1813, + "dstRIR" : "TEST", + "fileId" : [], "firstPacket" : 1463339353458, - "by1" : 389, - "fs" : [], - "ps" : [ + "ipProtocol" : 17, + "lastPacket" : 1463339353461, + "length" : 3, + "node" : "test", + "packetLen" : [ + 405, + 82 + ], + "packetPos" : [ 24, 429 ], - "rir2" : "TEST", - "fp" : 1463339353, + "protocol" : [ + "udp", + "radius" + ], + "protocolCnt" : 2, "radius" : { - "fip-cnt" : 1, - "fip-asn" : [ + "framedASN" : [ "AS0002 Hmm!@#$%^&*()" ], - "mac-term-cnt" : 1, - "fip-rir" : [ + "framedGEO" : [ + "---" + ], + "framedIp" : [ + "10.0.0.3" + ], + "framedIpCnt" : 1, + "framedRIR" : [ "" ], - "mac-term" : [ + "mac" : [ "98:5a:eb:89:80:00" ], - "fip" : [ - "10.0.0.3" - ], - "user-term" : [ + "macCnt" : 1, + "user" : [ "xxxxxxxxx" + ] + }, + "segmentCnt" : 1, + "srcASN" : "AS0000 This is neat", + "srcBytes" : 389, + "srcDataBytes" : 381, + "srcGEO" : "RU", + "srcIp" : "10.0.0.1", + "srcMac" : [ + "00:22:83:3f:17:cc" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Juniper Networks" + ], + "srcOuiCnt" : 1, + "srcPackets" : 1, + "srcPayload8" : "04e3015700000000", + "srcPort" : 45210, + "tags" : [ + "dstip", + "srcip" + ], + "tagsCnt" : 2, + "test" : { + "ASN" : [ + "AS0000 This is neat" ], - "fip-geo" : [ - "---" + "GEO" : [ + "RU" + ], + "RIR" : [ + "" + ], + "ip" : [ + "10.0.0.1" + ], + "number" : [ + 33554442 + ], + "string.snow" : [ + "16777226:45210,33554442:1813" ] }, - "ss" : 1, - "pa1" : 1, - "mac2-term-cnt" : 1, - "a2" : "10.0.0.2", - "vlan-cnt" : 1, - "sl" : 3, - "fb1" : "04e3015700000000", - "as2" : "AS0001 Cool Beans!", - "psl" : [ - 405, - 82 - ] + "timestamp" : "SET", + "totBytes" : 455, + "totDataBytes" : 439, + "totPackets" : 2, + "vlan" : [ + 50 + ], + "vlanCnt" : 1 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-160515", + "_type" : "session" + } } } ] diff --git a/tests/pcap/smb-port80.test b/tests/pcap/smb-port80.test index 52392c1574..9665cda084 100644 --- a/tests/pcap/smb-port80.test +++ b/tests/pcap/smb-port80.test @@ -1,74 +1,30 @@ { - "sessions" : [ + "sessions2" : [ { - "header" : { - "index" : { - "_type" : "session", - "_index" : "tests_sessions-130918" - } - }, "body" : { - "firstPacket" : 1379519109833, - "fb2" : "b378fd2aae2d4aee", - "prot-term" : [ - "smb", - "tcp" - ], - "pa" : 9, - "g1" : "CAN", - "p2" : 80, - "by1" : 506, - "rir1" : "TEST", - "fb1" : "000000a4ff534d42", - "g2" : "RUS", - "tcpflags" : { - "syn-ack" : 1, - "syn" : 1, - "fin" : 2, - "rst" : 0, - "psh" : 3, - "ack" : 3, - "urg" : 0 - }, - "fs" : [], - "pa1" : 5, - "ss" : 1, - "by" : 847, - "mac2-term-cnt" : 1, - "portDst" : 80, - "tacnt" : 2, - "pr" : 6, - "db2" : 69, - "mac2-term" : [ + "dstASN" : "AS0000 This is neat", + "dstBytes" : 341, + "dstDataBytes" : 69, + "dstGEO" : "RU", + "dstIp" : "10.0.0.1", + "dstMac" : [ "00:0b:45:b7:08:80" ], - "tags-term" : [ - "srcip", - "dstip" + "dstMacCnt" : 1, + "dstOui" : [ + "Cisco Systems, Inc" ], - "a2" : "10.0.0.1", - "ta" : [ - "dstip", - "srcip" - ], - "mac1-term" : [ - "00:26:88:df:17:c6", - "00:00:5e:00:01:02" - ], - "as2" : "AS0000 This is neat", - "timestamp" : "SET", - "ipDst" : "10.0.0.1", - "prot-term-cnt" : 2, - "fp" : 1379519109, - "lpd" : 1379519110616, + "dstOuiCnt" : 1, + "dstPackets" : 4, + "dstPayload8" : "b378fd2aae2d4aee", + "dstPort" : 80, + "fileId" : [], + "firstPacket" : 1379519109833, + "ipProtocol" : 6, "lastPacket" : 1379519110616, - "portSrc" : 46735, - "db1" : 168, - "no" : "test", - "p1" : 46735, - "by2" : 341, - "mac1-term-cnt" : 2, - "psl" : [ + "length" : 784, + "node" : "test", + "packetLen" : [ 90, 90, 82, @@ -79,14 +35,7 @@ 82, 82 ], - "lp" : 1379519110, - "fpd" : 1379519109833, - "db" : 237, - "pa2" : 4, - "a1" : "10.0.0.2", - "as1" : "AS0001 Cool Beans!", - "ipSrc" : "10.0.0.2", - "ps" : [ + "packetPos" : [ 24, 114, 204, @@ -97,7 +46,57 @@ 851, 933 ], - "sl" : 784 + "protocol" : [ + "smb", + "tcp" + ], + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcASN" : "AS0001 Cool Beans!", + "srcBytes" : 506, + "srcDataBytes" : 168, + "srcGEO" : "CA", + "srcIp" : "10.0.0.2", + "srcMac" : [ + "00:00:5e:00:01:02", + "00:26:88:df:17:c6" + ], + "srcMacCnt" : 2, + "srcOui" : [ + "ICANN, IANA Department", + "Juniper Networks" + ], + "srcOuiCnt" : 2, + "srcPackets" : 5, + "srcPayload8" : "000000a4ff534d42", + "srcPort" : 46735, + "srcRIR" : "TEST", + "tags" : [ + "dstip", + "srcip" + ], + "tagsCnt" : 2, + "tcpflags" : { + "ack" : 3, + "dstZero" : 0, + "fin" : 2, + "psh" : 3, + "rst" : 0, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, + "timestamp" : "SET", + "totBytes" : 847, + "totDataBytes" : 237, + "totPackets" : 9 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-130918", + "_type" : "session" + } } } ] diff --git a/tests/pcap/smb-smb1-ascii.test b/tests/pcap/smb-smb1-ascii.test index e10bd4069e..805ad6fb28 100644 --- a/tests/pcap/smb-smb1-ascii.test +++ b/tests/pcap/smb-smb1-ascii.test @@ -1,82 +1,28 @@ { - "sessions" : [ + "sessions2" : [ { - "header" : { - "index" : { - "_index" : "tests_sessions-150423", - "_type" : "session" - } - }, "body" : { - "smbfn" : [ - "XXXXXXXX.exe" - ], - "no" : "test", - "by2" : 1359, - "by" : 3188, - "p2" : 445, - "prot-term" : [ - "smb", - "tcp" - ], - "ps" : [ - 24, - 106, - 188, - 270, - 352, - 432, - 594, - 877, - 957, - 1214, - 1642, - 1722, - 2263, - 2481, - 2561, - 2706, - 2830, - 2910, - 3057, - 3184, - 3264, - 3421 - ], - "smbsh" : [ - "\\\\10.10.0.2\\IPC$", - "\\\\10.10.0.2\\ADMIN$" + "dstBytes" : 1359, + "dstDataBytes" : 879, + "dstIp" : "10.10.0.2", + "dstMac" : [ + "00:50:56:a8:45:c0" ], - "pr" : 6, - "smbdmcnt" : 1, - "vlan-cnt" : 1, - "smbuser" : [ - "Administrator" + "dstMacCnt" : 1, + "dstOui" : [ + "VMware, Inc." ], - "ipSrc" : "10.10.0.3", + "dstOuiCnt" : 1, + "dstPackets" : 8, + "dstPayload8" : "000000cdff534d42", + "dstPort" : 445, + "fileId" : [], + "firstPacket" : 1429783872765, + "ipProtocol" : 6, "lastPacket" : 1429783877023, - "smboscnt" : 1, - "fpd" : 1429783872765, - "smbver" : [ - "Windows 2000 5.0" - ], - "lp" : 1429783877, - "smbhocnt" : 1, - "ss" : 1, - "db1" : 965, - "smbho" : [ - "THEHOSTNAMEHEREX" - ], - "mac2-term-cnt" : 1, - "smbfncnt" : 1, - "sl" : 4258, - "a1" : "10.10.0.3", - "smbos" : [ - "Windows 2000 2195" - ], - "smbusercnt" : 1, - "prot-term-cnt" : 2, - "psl" : [ + "length" : 4258, + "node" : "test", + "packetLen" : [ 82, 82, 82, @@ -100,48 +46,106 @@ 157, 143 ], - "by1" : 1829, - "lpd" : 1429783877023, - "db" : 1844, - "p1" : 2204, - "pa" : 22, - "portSrc" : 2204, - "smbshcnt" : 2, - "pa1" : 14, - "mac1-term" : [ - "00:50:56:a8:1f:7c" + "packetPos" : [ + 24, + 106, + 188, + 270, + 352, + 432, + 594, + 877, + 957, + 1214, + 1642, + 1722, + 2263, + 2481, + 2561, + 2706, + 2830, + 2910, + 3057, + 3184, + 3264, + 3421 ], - "mac1-term-cnt" : 1, - "a2" : "10.10.0.2", - "db2" : 879, - "smbvercnt" : 1, - "fb2" : "000000cdff534d42", - "smbdm" : [ - "LAB" + "protocol" : [ + "smb", + "tcp" ], - "fp" : 1429783872, - "timestamp" : "SET", - "ipDst" : "10.10.0.2", - "pa2" : 8, - "vlan" : [ - 1113 + "protocolCnt" : 2, + "segmentCnt" : 1, + "smb" : { + "domain" : [ + "LAB" + ], + "domainCnt" : 1, + "filename" : [ + "XXXXXXXX.exe" + ], + "filenameCnt" : 1, + "host" : [ + "THEHOSTNAMEHEREX" + ], + "hostCnt" : 1, + "os" : [ + "Windows 2000 2195" + ], + "osCnt" : 1, + "share" : [ + "\\\\10.10.0.2\\IPC$", + "\\\\10.10.0.2\\ADMIN$" + ], + "shareCnt" : 2, + "user" : [ + "Administrator" + ], + "userCnt" : 1, + "version" : [ + "Windows 2000 5.0" + ], + "versionCnt" : 1 + }, + "srcBytes" : 1829, + "srcDataBytes" : 965, + "srcIp" : "10.10.0.3", + "srcMac" : [ + "00:50:56:a8:1f:7c" ], - "firstPacket" : 1429783872765, - "portDst" : 445, + "srcMacCnt" : 1, + "srcOui" : [ + "VMware, Inc." + ], + "srcOuiCnt" : 1, + "srcPackets" : 14, + "srcPayload8" : "00000054ff534d42", + "srcPort" : 2204, "tcpflags" : { "ack" : 6, + "dstZero" : 0, "fin" : 0, "psh" : 12, - "urg" : 0, "rst" : 0, + "srcZero" : 0, + "syn" : 2, "syn-ack" : 2, - "syn" : 2 + "urg" : 0 }, - "mac2-term" : [ - "00:50:56:a8:45:c0" + "timestamp" : "SET", + "totBytes" : 3188, + "totDataBytes" : 1844, + "totPackets" : 22, + "vlan" : [ + 1113 ], - "fb1" : "00000054ff534d42", - "fs" : [] + "vlanCnt" : 1 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-150423", + "_type" : "session" + } } } ] diff --git a/tests/pcap/smb-smbclient.test b/tests/pcap/smb-smbclient.test index e6b19eb9b0..24ae94aed4 100644 --- a/tests/pcap/smb-smbclient.test +++ b/tests/pcap/smb-smbclient.test @@ -1,45 +1,64 @@ { - "sessions" : [ + "sessions2" : [ { - "header" : { - "index" : { - "_index" : "tests_sessions-131219", - "_type" : "session" - } - }, "body" : { - "timestamp" : "SET", - "fp" : 1387494788, - "lastPacket" : 1387494791811, - "prot-term" : [ - "smb", - "tcp" - ], - "smbshcnt" : 1, - "portSrc" : 41823, - "mac2-term-cnt" : 1, - "smbuser" : [ - "user" - ], - "a2" : "127.0.0.1", - "smbvercnt" : 1, - "smbdmcnt" : 1, - "pa2" : 13, - "pa" : 34, - "smbsh" : [ - "\\\\LOCALHOST\\MP3" - ], - "mac2-term" : [ + "dstBytes" : 1667, + "dstDataBytes" : 801, + "dstIp" : "127.0.0.1", + "dstMac" : [ "00:00:00:00:00:00" ], - "lpd" : 1387494791811, - "fb1" : "000000beff534d42", - "prot-term-cnt" : 2, - "smbdm" : [ - "WORKGROUP" + "dstMacCnt" : 1, + "dstOui" : [ + "Officially Xerox, but 0:0:0:0:0:0 is more common" + ], + "dstOuiCnt" : 1, + "dstPackets" : 13, + "dstPayload8" : "00000061ff534d42", + "dstPort" : 445, + "fileId" : [], + "firstPacket" : 1387494788141, + "ipProtocol" : 6, + "lastPacket" : 1387494791811, + "length" : 3669, + "node" : "test", + "packetLen" : [ + 90, + 90, + 82, + 276, + 82, + 183, + 82, + 214, + 182, + 168, + 148, + 82, + 139, + 139, + 82, + 174, + 182, + 82, + 170, + 151, + 82, + 158, + 234, + 141, + 160, + 127, + 121, + 82, + 121, + 121, + 82, + 82, + 82, + 82 ], - "fb2" : "00000061ff534d42", - "ps" : [ + "packetPos" : [ 24, 114, 204, @@ -75,88 +94,73 @@ 4333, 4415 ], - "sl" : 3669, - "pr" : 6, - "ss" : 1, - "portDst" : 445, - "lp" : 1387494791, - "mac1-term" : [ + "protocol" : [ + "smb", + "tcp" + ], + "protocolCnt" : 2, + "segmentCnt" : 1, + "smb" : { + "domain" : [ + "WORKGROUP" + ], + "domainCnt" : 1, + "filename" : [ + "\\tmp\\foo" + ], + "filenameCnt" : 1, + "os" : [ + "Unix" + ], + "osCnt" : 1, + "share" : [ + "\\\\LOCALHOST\\MP3" + ], + "shareCnt" : 1, + "user" : [ + "user" + ], + "userCnt" : 1, + "version" : [ + "Samba 3.6.3" + ], + "versionCnt" : 1 + }, + "srcBytes" : 2262, + "srcDataBytes" : 868, + "srcIp" : "127.0.0.1", + "srcMac" : [ "00:00:00:00:00:00" ], - "smbos" : [ - "Unix" + "srcMacCnt" : 1, + "srcOui" : [ + "Officially Xerox, but 0:0:0:0:0:0 is more common" ], + "srcOuiCnt" : 1, + "srcPackets" : 21, + "srcPayload8" : "000000beff534d42", + "srcPort" : 41823, "tcpflags" : { - "psh" : 20, + "ack" : 10, + "dstZero" : 0, "fin" : 2, - "syn-ack" : 1, + "psh" : 20, "rst" : 0, + "srcZero" : 0, "syn" : 1, - "ack" : 10, + "syn-ack" : 1, "urg" : 0 }, - "smbfncnt" : 1, - "a1" : "127.0.0.1", - "no" : "test", - "db1" : 868, - "ipSrc" : "127.0.0.1", - "fs" : [], - "p1" : 41823, - "by2" : 1667, - "smbver" : [ - "Samba 3.6.3" - ], - "by" : 3929, - "ipDst" : "127.0.0.1", - "smbfn" : [ - "\\tmp\\foo" - ], - "by1" : 2262, - "firstPacket" : 1387494788141, - "db" : 1669, - "psl" : [ - 90, - 90, - 82, - 276, - 82, - 183, - 82, - 214, - 182, - 168, - 148, - 82, - 139, - 139, - 82, - 174, - 182, - 82, - 170, - 151, - 82, - 158, - 234, - 141, - 160, - 127, - 121, - 82, - 121, - 121, - 82, - 82, - 82, - 82 - ], - "smboscnt" : 1, - "mac1-term-cnt" : 1, - "p2" : 445, - "db2" : 801, - "pa1" : 21, - "fpd" : 1387494788141, - "smbusercnt" : 1 + "timestamp" : "SET", + "totBytes" : 3929, + "totDataBytes" : 1669, + "totPackets" : 34 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-131219", + "_type" : "session" + } } } ] diff --git a/tests/pcap/smbtorture-ntlmssp-moloch-crash.test b/tests/pcap/smbtorture-ntlmssp-moloch-crash.test index 185c3a43ef..c93df9a556 100644 --- a/tests/pcap/smbtorture-ntlmssp-moloch-crash.test +++ b/tests/pcap/smbtorture-ntlmssp-moloch-crash.test @@ -1,20 +1,29 @@ { - "sessions" : [ + "sessions2" : [ { - "header" : { - "index" : { - "_type" : "session", - "_index" : "tests_sessions-051119" - } - }, "body" : { - "prot-term-cnt" : 2, - "pa2" : 6, - "rir2" : "ARIN", - "ipSrc" : "192.168.114.1", - "tacnt" : 1, - "smboscnt" : 1, - "psl" : [ + "dstBytes" : 1326, + "dstDataBytes" : 922, + "dstIp" : "192.168.114.129", + "dstMac" : [ + "00:0c:29:30:60:27" + ], + "dstMacCnt" : 1, + "dstOui" : [ + "VMware, Inc." + ], + "dstOuiCnt" : 1, + "dstPackets" : 6, + "dstPayload8" : "00000095ff534d42", + "dstPort" : 445, + "dstRIR" : "ARIN", + "fileId" : [], + "firstPacket" : 1132371118535, + "ipProtocol" : 6, + "lastPacket" : 1132371118592, + "length" : 56, + "node" : "test", + "packetLen" : [ 90, 90, 82, @@ -30,9 +39,7 @@ 121, 131 ], - "p2" : 445, - "pa" : 14, - "ps" : [ + "packetPos" : [ 24, 114, 204, @@ -48,76 +55,70 @@ 2760, 2881 ], - "a1" : "192.168.114.1", - "tags-term" : [ + "protocol" : [ + "smb", + "tcp" + ], + "protocolCnt" : 2, + "segmentCnt" : 1, + "smb" : { + "domain" : [ + "VNET3" + ], + "domainCnt" : 1, + "os" : [ + "Unix" + ], + "osCnt" : 1, + "share" : [ + "\\\\192.168.114.129\\TEST" + ], + "shareCnt" : 1, + "version" : [ + "Samba 3.9.0-SVN-build-11572" + ], + "versionCnt" : 1 + }, + "srcBytes" : 1438, + "srcDataBytes" : 902, + "srcIp" : "192.168.114.1", + "srcMac" : [ + "00:50:56:c0:00:01" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "VMware, Inc." + ], + "srcOuiCnt" : 1, + "srcPackets" : 8, + "srcPayload8" : "000000dbff534d42", + "srcPort" : 52703, + "srcRIR" : "ARIN", + "tags" : [ "smb:bad-security-blob" ], - "mac1-term-cnt" : 1, - "portDst" : 445, + "tagsCnt" : 1, "tcpflags" : { + "ack" : 2, + "dstZero" : 0, "fin" : 0, - "syn-ack" : 1, - "rst" : 0, "psh" : 10, - "urg" : 0, + "rst" : 0, + "srcZero" : 0, "syn" : 1, - "ack" : 2 + "syn-ack" : 1, + "urg" : 0 }, "timestamp" : "SET", - "smbdmcnt" : 1, - "smbdm" : [ - "VNET3" - ], - "by" : 2764, - "smbvercnt" : 1, - "db1" : 902, - "fpd" : 1132371118535, - "lpd" : 1132371118592, - "portSrc" : 52703, - "p1" : 52703, - "prot-term" : [ - "smb", - "tcp" - ], - "smbver" : [ - "Samba 3.9.0-SVN-build-11572" - ], - "pr" : 6, - "by1" : 1438, - "pa1" : 8, - "no" : "test", - "mac2-term-cnt" : 1, - "fb1" : "000000dbff534d42", - "fs" : [], - "mac2-term" : [ - "00:0c:29:30:60:27" - ], - "ta" : [ - "smb:bad-security-blob" - ], - "db" : 1824, - "smbos" : [ - "Unix" - ], - "by2" : 1326, - "smbshcnt" : 1, - "fp" : 1132371118, - "db2" : 922, - "lastPacket" : 1132371118592, - "smbsh" : [ - "\\\\192.168.114.129\\TEST" - ], - "mac1-term" : [ - "00:50:56:c0:00:01" - ], - "lp" : 1132371118, - "a2" : "192.168.114.129", - "rir1" : "ARIN", - "fb2" : "00000095ff534d42", - "ss" : 1, - "firstPacket" : 1132371118535, - "ipDst" : "192.168.114.129", - "sl" : 56 + "totBytes" : 2764, + "totDataBytes" : 1824, + "totPackets" : 14 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-051119", + "_type" : "session" + } } } ] diff --git a/tests/pcap/smbtorture-ntlmssp.test b/tests/pcap/smbtorture-ntlmssp.test index 5da84ea870..f4b05658b6 100644 --- a/tests/pcap/smbtorture-ntlmssp.test +++ b/tests/pcap/smbtorture-ntlmssp.test @@ -1,78 +1,29 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "pa2" : 6, - "ipSrc" : "192.168.114.1", - "mac2-term" : [ + "dstBytes" : 1326, + "dstDataBytes" : 922, + "dstIp" : "192.168.114.129", + "dstMac" : [ "00:0c:29:30:60:27" ], - "rir2" : "ARIN", - "smbho" : [ - "BLU" + "dstMacCnt" : 1, + "dstOui" : [ + "VMware, Inc." ], - "smbdmcnt" : 1, - "smbusercnt" : 1, - "p1" : 52703, - "fp" : 1132371118, - "by2" : 1326, - "pa" : 14, - "portDst" : 445, - "timestamp" : "SET", - "mac1-term" : [ - "00:50:56:c0:00:01" - ], - "smbdm" : [ - "VNET3" - ], - "pa1" : 8, - "tcpflags" : { - "ack" : 2, - "psh" : 10, - "urg" : 0, - "fin" : 0, - "syn-ack" : 1, - "rst" : 0, - "syn" : 1 - }, - "fs" : [], - "smbsh" : [ - "\\\\192.168.114.129\\TEST" - ], - "db" : 1824, - "a2" : "192.168.114.129", - "ipDst" : "192.168.114.129", - "db2" : 922, - "portSrc" : 52703, - "smbuser" : [ - "administrator" - ], - "ss" : 1, - "no" : "test", - "mac1-term-cnt" : 1, - "smbvercnt" : 1, - "sl" : 56, - "a1" : "192.168.114.1", - "smboscnt" : 1, - "smbos" : [ - "Unix" - ], - "smbver" : [ - "Samba 3.9.0-SVN-build-11572" - ], - "fpd" : 1132371118535, - "prot-term-cnt" : 2, - "fb2" : "00000095ff534d42", - "smbhocnt" : 1, - "prot-term" : [ - "smb", - "tcp" - ], - "mac2-term-cnt" : 1, - "by1" : 1438, + "dstOuiCnt" : 1, + "dstPackets" : 6, + "dstPayload8" : "00000095ff534d42", + "dstPort" : 445, + "dstRIR" : "ARIN", + "fileId" : [], + "firstPacket" : 1132371118535, + "ipProtocol" : 6, "lastPacket" : 1132371118592, - "p2" : 445, - "psl" : [ + "length" : 56, + "node" : "test", + "packetLen" : [ 90, 90, 82, @@ -88,10 +39,7 @@ 121, 131 ], - "pr" : 6, - "lpd" : 1132371118592, - "firstPacket" : 1132371118535, - "ps" : [ + "packetPos" : [ 24, 114, 204, @@ -107,17 +55,73 @@ 2760, 2881 ], - "lp" : 1132371118, - "smbshcnt" : 1, - "by" : 2764, - "db1" : 902, - "rir1" : "ARIN", - "fb1" : "000000dbff534d42" + "protocol" : [ + "smb", + "tcp" + ], + "protocolCnt" : 2, + "segmentCnt" : 1, + "smb" : { + "domain" : [ + "VNET3" + ], + "domainCnt" : 1, + "host" : [ + "BLU" + ], + "hostCnt" : 1, + "os" : [ + "Unix" + ], + "osCnt" : 1, + "share" : [ + "\\\\192.168.114.129\\TEST" + ], + "shareCnt" : 1, + "user" : [ + "administrator" + ], + "userCnt" : 1, + "version" : [ + "Samba 3.9.0-SVN-build-11572" + ], + "versionCnt" : 1 + }, + "srcBytes" : 1438, + "srcDataBytes" : 902, + "srcIp" : "192.168.114.1", + "srcMac" : [ + "00:50:56:c0:00:01" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "VMware, Inc." + ], + "srcOuiCnt" : 1, + "srcPackets" : 8, + "srcPayload8" : "000000dbff534d42", + "srcPort" : 52703, + "srcRIR" : "ARIN", + "tcpflags" : { + "ack" : 2, + "dstZero" : 0, + "fin" : 0, + "psh" : 10, + "rst" : 0, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, + "timestamp" : "SET", + "totBytes" : 2764, + "totDataBytes" : 1824, + "totPackets" : 14 }, "header" : { "index" : { - "_type" : "session", - "_index" : "tests_sessions-051119" + "_index" : "tests_sessions2-051119", + "_type" : "session" } } } diff --git a/tests/pcap/smtp-data-250.test b/tests/pcap/smtp-data-250.test index 21d4857538..86d08e25d2 100644 --- a/tests/pcap/smtp-data-250.test +++ b/tests/pcap/smtp-data-250.test @@ -1,60 +1,70 @@ { - "sessions" : [ + "sessions2" : [ { - "header" : { - "index" : { - "_index" : "tests_sessions-131206", - "_type" : "session" - } - }, "body" : { - "ect" : [ - "text/plain; charset=utf-8" + "dstASN" : "AS1668 AOL Transit Data Network", + "dstBytes" : 1753, + "dstDataBytes" : 835, + "dstGEO" : "US", + "dstIp" : "64.12.168.40", + "dstMac" : [ + "00:00:5e:00:01:02", + "80:71:1f:82:cf:c6" ], - "tacnt" : 3, - "esrccnt" : 1, - "by2" : 1753, - "test" : { - "ip-rir" : [ - "" + "dstMacCnt" : 2, + "dstOui" : [ + "ICANN, IANA Department", + "Juniper Networks" + ], + "dstOuiCnt" : 2, + "dstPackets" : 16, + "dstPayload8" : "3232302d6d74616f", + "dstPort" : 587, + "dstRIR" : "ARIN", + "email" : { + "contentType" : [ + "text/plain; charset=utf-8" ], - "ip" : [ - 167772161 + "contentTypeCnt" : 1, + "dst" : [ + "xxxx.xxxxxx@xxxxx.com" ], - "string" : [ - "16777226:3293,682101824:587" + "dstCnt" : 1, + "header" : [ + "content-type", + "message-id", + "mime-version", + "from", + "to", + "content-transfer-encoding", + "date", + "subject" ], - "number" : [ - 682101824 + "headerCnt" : 8, + "id" : [ + "xxxxxxxxxxxxxxxxxxxxxxxx.xxxxxxxxxxxxx@xxxxx.xxxxxxx.xxx" ], - "ip-asn" : [ - "AS0000 This is neat" + "idCnt" : 1, + "mimeVersion" : [ + "1.0" ], - "ip-geo" : [ - "RUS" - ] + "mimeVersionCnt" : 1, + "src" : [ + "12345678@aol.com" + ], + "srcCnt" : 1, + "subject" : [ + "Re: xxx xxx xxxxx xxxxxx..." + ], + "subjectCnt" : 1 }, - "eid" : [ - "xxxxxxxxxxxxxxxxxxxxxxxx.xxxxxxxxxxxxx@xxxxx.xxxxxxx.xxx" - ], - "esub" : [ - "Re: xxx xxx xxxxx xxxxxx..." - ], - "prot-term-cnt" : 2, - "pr" : 6, - "no" : "test", - "tags-term" : [ - "smtp:statuscode:250", - "smtp:authplain", - "srcip" - ], - "fp" : 1386338020, - "prot-term" : [ - "smtp", - "tcp" - ], - "emvcnt" : 1, - "psl" : [ + "fileId" : [], + "firstPacket" : 1386338020379, + "ipProtocol" : 6, + "lastPacket" : 1386338052390, + "length" : 32010, + "node" : "test", + "packetLen" : [ 82, 82, 76, @@ -86,23 +96,7 @@ 76, 76 ], - "ehh" : [ - "content-type", - "message-id", - "mime-version", - "from", - "to", - "content-transfer-encoding", - "date", - "subject" - ], - "by" : 3620, - "fb2" : "3232302d6d74616f", - "fpd" : 1386338020379, - "mac1-term" : [ - "00:0a:f3:31:94:00" - ], - "ps" : [ + "packetPos" : [ 24, 106, 188, @@ -134,73 +128,79 @@ 3972, 4048 ], - "pa" : 30, + "protocol" : [ + "smtp", + "tcp" + ], + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcASN" : "AS0000 This is neat", + "srcBytes" : 1867, + "srcDataBytes" : 1063, + "srcGEO" : "RU", + "srcIp" : "10.0.0.1", + "srcMac" : [ + "00:0a:f3:31:94:00" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Cisco Systems, Inc" + ], + "srcOuiCnt" : 1, + "srcPackets" : 14, + "srcPayload8" : "45484c4f206c6f63", + "srcPort" : 3293, + "tags" : [ + "smtp:authplain", + "smtp:statuscode:250", + "srcip" + ], + "tagsCnt" : 3, "tcpflags" : { - "syn" : 1, - "psh" : 15, + "ack" : 12, + "dstZero" : 0, "fin" : 1, - "syn-ack" : 1, + "psh" : 15, "rst" : 0, - "urg" : 0, - "ack" : 12 + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, + "test" : { + "ASN" : [ + "AS0000 This is neat" + ], + "GEO" : [ + "RU" + ], + "RIR" : [ + "" + ], + "ip" : [ + "10.0.0.1" + ], + "number" : [ + 682101824 + ], + "string.snow" : [ + "16777226:3293,682101824:587" + ] }, - "lastPacket" : 1386338052390, - "lpd" : 1386338052390, - "lp" : 1386338052, - "a1" : "10.0.0.1", - "g1" : "RUS", - "ipSrc" : "10.0.0.1", "timestamp" : "SET", - "edst" : [ - "xxxx.xxxxxx@xxxxx.com" - ], - "mac1-term-cnt" : 1, - "sl" : 32010, - "db2" : 835, - "ss" : 1, - "pa2" : 16, + "totBytes" : 3620, + "totDataBytes" : 1898, + "totPackets" : 30, "user" : [ "12345678@aol.com" ], - "mac2-term-cnt" : 2, - "portDst" : 587, - "by1" : 1867, - "edstcnt" : 1, - "ta" : [ - "smtp:authplain", - "smtp:statuscode:250", - "srcip" - ], - "g2" : "USA", - "mac2-term" : [ - "00:00:5e:00:01:02", - "80:71:1f:82:cf:c6" - ], - "fs" : [], - "firstPacket" : 1386338020379, - "ehhcnt" : 8, - "p1" : 3293, - "ipDst" : "64.12.168.40", - "p2" : 587, - "as1" : "AS0000 This is neat", - "eidcnt" : 1, - "as2" : "AS1668 AOL Transit Data Network", - "emv" : [ - "1.0" - ], - "fb1" : "45484c4f206c6f63", - "esubcnt" : 1, - "db1" : 1063, - "portSrc" : 3293, - "db" : 1898, - "rir2" : "ARIN", - "pa1" : 14, - "ectcnt" : 1, - "usercnt" : 1, - "a2" : "64.12.168.40", - "esrc" : [ - "12345678@aol.com" - ] + "userCnt" : 1 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-131206", + "_type" : "session" + } } } ] diff --git a/tests/pcap/smtp-data-521.test b/tests/pcap/smtp-data-521.test index a2c3a16560..ab8f57745e 100644 --- a/tests/pcap/smtp-data-521.test +++ b/tests/pcap/smtp-data-521.test @@ -1,101 +1,105 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "ss" : 1, - "pa1" : 16, - "mac1-term" : [ - "00:0b:45:b7:08:80" - ], - "pa2" : 22, - "ipSrc" : "10.0.0.1", - "esrccnt" : 1, - "ehh" : [ - "reply-to", - "content-type", - "mime-version", - "from", - "to", - "date", - "subject" - ], - "ehhcnt" : 7, - "mac2-term-cnt" : 2, - "fp" : 1386251815, - "tags-term" : [ - "smtp:authplain", - "srcip", - "smtp:statuscode:521" - ], - "portDst" : 587, - "no" : "test", - "lpd" : 1386251823511, - "by" : 6494, - "ipDst" : "205.188.186.167", - "emv" : [ - "1.0" - ], - "lastPacket" : 1386251823511, - "ect" : [ - "multipart/alternative; boundary=\"xxxxxxxxxxxxxxxxxxx=xxxxxxxxxxxxxx\"" + "dstASN" : "AS1668 AOL Transit Data Network", + "dstBytes" : 2308, + "dstDataBytes" : 854, + "dstGEO" : "US", + "dstIp" : "205.188.186.167", + "dstMac" : [ + "00:00:5e:00:01:01", + "00:26:88:ca:1f:c6" ], - "pa" : 38, - "fb2" : "3232302d6d74616f", - "esrc" : [ - "1234567899@aol.com" + "dstMacCnt" : 2, + "dstOui" : [ + "ICANN, IANA Department", + "Juniper Networks" ], - "fb1" : "45484c4f20787878", - "firstPacket" : 1386251815266, - "tcpflags" : { - "fin" : 2, - "syn-ack" : 1, - "ack" : 12, - "urg" : 0, - "psh" : 21, - "syn" : 1, - "rst" : 1 - }, - "test" : { - "ip-rir" : [ - "" + "dstOuiCnt" : 2, + "dstPackets" : 22, + "dstPayload8" : "3232302d6d74616f", + "dstPort" : 587, + "dstRIR" : "ARIN", + "email" : { + "contentType" : [ + "multipart/alternative; boundary=\"xxxxxxxxxxxxxxxxxxx=xxxxxxxxxxxxxx\"" ], - "string" : [ - "16777226:56558,-1480934195:587" + "contentTypeCnt" : 1, + "dst" : [ + "xxxxx-xxxx@xxxx.xxx.xx.jp" ], - "ip-asn" : [ - "AS0000 This is neat" + "dstCnt" : 1, + "header" : [ + "reply-to", + "content-type", + "mime-version", + "from", + "to", + "date", + "subject" ], - "ip" : [ - 167772161 + "headerCnt" : 7, + "mimeVersion" : [ + "1.0" ], - "ip-geo" : [ - "RUS" + "mimeVersionCnt" : 1, + "src" : [ + "1234567899@aol.com" ], - "number" : [ - 2814033101 - ] + "srcCnt" : 1, + "subject" : [ + "Urgent Trip." + ], + "subjectCnt" : 1 }, - "ectcnt" : 1, - "g1" : "RUS", - "prot-term-cnt" : 2, - "db2" : 854, - "edst" : [ - "xxxxx-xxxx@xxxx.xxx.xx.jp" - ], - "a2" : "205.188.186.167", - "usercnt" : 1, - "lp" : 1386251823, - "by1" : 4186, - "as2" : "AS1668 AOL Transit Data Network", - "user" : [ - "123456789@aol.com" + "fileId" : [], + "firstPacket" : 1386251815266, + "ipProtocol" : 6, + "lastPacket" : 1386251823511, + "length" : 8244, + "node" : "test", + "packetLen" : [ + 90, + 90, + 82, + 504, + 109, + 82, + 291, + 94, + 88, + 120, + 82, + 119, + 88, + 82, + 96, + 115, + 82, + 96, + 119, + 82, + 96, + 88, + 82, + 119, + 397, + 82, + 1430, + 83, + 82, + 82, + 1375, + 82, + 183, + 82, + 82, + 88, + 82, + 76 ], - "sl" : 8244, - "fs" : [], - "fpd" : 1386251815266, - "rir2" : "ARIN", - "esubcnt" : 1, - "ps" : [ + "packetPos" : [ 24, 114, 204, @@ -135,81 +139,77 @@ 6968, 7050 ], - "mac1-term-cnt" : 1, - "esub" : [ - "Urgent Trip." + "protocol" : [ + "smtp", + "tcp" + ], + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcASN" : "AS0000 This is neat", + "srcBytes" : 4186, + "srcDataBytes" : 3122, + "srcGEO" : "RU", + "srcIp" : "10.0.0.1", + "srcMac" : [ + "00:0b:45:b7:08:80" ], - "pr" : 6, - "p1" : 56558, - "as1" : "AS0000 This is neat", - "db" : 3976, - "portSrc" : 56558, - "timestamp" : "SET", - "emvcnt" : 1, - "p2" : 587, - "psl" : [ - 90, - 90, - 82, - 504, - 109, - 82, - 291, - 94, - 88, - 120, - 82, - 119, - 88, - 82, - 96, - 115, - 82, - 96, - 119, - 82, - 96, - 88, - 82, - 119, - 397, - 82, - 1430, - 83, - 82, - 82, - 1375, - 82, - 183, - 82, - 82, - 88, - 82, - 76 + "srcMacCnt" : 1, + "srcOui" : [ + "Cisco Systems, Inc" ], - "by2" : 2308, - "db1" : 3122, - "ta" : [ + "srcOuiCnt" : 1, + "srcPackets" : 16, + "srcPayload8" : "45484c4f20787878", + "srcPort" : 56558, + "tags" : [ "smtp:authplain", "smtp:statuscode:521", "srcip" ], - "edstcnt" : 1, - "prot-term" : [ - "smtp", - "tcp" + "tagsCnt" : 3, + "tcpflags" : { + "ack" : 12, + "dstZero" : 0, + "fin" : 2, + "psh" : 21, + "rst" : 1, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, + "test" : { + "ASN" : [ + "AS0000 This is neat" + ], + "GEO" : [ + "RU" + ], + "RIR" : [ + "" + ], + "ip" : [ + "10.0.0.1" + ], + "number" : [ + 2814033101 + ], + "string.snow" : [ + "16777226:56558,-1480934195:587" + ] + }, + "timestamp" : "SET", + "totBytes" : 6494, + "totDataBytes" : 3976, + "totPackets" : 38, + "user" : [ + "123456789@aol.com" ], - "tacnt" : 3, - "g2" : "USA", - "a1" : "10.0.0.1", - "mac2-term" : [ - "00:00:5e:00:01:01", - "00:26:88:ca:1f:c6" - ] + "userCnt" : 1 }, "header" : { "index" : { - "_index" : "tests_sessions-131205", + "_index" : "tests_sessions2-131205", "_type" : "session" } } diff --git a/tests/pcap/smtp-moloch-bof.test b/tests/pcap/smtp-moloch-bof.test index 53eda7dd42..23f426bfe6 100644 --- a/tests/pcap/smtp-moloch-bof.test +++ b/tests/pcap/smtp-moloch-bof.test @@ -1,76 +1,63 @@ { - "sessions" : [ + "sessions2" : [ { - "header" : { - "index" : { - "_index" : "tests_sessions-151106", - "_type" : "session" - } - }, "body" : { - "esrccnt" : 2, - "db" : 9399, - "ectcnt" : 1, - "tags-term" : [ - "smtp:statuscode:250" - ], - "tacnt" : 1, - "edst" : [ - "mattenuttall@example.com" - ], - "timestamp" : "SET", - "mac2-term-cnt" : 1, - "mac2-term" : [ + "dstBytes" : 1305, + "dstDataBytes" : 439, + "dstIp" : "127.0.0.1", + "dstMac" : [ "00:00:00:00:00:00" ], - "ipSrc" : "127.0.0.1", - "mac1-term-cnt" : 1, - "pa2" : 13, - "prot-term-cnt" : 2, - "a2" : "127.0.0.1", - "a1" : "127.0.0.1", - "db1" : 8960, - "portDst" : 25, - "no" : "test", - "esub" : [ - " MIME test" - ], - "lastPacket" : 1446834929714, - "p2" : 25, - "fb1" : "48454c4f20414443", - "fpd" : 1446834929262, - "efncnt" : 1, - "sl" : 452, - "pa" : 26, - "emvcnt" : 1, - "ehh" : [ - "content-type", - "mime-version", - "from", - "to", - "subject" + "dstMacCnt" : 1, + "dstOui" : [ + "Officially Xerox, but 0:0:0:0:0:0 is more common" ], + "dstOuiCnt" : 1, + "dstPackets" : 13, + "dstPayload8" : "323230206c6f6361", + "dstPort" : 25, + "email" : { + "contentType" : [ + "multipart/mixed; boundary=\"=_frontier\"" + ], + "contentTypeCnt" : 1, + "dst" : [ + "mattenuttall@example.com" + ], + "dstCnt" : 1, + "filename" : [ + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + ], + "filenameCnt" : 1, + "header" : [ + "content-type", + "mime-version", + "from", + "to", + "subject" + ], + "headerCnt" : 5, + "mimeVersion" : [ + "1.0" + ], + "mimeVersionCnt" : 1, + "src" : [ + "matt@zpdt.example.com", + "ibmuser@zpdt.example.com" + ], + "srcCnt" : 2, + "subject" : [ + " MIME test" + ], + "subjectCnt" : 1 + }, + "fileId" : [], "firstPacket" : 1446834929262, - "lp" : 1446834929, - "pr" : 6, - "by1" : 9826, - "esubcnt" : 1, - "ta" : [ - "smtp:statuscode:250" - ], - "pa1" : 13, - "ss" : 1, - "fb2" : "323230206c6f6361", - "db2" : 439, - "edstcnt" : 1, - "prot-term" : [ - "smtp", - "tcp" - ], - "emv" : [ - "1.0" - ], - "psl" : [ + "ipProtocol" : 6, + "lastPacket" : 1446834929714, + "length" : 452, + "node" : "test", + "packetLen" : [ 90, 90, 82, @@ -98,32 +85,7 @@ 82, 82 ], - "tcpflags" : { - "urg" : 0, - "fin" : 1, - "ack" : 11, - "rst" : 1, - "psh" : 11, - "syn" : 1, - "syn-ack" : 1 - }, - "fp" : 1446834929, - "esrc" : [ - "matt@zpdt.example.com", - "ibmuser@zpdt.example.com" - ], - "p1" : 50328, - "fs" : [], - "lpd" : 1446834929714, - "mac1-term" : [ - "00:00:00:00:00:00" - ], - "ect" : [ - "multipart/mixed; boundary=\"=_frontier\"" - ], - "portSrc" : 50328, - "by2" : 1305, - "ps" : [ + "packetPos" : [ 24, 114, 204, @@ -151,12 +113,51 @@ 11407, 11489 ], - "ipDst" : "127.0.0.1", - "ehhcnt" : 5, - "by" : 11131, - "efn" : [ - "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" - ] + "protocol" : [ + "smtp", + "tcp" + ], + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 9826, + "srcDataBytes" : 8960, + "srcIp" : "127.0.0.1", + "srcMac" : [ + "00:00:00:00:00:00" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Officially Xerox, but 0:0:0:0:0:0 is more common" + ], + "srcOuiCnt" : 1, + "srcPackets" : 13, + "srcPayload8" : "48454c4f20414443", + "srcPort" : 50328, + "tags" : [ + "smtp:statuscode:250" + ], + "tagsCnt" : 1, + "tcpflags" : { + "ack" : 11, + "dstZero" : 0, + "fin" : 1, + "psh" : 11, + "rst" : 1, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, + "timestamp" : "SET", + "totBytes" : 11131, + "totDataBytes" : 9399, + "totPackets" : 26 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-151106", + "_type" : "session" + } } } ] diff --git a/tests/pcap/smtp-nospaces.test b/tests/pcap/smtp-nospaces.test index 85b9bf4b3f..326e7d44a8 100644 --- a/tests/pcap/smtp-nospaces.test +++ b/tests/pcap/smtp-nospaces.test @@ -1,37 +1,59 @@ { - "sessions" : [ + "sessions2" : [ { - "header" : { - "index" : { - "_index" : "tests_sessions-160120", - "_type" : "session" - } - }, "body" : { - "mac1-term-cnt" : 1, - "pr" : 6, - "db1" : 82, - "esrccnt" : 1, - "mac2-term-cnt" : 1, - "ipDst" : "127.0.0.1", - "pa2" : 9, - "lpd" : 1453298304047, - "by2" : 872, - "prot-term" : [ - "smtp", - "tcp" - ], - "fpd" : 1453298297881, - "timestamp" : "SET", - "p2" : 25, - "a1" : "127.0.0.1", - "portDst" : 25, - "mac1-term" : [ + "dstBytes" : 872, + "dstDataBytes" : 270, + "dstIp" : "127.0.0.1", + "dstMac" : [ "00:00:00:00:00:00" ], - "p1" : 42971, + "dstMacCnt" : 1, + "dstOui" : [ + "Officially Xerox, but 0:0:0:0:0:0 is more common" + ], + "dstOuiCnt" : 1, + "dstPackets" : 9, + "dstPayload8" : "323230206c6f6361", + "dstPort" : 25, + "email" : { + "dst" : [ + "reciever@example.com" + ], + "dstCnt" : 1, + "src" : [ + "random@example.com" + ], + "srcCnt" : 1 + }, + "fileId" : [], + "firstPacket" : 1453298297881, + "ipProtocol" : 6, "lastPacket" : 1453298304047, - "ps" : [ + "length" : 6167, + "node" : "test", + "packetLen" : [ + 90, + 90, + 82, + 171, + 82, + 88, + 82, + 112, + 82, + 112, + 82, + 88, + 82, + 88, + 82, + 263, + 82, + 86, + 82 + ], + "packetPos" : [ 24, 114, 204, @@ -52,65 +74,47 @@ 1782, 1868 ], - "by" : 1622, - "edst" : [ - "reciever@example.com" + "protocol" : [ + "smtp", + "tcp" ], - "mac2-term" : [ + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 750, + "srcDataBytes" : 82, + "srcIp" : "127.0.0.1", + "srcMac" : [ "00:00:00:00:00:00" ], - "prot-term-cnt" : 2, - "no" : "test", - "pa" : 19, - "fb1" : "48454c4f0d0a4d41", - "by1" : 750, - "a2" : "127.0.0.1", - "fb2" : "323230206c6f6361", - "pa1" : 10, + "srcMacCnt" : 1, + "srcOui" : [ + "Officially Xerox, but 0:0:0:0:0:0 is more common" + ], + "srcOuiCnt" : 1, + "srcPackets" : 10, + "srcPayload8" : "48454c4f0d0a4d41", + "srcPort" : 42971, "tcpflags" : { - "fin" : 0, - "urg" : 0, - "rst" : 0, "ack" : 9, + "dstZero" : 0, + "fin" : 0, "psh" : 8, + "rst" : 0, + "srcZero" : 0, + "syn" : 1, "syn-ack" : 1, - "syn" : 1 + "urg" : 0 }, - "firstPacket" : 1453298297881, - "db" : 352, - "esrc" : [ - "random@example.com" - ], - "sl" : 6167, - "edstcnt" : 1, - "portSrc" : 42971, - "psl" : [ - 90, - 90, - 82, - 171, - 82, - 88, - 82, - 112, - 82, - 112, - 82, - 88, - 82, - 88, - 82, - 263, - 82, - 86, - 82 - ], - "db2" : 270, - "lp" : 1453298304, - "ipSrc" : "127.0.0.1", - "fp" : 1453298297, - "ss" : 1, - "fs" : [] + "timestamp" : "SET", + "totBytes" : 1622, + "totDataBytes" : 352, + "totPackets" : 19 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-160120", + "_type" : "session" + } } } ] diff --git a/tests/pcap/smtp-originating.test b/tests/pcap/smtp-originating.test index a22229ddf2..8d6f870553 100644 --- a/tests/pcap/smtp-originating.test +++ b/tests/pcap/smtp-originating.test @@ -1,108 +1,102 @@ { - "sessions" : [ + "sessions2" : [ { - "header" : { - "index" : { - "_type" : "session", - "_index" : "tests_sessions-140113" - } - }, "body" : { - "p1" : 55713, - "ect" : [ - "text/plain; charset=us-ascii" + "dstBytes" : 1251, + "dstDataBytes" : 517, + "dstIp" : "10.0.0.6", + "dstMac" : [ + "00:25:90:a2:c2:6c" ], - "mac1-term-cnt" : 2, - "tcpflags" : { - "urg" : 0, - "syn-ack" : 1, - "syn" : 1, - "psh" : 11, - "fin" : 2, - "ack" : 8, - "rst" : 1 + "dstMacCnt" : 1, + "dstOui" : [ + "Super Micro Computer, Inc." + ], + "dstOuiCnt" : 1, + "dstPackets" : 11, + "dstPayload8" : "3232302078787878", + "dstPort" : 25, + "email" : { + "ASN" : [ + "AS0001 Cool Beans!", + "AS0002 Hmm!@#$%^&*()", + "---" + ], + "GEO" : [ + "CA", + "---", + "---" + ], + "RIR" : [ + "TEST", + "", + "" + ], + "contentType" : [ + "text/plain; charset=us-ascii" + ], + "contentTypeCnt" : 1, + "dst" : [ + "xxxxxxxx.xxxxxxx@xxxxxxxxxxxxxx.com" + ], + "dstCnt" : 1, + "header" : [ + "content-type", + "x-mailer", + "message-id", + "mime-version", + "from", + "to", + "x-elnk-trace", + "received", + "content-transfer-encoding", + "date", + "domainkey-signature", + "subject", + "x-originating-ip" + ], + "header-x-elnk-trace" : [ + "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + ], + "headerCnt" : 13, + "host" : [ + "xxxxxxxxxxxxxxxxxxxx.xxxxxxxxx.net" + ], + "hostCnt" : 1, + "id" : [ + "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx@xxxxxxxxx.net" + ], + "idCnt" : 1, + "ip" : [ + "10.0.0.2", + "10.0.0.3", + "10.0.0.4" + ], + "ipCnt" : 3, + "mimeVersion" : [ + "1.0 (Apple Message framework v1283)" + ], + "mimeVersionCnt" : 1, + "src" : [ + "xxxxxxxx@xxxxxxxxx.net" + ], + "srcCnt" : 1, + "subject" : [ + "xxxxxxxxxxxxxxxxxx" + ], + "subjectCnt" : 1, + "useragent" : [ + "Apple Mail (2.1283)" + ], + "useragentCnt" : 1 }, - "prot-term-cnt" : 2, - "aseip" : [ - "AS0002 Hmm!@#$%^&*()", - "---", - "AS0001 Cool Beans!" - ], - "eid" : [ - "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx@xxxxxxxxx.net" - ], - "a1" : "10.0.0.5", - "pa2" : 11, - "esrccnt" : 1, + "fileId" : [], + "firstPacket" : 1389625273848, + "ipProtocol" : 6, "lastPacket" : 1389625274669, - "mac2-term-cnt" : 1, - "eua" : [ - "Apple Mail (2.1283)" - ], - "ps" : [ - 24, - 114, - 204, - 286, - 439, - 521, - 644, - 726, - 988, - 1169, - 1291, - 1373, - 1511, - 1593, - 1721, - 1803, - 3333, - 3476, - 3558, - 3671, - 3759, - 3841, - 4016, - 4098 - ], - "emvcnt" : 1, - "eip" : [ - "10.0.0.3", - "10.0.0.4", - "10.0.0.2" - ], - "edst" : [ - "xxxxxxxx.xxxxxxx@xxxxxxxxxxxxxx.com" - ], - "eipcnt" : 3, - "by2" : 1251, - "geip" : [ - "---", - "---", - "CAN" - ], - "db1" : 1655, - "ehocnt" : 1, - "ipDst" : "10.0.0.6", - "fb1" : "45484c4f20787878", - "fs" : [], - "db" : 2172, - "no" : "test", - "ectcnt" : 1, - "lpd" : 1389625274669, - "pr" : 6, - "tags-term" : [ - "smtp:statuscode:250", - "dstip" - ], - "ss" : 1, - "fpd" : 1389625273848, - "esub" : [ - "xxxxxxxxxxxxxxxxxx" - ], - "esubcnt" : 1, - "ehhcnt" : 13, - "psl" : [ + "length" : 822, + "node" : "test", + "packetLen" : [ 90, 90, 82, @@ -128,70 +122,79 @@ 82, 76 ], - "mac2-term" : [ - "00:25:90:a2:c2:6c" - ], - "emv" : [ - "1.0 (Apple Message framework v1283)" - ], - "sl" : 822, - "edstcnt" : 1, - "fp" : 1389625273, - "by" : 3766, - "portSrc" : 55713, - "by1" : 2515, - "ta" : [ - "dstip", - "smtp:statuscode:250" + "packetPos" : [ + 24, + 114, + 204, + 286, + 439, + 521, + 644, + 726, + 988, + 1169, + 1291, + 1373, + 1511, + 1593, + 1721, + 1803, + 3333, + 3476, + 3558, + 3671, + 3759, + 3841, + 4016, + 4098 ], - "pa" : 24, - "prot-term" : [ + "protocol" : [ "smtp", "tcp" ], - "ipSrc" : "10.0.0.5", - "eho" : [ - "xxxxxxxxxxxxxxxxxxxx.xxxxxxxxx.net" - ], - "tacnt" : 2, - "mac1-term" : [ + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 2515, + "srcDataBytes" : 1655, + "srcIp" : "10.0.0.5", + "srcMac" : [ "00:00:5e:00:01:01", "82:71:1f:83:98:f6" ], - "p2" : 25, - "pa1" : 13, - "db2" : 517, - "euacnt" : 1, - "timestamp" : "SET", - "eidcnt" : 1, - "lp" : 1389625274, - "rireip" : [ - "", - "", - "TEST" - ], - "esrc" : [ - "xxxxxxxx@xxxxxxxxx.net" + "srcMacCnt" : 2, + "srcOui" : [ + "ICANN, IANA Department" ], - "fb2" : "3232302078787878", - "ehh" : [ - "content-type", - "x-mailer", - "message-id", - "mime-version", - "from", - "to", - "x-elnk-trace", - "received", - "content-transfer-encoding", - "date", - "domainkey-signature", - "subject", - "x-originating-ip" + "srcOuiCnt" : 1, + "srcPackets" : 13, + "srcPayload8" : "45484c4f20787878", + "srcPort" : 55713, + "tags" : [ + "dstip", + "smtp:statuscode:250" ], - "a2" : "10.0.0.6", - "firstPacket" : 1389625273848, - "portDst" : 25 + "tagsCnt" : 2, + "tcpflags" : { + "ack" : 8, + "dstZero" : 0, + "fin" : 2, + "psh" : 11, + "rst" : 1, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, + "timestamp" : "SET", + "totBytes" : 3766, + "totDataBytes" : 2172, + "totPackets" : 24 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-140113", + "_type" : "session" + } } } ] diff --git a/tests/pcap/smtp-rcpt-553.test b/tests/pcap/smtp-rcpt-553.test index 03c27c19c8..ff4c9d07ca 100644 --- a/tests/pcap/smtp-rcpt-553.test +++ b/tests/pcap/smtp-rcpt-553.test @@ -1,28 +1,43 @@ { - "sessions" : [ + "sessions2" : [ { - "header" : { - "index" : { - "_index" : "tests_sessions-131205", - "_type" : "session" - } - }, "body" : { - "no" : "test", - "fb1" : "45484c4f206c6f63", - "fp" : 1386252622, - "tacnt" : 1, - "sl" : 643, - "esrc" : [ - "12345678@aol.com" + "dstASN" : "AS1668 AOL Transit Data Network", + "dstBytes" : 1185, + "dstDataBytes" : 715, + "dstGEO" : "US", + "dstIp" : "64.12.168.40", + "dstMac" : [ + "00:00:5e:00:01:02", + "00:26:88:d8:bf:c1" + ], + "dstMacCnt" : 2, + "dstOui" : [ + "ICANN, IANA Department", + "Juniper Networks" ], - "fs" : [], - "ss" : 1, - "g2" : "USA", - "portSrc" : 48012, - "mac1-term-cnt" : 1, - "db1" : 74, - "psl" : [ + "dstOuiCnt" : 2, + "dstPackets" : 7, + "dstPayload8" : "3232302d6d74616f", + "dstPort" : 587, + "dstRIR" : "ARIN", + "email" : { + "dst" : [ + "123456@example" + ], + "dstCnt" : 1, + "src" : [ + "12345678@aol.com" + ], + "srcCnt" : 1 + }, + "fileId" : [], + "firstPacket" : 1386252622425, + "ipProtocol" : 6, + "lastPacket" : 1386252623067, + "length" : 643, + "node" : "test", + "packetLen" : [ 90, 90, 82, @@ -40,8 +55,7 @@ 82, 82 ], - "by2" : 1185, - "ps" : [ + "packetPos" : [ 24, 114, 204, @@ -59,85 +73,73 @@ 1977, 2059 ], - "g1" : "RUS", - "prot-term-cnt" : 2, - "esrccnt" : 1, - "mac2-term" : [ - "00:26:88:d8:bf:c1", - "00:00:5e:00:01:02" - ], - "pa2" : 7, - "p1" : 48012, - "lp" : 1386252623, - "p2" : 587, - "mac1-term" : [ - "00:0a:f3:31:84:00" - ], - "prot-term" : [ + "protocol" : [ "smtp", "tcp" ], - "ipDst" : "64.12.168.40", - "a2" : "64.12.168.40", - "edst" : [ - "123456@example" + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcASN" : "AS0000 This is neat", + "srcBytes" : 676, + "srcDataBytes" : 74, + "srcGEO" : "RU", + "srcIp" : "10.0.0.1", + "srcMac" : [ + "00:0a:f3:31:84:00" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Cisco Systems, Inc" ], - "ta" : [ + "srcOuiCnt" : 1, + "srcPackets" : 9, + "srcPayload8" : "45484c4f206c6f63", + "srcPort" : 48012, + "tags" : [ "srcip" ], - "fpd" : 1386252622425, - "lpd" : 1386252623067, - "by" : 1861, - "firstPacket" : 1386252622425, - "by1" : 676, - "db" : 789, - "pr" : 6, - "lastPacket" : 1386252623067, - "ipSrc" : "10.0.0.1", - "portDst" : 587, - "mac2-term-cnt" : 2, - "as1" : "AS0000 This is neat", - "timestamp" : "SET", + "tagsCnt" : 1, + "tcpflags" : { + "ack" : 5, + "dstZero" : 0, + "fin" : 2, + "psh" : 7, + "rst" : 0, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, "test" : { - "ip" : [ - 167772161 + "ASN" : [ + "AS0000 This is neat" ], - "ip-rir" : [ + "GEO" : [ + "RU" + ], + "RIR" : [ "" ], - "string" : [ - "16777226:48012,682101824:587" + "ip" : [ + "10.0.0.1" ], "number" : [ 682101824 ], - "ip-geo" : [ - "RUS" - ], - "ip-asn" : [ - "AS0000 This is neat" + "string.snow" : [ + "16777226:48012,682101824:587" ] }, - "pa" : 16, - "tags-term" : [ - "srcip" - ], - "rir2" : "ARIN", - "db2" : 715, - "edstcnt" : 1, - "a1" : "10.0.0.1", - "tcpflags" : { - "psh" : 7, - "rst" : 0, - "urg" : 0, - "fin" : 2, - "ack" : 5, - "syn-ack" : 1, - "syn" : 1 - }, - "fb2" : "3232302d6d74616f", - "as2" : "AS1668 AOL Transit Data Network", - "pa1" : 9 + "timestamp" : "SET", + "totBytes" : 1861, + "totDataBytes" : 789, + "totPackets" : 16 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-131205", + "_type" : "session" + } } } ] diff --git a/tests/pcap/smtp-starttls.test b/tests/pcap/smtp-starttls.test index 827c371cd5..26d03e8195 100644 --- a/tests/pcap/smtp-starttls.test +++ b/tests/pcap/smtp-starttls.test @@ -1,83 +1,139 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "pa" : 36, - "db" : 6011, - "mac2-term-cnt" : 2, - "as2" : "AS15169 Google LLC", - "by" : 8403, - "lpd" : 1388017125239, - "fb1" : "45484c4f20787878", - "tlsver-termcnt" : 1, - "fp" : 1388017124, - "tcpflags" : { - "syn-ack" : 1, - "fin" : 2, - "psh" : 20, - "syn" : 1, - "rst" : 0, - "urg" : 0, - "ack" : 12 - }, - "portDst" : 25, - "mac1-term-cnt" : 1, - "lastPacket" : 1388017125239, - "tlscnt" : 3, - "fb2" : "323230206d782e67", - "as1" : "AS0000 This is neat", - "rir2" : "ARIN", - "test" : { - "number" : [ - 440713901 - ], - "ip-rir" : [ - "" - ], - "ip" : [ - 167772161 - ], - "ip-asn" : [ - "AS0000 This is neat" - ], - "string" : [ - "16777226:57406,440713901:25" - ], - "ip-geo" : [ - "RUS" - ] - }, - "pa2" : 19, - "fs" : [], - "mac1-term" : [ - "00:13:72:c4:f1:e1" - ], - "db1" : 1384, - "prot-term-cnt" : 3, - "pr" : 6, - "fpd" : 1388017124762, - "portSrc" : 57406, - "tlscipher-termcnt" : 1, - "prot-term" : [ - "tls", - "smtp", - "tcp" + "cert" : [ + { + "hash" : "73:59:75:5c:6d:f9:a0:ab:c3:06:0b:ce:36:95:64:c8:ec:45:42:a3", + "issuerON" : "Equifax", + "notAfter" : 1534824000000, + "notBefore" : 1021953600000, + "serial" : "12bbe6", + "subjectCN" : [ + "geotrust global ca" + ], + "subjectON" : "GeoTrust Inc.", + "validDays" : 5936 + }, + { + "hash" : "d8:3c:1a:7f:4d:04:46:bb:20:81:b8:1a:16:70:f8:18:34:51:ca:24", + "issuerCN" : [ + "geotrust global ca" + ], + "issuerON" : "GeoTrust Inc.", + "notAfter" : 1428160555000, + "notBefore" : 1365174955000, + "serial" : "023a69", + "subjectCN" : [ + "google internet authority g2" + ], + "subjectON" : "Google Inc", + "validDays" : 729 + }, + { + "alt" : [ + "aspmx.l.google.com", + "alt1.aspmx.l.google.com", + "alt2.aspmx.l.google.com", + "alt3.aspmx.l.google.com", + "alt4.aspmx.l.google.com", + "gmail-smtp-in.l.google.com", + "alt1.gmail-smtp-in.l.google.com", + "alt2.gmail-smtp-in.l.google.com", + "alt3.gmail-smtp-in.l.google.com", + "alt4.gmail-smtp-in.l.google.com", + "gmr-smtp-in.l.google.com", + "alt1.gmr-smtp-in.l.google.com", + "alt2.gmr-smtp-in.l.google.com", + "alt3.gmr-smtp-in.l.google.com", + "alt4.gmr-smtp-in.l.google.com", + "mx.google.com", + "aspmx2.googlemail.com", + "aspmx3.googlemail.com", + "aspmx4.googlemail.com", + "aspmx5.googlemail.com" + ], + "altCnt" : 20, + "hash" : "45:15:6a:e7:49:63:40:94:f9:ab:09:1e:f5:a7:33:6d:f3:7b:28:fc", + "issuerCN" : [ + "google internet authority g2" + ], + "issuerON" : "Google Inc", + "notAfter" : 1410262355000, + "notBefore" : 1378726355000, + "serial" : "325d8297987d50b0", + "subjectCN" : [ + "mx.google.com" + ], + "subjectON" : "Google Inc", + "validDays" : 365 + } ], - "tlscipher-term" : [ - "TLS_RSA_WITH_RC4_128_SHA" + "certCnt" : 3, + "dstASN" : "AS15169 Google LLC", + "dstBytes" : 5889, + "dstDataBytes" : 4627, + "dstGEO" : "US", + "dstIp" : "173.194.68.26", + "dstMac" : [ + "00:00:0c:07:ac:01", + "00:d0:2b:d1:76:00" ], - "tacnt" : 2, - "tags-term" : [ - "srcip", - "smtp:starttls" + "dstMacCnt" : 2, + "dstOui" : [ + "Cisco", + "Jetcell, Inc." ], - "g2" : "USA", - "sl" : 478, - "ta" : [ - "smtp:starttls", - "srcip" + "dstOuiCnt" : 2, + "dstPackets" : 19, + "dstPayload8" : "323230206d782e67", + "dstPort" : 25, + "dstRIR" : "ARIN", + "fileId" : [], + "firstPacket" : 1388017124762, + "ipProtocol" : 6, + "lastPacket" : 1388017125239, + "length" : 478, + "node" : "test", + "packetLen" : [ + 90, + 90, + 82, + 133, + 82, + 110, + 82, + 236, + 92, + 112, + 194, + 1500, + 1500, + 82, + 935, + 396, + 292, + 135, + 247, + 143, + 147, + 143, + 82, + 188, + 768, + 82, + 110, + 82, + 158, + 82, + 113, + 163, + 82, + 82, + 82, + 82 ], - "ps" : [ + "packetPos" : [ 24, 114, 204, @@ -115,138 +171,83 @@ 8839, 8921 ], - "by2" : 5889, - "by1" : 2514, - "ss" : 1, - "no" : "test", - "a2" : "173.194.68.26", - "psl" : [ - 90, - 90, - 82, - 133, - 82, - 110, - 82, - 236, - 92, - 112, - 194, - 1500, - 1500, - 82, - 935, - 396, - 292, - 135, - 247, - 143, - 147, - 143, - 82, - 188, - 768, - 82, - 110, - 82, - 158, - 82, - 113, - 163, - 82, - 82, - 82, - 82 + "protocol" : [ + "tls", + "smtp", + "tcp" ], - "firstPacket" : 1388017124762, - "p1" : 57406, - "a1" : "10.0.0.1", - "db2" : 4627, - "mac2-term" : [ - "00:00:0c:07:ac:01", - "00:d0:2b:d1:76:00" + "protocolCnt" : 3, + "segmentCnt" : 1, + "srcASN" : "AS0000 This is neat", + "srcBytes" : 2514, + "srcDataBytes" : 1384, + "srcGEO" : "RU", + "srcIp" : "10.0.0.1", + "srcMac" : [ + "00:13:72:c4:f1:e1" ], - "ipSrc" : "10.0.0.1", - "timestamp" : "SET", - "ipDst" : "173.194.68.26", - "lp" : 1388017125, - "g1" : "RUS", - "p2" : 25, - "tlsver-term" : [ - "TLSv1" + "srcMacCnt" : 1, + "srcOui" : [ + "Dell Inc." ], - "tls" : [ - { - "notBefore" : 1021953600, - "notAfter" : 1534824000, - "diffDays" : 5936, - "sCn" : [ - "geotrust global ca" - ], - "hash" : "73:59:75:5c:6d:f9:a0:ab:c3:06:0b:ce:36:95:64:c8:ec:45:42:a3", - "sOn" : "GeoTrust Inc.", - "sn" : "12bbe6", - "iOn" : "Equifax" - }, - { - "iCn" : [ - "geotrust global ca" - ], - "iOn" : "GeoTrust Inc.", - "sn" : "023a69", - "hash" : "d8:3c:1a:7f:4d:04:46:bb:20:81:b8:1a:16:70:f8:18:34:51:ca:24", - "sOn" : "Google Inc", - "notAfter" : 1428160555, - "notBefore" : 1365174955, - "diffDays" : 729, - "sCn" : [ - "google internet authority g2" - ] - }, - { - "iOn" : "Google Inc", - "sn" : "325d8297987d50b0", - "iCn" : [ - "google internet authority g2" - ], - "hash" : "45:15:6a:e7:49:63:40:94:f9:ab:09:1e:f5:a7:33:6d:f3:7b:28:fc", - "sOn" : "Google Inc", - "alt" : [ - "aspmx.l.google.com", - "alt1.aspmx.l.google.com", - "alt2.aspmx.l.google.com", - "alt3.aspmx.l.google.com", - "alt4.aspmx.l.google.com", - "gmail-smtp-in.l.google.com", - "alt1.gmail-smtp-in.l.google.com", - "alt2.gmail-smtp-in.l.google.com", - "alt3.gmail-smtp-in.l.google.com", - "alt4.gmail-smtp-in.l.google.com", - "gmr-smtp-in.l.google.com", - "alt1.gmr-smtp-in.l.google.com", - "alt2.gmr-smtp-in.l.google.com", - "alt3.gmr-smtp-in.l.google.com", - "alt4.gmr-smtp-in.l.google.com", - "mx.google.com", - "aspmx2.googlemail.com", - "aspmx3.googlemail.com", - "aspmx4.googlemail.com", - "aspmx5.googlemail.com" - ], - "sCn" : [ - "mx.google.com" - ], - "diffDays" : 365, - "altcnt" : 20, - "notBefore" : 1378726355, - "notAfter" : 1410262355 - } + "srcOuiCnt" : 1, + "srcPackets" : 17, + "srcPayload8" : "45484c4f20787878", + "srcPort" : 57406, + "tags" : [ + "smtp:starttls", + "srcip" ], - "pa1" : 17 + "tagsCnt" : 2, + "tcpflags" : { + "ack" : 12, + "dstZero" : 0, + "fin" : 2, + "psh" : 20, + "rst" : 0, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, + "test" : { + "ASN" : [ + "AS0000 This is neat" + ], + "GEO" : [ + "RU" + ], + "RIR" : [ + "" + ], + "ip" : [ + "10.0.0.1" + ], + "number" : [ + 440713901 + ], + "string.snow" : [ + "16777226:57406,440713901:25" + ] + }, + "timestamp" : "SET", + "tls" : { + "cipher" : [ + "TLS_RSA_WITH_RC4_128_SHA" + ], + "cipherCnt" : 1, + "version" : [ + "TLSv1" + ], + "versionCnt" : 1 + }, + "totBytes" : 8403, + "totDataBytes" : 6011, + "totPackets" : 36 }, "header" : { "index" : { - "_index" : "tests_sessions-131226", + "_index" : "tests_sessions2-131226", "_type" : "session" } } diff --git a/tests/pcap/smtp-subject-8859-b.test b/tests/pcap/smtp-subject-8859-b.test index cd8b93d157..0919a72d6c 100644 --- a/tests/pcap/smtp-subject-8859-b.test +++ b/tests/pcap/smtp-subject-8859-b.test @@ -1,116 +1,106 @@ { - "sessions" : [ + "sessions2" : [ { - "header" : { - "index" : { - "_index" : "tests_sessions-131206", - "_type" : "session" - } - }, "body" : { - "fb2" : "3232302078787878", - "esub" : [ - "test å é î ø ü" - ], - "esrc" : [ - "xxxxxxxxx@xxxxxxx.com" - ], - "g2" : "USA", - "sl" : 518, - "p2" : 25, - "mac2-term-cnt" : 1, - "no" : "test", - "tags-term" : [ - "smtp:statuscode:250" - ], - "ectcnt" : 1, - "timestamp" : "SET", - "pr" : 6, - "fs" : [], - "eid" : [ - "CEC7902D.1A0ED%xxxxxxxxx@xxxxxxx.com" - ], - "mac2-term" : [ + "dstASN" : "AS1668 AOL Transit Data Network", + "dstBytes" : 1169, + "dstDataBytes" : 485, + "dstGEO" : "US", + "dstIp" : "64.236.55.18", + "dstMac" : [ "00:25:90:a2:c2:52" ], - "ss" : 1, - "a2" : "64.236.55.18", - "db1" : 1666, - "as2" : "AS1668 AOL Transit Data Network", - "lp" : 1386358271, - "emvcnt" : 1, - "ipSrc" : "10.180.121.109", - "rireip" : [ - "", - "", - "ARIN" - ], - "edstcnt" : 1, - "eua" : [ - "Microsoft-MacOutlook/14.3.9.131030" - ], - "eipcnt" : 3, - "ect" : [ - "multipart/alternative; boundary=\"_000_CEC7902D1A0EDxxxxxxxxxxxxxxxxxx_\"" - ], - "fpd" : 1386358271061, - "eip" : [ - "10.0.0.4", - "10.180.121.108", - "169.254.4.195" - ], - "mac1-term" : [ - "00:00:0c:07:ac:02", - "00:0b:5f:6b:5c:00" - ], - "ta" : [ - "smtp:statuscode:250" - ], - "portDst" : 25, - "geip" : [ - "---", - "USA", - "---" - ], - "tcpflags" : { - "ack" : 7, - "fin" : 2, - "urg" : 0, - "rst" : 0, - "syn-ack" : 1, - "syn" : 1, - "psh" : 12 + "dstMacCnt" : 1, + "dstOui" : [ + "Super Micro Computer, Inc." + ], + "dstOuiCnt" : 1, + "dstPackets" : 12, + "dstPayload8" : "3232302078787878", + "dstPort" : 25, + "dstRIR" : "ARIN", + "email" : { + "ASN" : [ + "---", + "---", + "---" + ], + "GEO" : [ + "---", + "US", + "---" + ], + "RIR" : [ + "ARIN", + "", + "" + ], + "contentType" : [ + "multipart/alternative; boundary=\"_000_CEC7902D1A0EDxxxxxxxxxxxxxxxxxx_\"" + ], + "contentTypeCnt" : 1, + "dst" : [ + "xxxxxxxx@gmail.com" + ], + "dstCnt" : 1, + "header" : [ + "content-type", + "message-id", + "mime-version", + "from", + "to", + "user-agent", + "x-ms-tnef-correlator", + "received", + "thread-index", + "date", + "subject", + "content-language", + "x-originating-ip", + "x-ms-has-attach", + "accept-language", + "thread-topic" + ], + "headerCnt" : 16, + "host" : [ + "xxxxxxxxxxxxxxxxx2.xxxxx.net", + "xxxxxxxxxxxxxxxxx1.xxxxx.net" + ], + "hostCnt" : 2, + "id" : [ + "CEC7902D.1A0ED%xxxxxxxxx@xxxxxxx.com" + ], + "idCnt" : 1, + "ip" : [ + "169.254.4.195", + "10.180.121.108", + "10.0.0.4" + ], + "ipCnt" : 3, + "mimeVersion" : [ + "1.0" + ], + "mimeVersionCnt" : 1, + "src" : [ + "xxxxxxxxx@xxxxxxx.com" + ], + "srcCnt" : 1, + "subject" : [ + "test å é î ø ü" + ], + "subjectCnt" : 1, + "useragent" : [ + "Microsoft-MacOutlook/14.3.9.131030" + ], + "useragentCnt" : 1 }, - "pa2" : 12, - "db" : 2151, - "db2" : 485, - "a1" : "10.180.121.109", - "ps" : [ - 24, - 106, - 188, - 264, - 405, - 510, - 586, - 830, - 975, - 1084, - 1160, - 1269, - 1345, - 1461, - 2991, - 3145, - 3221, - 3322, - 3398, - 3553, - 3629, - 3705, - 3781 - ], - "psl" : [ + "fileId" : [], + "firstPacket" : 1386358271061, + "ipProtocol" : 6, + "lastPacket" : 1386358271580, + "length" : 518, + "node" : "test", + "packetLen" : [ 82, 82, 76, @@ -135,67 +125,79 @@ 76, 76 ], - "esubcnt" : 1, - "fp" : 1386358271, - "firstPacket" : 1386358271061, - "by" : 3465, - "prot-term-cnt" : 2, - "pa" : 23, - "mac1-term-cnt" : 2, - "ipDst" : "64.236.55.18", - "fb1" : "45484c4f20787878", - "prot-term" : [ + "packetPos" : [ + 24, + 106, + 188, + 264, + 405, + 510, + 586, + 830, + 975, + 1084, + 1160, + 1269, + 1345, + 1461, + 2991, + 3145, + 3221, + 3322, + 3398, + 3553, + 3629, + 3705, + 3781 + ], + "protocol" : [ "smtp", "tcp" ], - "aseip" : [ - "---", - "---", - "---" - ], - "by2" : 1169, - "euacnt" : 1, - "emv" : [ - "1.0" + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 2296, + "srcDataBytes" : 1666, + "srcGEO" : "US", + "srcIp" : "10.180.121.109", + "srcMac" : [ + "00:00:0c:07:ac:02", + "00:0b:5f:6b:5c:00" ], - "rir2" : "ARIN", - "edst" : [ - "xxxxxxxx@gmail.com" + "srcMacCnt" : 2, + "srcOui" : [ + "Cisco", + "Cisco Systems, Inc" ], - "lpd" : 1386358271580, - "tacnt" : 1, - "ehocnt" : 2, - "pa1" : 11, - "ehh" : [ - "content-type", - "message-id", - "mime-version", - "from", - "to", - "user-agent", - "x-ms-tnef-correlator", - "received", - "thread-index", - "date", - "subject", - "content-language", - "x-originating-ip", - "x-ms-has-attach", - "accept-language", - "thread-topic" + "srcOuiCnt" : 2, + "srcPackets" : 11, + "srcPayload8" : "45484c4f20787878", + "srcPort" : 11084, + "tags" : [ + "smtp:statuscode:250" ], - "p1" : 11084, - "ehhcnt" : 16, - "esrccnt" : 1, - "eidcnt" : 1, - "lastPacket" : 1386358271580, - "portSrc" : 11084, - "by1" : 2296, - "g1" : "USA", - "eho" : [ - "xxxxxxxxxxxxxxxxx2.xxxxx.net", - "xxxxxxxxxxxxxxxxx1.xxxxx.net" - ] + "tagsCnt" : 1, + "tcpflags" : { + "ack" : 7, + "dstZero" : 0, + "fin" : 2, + "psh" : 12, + "rst" : 0, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, + "timestamp" : "SET", + "totBytes" : 3465, + "totDataBytes" : 2151, + "totPackets" : 23 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-131206", + "_type" : "session" + } } } ] diff --git a/tests/pcap/smtp-subject-8859-multi.test b/tests/pcap/smtp-subject-8859-multi.test index aaae42eea5..3cb803ca49 100644 --- a/tests/pcap/smtp-subject-8859-multi.test +++ b/tests/pcap/smtp-subject-8859-multi.test @@ -1,33 +1,94 @@ { - "sessions" : [ + "sessions2" : [ { - "header" : { - "index" : { - "_index" : "tests_sessions-131212", - "_type" : "session" - } - }, "body" : { - "rireip" : [ - "" - ], - "no" : "test", - "p2" : 25, - "fb1" : "45484c4f20787878", - "eip" : [ - "127.0.0.1" - ], - "a2" : "64.236.64.225", - "ta" : [ - "smtp:statuscode:250" + "dstASN" : "AS1668 AOL Transit Data Network", + "dstBytes" : 1425, + "dstDataBytes" : 493, + "dstGEO" : "US", + "dstIp" : "64.236.64.225", + "dstMac" : [ + "00:00:0c:07:ac:01", + "00:0e:d6:0b:98:80" ], - "eipcnt" : 1, - "fpd" : 1386877497906, - "euacnt" : 1, - "as2" : "AS1668 AOL Transit Data Network", - "prot-term-cnt" : 2, - "lp" : 1386877498, - "psl" : [ + "dstMacCnt" : 2, + "dstOui" : [ + "Cisco", + "Cisco Systems, Inc" + ], + "dstOuiCnt" : 2, + "dstPackets" : 14, + "dstPayload8" : "3232302078787878", + "dstPort" : 25, + "dstRIR" : "ARIN", + "email" : { + "ASN" : [ + "---" + ], + "GEO" : [ + "---" + ], + "RIR" : [ + "" + ], + "contentType" : [ + "text/plain; charset=us-ascii" + ], + "contentTypeCnt" : 1, + "dst" : [ + "xxxxxxxxx@xxxxxxx.com" + ], + "dstCnt" : 1, + "header" : [ + "content-type", + "message-id", + "mime-version", + "from", + "to", + "user-agent", + "received", + "content-disposition", + "date", + "subject" + ], + "headerCnt" : 10, + "host" : [ + "localhost", + "xxxxxxxxxxxxx.xxx.com" + ], + "hostCnt" : 2, + "id" : [ + "20131212194457.GA7990@xxx.net" + ], + "idCnt" : 1, + "ip" : [ + "127.0.0.1" + ], + "ipCnt" : 1, + "mimeVersion" : [ + "1.0" + ], + "mimeVersionCnt" : 1, + "src" : [ + "xxxxx@xxx.net" + ], + "srcCnt" : 1, + "subject" : [ + "4spaces å é î ø ü 5spaces" + ], + "subjectCnt" : 1, + "useragent" : [ + "Mutt/1.5.20 (2009-12-10)" + ], + "useragentCnt" : 1 + }, + "fileId" : [], + "firstPacket" : 1386877497906, + "ipProtocol" : 6, + "lastPacket" : 1386877498208, + "length" : 302, + "node" : "test", + "packetLen" : [ 90, 90, 82, @@ -57,70 +118,7 @@ 82, 82 ], - "fs" : [], - "eid" : [ - "20131212194457.GA7990@xxx.net" - ], - "ehocnt" : 2, - "esubcnt" : 1, - "a1" : "10.180.156.249", - "edstcnt" : 1, - "pa1" : 14, - "lpd" : 1386877498208, - "esrccnt" : 1, - "ehh" : [ - "content-type", - "message-id", - "mime-version", - "from", - "to", - "user-agent", - "received", - "content-disposition", - "date", - "subject" - ], - "esub" : [ - "4spaces å é î ø ü 5spaces" - ], - "ect" : [ - "text/plain; charset=us-ascii" - ], - "g2" : "USA", - "p1" : 51650, - "edst" : [ - "xxxxxxxxx@xxxxxxx.com" - ], - "ipSrc" : "10.180.156.249", - "db2" : 493, - "eho" : [ - "localhost", - "xxxxxxxxxxxxx.xxx.com" - ], - "tacnt" : 1, - "aseip" : [ - "---" - ], - "ehhcnt" : 10, - "tags-term" : [ - "smtp:statuscode:250" - ], - "emvcnt" : 1, - "tcpflags" : { - "fin" : 2, - "rst" : 0, - "psh" : 13, - "ack" : 11, - "syn" : 1, - "syn-ack" : 1, - "urg" : 0 - }, - "pa2" : 14, - "ipDst" : "64.236.64.225", - "esrc" : [ - "xxxxx@xxx.net" - ], - "ps" : [ + "packetPos" : [ 24, 114, 204, @@ -150,48 +148,52 @@ 3513, 3595 ], - "ss" : 1, - "by1" : 1780, - "pr" : 6, - "by" : 3205, - "prot-term" : [ + "protocol" : [ "smtp", "tcp" ], - "mac1-term" : [ + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 1780, + "srcDataBytes" : 848, + "srcGEO" : "US", + "srcIp" : "10.180.156.249", + "srcMac" : [ "00:13:72:c4:f1:e1" ], - "eidcnt" : 1, - "rir2" : "ARIN", - "g1" : "USA", - "eua" : [ - "Mutt/1.5.20 (2009-12-10)" + "srcMacCnt" : 1, + "srcOui" : [ + "Dell Inc." ], - "emv" : [ - "1.0" + "srcOuiCnt" : 1, + "srcPackets" : 14, + "srcPayload8" : "45484c4f20787878", + "srcPort" : 51650, + "tags" : [ + "smtp:statuscode:250" ], - "pa" : 28, - "ectcnt" : 1, - "fp" : 1386877497, + "tagsCnt" : 1, + "tcpflags" : { + "ack" : 11, + "dstZero" : 0, + "fin" : 2, + "psh" : 13, + "rst" : 0, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, "timestamp" : "SET", - "sl" : 302, - "mac2-term" : [ - "00:00:0c:07:ac:01", - "00:0e:d6:0b:98:80" - ], - "fb2" : "3232302078787878", - "mac1-term-cnt" : 1, - "db" : 1341, - "firstPacket" : 1386877497906, - "db1" : 848, - "lastPacket" : 1386877498208, - "portSrc" : 51650, - "geip" : [ - "---" - ], - "portDst" : 25, - "mac2-term-cnt" : 2, - "by2" : 1425 + "totBytes" : 3205, + "totDataBytes" : 1341, + "totPackets" : 28 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-131212", + "_type" : "session" + } } } ] diff --git a/tests/pcap/smtp-subject-8859-q.test b/tests/pcap/smtp-subject-8859-q.test index 579f490c5a..a899e75aba 100644 --- a/tests/pcap/smtp-subject-8859-q.test +++ b/tests/pcap/smtp-subject-8859-q.test @@ -1,57 +1,94 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "mac2-term-cnt" : 2, - "pa1" : 15, - "timestamp" : "SET", - "fs" : [], - "db" : 1570, - "emvcnt" : 1, - "eid" : [ - "20131212030845.GA715@xxx.net" + "dstASN" : "AS1668 AOL Transit Data Network", + "dstBytes" : 1425, + "dstDataBytes" : 493, + "dstGEO" : "US", + "dstIp" : "64.236.55.17", + "dstMac" : [ + "00:00:0c:07:ac:01", + "00:0e:d6:0b:98:80" ], - "edstcnt" : 1, - "p2" : 25, - "esrccnt" : 1, - "ipDst" : "64.236.55.17", - "by" : 3500, - "tcpflags" : { - "urg" : 0, - "syn" : 1, - "ack" : 12, - "fin" : 2, - "rst" : 0, - "psh" : 13, - "syn-ack" : 1 + "dstMacCnt" : 2, + "dstOui" : [ + "Cisco", + "Cisco Systems, Inc" + ], + "dstOuiCnt" : 2, + "dstPackets" : 14, + "dstPayload8" : "3232302078787878", + "dstPort" : 25, + "dstRIR" : "ARIN", + "email" : { + "ASN" : [ + "---" + ], + "GEO" : [ + "---" + ], + "RIR" : [ + "" + ], + "contentType" : [ + "text/plain; charset=us-ascii" + ], + "contentTypeCnt" : 1, + "dst" : [ + "xxxxxxxxx@xxxxxxx.com" + ], + "dstCnt" : 1, + "header" : [ + "content-type", + "message-id", + "mime-version", + "from", + "to", + "user-agent", + "received", + "content-disposition", + "date", + "subject" + ], + "headerCnt" : 10, + "host" : [ + "localhost", + "xxxxxxxxxxxxx.xxx.com" + ], + "hostCnt" : 2, + "id" : [ + "20131212030845.GA715@xxx.net" + ], + "idCnt" : 1, + "ip" : [ + "127.0.0.1" + ], + "ipCnt" : 1, + "mimeVersion" : [ + "1.0" + ], + "mimeVersionCnt" : 1, + "src" : [ + "xxxxx@xxx.net" + ], + "srcCnt" : 1, + "subject" : [ + "é î ø ü test" + ], + "subjectCnt" : 1, + "useragent" : [ + "Mutt/1.5.20 (2009-12-10)" + ], + "useragentCnt" : 1 }, - "eho" : [ - "localhost", - "xxxxxxxxxxxxx.xxx.com" - ], - "prot-term-cnt" : 2, - "fpd" : 1386817725755, - "fb1" : "45484c4f20787878", - "sl" : 401, - "p1" : 35796, - "pa" : 29, - "ipSrc" : "10.180.156.249", - "aseip" : [ - "---" - ], - "rir2" : "ARIN", - "by2" : 1425, - "pa2" : 14, - "esub" : [ - "é î ø ü test" - ], - "fp" : 1386817725, - "eipcnt" : 1, - "ectcnt" : 1, - "eip" : [ - "127.0.0.1" - ], - "psl" : [ + "fileId" : [], + "firstPacket" : 1386817725755, + "ipProtocol" : 6, + "lastPacket" : 1386817726156, + "length" : 401, + "node" : "test", + "packetLen" : [ 90, 90, 82, @@ -82,67 +119,7 @@ 82, 82 ], - "a2" : "64.236.55.17", - "emv" : [ - "1.0" - ], - "db1" : 1077, - "mac1-term-cnt" : 1, - "lpd" : 1386817726156, - "firstPacket" : 1386817725755, - "portDst" : 25, - "rireip" : [ - "" - ], - "mac1-term" : [ - "00:13:72:c4:f1:e1" - ], - "tacnt" : 1, - "eua" : [ - "Mutt/1.5.20 (2009-12-10)" - ], - "euacnt" : 1, - "ect" : [ - "text/plain; charset=us-ascii" - ], - "geip" : [ - "---" - ], - "ehocnt" : 2, - "portSrc" : 35796, - "ss" : 1, - "ehh" : [ - "content-type", - "message-id", - "mime-version", - "from", - "to", - "user-agent", - "received", - "content-disposition", - "date", - "subject" - ], - "by1" : 2075, - "no" : "test", - "ta" : [ - "smtp:statuscode:250" - ], - "esrc" : [ - "xxxxx@xxx.net" - ], - "eidcnt" : 1, - "db2" : 493, - "edst" : [ - "xxxxxxxxx@xxxxxxx.com" - ], - "mac2-term" : [ - "00:00:0c:07:ac:01", - "00:0e:d6:0b:98:80" - ], - "esubcnt" : 1, - "ehhcnt" : 10, - "ps" : [ + "packetPos" : [ 24, 114, 204, @@ -173,26 +150,51 @@ 3824, 3906 ], - "fb2" : "3232302078787878", - "prot-term" : [ + "protocol" : [ "smtp", "tcp" ], - "as2" : "AS1668 AOL Transit Data Network", - "lp" : 1386817726, - "lastPacket" : 1386817726156, - "pr" : 6, - "g2" : "USA", - "a1" : "10.180.156.249", - "g1" : "USA", - "tags-term" : [ + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 2075, + "srcDataBytes" : 1077, + "srcGEO" : "US", + "srcIp" : "10.180.156.249", + "srcMac" : [ + "00:13:72:c4:f1:e1" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Dell Inc." + ], + "srcOuiCnt" : 1, + "srcPackets" : 15, + "srcPayload8" : "45484c4f20787878", + "srcPort" : 35796, + "tags" : [ "smtp:statuscode:250" - ] + ], + "tagsCnt" : 1, + "tcpflags" : { + "ack" : 12, + "dstZero" : 0, + "fin" : 2, + "psh" : 13, + "rst" : 0, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, + "timestamp" : "SET", + "totBytes" : 3500, + "totDataBytes" : 1570, + "totPackets" : 29 }, "header" : { "index" : { - "_type" : "session", - "_index" : "tests_sessions-131212" + "_index" : "tests_sessions2-131212", + "_type" : "session" } } } diff --git a/tests/pcap/smtp-subject-encoded-empty.test b/tests/pcap/smtp-subject-encoded-empty.test index b27a1789b9..0ad762ed03 100644 --- a/tests/pcap/smtp-subject-encoded-empty.test +++ b/tests/pcap/smtp-subject-encoded-empty.test @@ -1,54 +1,53 @@ { - "sessions" : [ + "sessions2" : [ { - "header" : { - "index" : { - "_index" : "tests_sessions-140425", - "_type" : "session" - } - }, "body" : { - "sl" : 30678, - "lp" : 1398431453, - "g2" : "CAN", - "by" : 3304, - "ehhcnt" : 3, - "ta" : [ - "dstip", - "smtp:statuscode:250", - "srcip" - ], - "db1" : 350, - "p1" : 62855, - "db2" : 756, - "no" : "test", - "esub" : [ - "Can this 10 Second Trick Help Prevent YOUR Heart Attack?" - ], - "lpd" : 1398431453159, - "as2" : "AS0001 Cool Beans!", - "ehh" : [ - "from", - "to", - "subject" - ], - "edstcnt" : 1, - "pr" : 6, - "a1" : "10.0.0.1", - "as1" : "AS0000 This is neat", - "timestamp" : "SET", - "g1" : "RUS", - "portSrc" : 62855, - "esrc" : [ - "user1@xxx.net" + "dstASN" : "AS0001 Cool Beans!", + "dstBytes" : 1688, + "dstDataBytes" : 756, + "dstGEO" : "CA", + "dstIp" : "10.0.0.2", + "dstMac" : [ + "00:00:0c:07:ac:01", + "00:d0:2b:d1:76:00" ], - "db" : 1106, - "ss" : 1, - "mac1-term" : [ - "00:1f:5b:ff:51:cb" + "dstMacCnt" : 2, + "dstOui" : [ + "Cisco", + "Jetcell, Inc." ], - "fb1" : "45484c4f20787878", - "psl" : [ + "dstOuiCnt" : 2, + "dstPackets" : 14, + "dstPayload8" : "3232302d78787878", + "dstPort" : 25, + "dstRIR" : "TEST", + "email" : { + "dst" : [ + "user2@xxx.net" + ], + "dstCnt" : 1, + "header" : [ + "from", + "to", + "subject" + ], + "headerCnt" : 3, + "src" : [ + "user1@xxx.net" + ], + "srcCnt" : 1, + "subject" : [ + "Can this 10 Second Trick Help Prevent YOUR Heart Attack?" + ], + "subjectCnt" : 1 + }, + "fileId" : [], + "firstPacket" : 1398431422481, + "ipProtocol" : 6, + "lastPacket" : 1398431453159, + "length" : 30678, + "node" : "test", + "packetLen" : [ 94, 90, 82, @@ -83,75 +82,7 @@ 82, 82 ], - "a2" : "10.0.0.2", - "esrccnt" : 1, - "mac2-term-cnt" : 2, - "by2" : 1688, - "test" : { - "number" : [ - 33554442 - ], - "ip-asn" : [ - "AS0000 This is neat" - ], - "string" : [ - "16777226:62855,33554442:25" - ], - "ip" : [ - 167772161 - ], - "ip-geo" : [ - "RUS" - ], - "ip-rir" : [ - "" - ] - }, - "mac2-term" : [ - "00:00:0c:07:ac:01", - "00:d0:2b:d1:76:00" - ], - "ipSrc" : "10.0.0.1", - "portDst" : 25, - "fb2" : "3232302d78787878", - "fpd" : 1398431422481, - "p2" : 25, - "tacnt" : 3, - "prot-term-cnt" : 2, - "mac1-term-cnt" : 1, - "pa1" : 19, - "tcpflags" : { - "fin" : 2, - "ack" : 14, - "urg" : 0, - "psh" : 15, - "rst" : 0, - "syn-ack" : 1, - "syn" : 1 - }, - "by1" : 1616, - "fs" : [], - "fp" : 1398431422, - "pa" : 33, - "ipDst" : "10.0.0.2", - "rir2" : "TEST", - "prot-term" : [ - "smtp", - "tcp" - ], - "lastPacket" : 1398431453159, - "esubcnt" : 1, - "edst" : [ - "user2@xxx.net" - ], - "tags-term" : [ - "smtp:statuscode:250", - "srcip", - "dstip" - ], - "pa2" : 14, - "firstPacket" : 1398431422481, - "ps" : [ + "packetPos" : [ 24, 118, 208, @@ -185,7 +116,76 @@ 3610, 3692, 3774 - ] + ], + "protocol" : [ + "smtp", + "tcp" + ], + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcASN" : "AS0000 This is neat", + "srcBytes" : 1616, + "srcDataBytes" : 350, + "srcGEO" : "RU", + "srcIp" : "10.0.0.1", + "srcMac" : [ + "00:1f:5b:ff:51:cb" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Apple, Inc." + ], + "srcOuiCnt" : 1, + "srcPackets" : 19, + "srcPayload8" : "45484c4f20787878", + "srcPort" : 62855, + "tags" : [ + "dstip", + "smtp:statuscode:250", + "srcip" + ], + "tagsCnt" : 3, + "tcpflags" : { + "ack" : 14, + "dstZero" : 0, + "fin" : 2, + "psh" : 15, + "rst" : 0, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, + "test" : { + "ASN" : [ + "AS0000 This is neat" + ], + "GEO" : [ + "RU" + ], + "RIR" : [ + "" + ], + "ip" : [ + "10.0.0.1" + ], + "number" : [ + 33554442 + ], + "string.snow" : [ + "16777226:62855,33554442:25" + ] + }, + "timestamp" : "SET", + "totBytes" : 3304, + "totDataBytes" : 1106, + "totPackets" : 33 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-140425", + "_type" : "session" + } } } ] diff --git a/tests/pcap/smtp-subject-gb2312-b.test b/tests/pcap/smtp-subject-gb2312-b.test index 53b62d017e..3e152e5428 100644 --- a/tests/pcap/smtp-subject-gb2312-b.test +++ b/tests/pcap/smtp-subject-gb2312-b.test @@ -1,106 +1,140 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "lp" : 1386358429, - "ehhcnt" : 16, - "ipSrc" : "10.180.121.151", - "ipDst" : "64.236.64.225", - "eid" : [ - "CEC790C3.1A0F0%xxxxxxxxx@xxxxxxx.com" - ], - "timestamp" : "SET", - "db" : 2059, - "eho" : [ - "xxxxxxxxxxxxxxxxx2.xxxxx.net", - "xxxxxxxxxxxxxxxxx4.xxxxx.net" - ], - "p2" : 25, - "emd5" : [ - "9f06243abcb89c70e0c331c61d871fa7" - ], - "ectcnt" : 1, - "firstPacket" : 1386358422209, - "as2" : "AS1668 AOL Transit Data Network", - "db1" : 1573, - "pa" : 24, - "ta" : [ - "smtp:statuscode:250" - ], - "pr" : 6, - "fb1" : "45484c4f20787878", - "prot-term-cnt" : 2, - "eidcnt" : 1, - "eua" : [ - "Microsoft-MacOutlook/14.3.9.131030" - ], - "mac1-term-cnt" : 2, - "edstcnt" : 1, - "by" : 3439, - "ss" : 1, - "fs" : [], - "euacnt" : 1, - "tcpflags" : { - "syn-ack" : 1, - "ack" : 8, - "rst" : 0, - "fin" : 2, - "urg" : 0, - "psh" : 12, - "syn" : 1 - }, - "lpd" : 1386358429964, - "fp" : 1386358422, - "fb2" : "3232302078787878", - "sl" : 7755, - "by2" : 1290, - "rireip" : [ - "", - "ARIN", - "" - ], - "a2" : "64.236.64.225", - "esrc" : [ - "xxxxxxxxx@xxxxxxx.com" - ], - "pa1" : 10, - "mac1-term" : [ - "00:00:0c:07:ac:01", - "00:0b:5f:6b:5d:40" - ], - "db2" : 486, - "aseip" : [ - "---", - "---", - "---" - ], - "esubcnt" : 1, - "tags-term" : [ - "smtp:statuscode:250" - ], - "portSrc" : 24962, - "mac2-term" : [ + "dstASN" : "AS1668 AOL Transit Data Network", + "dstBytes" : 1290, + "dstDataBytes" : 486, + "dstGEO" : "US", + "dstIp" : "64.236.64.225", + "dstMac" : [ "00:25:90:7e:28:f6" ], - "rir2" : "ARIN", - "prot-term" : [ - "smtp", - "tcp" - ], - "edst" : [ - "xxxxxxxx@gmail.com" - ], - "esrccnt" : 1, - "emv" : [ - "1.0" + "dstMacCnt" : 1, + "dstOui" : [ + "Super Micro Computer, Inc." + ], + "dstOuiCnt" : 1, + "dstPackets" : 14, + "dstPayload8" : "3232302078787878", + "dstPort" : 25, + "dstRIR" : "ARIN", + "email" : { + "ASN" : [ + "---", + "---", + "---" + ], + "GEO" : [ + "---", + "US", + "---" + ], + "RIR" : [ + "ARIN", + "", + "" + ], + "contentType" : [ + "multipart/alternative; boundary=\"_000_CEC790C31A0F0xxxxxxxxxxxxxxxxxx_\"" + ], + "contentTypeCnt" : 1, + "dst" : [ + "xxxxxxxx@gmail.com" + ], + "dstCnt" : 1, + "header" : [ + "content-type", + "message-id", + "mime-version", + "from", + "to", + "user-agent", + "x-ms-tnef-correlator", + "received", + "thread-index", + "date", + "subject", + "content-language", + "x-originating-ip", + "x-ms-has-attach", + "accept-language", + "thread-topic" + ], + "headerCnt" : 16, + "host" : [ + "xxxxxxxxxxxxxxxxx2.xxxxx.net", + "xxxxxxxxxxxxxxxxx4.xxxxx.net" + ], + "hostCnt" : 2, + "id" : [ + "CEC790C3.1A0F0%xxxxxxxxx@xxxxxxx.com" + ], + "idCnt" : 1, + "ip" : [ + "169.254.4.195", + "10.180.121.151", + "10.0.0.4" + ], + "ipCnt" : 3, + "md5" : [ + "9f06243abcb89c70e0c331c61d871fa7" + ], + "md5Cnt" : 1, + "mimeVersion" : [ + "1.0" + ], + "mimeVersionCnt" : 1, + "sha256" : [ + "837ccb607e312b170fac7383d7ccfd61fa5072793f19a25e75fbacb56539b86b" + ], + "sha256Cnt" : 1, + "src" : [ + "xxxxxxxxx@xxxxxxx.com" + ], + "srcCnt" : 1, + "subject" : [ + "test 测试" + ], + "subjectCnt" : 1, + "useragent" : [ + "Microsoft-MacOutlook/14.3.9.131030" + ], + "useragentCnt" : 1 + }, + "fileId" : [], + "firstPacket" : 1386358422209, + "ipProtocol" : 6, + "lastPacket" : 1386358429964, + "length" : 7755, + "node" : "test", + "packetLen" : [ + 82, + 82, + 76, + 141, + 105, + 76, + 244, + 145, + 109, + 76, + 109, + 76, + 76, + 116, + 1521, + 76, + 102, + 76, + 76, + 155, + 76, + 76, + 76, + 76 ], - "mac2-term-cnt" : 1, - "by1" : 2149, - "pa2" : 14, - "ehocnt" : 2, - "p1" : 24962, - "eipcnt" : 3, - "ps" : [ + "packetPos" : [ 24, 106, 188, @@ -126,80 +160,52 @@ 3695, 3771 ], - "g1" : "USA", - "emd5cnt" : 1, - "eip" : [ - "10.0.0.4", - "169.254.4.195", - "10.180.121.151" - ], - "psl" : [ - 82, - 82, - 76, - 141, - 105, - 76, - 244, - 145, - 109, - 76, - 109, - 76, - 76, - 116, - 1521, - 76, - 102, - 76, - 76, - 155, - 76, - 76, - 76, - 76 + "protocol" : [ + "smtp", + "tcp" ], - "lastPacket" : 1386358429964, - "tacnt" : 1, - "ect" : [ - "multipart/alternative; boundary=\"_000_CEC790C31A0F0xxxxxxxxxxxxxxxxxx_\"" + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 2149, + "srcDataBytes" : 1573, + "srcGEO" : "US", + "srcIp" : "10.180.121.151", + "srcMac" : [ + "00:00:0c:07:ac:01", + "00:0b:5f:6b:5d:40" ], - "portDst" : 25, - "emvcnt" : 1, - "ehh" : [ - "content-type", - "message-id", - "mime-version", - "from", - "to", - "user-agent", - "x-ms-tnef-correlator", - "received", - "thread-index", - "date", - "subject", - "content-language", - "x-originating-ip", - "x-ms-has-attach", - "accept-language", - "thread-topic" + "srcMacCnt" : 2, + "srcOui" : [ + "Cisco", + "Cisco Systems, Inc" ], - "a1" : "10.180.121.151", - "g2" : "USA", - "esub" : [ - "test 测试" + "srcOuiCnt" : 2, + "srcPackets" : 10, + "srcPayload8" : "45484c4f20787878", + "srcPort" : 24962, + "tags" : [ + "smtp:statuscode:250" ], - "fpd" : 1386358422209, - "no" : "test", - "geip" : [ - "---", - "---", - "USA" - ] + "tagsCnt" : 1, + "tcpflags" : { + "ack" : 8, + "dstZero" : 0, + "fin" : 2, + "psh" : 12, + "rst" : 0, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, + "timestamp" : "SET", + "totBytes" : 3439, + "totDataBytes" : 2059, + "totPackets" : 24 }, "header" : { "index" : { - "_index" : "tests_sessions-131206", + "_index" : "tests_sessions2-131206", "_type" : "session" } } diff --git a/tests/pcap/smtp-subject-multi-nospace.test b/tests/pcap/smtp-subject-multi-nospace.test index 01b31bc160..6940fcb895 100644 --- a/tests/pcap/smtp-subject-multi-nospace.test +++ b/tests/pcap/smtp-subject-multi-nospace.test @@ -1,132 +1,133 @@ { - "sessions" : [ + "sessions2" : [ { - "header" : { - "index" : { - "_type" : "session", - "_index" : "tests_sessions-140313" - } - }, "body" : { - "pa2" : 1, - "ps" : [ - 24, - 114, - 190, - 260, - 655 + "dstASN" : "AS0001 Cool Beans!", + "dstBytes" : 60, + "dstDataBytes" : 0, + "dstGEO" : "CA", + "dstIp" : "10.0.0.2", + "dstMac" : [ + "00:00:0c:07:ac:01", + "00:d0:2b:d1:76:00" ], - "ehh" : [ - "from", - "to", - "subject" + "dstMacCnt" : 2, + "dstOui" : [ + "Cisco", + "Jetcell, Inc." ], - "test" : { - "string" : [ - "16777226:58802,33554442:25" + "dstOuiCnt" : 2, + "dstPackets" : 1, + "dstPort" : 25, + "dstRIR" : "TEST", + "email" : { + "dst" : [ + "xxxxx@xxx.net" ], - "ip-asn" : [ - "AS0000 This is neat" + "dstCnt" : 1, + "header" : [ + "from", + "to", + "subject" ], - "ip-geo" : [ - "RUS" + "headerCnt" : 3, + "src" : [ + "xxxxx@xxx.net" ], - "ip" : [ - 167772161 + "srcCnt" : 1, + "subject" : [ + "xxxxxxxxxxxxx: xxxxêxx xxéxxxxxxx xx xxx xxx xxxxxxxxxx x xx xxxxxxxxxxxxxxx - xxxx" ], - "ip-rir" : [ - "" - ], - "number" : [ - 33554442 - ] + "subjectCnt" : 1 }, - "lpd" : 1394730057312, - "db2" : 0, - "ipDst" : "10.0.0.2", - "p1" : 58802, - "by2" : 60, + "fileId" : [], + "firstPacket" : 1394730057309, + "ipProtocol" : 6, "lastPacket" : 1394730057312, - "mac1-term-cnt" : 1, - "ta" : [ - "dstip", - "srcip" + "length" : 2, + "node" : "test", + "packetLen" : [ + 90, + 76, + 70, + 395, + 70 ], - "prot-term" : [ + "packetPos" : [ + 24, + 114, + 190, + 260, + 655 + ], + "protocol" : [ "smtp", "tcp" ], - "g1" : "RUS", - "by1" : 561, - "p2" : 25, - "esub" : [ - "xxxxxxxxxxxxx: xxxxêxx xxéxxxxxxx xx xxx xxx xxxxxxxxxx x xx xxxxxxxxxxxxxxx - xxxx" - ], - "by" : 621, - "edst" : [ - "xxxxx@xxx.net" - ], - "esrccnt" : 1, - "g2" : "CAN", - "mac2-term-cnt" : 2, - "db1" : 325, - "pa" : 5, - "no" : "test", - "fs" : [], - "as1" : "AS0000 This is neat", - "db" : 325, - "as2" : "AS0001 Cool Beans!", - "a1" : "10.0.0.1", - "mac2-term" : [ - "00:00:0c:07:ac:01", - "00:d0:2b:d1:76:00" - ], - "firstPacket" : 1394730057309, - "sl" : 2, - "portDst" : 25, - "portSrc" : 58802, - "tags-term" : [ - "srcip", - "dstip" + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcASN" : "AS0000 This is neat", + "srcBytes" : 561, + "srcDataBytes" : 325, + "srcGEO" : "RU", + "srcIp" : "10.0.0.1", + "srcMac" : [ + "00:13:72:c4:f1:e1" ], - "esubcnt" : 1, - "edstcnt" : 1, - "esrc" : [ - "xxxxx@xxx.net" + "srcMacCnt" : 1, + "srcOui" : [ + "Dell Inc." ], - "mac1-term" : [ - "00:13:72:c4:f1:e1" + "srcOuiCnt" : 1, + "srcPackets" : 4, + "srcPayload8" : "45484c4f20787878", + "srcPort" : 58802, + "tags" : [ + "dstip", + "srcip" ], - "rir2" : "TEST", + "tagsCnt" : 2, "tcpflags" : { - "syn" : 1, - "urg" : 0, - "syn-ack" : 1, - "psh" : 1, + "ack" : 1, + "dstZero" : 0, "fin" : 1, + "psh" : 1, "rst" : 0, - "ack" : 1 + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, + "test" : { + "ASN" : [ + "AS0000 This is neat" + ], + "GEO" : [ + "RU" + ], + "RIR" : [ + "" + ], + "ip" : [ + "10.0.0.1" + ], + "number" : [ + 33554442 + ], + "string.snow" : [ + "16777226:58802,33554442:25" + ] }, - "pa1" : 4, - "a2" : "10.0.0.2", - "ehhcnt" : 3, - "ipSrc" : "10.0.0.1", - "fb1" : "45484c4f20787878", - "psl" : [ - 90, - 76, - 70, - 395, - 70 - ], - "pr" : 6, - "tacnt" : 2, - "fpd" : 1394730057309, "timestamp" : "SET", - "lp" : 1394730057, - "prot-term-cnt" : 2, - "ss" : 1, - "fp" : 1394730057 + "totBytes" : 621, + "totDataBytes" : 325, + "totPackets" : 5 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-140313", + "_type" : "session" + } } } ] diff --git a/tests/pcap/smtp-subject-utf8-mixed.test b/tests/pcap/smtp-subject-utf8-mixed.test index ddf8a25fac..212fc57d8d 100644 --- a/tests/pcap/smtp-subject-utf8-mixed.test +++ b/tests/pcap/smtp-subject-utf8-mixed.test @@ -1,62 +1,94 @@ { - "sessions" : [ + "sessions2" : [ { - "header" : { - "index" : { - "_type" : "session", - "_index" : "tests_sessions-131212" - } - }, "body" : { - "p2" : 25, - "geip" : [ - "---" - ], - "emv" : [ - "1.0" - ], - "esrc" : [ - "xxxxx@xxx.net" - ], - "mac1-term" : [ - "00:13:72:c4:f1:e1" - ], - "pr" : 6, - "fp" : 1386817958, - "esub" : [ - "这是用空格测试 å é î ø ü test" - ], - "ipDst" : "64.236.55.17", - "esrccnt" : 1, - "fb1" : "45484c4f20787878", - "by" : 3458, - "ehh" : [ - "content-type", - "message-id", - "mime-version", - "from", - "to", - "user-agent", - "received", - "content-disposition", - "date", - "subject" + "dstASN" : "AS1668 AOL Transit Data Network", + "dstBytes" : 1425, + "dstDataBytes" : 493, + "dstGEO" : "US", + "dstIp" : "64.236.55.17", + "dstMac" : [ + "00:00:0c:07:ac:01", + "00:0e:d6:0b:98:80" ], - "fs" : [], - "db" : 1594, - "tcpflags" : { - "ack" : 11, - "psh" : 13, - "urg" : 0, - "syn-ack" : 1, - "syn" : 1, - "rst" : 0, - "fin" : 2 + "dstMacCnt" : 2, + "dstOui" : [ + "Cisco", + "Cisco Systems, Inc" + ], + "dstOuiCnt" : 2, + "dstPackets" : 14, + "dstPayload8" : "3232302078787878", + "dstPort" : 25, + "dstRIR" : "ARIN", + "email" : { + "ASN" : [ + "---" + ], + "GEO" : [ + "---" + ], + "RIR" : [ + "" + ], + "contentType" : [ + "text/plain; charset=us-ascii" + ], + "contentTypeCnt" : 1, + "dst" : [ + "xxxxxxxxx@xxxxxxx.com" + ], + "dstCnt" : 1, + "header" : [ + "content-type", + "message-id", + "mime-version", + "from", + "to", + "user-agent", + "received", + "content-disposition", + "date", + "subject" + ], + "headerCnt" : 10, + "host" : [ + "localhost", + "xxxxxxxxxxxxx.xxx.com" + ], + "hostCnt" : 2, + "id" : [ + "20131212031238.GA1705@xxx.net" + ], + "idCnt" : 1, + "ip" : [ + "127.0.0.1" + ], + "ipCnt" : 1, + "mimeVersion" : [ + "1.0" + ], + "mimeVersionCnt" : 1, + "src" : [ + "xxxxx@xxx.net" + ], + "srcCnt" : 1, + "subject" : [ + "这是用空格测试 å é î ø ü test" + ], + "subjectCnt" : 1, + "useragent" : [ + "Mutt/1.5.20 (2009-12-10)" + ], + "useragentCnt" : 1 }, - "eid" : [ - "20131212031238.GA1705@xxx.net" - ], - "psl" : [ + "fileId" : [], + "firstPacket" : 1386817958900, + "ipProtocol" : 6, + "lastPacket" : 1386817959201, + "length" : 302, + "node" : "test", + "packetLen" : [ 90, 90, 82, @@ -86,37 +118,7 @@ 82, 82 ], - "mac2-term-cnt" : 2, - "rireip" : [ - "" - ], - "sl" : 302, - "rir2" : "ARIN", - "eidcnt" : 1, - "pa2" : 14, - "esubcnt" : 1, - "g1" : "USA", - "db2" : 493, - "mac1-term-cnt" : 1, - "p1" : 35803, - "ehhcnt" : 10, - "firstPacket" : 1386817958900, - "pa" : 28, - "prot-term" : [ - "smtp", - "tcp" - ], - "db1" : 1101, - "aseip" : [ - "---" - ], - "lastPacket" : 1386817959201, - "a2" : "64.236.55.17", - "portSrc" : 35803, - "pa1" : 14, - "fpd" : 1386817958900, - "emvcnt" : 1, - "ps" : [ + "packetPos" : [ 24, 114, 204, @@ -146,52 +148,52 @@ 3766, 3848 ], - "ect" : [ - "text/plain; charset=us-ascii" + "protocol" : [ + "smtp", + "tcp" ], - "eho" : [ - "localhost", - "xxxxxxxxxxxxx.xxx.com" + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 2033, + "srcDataBytes" : 1101, + "srcGEO" : "US", + "srcIp" : "10.180.156.249", + "srcMac" : [ + "00:13:72:c4:f1:e1" ], - "portDst" : 25, - "eua" : [ - "Mutt/1.5.20 (2009-12-10)" + "srcMacCnt" : 1, + "srcOui" : [ + "Dell Inc." ], - "tacnt" : 1, - "ta" : [ + "srcOuiCnt" : 1, + "srcPackets" : 14, + "srcPayload8" : "45484c4f20787878", + "srcPort" : 35803, + "tags" : [ "smtp:statuscode:250" ], - "no" : "test", - "lpd" : 1386817959201, - "as2" : "AS1668 AOL Transit Data Network", - "edst" : [ - "xxxxxxxxx@xxxxxxx.com" - ], - "g2" : "USA", - "ectcnt" : 1, - "ehocnt" : 2, + "tagsCnt" : 1, + "tcpflags" : { + "ack" : 11, + "dstZero" : 0, + "fin" : 2, + "psh" : 13, + "rst" : 0, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, "timestamp" : "SET", - "a1" : "10.180.156.249", - "mac2-term" : [ - "00:00:0c:07:ac:01", - "00:0e:d6:0b:98:80" - ], - "fb2" : "3232302078787878", - "eip" : [ - "127.0.0.1" - ], - "by1" : 2033, - "prot-term-cnt" : 2, - "lp" : 1386817959, - "ipSrc" : "10.180.156.249", - "edstcnt" : 1, - "tags-term" : [ - "smtp:statuscode:250" - ], - "eipcnt" : 1, - "ss" : 1, - "by2" : 1425, - "euacnt" : 1 + "totBytes" : 3458, + "totDataBytes" : 1594, + "totPackets" : 28 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-131212", + "_type" : "session" + } } } ] diff --git a/tests/pcap/smtp-subject-utf8-q.test b/tests/pcap/smtp-subject-utf8-q.test index 5bf00adc6f..3bf091d858 100644 --- a/tests/pcap/smtp-subject-utf8-q.test +++ b/tests/pcap/smtp-subject-utf8-q.test @@ -1,73 +1,93 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "ectcnt" : 1, - "as1" : "AS0000 This is neat", - "emvcnt" : 1, - "db1" : 1834, - "ipDst" : "64.236.64.226", - "portSrc" : 20720, - "emd5" : [ - "5b153a606bea42005e1eedb5ddeabcf0" - ], - "ehocnt" : 3, - "prot-term-cnt" : 2, - "ps" : [ - 24, - 114, - 204, - 286, - 439, - 549, - 631, - 882, - 1064, - 1187, - 1269, - 1453, - 2983, - 3323, - 3405, - 3519, - 3681, - 3763, - 3845 + "dstASN" : "AS1668 AOL Transit Data Network", + "dstBytes" : 1163, + "dstDataBytes" : 495, + "dstGEO" : "US", + "dstIp" : "64.236.64.226", + "dstMac" : [ + "00:25:90:ac:d0:6a" ], - "test" : { + "dstMacCnt" : 1, + "dstOui" : [ + "Super Micro Computer, Inc." + ], + "dstOuiCnt" : 1, + "dstPackets" : 10, + "dstPayload8" : "32323020616f6c6d", + "dstPort" : 25, + "dstRIR" : "ARIN", + "email" : { + "ASN" : [ + "---", + "---" + ], + "GEO" : [ + "---", + "---" + ], + "RIR" : [ + "", + "" + ], + "contentType" : [ + "multipart/mixed; boundary=\"===============1250870193309395048==\"" + ], + "contentTypeCnt" : 1, + "dst" : [ + "xxxxxx.xxxxxxxxx@xx.xxxxxxxxxxx.com" + ], + "dstCnt" : 1, + "header" : [ + "content-type", + "mime-version", + "from", + "to", + "received", + "subject" + ], + "headerCnt" : 6, + "host" : [ + "xxxx.xxxx.xxxxxxxxx.com", + "smtp-02.xxxxxxxxx.com", + "smtp-01.xxxxxxxxx.com" + ], + "hostCnt" : 3, "ip" : [ - 167772161 + "10.168.0.1", + "10.168.0.2" ], - "string" : [ - "16777226:20720,-499061696:25" + "ipCnt" : 2, + "md5" : [ + "5b153a606bea42005e1eedb5ddeabcf0" ], - "ip-geo" : [ - "RUS" + "md5Cnt" : 1, + "mimeVersion" : [ + "1.0" ], - "number" : [ - 3795905600 + "mimeVersionCnt" : 1, + "sha256" : [ + "8d4b2e39ccf34cff2147c7b6896f4bd5ce0a209d0cd87ca75035b4c8243bf865" ], - "ip-asn" : [ - "AS0000 This is neat" + "sha256Cnt" : 1, + "src" : [ + "xxxxxxxxx@xxxxxxxxx.com" ], - "ip-rir" : [ - "" - ] + "srcCnt" : 1, + "subject" : [ + "xxxxxx xxxxxx xx xxxxxxxxx xx-xxxxxxx（ANA）GGL_xxxxxxxxxxxxx_xxxxxx.zip" + ], + "subjectCnt" : 1 }, - "lp" : 1384762016, - "ect" : [ - "multipart/mixed; boundary=\"===============1250870193309395048==\"" - ], - "by2" : 1163, - "mac2-term" : [ - "00:25:90:ac:d0:6a" - ], - "eip" : [ - "10.168.0.1", - "10.168.0.2" - ], - "fb2" : "32323020616f6c6d", - "psl" : [ + "fileId" : [], + "firstPacket" : 1384762012670, + "ipProtocol" : 6, + "lastPacket" : 1384762016457, + "length" : 3788, + "node" : "test", + "packetLen" : [ 90, 90, 82, @@ -88,111 +108,96 @@ 82, 82 ], - "by1" : 2436, - "emv" : [ - "1.0" + "packetPos" : [ + 24, + 114, + 204, + 286, + 439, + 549, + 631, + 882, + 1064, + 1187, + 1269, + 1453, + 2983, + 3323, + 3405, + 3519, + 3681, + 3763, + 3845 ], - "pa2" : 10, - "mac1-term-cnt" : 2, - "lastPacket" : 1384762016457, - "mac1-term" : [ + "protocol" : [ + "smtp", + "tcp" + ], + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcASN" : "AS0000 This is neat", + "srcBytes" : 2436, + "srcDataBytes" : 1834, + "srcGEO" : "RU", + "srcIp" : "10.0.0.1", + "srcMac" : [ "00:00:0c:07:ac:01", "00:0b:5f:6b:5d:40" ], - "db2" : 495, - "esub" : [ - "xxxxxx xxxxxx xx xxxxxxxxx xx-xxxxxxx（ANA）GGL_xxxxxxxxxxxxx_xxxxxx.zip" + "srcMacCnt" : 2, + "srcOui" : [ + "Cisco", + "Cisco Systems, Inc" ], - "eipcnt" : 2, - "g2" : "USA", - "rireip" : [ - "", - "" - ], - "by" : 3599, - "firstPacket" : 1384762012670, - "rir2" : "ARIN", - "pa" : 19, - "no" : "test", - "tags-term" : [ + "srcOuiCnt" : 2, + "srcPackets" : 9, + "srcPayload8" : "45484c4f20736d74", + "srcPort" : 20720, + "tags" : [ "smtp:statuscode:250", "srcip" ], - "edstcnt" : 1, - "g1" : "RUS", - "lpd" : 1384762016457, - "fpd" : 1384762012670, - "esrc" : [ - "xxxxxxxxx@xxxxxxxxx.com" - ], + "tagsCnt" : 2, "tcpflags" : { - "syn-ack" : 1, - "rst" : 0, + "ack" : 7, + "dstZero" : 0, + "fin" : 2, "psh" : 9, + "rst" : 0, + "srcZero" : 0, "syn" : 1, - "urg" : 0, - "ack" : 7, - "fin" : 2 + "syn-ack" : 1, + "urg" : 0 + }, + "test" : { + "ASN" : [ + "AS0000 This is neat" + ], + "GEO" : [ + "RU" + ], + "RIR" : [ + "" + ], + "ip" : [ + "10.0.0.1" + ], + "number" : [ + 3795905600 + ], + "string.snow" : [ + "16777226:20720,-499061696:25" + ] }, - "edst" : [ - "xxxxxx.xxxxxxxxx@xx.xxxxxxxxxxx.com" - ], - "mac2-term-cnt" : 1, - "a1" : "10.0.0.1", - "fp" : 1384762012, - "emd5cnt" : 1, - "p1" : 20720, - "prot-term" : [ - "smtp", - "tcp" - ], - "pr" : 6, - "ta" : [ - "smtp:statuscode:250", - "srcip" - ], - "fs" : [], - "pa1" : 9, - "ipSrc" : "10.0.0.1", - "ss" : 1, - "esrccnt" : 1, "timestamp" : "SET", - "tacnt" : 2, - "ehh" : [ - "content-type", - "mime-version", - "from", - "to", - "received", - "subject" - ], - "fb1" : "45484c4f20736d74", - "esubcnt" : 1, - "a2" : "64.236.64.226", - "as2" : "AS1668 AOL Transit Data Network", - "portDst" : 25, - "db" : 2329, - "p2" : 25, - "geip" : [ - "---", - "---" - ], - "sl" : 3788, - "aseip" : [ - "---", - "---" - ], - "ehhcnt" : 6, - "eho" : [ - "xxxx.xxxx.xxxxxxxxx.com", - "smtp-02.xxxxxxxxx.com", - "smtp-01.xxxxxxxxx.com" - ] + "totBytes" : 3599, + "totDataBytes" : 2329, + "totPackets" : 19 }, "header" : { "index" : { - "_type" : "session", - "_index" : "tests_sessions-131118" + "_index" : "tests_sessions2-131118", + "_type" : "session" } } } diff --git a/tests/pcap/smtp-subject-windows.test b/tests/pcap/smtp-subject-windows.test index 1ccf13678f..546ef2156e 100644 --- a/tests/pcap/smtp-subject-windows.test +++ b/tests/pcap/smtp-subject-windows.test @@ -1,31 +1,78 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "ss" : 1, - "mac1-term" : [ - "00:13:72:c4:f1:e1" + "dstASN" : "AS0001 Cool Beans!", + "dstBytes" : 1487, + "dstDataBytes" : 749, + "dstGEO" : "CA", + "dstIp" : "10.0.0.2", + "dstMac" : [ + "00:00:0c:07:ac:01", + "00:d0:2b:d1:76:00" ], - "tcpflags" : { - "syn" : 1, - "syn-ack" : 1, - "ack" : 11, - "rst" : 0, - "fin" : 2, - "psh" : 8, - "urg" : 0 + "dstMacCnt" : 2, + "dstOui" : [ + "Cisco", + "Jetcell, Inc." + ], + "dstOuiCnt" : 2, + "dstPackets" : 11, + "dstPayload8" : "3232302d78787878", + "dstPort" : 25, + "dstRIR" : "TEST", + "email" : { + "dst" : [ + "xxxxx@xxx.net" + ], + "dstCnt" : 1, + "header" : [ + "from", + "to", + "subject" + ], + "headerCnt" : 3, + "src" : [ + "xxxxx@xxx.net" + ], + "srcCnt" : 1, + "subject" : [ + "Awesome windows: See What’s Next in charset decoding!" + ], + "subjectCnt" : 1 }, - "mac1-term-cnt" : 1, - "g1" : "RUS", - "as1" : "AS0000 This is neat", - "lp" : 1394118475, - "ipDst" : "10.0.0.2", - "a1" : "10.0.0.1", - "db2" : 749, - "db" : 960, - "a2" : "10.0.0.2", - "esubcnt" : 1, - "ps" : [ + "fileId" : [], + "firstPacket" : 1394118475009, + "ipProtocol" : 6, + "lastPacket" : 1394118475137, + "length" : 128, + "node" : "test", + "packetLen" : [ + 90, + 94, + 82, + 293, + 82, + 82, + 82, + 173, + 82, + 515, + 82, + 122, + 82, + 125, + 82, + 132, + 82, + 138, + 82, + 118, + 82, + 82, + 82 + ], + "packetPos" : [ 24, 114, 208, @@ -50,118 +97,72 @@ 2726, 2808 ], - "esub" : [ - "Awesome windows: See What’s Next in charset decoding!" + "protocol" : [ + "smtp", + "tcp" + ], + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcASN" : "AS0000 This is neat", + "srcBytes" : 1011, + "srcDataBytes" : 211, + "srcGEO" : "RU", + "srcIp" : "10.0.0.1", + "srcMac" : [ + "00:13:72:c4:f1:e1" ], - "lpd" : 1394118475137, - "p1" : 40531, - "p2" : 25, - "edstcnt" : 1, - "prot-term-cnt" : 2, - "as2" : "AS0001 Cool Beans!", - "by2" : 1487, - "pa1" : 12, + "srcMacCnt" : 1, + "srcOui" : [ + "Dell Inc." + ], + "srcOuiCnt" : 1, + "srcPackets" : 12, + "srcPayload8" : "45484c4f20787878", + "srcPort" : 40531, + "tags" : [ + "dstip", + "srcip" + ], + "tagsCnt" : 2, + "tcpflags" : { + "ack" : 11, + "dstZero" : 0, + "fin" : 2, + "psh" : 8, + "rst" : 0, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, "test" : { - "string" : [ - "16777226:40531,33554442:25" - ], - "ip-asn" : [ + "ASN" : [ "AS0000 This is neat" ], - "ip-rir" : [ + "GEO" : [ + "RU" + ], + "RIR" : [ "" ], - "ip-geo" : [ - "RUS" + "ip" : [ + "10.0.0.1" ], "number" : [ 33554442 ], - "ip" : [ - 167772161 + "string.snow" : [ + "16777226:40531,33554442:25" ] }, - "db1" : 211, - "esrccnt" : 1, - "esrc" : [ - "xxxxx@xxx.net" - ], - "tags-term" : [ - "srcip", - "dstip" - ], - "by1" : 1011, - "fb2" : "3232302d78787878", - "g2" : "CAN", - "pa" : 23, - "tacnt" : 2, - "no" : "test", - "lastPacket" : 1394118475137, - "prot-term" : [ - "smtp", - "tcp" - ], - "pa2" : 11, - "pr" : 6, - "ehh" : [ - "from", - "to", - "subject" - ], - "firstPacket" : 1394118475009, - "fs" : [], - "fpd" : 1394118475009, - "sl" : 128, - "ipSrc" : "10.0.0.1", "timestamp" : "SET", - "by" : 2498, - "mac2-term-cnt" : 2, - "portSrc" : 40531, - "ta" : [ - "dstip", - "srcip" - ], - "portDst" : 25, - "ehhcnt" : 3, - "fp" : 1394118475, - "edst" : [ - "xxxxx@xxx.net" - ], - "rir2" : "TEST", - "mac2-term" : [ - "00:00:0c:07:ac:01", - "00:d0:2b:d1:76:00" - ], - "psl" : [ - 90, - 94, - 82, - 293, - 82, - 82, - 82, - 173, - 82, - 515, - 82, - 122, - 82, - 125, - 82, - 132, - 82, - 138, - 82, - 118, - 82, - 82, - 82 - ], - "fb1" : "45484c4f20787878" + "totBytes" : 2498, + "totDataBytes" : 960, + "totPackets" : 23 }, "header" : { "index" : { - "_index" : "tests_sessions-140306", + "_index" : "tests_sessions2-140306", "_type" : "session" } } diff --git a/tests/pcap/smtp-zip.test b/tests/pcap/smtp-zip.test index f03cda6020..b5093fd57d 100644 --- a/tests/pcap/smtp-zip.test +++ b/tests/pcap/smtp-zip.test @@ -1,31 +1,140 @@ { - "sessions" : [ + "sessions2" : [ { - "header" : { - "index" : { - "_index" : "tests_sessions-131217", - "_type" : "session" - } - }, "body" : { - "euacnt" : 1, - "pr" : 6, - "by1" : 2431, - "eua" : [ - "Mutt/1.5.20 (2009-12-10)" + "dstASN" : "AS1668 AOL Transit Data Network", + "dstBytes" : 1402, + "dstDataBytes" : 470, + "dstGEO" : "US", + "dstIp" : "64.236.64.225", + "dstMac" : [ + "00:00:0c:07:ac:01", + "00:0e:d6:0b:98:80" ], - "ectcnt" : 1, - "ss" : 1, - "p2" : 25, - "efn" : [ - "a.zip" + "dstMacCnt" : 2, + "dstOui" : [ + "Cisco", + "Cisco Systems, Inc" ], - "portDst" : 25, - "pa1" : 14, - "emv" : [ - "1.0" + "dstOuiCnt" : 2, + "dstPackets" : 14, + "dstPayload8" : "3232302078787878", + "dstPort" : 25, + "dstRIR" : "ARIN", + "email" : { + "ASN" : [ + "---" + ], + "GEO" : [ + "---" + ], + "RIR" : [ + "" + ], + "bodyMagic" : [ + "application/zip" + ], + "bodyMagicCnt" : 1, + "contentType" : [ + "multipart/mixed; boundary=\"HcAYCG3uE/tztfnV\"" + ], + "contentTypeCnt" : 1, + "dst" : [ + "xxxxxxxxx@xxxxxxx.com" + ], + "dstCnt" : 1, + "filename" : [ + "a.zip" + ], + "filenameCnt" : 1, + "header" : [ + "content-type", + "message-id", + "mime-version", + "from", + "to", + "user-agent", + "received", + "content-disposition", + "date", + "subject" + ], + "headerCnt" : 10, + "host" : [ + "localhost", + "xxxxxxxxxxxxx.xxx.com" + ], + "hostCnt" : 2, + "id" : [ + "20131217145016.GA29077@xxx.net" + ], + "idCnt" : 1, + "ip" : [ + "127.0.0.1" + ], + "ipCnt" : 1, + "md5" : [ + "40be8f5100e9beabab293c9d7bacaff0" + ], + "md5Cnt" : 1, + "mimeVersion" : [ + "1.0" + ], + "mimeVersionCnt" : 1, + "sha256" : [ + "61479904e443b354d4427a51b990c696f731e341e4c63328e07c1a92658ba591" + ], + "sha256Cnt" : 1, + "src" : [ + "xxxxx@xxx.net" + ], + "srcCnt" : 1, + "subject" : [ + "zip test" + ], + "subjectCnt" : 1, + "useragent" : [ + "Mutt/1.5.20 (2009-12-10)" + ], + "useragentCnt" : 1 + }, + "fileId" : [], + "firstPacket" : 1387291817187, + "ipProtocol" : 6, + "lastPacket" : 1387291817565, + "length" : 377, + "node" : "test", + "packetLen" : [ + 90, + 90, + 82, + 153, + 82, + 110, + 82, + 251, + 119, + 113, + 121, + 124, + 82, + 128, + 82, + 1468, + 82, + 85, + 82, + 113, + 82, + 88, + 82, + 162, + 82, + 82, + 82, + 82 ], - "ps" : [ + "packetPos" : [ 24, 114, 204, @@ -55,157 +164,52 @@ 4141, 4223 ], - "db" : 1969, - "prot-term-cnt" : 2, - "ehh" : [ - "content-type", - "message-id", - "mime-version", - "from", - "to", - "user-agent", - "received", - "content-disposition", - "date", - "subject" + "protocol" : [ + "smtp", + "tcp" + ], + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 2431, + "srcDataBytes" : 1499, + "srcGEO" : "US", + "srcIp" : "10.180.156.249", + "srcMac" : [ + "00:13:72:c4:f1:e1" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Dell Inc." ], - "fp" : 1387291817, - "no" : "test", - "p1" : 46671, - "ipSrc" : "10.180.156.249", - "lpd" : 1387291817565, - "ehocnt" : 2, - "a2" : "64.236.64.225", + "srcOuiCnt" : 1, + "srcPackets" : 14, + "srcPayload8" : "45484c4f20787878", + "srcPort" : 46671, + "tags" : [ + "smtp:statuscode:250" + ], + "tagsCnt" : 1, "tcpflags" : { "ack" : 11, + "dstZero" : 0, + "fin" : 2, "psh" : 13, "rst" : 0, + "srcZero" : 0, "syn" : 1, - "fin" : 2, "syn-ack" : 1, "urg" : 0 }, - "eidcnt" : 1, - "psl" : [ - 90, - 90, - 82, - 153, - 82, - 110, - 82, - 251, - 119, - 113, - 121, - 124, - 82, - 128, - 82, - 1468, - 82, - 85, - 82, - 113, - 82, - 88, - 82, - 162, - 82, - 82, - 82, - 82 - ], - "emd5cnt" : 1, - "a1" : "10.180.156.249", - "tags-term" : [ - "smtp:statuscode:250" - ], - "ipDst" : "64.236.64.225", - "pa" : 28, - "portSrc" : 46671, - "ta" : [ - "smtp:statuscode:250" - ], - "rireip" : [ - "" - ], - "esubcnt" : 1, - "g1" : "USA", - "by2" : 1402, - "lp" : 1387291817, - "lastPacket" : 1387291817565, - "mac2-term-cnt" : 2, - "mac1-term-cnt" : 1, - "mac1-term" : [ - "00:13:72:c4:f1:e1" - ], - "edstcnt" : 1, - "sl" : 377, - "db1" : 1499, - "edst" : [ - "xxxxxxxxx@xxxxxxx.com" - ], - "as2" : "AS1668 AOL Transit Data Network", - "esrccnt" : 1, - "g2" : "USA", - "geip" : [ - "---" - ], "timestamp" : "SET", - "firstPacket" : 1387291817187, - "pa2" : 14, - "emvcnt" : 1, - "mac2-term" : [ - "00:00:0c:07:ac:01", - "00:0e:d6:0b:98:80" - ], - "eid" : [ - "20131217145016.GA29077@xxx.net" - ], - "tacnt" : 1, - "db2" : 470, - "eip" : [ - "127.0.0.1" - ], - "eipcnt" : 1, - "email" : { - "bodymagic-term-cnt" : 1, - "bodymagic-term" : [ - "application/zip" - ] - }, - "fb1" : "45484c4f20787878", - "esub" : [ - "zip test" - ], - "eho" : [ - "localhost", - "xxxxxxxxxxxxx.xxx.com" - ], - "by" : 3833, - "fb2" : "3232302078787878", - "esrc" : [ - "xxxxx@xxx.net" - ], - "ehhcnt" : 10, - "rir2" : "ARIN", - "prot-term" : [ - "smtp", - "tcp" - ], - "aseip" : [ - "---" - ], - "ect" : [ - "multipart/mixed; boundary=\"HcAYCG3uE/tztfnV\"" - ], - "efncnt" : 1, - "emd5" : [ - "40be8f5100e9beabab293c9d7bacaff0" - ], - "fpd" : 1387291817187, - "fs" : [] + "totBytes" : 3833, + "totDataBytes" : 1969, + "totPackets" : 28 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-131217", + "_type" : "session" + } } } ] diff --git a/tests/pcap/socks-http-example.test b/tests/pcap/socks-http-example.test index ceb38a9660..8ed77255a9 100644 --- a/tests/pcap/socks-http-example.test +++ b/tests/pcap/socks-http-example.test @@ -1,108 +1,111 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "a1" : "10.180.156.185", - "a2" : "10.180.156.249", - "assocksip" : "AS15133 MCI Communications Services, Inc. d/b/a Verizon Business", - "by" : 2698, - "by1" : 695, - "by2" : 2003, - "db" : 1754, - "db1" : 155, - "db2" : 1599, - "fb1" : "040100505db8d877", - "fb2" : "005adfb20ab49cf9", - "firstPacket" : 1386004309468, - "fp" : 1386004309, - "fpd" : 1386004309468, - "fs" : [], - "g1" : "USA", - "g2" : "USA", - "gsocksip" : "USA", - "hdver" : [ - "1.1" - ], - "hdvercnt" : 1, - "hh1" : [ - "http:header:accept", - "http:header:host", - "http:header:user-agent" - ], - "hh1cnt" : 3, - "hh2" : [ - "http:header:accept-ranges", - "http:header:cache-control", - "http:header:content-length", - "http:header:content-type", - "http:header:date", - "http:header:etag", - "http:header:expires", - "http:header:last-modified", - "http:header:server", - "http:header:x-cache", - "http:header:x-ec-custom-error" - ], - "hh2cnt" : 11, - "hmd5" : [ - "09b9c392dc1f6e914cea287cb6be34b0" - ], - "hmd5cnt" : 1, - "ho" : [ - "www.example.com" - ], - "hocnt" : 1, - "hpath" : [ - "/" + "dstBytes" : 2003, + "dstDataBytes" : 1599, + "dstGEO" : "US", + "dstIp" : "10.180.156.249", + "dstMac" : [ + "00:13:72:c4:f1:e1" ], - "hpathcnt" : 1, - "hsver" : [ - "1.1" + "dstMacCnt" : 1, + "dstOui" : [ + "Dell Inc." ], - "hsvercnt" : 1, + "dstOuiCnt" : 1, + "dstPackets" : 6, + "dstPayload8" : "005adfb20ab49cf9", + "dstPort" : 1080, + "fileId" : [], + "firstPacket" : 1386004309468, "http" : { - "bodymagic-term" : [ + "bodyMagic" : [ "text/html" ], - "bodymagic-term-cnt" : 1, - "method-term" : [ + "bodyMagicCnt" : 1, + "clientVersion" : [ + "1.1" + ], + "clientVersionCnt" : 1, + "host" : [ + "www.example.com" + ], + "hostCnt" : 1, + "md5" : [ + "09b9c392dc1f6e914cea287cb6be34b0" + ], + "md5Cnt" : 1, + "method" : [ "GET" ], - "method-term-cnt" : 1, + "methodCnt" : 1, + "path" : [ + "/" + ], + "pathCnt" : 1, + "requestHeader" : [ + "accept", + "user-agent", + "host" + ], + "requestHeaderCnt" : 3, + "responseHeader" : [ + "expires", + "content-type", + "x-cache", + "cache-control", + "accept-ranges", + "content-length", + "etag", + "date", + "last-modified", + "x-ec-custom-error", + "server" + ], + "responseHeaderCnt" : 11, + "serverVersion" : [ + "1.1" + ], + "serverVersionCnt" : 1, + "sha256" : [ + "3587cb776ce0e4e8237f215800b7dffba0f25865cb84550e87ea8bbac838c423" + ], + "sha256Cnt" : 1, "statuscode" : [ 200 ], - "statuscode-cnt" : 1 + "statuscodeCnt" : 1, + "uri" : [ + "www.example.com/" + ], + "uriCnt" : 1, + "useragent" : [ + "curl/7.24.0 (x86_64-apple-darwin12.0) libcurl/7.24.0 OpenSSL/0.9.8y zlib/1.2.5" + ], + "useragentCnt" : 1 }, - "ipDst" : "10.180.156.249", - "ipSrc" : "10.180.156.185", + "ipProtocol" : 6, "lastPacket" : 1386004309478, - "lp" : 1386004309, - "lpd" : 1386004309478, - "mac1-term" : [ - "00:1f:5b:ff:51:cb" - ], - "mac1-term-cnt" : 1, - "mac2-term" : [ - "00:13:72:c4:f1:e1" - ], - "mac2-term-cnt" : 1, - "no" : "test", - "p1" : 53533, - "p2" : 1080, - "pa" : 14, - "pa1" : 8, - "pa2" : 6, - "portDst" : 1080, - "portSrc" : 53533, - "pr" : 6, - "prot-term" : [ - "http", - "socks", - "tcp" + "length" : 10, + "node" : "test", + "packetLen" : [ + 94, + 90, + 82, + 91, + 82, + 90, + 82, + 228, + 1530, + 225, + 82, + 82, + 82, + 82 ], - "prot-term-cnt" : 3, - "ps" : [ + "packetPos" : [ 24, 118, 208, @@ -118,154 +121,165 @@ 2782, 2864 ], - "psl" : [ - 94, - 90, - 82, - 91, - 82, - 90, - 82, - 228, - 1530, - 225, - 82, - 82, - 82, - 82 + "protocol" : [ + "http", + "socks", + "tcp" + ], + "protocolCnt" : 3, + "segmentCnt" : 1, + "socks" : { + "ASN" : "AS15133 MCI Communications Services, Inc. d/b/a Verizon Business", + "GEO" : "US", + "RIR" : "RIPE", + "ip" : "93.184.216.119", + "port" : 80 + }, + "srcBytes" : 695, + "srcDataBytes" : 155, + "srcGEO" : "US", + "srcIp" : "10.180.156.185", + "srcMac" : [ + "00:1f:5b:ff:51:cb" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Apple, Inc." ], - "rirsocksip" : "RIPE", - "sl" : 10, - "socksip" : "93.184.216.119", - "sockspo" : 80, - "ss" : 1, + "srcOuiCnt" : 1, + "srcPackets" : 8, + "srcPayload8" : "040100505db8d877", + "srcPort" : 53533, "tcpflags" : { "ack" : 6, + "dstZero" : 0, "fin" : 2, "psh" : 4, "rst" : 0, + "srcZero" : 0, "syn" : 1, "syn-ack" : 1, "urg" : 0 }, "timestamp" : "SET", - "ua" : [ - "curl/7.24.0 (x86_64-apple-darwin12.0) libcurl/7.24.0 OpenSSL/0.9.8y zlib/1.2.5" - ], - "uacnt" : 1, - "us" : [ - "//www.example.com/" - ], - "uscnt" : 1 + "totBytes" : 2698, + "totDataBytes" : 1754, + "totPackets" : 14 }, "header" : { "index" : { - "_index" : "tests_sessions-131202", + "_index" : "tests_sessions2-131202", "_type" : "session" } } }, { "body" : { - "a1" : "10.180.156.185", - "a2" : "10.180.156.249", - "by" : 2780, - "by1" : 711, - "by2" : 2069, - "db" : 1770, - "db1" : 171, - "db2" : 1599, - "fb1" : "0401005000000001", - "fb2" : "005adfb30ab49cf9", - "firstPacket" : 1386004312331, - "fp" : 1386004312, - "fpd" : 1386004312331, - "fs" : [], - "g1" : "USA", - "g2" : "USA", - "hdver" : [ - "1.1" - ], - "hdvercnt" : 1, - "hh1" : [ - "http:header:accept", - "http:header:host", - "http:header:user-agent" - ], - "hh1cnt" : 3, - "hh2" : [ - "http:header:accept-ranges", - "http:header:cache-control", - "http:header:content-length", - "http:header:content-type", - "http:header:date", - "http:header:etag", - "http:header:expires", - "http:header:last-modified", - "http:header:server", - "http:header:x-cache", - "http:header:x-ec-custom-error" - ], - "hh2cnt" : 11, - "hmd5" : [ - "09b9c392dc1f6e914cea287cb6be34b0" - ], - "hmd5cnt" : 1, - "ho" : [ - "www.example.com" - ], - "hocnt" : 1, - "hpath" : [ - "/" + "dstBytes" : 2069, + "dstDataBytes" : 1599, + "dstGEO" : "US", + "dstIp" : "10.180.156.249", + "dstMac" : [ + "00:13:72:c4:f1:e1" ], - "hpathcnt" : 1, - "hsver" : [ - "1.1" + "dstMacCnt" : 1, + "dstOui" : [ + "Dell Inc." ], - "hsvercnt" : 1, + "dstOuiCnt" : 1, + "dstPackets" : 7, + "dstPayload8" : "005adfb30ab49cf9", + "dstPort" : 1080, + "fileId" : [], + "firstPacket" : 1386004312331, "http" : { - "bodymagic-term" : [ + "bodyMagic" : [ "text/html" ], - "bodymagic-term-cnt" : 1, - "method-term" : [ + "bodyMagicCnt" : 1, + "clientVersion" : [ + "1.1" + ], + "clientVersionCnt" : 1, + "host" : [ + "www.example.com" + ], + "hostCnt" : 1, + "md5" : [ + "09b9c392dc1f6e914cea287cb6be34b0" + ], + "md5Cnt" : 1, + "method" : [ "GET" ], - "method-term-cnt" : 1, + "methodCnt" : 1, + "path" : [ + "/" + ], + "pathCnt" : 1, + "requestHeader" : [ + "accept", + "user-agent", + "host" + ], + "requestHeaderCnt" : 3, + "responseHeader" : [ + "expires", + "content-type", + "x-cache", + "cache-control", + "accept-ranges", + "content-length", + "etag", + "date", + "last-modified", + "x-ec-custom-error", + "server" + ], + "responseHeaderCnt" : 11, + "serverVersion" : [ + "1.1" + ], + "serverVersionCnt" : 1, + "sha256" : [ + "3587cb776ce0e4e8237f215800b7dffba0f25865cb84550e87ea8bbac838c423" + ], + "sha256Cnt" : 1, "statuscode" : [ 200 ], - "statuscode-cnt" : 1 + "statuscodeCnt" : 1, + "uri" : [ + "www.example.com/" + ], + "uriCnt" : 1, + "useragent" : [ + "curl/7.24.0 (x86_64-apple-darwin12.0) libcurl/7.24.0 OpenSSL/0.9.8y zlib/1.2.5" + ], + "useragentCnt" : 1 }, - "ipDst" : "10.180.156.249", - "ipSrc" : "10.180.156.185", + "ipProtocol" : 6, "lastPacket" : 1386004312384, - "lp" : 1386004312, - "lpd" : 1386004312384, - "mac1-term" : [ - "00:1f:5b:ff:51:cb" - ], - "mac1-term-cnt" : 1, - "mac2-term" : [ - "00:13:72:c4:f1:e1" - ], - "mac2-term-cnt" : 1, - "no" : "test", - "p1" : 53534, - "p2" : 1080, - "pa" : 15, - "pa1" : 8, - "pa2" : 7, - "portDst" : 1080, - "portSrc" : 53534, - "pr" : 6, - "prot-term" : [ - "http", - "socks", - "tcp" + "length" : 53, + "node" : "test", + "packetLen" : [ + 94, + 90, + 82, + 107, + 82, + 90, + 82, + 228, + 82, + 1530, + 225, + 82, + 82, + 82, + 82 ], - "prot-term-cnt" : 3, - "ps" : [ + "packetPos" : [ 2946, 3040, 3130, @@ -282,156 +296,164 @@ 5802, 5884 ], - "psl" : [ - 94, - 90, - 82, - 107, - 82, - 90, - 82, - 228, - 82, - 1530, - 225, - 82, - 82, - 82, - 82 + "protocol" : [ + "http", + "socks", + "tcp" ], - "sl" : 53, - "socksho" : "www.example.com", - "sockspo" : 80, - "ss" : 1, + "protocolCnt" : 3, + "segmentCnt" : 1, + "socks" : { + "host" : "www.example.com", + "port" : 80 + }, + "srcBytes" : 711, + "srcDataBytes" : 171, + "srcGEO" : "US", + "srcIp" : "10.180.156.185", + "srcMac" : [ + "00:1f:5b:ff:51:cb" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Apple, Inc." + ], + "srcOuiCnt" : 1, + "srcPackets" : 8, + "srcPayload8" : "0401005000000001", + "srcPort" : 53534, "tcpflags" : { "ack" : 7, + "dstZero" : 0, "fin" : 2, "psh" : 4, "rst" : 0, + "srcZero" : 0, "syn" : 1, "syn-ack" : 1, "urg" : 0 }, "timestamp" : "SET", - "ua" : [ - "curl/7.24.0 (x86_64-apple-darwin12.0) libcurl/7.24.0 OpenSSL/0.9.8y zlib/1.2.5" - ], - "uacnt" : 1, - "us" : [ - "//www.example.com/" - ], - "uscnt" : 1 + "totBytes" : 2780, + "totDataBytes" : 1770, + "totPackets" : 15 }, "header" : { "index" : { - "_index" : "tests_sessions-131202", + "_index" : "tests_sessions2-131202", "_type" : "session" } } }, { "body" : { - "a1" : "10.180.156.185", - "a2" : "10.180.156.249", - "assocksip" : "AS15133 MCI Communications Services, Inc. d/b/a Verizon Business", - "by" : 2905, - "by1" : 832, - "by2" : 2073, - "db" : 1763, - "db1" : 160, - "db2" : 1603, - "fb1" : "0502000105010001", - "fb2" : "0500050000010ab4", - "firstPacket" : 1386004317979, - "fp" : 1386004317, - "fpd" : 1386004317979, - "fs" : [], - "g1" : "USA", - "g2" : "USA", - "gsocksip" : "USA", - "hdver" : [ - "1.1" - ], - "hdvercnt" : 1, - "hh1" : [ - "http:header:accept", - "http:header:host", - "http:header:user-agent" - ], - "hh1cnt" : 3, - "hh2" : [ - "http:header:accept-ranges", - "http:header:cache-control", - "http:header:content-length", - "http:header:content-type", - "http:header:date", - "http:header:etag", - "http:header:expires", - "http:header:last-modified", - "http:header:server", - "http:header:x-cache", - "http:header:x-ec-custom-error" - ], - "hh2cnt" : 11, - "hmd5" : [ - "09b9c392dc1f6e914cea287cb6be34b0" - ], - "hmd5cnt" : 1, - "ho" : [ - "www.example.com" - ], - "hocnt" : 1, - "hpath" : [ - "/" + "dstBytes" : 2073, + "dstDataBytes" : 1603, + "dstGEO" : "US", + "dstIp" : "10.180.156.249", + "dstMac" : [ + "00:13:72:c4:f1:e1" ], - "hpathcnt" : 1, - "hsver" : [ - "1.1" + "dstMacCnt" : 1, + "dstOui" : [ + "Dell Inc." ], - "hsvercnt" : 1, + "dstOuiCnt" : 1, + "dstPackets" : 7, + "dstPayload8" : "0500050000010ab4", + "dstPort" : 1080, + "fileId" : [], + "firstPacket" : 1386004317979, "http" : { - "bodymagic-term" : [ + "bodyMagic" : [ "text/html" ], - "bodymagic-term-cnt" : 1, - "method-term" : [ + "bodyMagicCnt" : 1, + "clientVersion" : [ + "1.1" + ], + "clientVersionCnt" : 1, + "host" : [ + "www.example.com" + ], + "hostCnt" : 1, + "md5" : [ + "09b9c392dc1f6e914cea287cb6be34b0" + ], + "md5Cnt" : 1, + "method" : [ "GET" ], - "method-term-cnt" : 1, + "methodCnt" : 1, + "path" : [ + "/" + ], + "pathCnt" : 1, + "requestHeader" : [ + "accept", + "user-agent", + "host" + ], + "requestHeaderCnt" : 3, + "responseHeader" : [ + "expires", + "content-type", + "x-cache", + "cache-control", + "accept-ranges", + "content-length", + "etag", + "date", + "last-modified", + "x-ec-custom-error", + "server" + ], + "responseHeaderCnt" : 11, + "serverVersion" : [ + "1.1" + ], + "serverVersionCnt" : 1, + "sha256" : [ + "3587cb776ce0e4e8237f215800b7dffba0f25865cb84550e87ea8bbac838c423" + ], + "sha256Cnt" : 1, "statuscode" : [ 200 ], - "statuscode-cnt" : 1 + "statuscodeCnt" : 1, + "uri" : [ + "www.example.com/" + ], + "uriCnt" : 1, + "useragent" : [ + "curl/7.24.0 (x86_64-apple-darwin12.0) libcurl/7.24.0 OpenSSL/0.9.8y zlib/1.2.5" + ], + "useragentCnt" : 1 }, - "ipDst" : "10.180.156.249", - "ipSrc" : "10.180.156.185", + "ipProtocol" : 6, "lastPacket" : 1386004317989, - "lp" : 1386004317, - "lpd" : 1386004317989, - "mac1-term" : [ - "00:1f:5b:ff:51:cb" - ], - "mac1-term-cnt" : 1, - "mac2-term" : [ - "00:13:72:c4:f1:e1" - ], - "mac2-term-cnt" : 1, - "no" : "test", - "p1" : 53535, - "p2" : 1080, - "pa" : 17, - "pa1" : 10, - "pa2" : 7, - "portDst" : 1080, - "portSrc" : 53535, - "pr" : 6, - "prot-term" : [ - "http", - "socks", - "tcp" + "length" : 9, + "node" : "test", + "packetLen" : [ + 94, + 90, + 82, + 86, + 82, + 84, + 82, + 92, + 92, + 82, + 228, + 1530, + 225, + 82, + 82, + 82, + 82 ], - "prot-term-cnt" : 3, - "ps" : [ + "packetPos" : [ 5966, 6060, 6150, @@ -450,52 +472,54 @@ 8979, 9061 ], - "psl" : [ - 94, - 90, - 82, - 86, - 82, - 84, - 82, - 92, - 92, - 82, - 228, - 1530, - 225, - 82, - 82, - 82, - 82 + "protocol" : [ + "http", + "socks", + "tcp" ], - "rirsocksip" : "RIPE", - "sl" : 9, - "socksip" : "93.184.216.119", - "sockspo" : 80, - "ss" : 1, + "protocolCnt" : 3, + "segmentCnt" : 1, + "socks" : { + "ASN" : "AS15133 MCI Communications Services, Inc. d/b/a Verizon Business", + "GEO" : "US", + "RIR" : "RIPE", + "ip" : "93.184.216.119", + "port" : 80 + }, + "srcBytes" : 832, + "srcDataBytes" : 160, + "srcGEO" : "US", + "srcIp" : "10.180.156.185", + "srcMac" : [ + "00:1f:5b:ff:51:cb" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Apple, Inc." + ], + "srcOuiCnt" : 1, + "srcPackets" : 10, + "srcPayload8" : "0502000105010001", + "srcPort" : 53535, "tcpflags" : { "ack" : 7, + "dstZero" : 0, "fin" : 2, "psh" : 6, "rst" : 0, + "srcZero" : 0, "syn" : 1, "syn-ack" : 1, "urg" : 0 }, "timestamp" : "SET", - "ua" : [ - "curl/7.24.0 (x86_64-apple-darwin12.0) libcurl/7.24.0 OpenSSL/0.9.8y zlib/1.2.5" - ], - "uacnt" : 1, - "us" : [ - "//www.example.com/" - ], - "uscnt" : 1 + "totBytes" : 2905, + "totDataBytes" : 1763, + "totPackets" : 17 }, "header" : { "index" : { - "_index" : "tests_sessions-131202", + "_index" : "tests_sessions2-131202", "_type" : "session" } } diff --git a/tests/pcap/socks-http-pass.test b/tests/pcap/socks-http-pass.test index a385fbfb58..4e118ffe16 100644 --- a/tests/pcap/socks-http-pass.test +++ b/tests/pcap/socks-http-pass.test @@ -1,50 +1,42 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "a1" : "10.180.156.185", - "a2" : "10.180.156.249", - "by" : 752, - "by1" : 412, - "by2" : 340, - "db" : 6, - "db1" : 4, - "db2" : 2, - "fb1" : "05020001", - "fb2" : "05ff", - "firstPacket" : 1386090517357, - "fp" : 1386090517, - "fpd" : 1386090517357, - "fs" : [], - "g1" : "USA", - "g2" : "USA", - "ipDst" : "10.180.156.249", - "ipSrc" : "10.180.156.185", - "lastPacket" : 1386090517358, - "lp" : 1386090517, - "lpd" : 1386090517358, - "mac1-term" : [ - "00:1f:5b:ff:51:cb" - ], - "mac1-term-cnt" : 1, - "mac2-term" : [ + "dstBytes" : 340, + "dstDataBytes" : 2, + "dstGEO" : "US", + "dstIp" : "10.180.156.249", + "dstMac" : [ "00:13:72:c4:f1:e1" ], - "mac2-term-cnt" : 1, - "no" : "test", - "p1" : 54068, - "p2" : 1080, - "pa" : 11, - "pa1" : 6, - "pa2" : 5, - "portDst" : 1080, - "portSrc" : 54068, - "pr" : 6, - "prot-term" : [ - "tcp" + "dstMacCnt" : 1, + "dstOui" : [ + "Dell Inc." + ], + "dstOuiCnt" : 1, + "dstPackets" : 5, + "dstPayload8" : "05ff", + "dstPort" : 1080, + "fileId" : [], + "firstPacket" : 1386090517357, + "ipProtocol" : 6, + "lastPacket" : 1386090517358, + "length" : 1, + "node" : "test", + "packetLen" : [ + 94, + 90, + 82, + 86, + 82, + 84, + 82, + 82, + 82, + 82, + 82 ], - "prot-term-cnt" : 1, - "ps" : [ + "packetPos" : [ 24, 118, 208, @@ -57,85 +49,89 @@ 788, 870 ], - "psl" : [ - 94, - 90, - 82, - 86, - 82, - 84, - 82, - 82, - 82, - 82, - 82 + "protocol" : [ + "tcp" + ], + "protocolCnt" : 1, + "segmentCnt" : 1, + "srcBytes" : 412, + "srcDataBytes" : 4, + "srcGEO" : "US", + "srcIp" : "10.180.156.185", + "srcMac" : [ + "00:1f:5b:ff:51:cb" ], - "sl" : 1, - "ss" : 1, + "srcMacCnt" : 1, + "srcOui" : [ + "Apple, Inc." + ], + "srcOuiCnt" : 1, + "srcPackets" : 6, + "srcPayload8" : "05020001", + "srcPort" : 54068, "tcpflags" : { "ack" : 5, + "dstZero" : 0, "fin" : 2, "psh" : 2, "rst" : 0, + "srcZero" : 0, "syn" : 1, "syn-ack" : 1, "urg" : 0 }, - "timestamp" : "SET" + "timestamp" : "SET", + "totBytes" : 752, + "totDataBytes" : 6, + "totPackets" : 11 }, "header" : { "index" : { - "_index" : "tests_sessions-131203", + "_index" : "tests_sessions2-131203", "_type" : "session" } } }, { "body" : { - "a1" : "10.180.156.185", - "a2" : "10.180.156.249", - "by" : 972, - "by1" : 564, - "by2" : 408, - "db" : 28, - "db1" : 24, - "db2" : 4, - "fb1" : "0503000102010874", - "fb2" : "050201ff", - "firstPacket" : 1386090528538, - "fp" : 1386090528, - "fpd" : 1386090528538, - "fs" : [], - "g1" : "USA", - "g2" : "USA", - "ipDst" : "10.180.156.249", - "ipSrc" : "10.180.156.185", - "lastPacket" : 1386090528547, - "lp" : 1386090528, - "lpd" : 1386090528547, - "mac1-term" : [ - "00:1f:5b:ff:51:cb" - ], - "mac1-term-cnt" : 1, - "mac2-term" : [ + "dstBytes" : 408, + "dstDataBytes" : 4, + "dstGEO" : "US", + "dstIp" : "10.180.156.249", + "dstMac" : [ "00:13:72:c4:f1:e1" ], - "mac2-term-cnt" : 1, - "no" : "test", - "p1" : 54069, - "p2" : 1080, - "pa" : 14, - "pa1" : 8, - "pa2" : 6, - "portDst" : 1080, - "portSrc" : 54069, - "pr" : 6, - "prot-term" : [ - "socks", - "tcp" + "dstMacCnt" : 1, + "dstOui" : [ + "Dell Inc." + ], + "dstOuiCnt" : 1, + "dstPackets" : 6, + "dstPayload8" : "050201ff", + "dstPort" : 1080, + "fileId" : [], + "firstPacket" : 1386090528538, + "ipProtocol" : 6, + "lastPacket" : 1386090528547, + "length" : 8, + "node" : "test", + "packetLen" : [ + 94, + 90, + 82, + 87, + 82, + 84, + 82, + 101, + 84, + 82, + 82, + 82, + 82, + 82 ], - "prot-term-cnt" : 2, - "ps" : [ + "packetPos" : [ 952, 1046, 1136, @@ -151,153 +147,169 @@ 1984, 2066 ], - "psl" : [ - 94, - 90, - 82, - 87, - 82, - 84, - 82, - 101, - 84, - 82, - 82, - 82, - 82, - 82 + "protocol" : [ + "socks", + "tcp" ], - "sl" : 8, - "socksuser" : "testuser", - "ss" : 1, - "ta" : [ - "socks:password" + "protocolCnt" : 2, + "segmentCnt" : 1, + "socks" : { + "user" : "testuser" + }, + "srcBytes" : 564, + "srcDataBytes" : 24, + "srcGEO" : "US", + "srcIp" : "10.180.156.185", + "srcMac" : [ + "00:1f:5b:ff:51:cb" ], - "tacnt" : 1, - "tags-term" : [ + "srcMacCnt" : 1, + "srcOui" : [ + "Apple, Inc." + ], + "srcOuiCnt" : 1, + "srcPackets" : 8, + "srcPayload8" : "0503000102010874", + "srcPort" : 54069, + "tags" : [ "socks:password" ], + "tagsCnt" : 1, "tcpflags" : { "ack" : 6, + "dstZero" : 0, "fin" : 2, "psh" : 4, "rst" : 0, + "srcZero" : 0, "syn" : 1, "syn-ack" : 1, "urg" : 0 }, - "timestamp" : "SET" + "timestamp" : "SET", + "totBytes" : 972, + "totDataBytes" : 28, + "totPackets" : 14 }, "header" : { "index" : { - "_index" : "tests_sessions-131203", + "_index" : "tests_sessions2-131203", "_type" : "session" } } }, { "body" : { - "a1" : "10.180.156.185", - "a2" : "10.180.156.249", - "assocksip" : "AS15133 MCI Communications Services, Inc. d/b/a Verizon Business", - "by" : 3125, - "by1" : 984, - "by2" : 2141, - "db" : 1785, - "db1" : 180, - "db2" : 1605, - "fb1" : "0503000102010874", - "fb2" : "0502010005000001", - "firstPacket" : 1386090534425, - "fp" : 1386090534, - "fpd" : 1386090534425, - "fs" : [], - "g1" : "USA", - "g2" : "USA", - "gsocksip" : "USA", - "hdver" : [ - "1.1" - ], - "hdvercnt" : 1, - "hh1" : [ - "http:header:accept", - "http:header:host", - "http:header:user-agent" - ], - "hh1cnt" : 3, - "hh2" : [ - "http:header:accept-ranges", - "http:header:cache-control", - "http:header:content-length", - "http:header:content-type", - "http:header:date", - "http:header:etag", - "http:header:expires", - "http:header:last-modified", - "http:header:server", - "http:header:x-cache", - "http:header:x-ec-custom-error" - ], - "hh2cnt" : 11, - "hmd5" : [ - "09b9c392dc1f6e914cea287cb6be34b0" - ], - "hmd5cnt" : 1, - "ho" : [ - "www.example.com" - ], - "hocnt" : 1, - "hpath" : [ - "/" + "dstBytes" : 2141, + "dstDataBytes" : 1605, + "dstGEO" : "US", + "dstIp" : "10.180.156.249", + "dstMac" : [ + "00:13:72:c4:f1:e1" ], - "hpathcnt" : 1, - "hsver" : [ - "1.1" + "dstMacCnt" : 1, + "dstOui" : [ + "Dell Inc." ], - "hsvercnt" : 1, + "dstOuiCnt" : 1, + "dstPackets" : 8, + "dstPayload8" : "0502010005000001", + "dstPort" : 1080, + "fileId" : [], + "firstPacket" : 1386090534425, "http" : { - "bodymagic-term" : [ + "bodyMagic" : [ "text/html" ], - "bodymagic-term-cnt" : 1, - "method-term" : [ + "bodyMagicCnt" : 1, + "clientVersion" : [ + "1.1" + ], + "clientVersionCnt" : 1, + "host" : [ + "www.example.com" + ], + "hostCnt" : 1, + "md5" : [ + "09b9c392dc1f6e914cea287cb6be34b0" + ], + "md5Cnt" : 1, + "method" : [ "GET" ], - "method-term-cnt" : 1, + "methodCnt" : 1, + "path" : [ + "/" + ], + "pathCnt" : 1, + "requestHeader" : [ + "accept", + "user-agent", + "host" + ], + "requestHeaderCnt" : 3, + "responseHeader" : [ + "expires", + "content-type", + "x-cache", + "cache-control", + "accept-ranges", + "content-length", + "etag", + "date", + "last-modified", + "x-ec-custom-error", + "server" + ], + "responseHeaderCnt" : 11, + "serverVersion" : [ + "1.1" + ], + "serverVersionCnt" : 1, + "sha256" : [ + "3587cb776ce0e4e8237f215800b7dffba0f25865cb84550e87ea8bbac838c423" + ], + "sha256Cnt" : 1, "statuscode" : [ 200 ], - "statuscode-cnt" : 1 + "statuscodeCnt" : 1, + "uri" : [ + "www.example.com/" + ], + "uriCnt" : 1, + "useragent" : [ + "curl/7.24.0 (x86_64-apple-darwin12.0) libcurl/7.24.0 OpenSSL/0.9.8y zlib/1.2.5" + ], + "useragentCnt" : 1 }, - "ipDst" : "10.180.156.249", - "ipSrc" : "10.180.156.185", + "ipProtocol" : 6, "lastPacket" : 1386090534579, - "lp" : 1386090534, - "lpd" : 1386090534579, - "mac1-term" : [ - "00:1f:5b:ff:51:cb" - ], - "mac1-term-cnt" : 1, - "mac2-term" : [ - "00:13:72:c4:f1:e1" - ], - "mac2-term-cnt" : 1, - "no" : "test", - "p1" : 54072, - "p2" : 1080, - "pa" : 20, - "pa1" : 12, - "pa2" : 8, - "portDst" : 1080, - "portSrc" : 54072, - "pr" : 6, - "prot-term" : [ - "http", - "socks", - "tcp" + "length" : 153, + "node" : "test", + "packetLen" : [ + 94, + 90, + 82, + 87, + 82, + 84, + 82, + 101, + 84, + 82, + 92, + 92, + 82, + 228, + 1530, + 225, + 82, + 82, + 82, + 82 ], - "prot-term-cnt" : 3, - "ps" : [ + "packetPos" : [ 2148, 2242, 2332, @@ -319,63 +331,59 @@ 5429, 5511 ], - "psl" : [ - 94, - 90, - 82, - 87, - 82, - 84, - 82, - 101, - 84, - 82, - 92, - 92, - 82, - 228, - 1530, - 225, - 82, - 82, - 82, - 82 + "protocol" : [ + "http", + "socks", + "tcp" ], - "rirsocksip" : "RIPE", - "sl" : 153, - "socksip" : "93.184.216.119", - "sockspo" : 80, - "socksuser" : "testuser", - "ss" : 1, - "ta" : [ - "socks:password" + "protocolCnt" : 3, + "segmentCnt" : 1, + "socks" : { + "ASN" : "AS15133 MCI Communications Services, Inc. d/b/a Verizon Business", + "GEO" : "US", + "RIR" : "RIPE", + "ip" : "93.184.216.119", + "port" : 80, + "user" : "testuser" + }, + "srcBytes" : 984, + "srcDataBytes" : 180, + "srcGEO" : "US", + "srcIp" : "10.180.156.185", + "srcMac" : [ + "00:1f:5b:ff:51:cb" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Apple, Inc." ], - "tacnt" : 1, - "tags-term" : [ + "srcOuiCnt" : 1, + "srcPackets" : 12, + "srcPayload8" : "0503000102010874", + "srcPort" : 54072, + "tags" : [ "socks:password" ], + "tagsCnt" : 1, "tcpflags" : { "ack" : 8, + "dstZero" : 0, "fin" : 2, "psh" : 8, "rst" : 0, + "srcZero" : 0, "syn" : 1, "syn-ack" : 1, "urg" : 0 }, "timestamp" : "SET", - "ua" : [ - "curl/7.24.0 (x86_64-apple-darwin12.0) libcurl/7.24.0 OpenSSL/0.9.8y zlib/1.2.5" - ], - "uacnt" : 1, - "us" : [ - "//www.example.com/" - ], - "uscnt" : 1 + "totBytes" : 3125, + "totDataBytes" : 1785, + "totPackets" : 20 }, "header" : { "index" : { - "_index" : "tests_sessions-131203", + "_index" : "tests_sessions2-131203", "_type" : "session" } } diff --git a/tests/pcap/socks-https-example.test b/tests/pcap/socks-https-example.test index 440255a3a9..3ac47d0fa5 100644 --- a/tests/pcap/socks-https-example.test +++ b/tests/pcap/socks-https-example.test @@ -1,137 +1,8 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "a1" : "10.180.156.185", - "a2" : "10.180.156.249", - "assocksip" : "AS15133 MCI Communications Services, Inc. d/b/a Verizon Business", - "by" : 10920, - "by1" : 1775, - "by2" : 9145, - "db" : 8920, - "db1" : 641, - "db2" : 8279, - "fb1" : "040101bb5db8d877", - "fb2" : "005a99b40ab49cf9", - "firstPacket" : 1386004472572, - "fp" : 1386004472, - "fpd" : 1386004472572, - "fs" : [], - "g1" : "USA", - "g2" : "USA", - "gsocksip" : "USA", - "ho" : [ - "www.example.com" - ], - "hocnt" : 1, - "ipDst" : "10.180.156.249", - "ipSrc" : "10.180.156.185", - "lastPacket" : 1386004472629, - "lp" : 1386004472, - "lpd" : 1386004472629, - "mac1-term" : [ - "00:1f:5b:ff:51:cb" - ], - "mac1-term-cnt" : 1, - "mac2-term" : [ - "00:13:72:c4:f1:e1" - ], - "mac2-term-cnt" : 1, - "no" : "test", - "p1" : 53554, - "p2" : 1080, - "pa" : 30, - "pa1" : 17, - "pa2" : 13, - "portDst" : 1080, - "portSrc" : 53554, - "pr" : 6, - "prot-term" : [ - "tls", - "socks", - "tcp" - ], - "prot-term-cnt" : 3, - "ps" : [ - 24, - 118, - 208, - 290, - 381, - 463, - 553, - 635, - 837, - 2367, - 2461, - 2543, - 4073, - 5603, - 5685, - 7215, - 7297, - 8133, - 8215, - 8611, - 8740, - 8822, - 9075, - 9503, - 9585, - 10987, - 11069, - 11178, - 11260, - 11342 - ], - "psl" : [ - 94, - 90, - 82, - 91, - 82, - 90, - 82, - 202, - 1530, - 94, - 82, - 1530, - 1530, - 82, - 1530, - 82, - 836, - 82, - 396, - 129, - 82, - 253, - 428, - 82, - 1402, - 82, - 109, - 82, - 82, - 82 - ], - "rirsocksip" : "RIPE", - "sl" : 57, - "socksip" : "93.184.216.119", - "sockspo" : 443, - "ss" : 1, - "tcpflags" : { - "ack" : 15, - "fin" : 2, - "psh" : 11, - "rst" : 0, - "syn" : 1, - "syn-ack" : 1, - "urg" : 0 - }, - "timestamp" : "SET", - "tls" : [ + "cert" : [ { "alt" : [ "gp1.wac.edgecastcdn.net", @@ -284,148 +155,70 @@ "static.teamtreehouse.com", "wac.a8b5.edgecastcdn.net" ], - "altcnt" : 149, - "diffDays" : 1164, + "altCnt" : 149, "hash" : "d8:af:99:8d:b5:e0:42:a7:b4:7b:6d:41:62:75:00:a7:f7:ed:96:5c", - "iCn" : [ + "issuerCN" : [ "digicert high assurance ca-3" ], - "iOn" : "DigiCert Inc", - "notAfter" : 1418212800, - "notBefore" : 1317600000, - "sCn" : [ + "issuerON" : "DigiCert Inc", + "notAfter" : 1418212800000, + "notBefore" : 1317600000000, + "serial" : "062d488986c9a6d7f94901c2b5906882", + "subjectCN" : [ "gp1.wac.edgecastcdn.net" ], - "sOn" : "EdgeCast Networks, Inc.", - "sn" : "062d488986c9a6d7f94901c2b5906882" + "subjectON" : "EdgeCast Networks, Inc.", + "validDays" : 1164 }, { - "diffDays" : 5113, "hash" : "42:85:78:55:fb:0e:a4:3f:54:c9:91:1e:30:e7:79:1d:8c:e8:27:05", - "iCn" : [ + "issuerCN" : [ "digicert high assurance ev root ca" ], - "iOn" : "DigiCert Inc", - "notAfter" : 1648944000, - "notBefore" : 1207137600, - "sCn" : [ + "issuerON" : "DigiCert Inc", + "notAfter" : 1648944000000, + "notBefore" : 1207137600000, + "serial" : "0a5f114d035b179117d2efd4038c3f3b", + "subjectCN" : [ "digicert high assurance ca-3" ], - "sOn" : "DigiCert Inc", - "sn" : "0a5f114d035b179117d2efd4038c3f3b" + "subjectON" : "DigiCert Inc", + "validDays" : 5113 } ], - "tlscipher-term" : [ - "TLS_RSA_WITH_RC4_128_SHA" - ], - "tlscipher-termcnt" : 1, - "tlscnt" : 2, - "tlsdstid-term" : [ - "85bbc584132410aa03c3d6aac195e2d81e7b8d24e63b314a4a5d214cbcf0080a" - ], - "tlsja3-term" : [ - "06a92bf69b367389d2feb0d70501ddfe" - ], - "tlsja3-termcnt" : 1, - "tlsver-term" : [ - "TLSv1" - ], - "tlsver-termcnt" : 1 - }, - "header" : { - "index" : { - "_index" : "tests_sessions-131202", - "_type" : "session" - } - } - }, - { - "body" : { - "a1" : "10.180.156.185", - "a2" : "10.180.156.249", - "by" : 10911, - "by1" : 1791, - "by2" : 9120, - "db" : 8911, - "db1" : 657, - "db2" : 8254, - "fb1" : "040101bb00000001", - "fb2" : "005a99b50ab49cf9", - "firstPacket" : 1386004475691, - "fp" : 1386004475, - "fpd" : 1386004475691, - "fs" : [], - "g1" : "USA", - "g2" : "USA", - "ho" : [ - "www.example.com" - ], - "hocnt" : 1, - "ipDst" : "10.180.156.249", - "ipSrc" : "10.180.156.185", - "lastPacket" : 1386004475761, - "lp" : 1386004475, - "lpd" : 1386004475761, - "mac1-term" : [ - "00:1f:5b:ff:51:cb" - ], - "mac1-term-cnt" : 1, - "mac2-term" : [ + "certCnt" : 2, + "dstBytes" : 9145, + "dstDataBytes" : 8279, + "dstGEO" : "US", + "dstIp" : "10.180.156.249", + "dstMac" : [ "00:13:72:c4:f1:e1" ], - "mac2-term-cnt" : 1, - "no" : "test", - "p1" : 53555, - "p2" : 1080, - "pa" : 30, - "pa1" : 17, - "pa2" : 13, - "portDst" : 1080, - "portSrc" : 53555, - "pr" : 6, - "prot-term" : [ - "tls", - "socks", - "tcp" - ], - "prot-term-cnt" : 3, - "ps" : [ - 11424, - 11518, - 11608, - 11690, - 11797, - 11879, - 11969, - 12051, - 12253, - 13783, - 13877, - 13959, - 15489, - 17019, - 18549, - 18631, - 18713, - 19549, - 19631, - 20027, - 20156, - 20238, - 20491, - 20919, - 21001, - 22378, - 22460, - 22569, - 22651, - 22733 + "dstMacCnt" : 1, + "dstOui" : [ + "Dell Inc." ], - "psl" : [ + "dstOuiCnt" : 1, + "dstPackets" : 13, + "dstPayload8" : "005a99b40ab49cf9", + "dstPort" : 1080, + "fileId" : [], + "firstPacket" : 1386004472572, + "http" : { + "host" : [ + "www.example.com" + ], + "hostCnt" : 1 + }, + "ipProtocol" : 6, + "lastPacket" : 1386004472629, + "length" : 57, + "node" : "test", + "packetLen" : [ 94, 90, 82, - 107, + 91, 82, 90, 82, @@ -435,8 +228,8 @@ 82, 1530, 1530, - 1530, 82, + 1530, 82, 836, 82, @@ -446,28 +239,117 @@ 253, 428, 82, - 1377, + 1402, 82, 109, 82, 82, 82 ], - "sl" : 69, - "socksho" : "www.example.com", - "sockspo" : 443, - "ss" : 1, + "packetPos" : [ + 24, + 118, + 208, + 290, + 381, + 463, + 553, + 635, + 837, + 2367, + 2461, + 2543, + 4073, + 5603, + 5685, + 7215, + 7297, + 8133, + 8215, + 8611, + 8740, + 8822, + 9075, + 9503, + 9585, + 10987, + 11069, + 11178, + 11260, + 11342 + ], + "protocol" : [ + "tls", + "socks", + "tcp" + ], + "protocolCnt" : 3, + "segmentCnt" : 1, + "socks" : { + "ASN" : "AS15133 MCI Communications Services, Inc. d/b/a Verizon Business", + "GEO" : "US", + "RIR" : "RIPE", + "ip" : "93.184.216.119", + "port" : 443 + }, + "srcBytes" : 1775, + "srcDataBytes" : 641, + "srcGEO" : "US", + "srcIp" : "10.180.156.185", + "srcMac" : [ + "00:1f:5b:ff:51:cb" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Apple, Inc." + ], + "srcOuiCnt" : 1, + "srcPackets" : 17, + "srcPayload8" : "040101bb5db8d877", + "srcPort" : 53554, "tcpflags" : { "ack" : 15, + "dstZero" : 0, "fin" : 2, "psh" : 11, "rst" : 0, + "srcZero" : 0, "syn" : 1, "syn-ack" : 1, "urg" : 0 }, "timestamp" : "SET", - "tls" : [ + "tls" : { + "cipher" : [ + "TLS_RSA_WITH_RC4_128_SHA" + ], + "cipherCnt" : 1, + "dstSessionId" : [ + "85bbc584132410aa03c3d6aac195e2d81e7b8d24e63b314a4a5d214cbcf0080a" + ], + "ja3" : [ + "06a92bf69b367389d2feb0d70501ddfe" + ], + "ja3Cnt" : 1, + "version" : [ + "TLSv1" + ], + "versionCnt" : 1 + }, + "totBytes" : 10920, + "totDataBytes" : 8920, + "totPackets" : 30 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-131202", + "_type" : "session" + } + } + }, + { + "body" : { + "cert" : [ { "alt" : [ "gp1.wac.edgecastcdn.net", @@ -620,169 +502,83 @@ "static.teamtreehouse.com", "wac.a8b5.edgecastcdn.net" ], - "altcnt" : 149, - "diffDays" : 1164, + "altCnt" : 149, "hash" : "d8:af:99:8d:b5:e0:42:a7:b4:7b:6d:41:62:75:00:a7:f7:ed:96:5c", - "iCn" : [ + "issuerCN" : [ "digicert high assurance ca-3" ], - "iOn" : "DigiCert Inc", - "notAfter" : 1418212800, - "notBefore" : 1317600000, - "sCn" : [ + "issuerON" : "DigiCert Inc", + "notAfter" : 1418212800000, + "notBefore" : 1317600000000, + "serial" : "062d488986c9a6d7f94901c2b5906882", + "subjectCN" : [ "gp1.wac.edgecastcdn.net" ], - "sOn" : "EdgeCast Networks, Inc.", - "sn" : "062d488986c9a6d7f94901c2b5906882" + "subjectON" : "EdgeCast Networks, Inc.", + "validDays" : 1164 }, { - "diffDays" : 5113, "hash" : "42:85:78:55:fb:0e:a4:3f:54:c9:91:1e:30:e7:79:1d:8c:e8:27:05", - "iCn" : [ + "issuerCN" : [ "digicert high assurance ev root ca" ], - "iOn" : "DigiCert Inc", - "notAfter" : 1648944000, - "notBefore" : 1207137600, - "sCn" : [ + "issuerON" : "DigiCert Inc", + "notAfter" : 1648944000000, + "notBefore" : 1207137600000, + "serial" : "0a5f114d035b179117d2efd4038c3f3b", + "subjectCN" : [ "digicert high assurance ca-3" ], - "sOn" : "DigiCert Inc", - "sn" : "0a5f114d035b179117d2efd4038c3f3b" + "subjectON" : "DigiCert Inc", + "validDays" : 5113 } ], - "tlscipher-term" : [ - "TLS_RSA_WITH_RC4_128_SHA" - ], - "tlscipher-termcnt" : 1, - "tlscnt" : 2, - "tlsdstid-term" : [ - "8142ff7f5af97c1486dc2addaf9cb504fdfb6c26df171cbd15bd29551adacd03" - ], - "tlsja3-term" : [ - "06a92bf69b367389d2feb0d70501ddfe" - ], - "tlsja3-termcnt" : 1, - "tlsver-term" : [ - "TLSv1" - ], - "tlsver-termcnt" : 1 - }, - "header" : { - "index" : { - "_index" : "tests_sessions-131202", - "_type" : "session" - } - } - }, - { - "body" : { - "a1" : "10.180.156.185", - "a2" : "10.180.156.249", - "assocksip" : "AS15133 MCI Communications Services, Inc. d/b/a Verizon Business", - "by" : 11127, - "by1" : 1912, - "by2" : 9215, - "db" : 8929, - "db1" : 646, - "db2" : 8283, - "fb1" : "0502000105010001", - "fb2" : "0500050000010ab4", - "firstPacket" : 1386004480852, - "fp" : 1386004480, - "fpd" : 1386004480852, - "fs" : [], - "g1" : "USA", - "g2" : "USA", - "gsocksip" : "USA", - "ho" : [ - "www.example.com" - ], - "hocnt" : 1, - "ipDst" : "10.180.156.249", - "ipSrc" : "10.180.156.185", - "lastPacket" : 1386004480888, - "lp" : 1386004480, - "lpd" : 1386004480888, - "mac1-term" : [ - "00:1f:5b:ff:51:cb" - ], - "mac1-term-cnt" : 1, - "mac2-term" : [ + "certCnt" : 2, + "dstBytes" : 9120, + "dstDataBytes" : 8254, + "dstGEO" : "US", + "dstIp" : "10.180.156.249", + "dstMac" : [ "00:13:72:c4:f1:e1" ], - "mac2-term-cnt" : 1, - "no" : "test", - "p1" : 53556, - "p2" : 1080, - "pa" : 33, - "pa1" : 19, - "pa2" : 14, - "portDst" : 1080, - "portSrc" : 53556, - "pr" : 6, - "prot-term" : [ - "tls", - "socks", - "tcp" - ], - "prot-term-cnt" : 3, - "ps" : [ - 22815, - 22909, - 22999, - 23081, - 23167, - 23249, - 23333, - 23415, - 23507, - 23599, - 23681, - 23883, - 25413, - 26943, - 27049, - 27131, - 27213, - 28743, - 28825, - 30355, - 31179, - 31261, - 31657, - 31786, - 31868, - 32121, - 32549, - 32631, - 34033, - 34115, - 34224, - 34306, - 34388 + "dstMacCnt" : 1, + "dstOui" : [ + "Dell Inc." ], - "psl" : [ + "dstOuiCnt" : 1, + "dstPackets" : 13, + "dstPayload8" : "005a99b50ab49cf9", + "dstPort" : 1080, + "fileId" : [], + "firstPacket" : 1386004475691, + "http" : { + "host" : [ + "www.example.com" + ], + "hostCnt" : 1 + }, + "ipProtocol" : 6, + "lastPacket" : 1386004475761, + "length" : 69, + "node" : "test", + "packetLen" : [ 94, 90, 82, - 86, - 82, - 84, + 107, 82, - 92, - 92, + 90, 82, 202, 1530, - 1530, - 106, - 82, + 94, 82, 1530, - 82, 1530, - 824, + 1530, + 82, + 82, + 836, 82, 396, 129, @@ -790,29 +586,114 @@ 253, 428, 82, - 1402, + 1377, 82, 109, 82, 82, 82 ], - "rirsocksip" : "RIPE", - "sl" : 35, - "socksip" : "93.184.216.119", - "sockspo" : 443, - "ss" : 1, + "packetPos" : [ + 11424, + 11518, + 11608, + 11690, + 11797, + 11879, + 11969, + 12051, + 12253, + 13783, + 13877, + 13959, + 15489, + 17019, + 18549, + 18631, + 18713, + 19549, + 19631, + 20027, + 20156, + 20238, + 20491, + 20919, + 21001, + 22378, + 22460, + 22569, + 22651, + 22733 + ], + "protocol" : [ + "tls", + "socks", + "tcp" + ], + "protocolCnt" : 3, + "segmentCnt" : 1, + "socks" : { + "host" : "www.example.com", + "port" : 443 + }, + "srcBytes" : 1791, + "srcDataBytes" : 657, + "srcGEO" : "US", + "srcIp" : "10.180.156.185", + "srcMac" : [ + "00:1f:5b:ff:51:cb" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Apple, Inc." + ], + "srcOuiCnt" : 1, + "srcPackets" : 17, + "srcPayload8" : "040101bb00000001", + "srcPort" : 53555, "tcpflags" : { - "ack" : 16, + "ack" : 15, + "dstZero" : 0, "fin" : 2, - "psh" : 13, + "psh" : 11, "rst" : 0, + "srcZero" : 0, "syn" : 1, "syn-ack" : 1, "urg" : 0 }, "timestamp" : "SET", - "tls" : [ + "tls" : { + "cipher" : [ + "TLS_RSA_WITH_RC4_128_SHA" + ], + "cipherCnt" : 1, + "dstSessionId" : [ + "8142ff7f5af97c1486dc2addaf9cb504fdfb6c26df171cbd15bd29551adacd03" + ], + "ja3" : [ + "06a92bf69b367389d2feb0d70501ddfe" + ], + "ja3Cnt" : 1, + "version" : [ + "TLSv1" + ], + "versionCnt" : 1 + }, + "totBytes" : 10911, + "totDataBytes" : 8911, + "totPackets" : 30 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-131202", + "_type" : "session" + } + } + }, + { + "body" : { + "cert" : [ { "alt" : [ "gp1.wac.edgecastcdn.net", @@ -965,57 +846,200 @@ "static.teamtreehouse.com", "wac.a8b5.edgecastcdn.net" ], - "altcnt" : 149, - "diffDays" : 1164, + "altCnt" : 149, "hash" : "d8:af:99:8d:b5:e0:42:a7:b4:7b:6d:41:62:75:00:a7:f7:ed:96:5c", - "iCn" : [ + "issuerCN" : [ "digicert high assurance ca-3" ], - "iOn" : "DigiCert Inc", - "notAfter" : 1418212800, - "notBefore" : 1317600000, - "sCn" : [ + "issuerON" : "DigiCert Inc", + "notAfter" : 1418212800000, + "notBefore" : 1317600000000, + "serial" : "062d488986c9a6d7f94901c2b5906882", + "subjectCN" : [ "gp1.wac.edgecastcdn.net" ], - "sOn" : "EdgeCast Networks, Inc.", - "sn" : "062d488986c9a6d7f94901c2b5906882" + "subjectON" : "EdgeCast Networks, Inc.", + "validDays" : 1164 }, { - "diffDays" : 5113, "hash" : "42:85:78:55:fb:0e:a4:3f:54:c9:91:1e:30:e7:79:1d:8c:e8:27:05", - "iCn" : [ + "issuerCN" : [ "digicert high assurance ev root ca" ], - "iOn" : "DigiCert Inc", - "notAfter" : 1648944000, - "notBefore" : 1207137600, - "sCn" : [ + "issuerON" : "DigiCert Inc", + "notAfter" : 1648944000000, + "notBefore" : 1207137600000, + "serial" : "0a5f114d035b179117d2efd4038c3f3b", + "subjectCN" : [ "digicert high assurance ca-3" ], - "sOn" : "DigiCert Inc", - "sn" : "0a5f114d035b179117d2efd4038c3f3b" + "subjectON" : "DigiCert Inc", + "validDays" : 5113 } ], - "tlscipher-term" : [ - "TLS_RSA_WITH_RC4_128_SHA" + "certCnt" : 2, + "dstBytes" : 9215, + "dstDataBytes" : 8283, + "dstGEO" : "US", + "dstIp" : "10.180.156.249", + "dstMac" : [ + "00:13:72:c4:f1:e1" ], - "tlscipher-termcnt" : 1, - "tlscnt" : 2, - "tlsdstid-term" : [ - "f7186f0670e348e24ec9b816eb2a2832c21ce3a14ea81aa1a388f91c3cd014b1" + "dstMacCnt" : 1, + "dstOui" : [ + "Dell Inc." ], - "tlsja3-term" : [ - "06a92bf69b367389d2feb0d70501ddfe" + "dstOuiCnt" : 1, + "dstPackets" : 14, + "dstPayload8" : "0500050000010ab4", + "dstPort" : 1080, + "fileId" : [], + "firstPacket" : 1386004480852, + "http" : { + "host" : [ + "www.example.com" + ], + "hostCnt" : 1 + }, + "ipProtocol" : 6, + "lastPacket" : 1386004480888, + "length" : 35, + "node" : "test", + "packetLen" : [ + 94, + 90, + 82, + 86, + 82, + 84, + 82, + 92, + 92, + 82, + 202, + 1530, + 1530, + 106, + 82, + 82, + 1530, + 82, + 1530, + 824, + 82, + 396, + 129, + 82, + 253, + 428, + 82, + 1402, + 82, + 109, + 82, + 82, + 82 ], - "tlsja3-termcnt" : 1, - "tlsver-term" : [ - "TLSv1" + "packetPos" : [ + 22815, + 22909, + 22999, + 23081, + 23167, + 23249, + 23333, + 23415, + 23507, + 23599, + 23681, + 23883, + 25413, + 26943, + 27049, + 27131, + 27213, + 28743, + 28825, + 30355, + 31179, + 31261, + 31657, + 31786, + 31868, + 32121, + 32549, + 32631, + 34033, + 34115, + 34224, + 34306, + 34388 + ], + "protocol" : [ + "tls", + "socks", + "tcp" + ], + "protocolCnt" : 3, + "segmentCnt" : 1, + "socks" : { + "ASN" : "AS15133 MCI Communications Services, Inc. d/b/a Verizon Business", + "GEO" : "US", + "RIR" : "RIPE", + "ip" : "93.184.216.119", + "port" : 443 + }, + "srcBytes" : 1912, + "srcDataBytes" : 646, + "srcGEO" : "US", + "srcIp" : "10.180.156.185", + "srcMac" : [ + "00:1f:5b:ff:51:cb" ], - "tlsver-termcnt" : 1 + "srcMacCnt" : 1, + "srcOui" : [ + "Apple, Inc." + ], + "srcOuiCnt" : 1, + "srcPackets" : 19, + "srcPayload8" : "0502000105010001", + "srcPort" : 53556, + "tcpflags" : { + "ack" : 16, + "dstZero" : 0, + "fin" : 2, + "psh" : 13, + "rst" : 0, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, + "timestamp" : "SET", + "tls" : { + "cipher" : [ + "TLS_RSA_WITH_RC4_128_SHA" + ], + "cipherCnt" : 1, + "dstSessionId" : [ + "f7186f0670e348e24ec9b816eb2a2832c21ce3a14ea81aa1a388f91c3cd014b1" + ], + "ja3" : [ + "06a92bf69b367389d2feb0d70501ddfe" + ], + "ja3Cnt" : 1, + "version" : [ + "TLSv1" + ], + "versionCnt" : 1 + }, + "totBytes" : 11127, + "totDataBytes" : 8929, + "totPackets" : 33 }, "header" : { "index" : { - "_index" : "tests_sessions-131202", + "_index" : "tests_sessions2-131202", "_type" : "session" } } diff --git a/tests/pcap/socks4-https.test b/tests/pcap/socks4-https.test index a6904600d7..2a4da5ab61 100644 --- a/tests/pcap/socks4-https.test +++ b/tests/pcap/socks4-https.test @@ -1,55 +1,109 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "a1" : "10.0.0.1", - "a2" : "10.0.0.2", - "as1" : "AS0000 This is neat", - "as2" : "AS0001 Cool Beans!", - "by" : 7752, - "by1" : 404, - "by2" : 7348, - "db" : 6822, - "db1" : 38, - "db2" : 6784, - "fb1" : "040101bb00000001", - "fb2" : "005a01bb83fd3d50", - "firstPacket" : 1415053640836, - "fp" : 1415053640, - "fpd" : 1415053640836, - "fs" : [], - "g1" : "RUS", - "g2" : "CAN", - "ipDst" : "10.0.0.2", - "ipSrc" : "10.0.0.1", - "lastPacket" : 1415053642702, - "lp" : 1415053642, - "lpd" : 1415053642702, - "mac1-term" : [ - "00:0a:f3:31:84:00" + "cert" : [ + { + "alt" : [ + "login.live.com", + "loginnet.passport.com", + "msnia.login.live.com", + "pst.microsoftpassportsupport.net", + "api.login.live.com", + "tools.login.live.com", + "xml.login.live.com", + "nexus.passport.com", + "login.passport.com", + "msnialogin.passport.com" + ], + "altCnt" : 10, + "hash" : "d4:18:6b:6e:6d:82:6a:53:c9:a6:2e:f2:c0:cd:1b:45:c0:e7:e6:c4", + "issuerCN" : [ + "verisign class 3 extended validation ssl sgc ca" + ], + "issuerON" : "VeriSign, Inc.", + "notAfter" : 1448927999000, + "notBefore" : 1413504000000, + "serial" : "0dc04875701d090d9f3645da26f51fe7", + "subjectCN" : [ + "login.live.com" + ], + "subjectON" : "Microsoft Corporation", + "validDays" : 409 + }, + { + "hash" : "f4:a8:0a:0c:d1:e6:cf:19:0b:8c:bc:6f:bc:99:17:11:d4:82:c9:d0", + "issuerON" : "VeriSign, Inc.", + "notAfter" : 1636329599000, + "notBefore" : 1162944000000, + "serial" : "35973187f3873a07327ece580c9b7eda", + "subjectCN" : [ + "verisign class 3 public primary certification authority - g5" + ], + "subjectON" : "VeriSign, Inc.", + "validDays" : 5478 + }, + { + "hash" : "4a:8a:2a:0e:27:6f:f3:3b:5d:d8:8a:36:21:46:01:0f:2a:8b:6a:ee", + "issuerCN" : [ + "verisign class 3 public primary certification authority - g5" + ], + "issuerON" : "VeriSign, Inc.", + "notAfter" : 1478563199000, + "notBefore" : 1162944000000, + "serial" : "112a006d37e5106fd6ca7cc3efbacc18", + "subjectCN" : [ + "verisign class 3 extended validation ssl sgc ca" + ], + "subjectON" : "VeriSign, Inc.", + "validDays" : 3652 + } ], - "mac1-term-cnt" : 1, - "mac2-term" : [ + "certCnt" : 3, + "dstASN" : "AS0001 Cool Beans!", + "dstBytes" : 7348, + "dstDataBytes" : 6784, + "dstGEO" : "CA", + "dstIp" : "10.0.0.2", + "dstMac" : [ "00:00:5e:00:01:01", "00:26:88:d8:bf:c1" ], - "mac2-term-cnt" : 2, - "no" : "test", - "p1" : 50606, - "p2" : 9901, - "pa" : 16, - "pa1" : 6, - "pa2" : 10, - "portDst" : 9901, - "portSrc" : 50606, - "pr" : 6, - "prot-term" : [ - "tls", - "socks", - "tcp" + "dstMacCnt" : 2, + "dstOui" : [ + "ICANN, IANA Department", + "Juniper Networks" ], - "prot-term-cnt" : 3, - "ps" : [ + "dstOuiCnt" : 2, + "dstPackets" : 10, + "dstPayload8" : "005a01bb83fd3d50", + "dstPort" : 9901, + "dstRIR" : "TEST", + "fileId" : [], + "firstPacket" : 1415053640836, + "ipProtocol" : 6, + "lastPacket" : 1415053642702, + "length" : 1867, + "node" : "test", + "packetLen" : [ + 82, + 82, + 76, + 108, + 76, + 78, + 76, + 110, + 1430, + 82, + 76, + 1430, + 1430, + 1430, + 76, + 1366 + ], + "packetPos" : [ 24, 106, 188, @@ -67,146 +121,93 @@ 6590, 6666 ], - "psl" : [ - 82, - 82, - 76, - 108, - 76, - 78, - 76, - 110, - 1430, - 82, - 76, - 1430, - 1430, - 1430, - 76, - 1366 + "protocol" : [ + "tls", + "socks", + "tcp" + ], + "protocolCnt" : 3, + "segmentCnt" : 1, + "socks" : { + "host" : "login.live.com", + "port" : 443, + "user" : "fd7bcf7b5c9e1d" + }, + "srcASN" : "AS0000 This is neat", + "srcBytes" : 404, + "srcDataBytes" : 38, + "srcGEO" : "RU", + "srcIp" : "10.0.0.1", + "srcMac" : [ + "00:0a:f3:31:84:00" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Cisco Systems, Inc" ], - "rir2" : "TEST", - "sl" : 1867, - "socksho" : "login.live.com", - "sockspo" : 443, - "socksuser" : "fd7bcf7b5c9e1d", - "ss" : 1, - "ta" : [ + "srcOuiCnt" : 1, + "srcPackets" : 6, + "srcPayload8" : "040101bb00000001", + "srcPort" : 50606, + "tags" : [ "acked-unseen-segment-dst", "dstip", "out-of-order-dst", "srcip" ], - "tacnt" : 4, - "tags-term" : [ - "srcip", - "dstip", - "out-of-order-dst", - "acked-unseen-segment-dst" - ], + "tagsCnt" : 4, "tcpflags" : { "ack" : 10, + "dstZero" : 0, "fin" : 0, "psh" : 4, "rst" : 0, + "srcZero" : 0, "syn" : 1, "syn-ack" : 1, "urg" : 0 }, "test" : { - "ip" : [ - 167772161 - ], - "ip-asn" : [ + "ASN" : [ "AS0000 This is neat" ], - "ip-geo" : [ - "RUS" + "GEO" : [ + "RU" ], - "ip-rir" : [ + "RIR" : [ "" ], + "ip" : [ + "10.0.0.1" + ], "number" : [ 33554442 ], - "string" : [ + "string.snow" : [ "16777226:50606,33554442:9901" ] }, "timestamp" : "SET", - "tls" : [ - { - "alt" : [ - "login.live.com", - "loginnet.passport.com", - "msnia.login.live.com", - "pst.microsoftpassportsupport.net", - "api.login.live.com", - "tools.login.live.com", - "xml.login.live.com", - "nexus.passport.com", - "login.passport.com", - "msnialogin.passport.com" - ], - "altcnt" : 10, - "diffDays" : 409, - "hash" : "d4:18:6b:6e:6d:82:6a:53:c9:a6:2e:f2:c0:cd:1b:45:c0:e7:e6:c4", - "iCn" : [ - "verisign class 3 extended validation ssl sgc ca" - ], - "iOn" : "VeriSign, Inc.", - "notAfter" : 1448927999, - "notBefore" : 1413504000, - "sCn" : [ - "login.live.com" - ], - "sOn" : "Microsoft Corporation", - "sn" : "0dc04875701d090d9f3645da26f51fe7" - }, - { - "diffDays" : 5478, - "hash" : "f4:a8:0a:0c:d1:e6:cf:19:0b:8c:bc:6f:bc:99:17:11:d4:82:c9:d0", - "iOn" : "VeriSign, Inc.", - "notAfter" : 1636329599, - "notBefore" : 1162944000, - "sCn" : [ - "verisign class 3 public primary certification authority - g5" - ], - "sOn" : "VeriSign, Inc.", - "sn" : "35973187f3873a07327ece580c9b7eda" - }, - { - "diffDays" : 3652, - "hash" : "4a:8a:2a:0e:27:6f:f3:3b:5d:d8:8a:36:21:46:01:0f:2a:8b:6a:ee", - "iCn" : [ - "verisign class 3 public primary certification authority - g5" - ], - "iOn" : "VeriSign, Inc.", - "notAfter" : 1478563199, - "notBefore" : 1162944000, - "sCn" : [ - "verisign class 3 extended validation ssl sgc ca" - ], - "sOn" : "VeriSign, Inc.", - "sn" : "112a006d37e5106fd6ca7cc3efbacc18" - } - ], - "tlscipher-term" : [ - "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256" - ], - "tlscipher-termcnt" : 1, - "tlscnt" : 3, - "tlsdstid-term" : [ - "972d0000e526f027ac3e73bab09d466f73291c310391f9b105146302a6ffab30" - ], - "tlsver-term" : [ - "TLSv1.2" - ], - "tlsver-termcnt" : 1 + "tls" : { + "cipher" : [ + "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256" + ], + "cipherCnt" : 1, + "dstSessionId" : [ + "972d0000e526f027ac3e73bab09d466f73291c310391f9b105146302a6ffab30" + ], + "version" : [ + "TLSv1.2" + ], + "versionCnt" : 1 + }, + "totBytes" : 7752, + "totDataBytes" : 6822, + "totPackets" : 16 }, "header" : { "index" : { - "_index" : "tests_sessions-141103", + "_index" : "tests_sessions2-141103", "_type" : "session" } } diff --git a/tests/pcap/socks5-http-302-frag.test b/tests/pcap/socks5-http-302-frag.test index 56181057e6..fbcb479736 100644 --- a/tests/pcap/socks5-http-302-frag.test +++ b/tests/pcap/socks5-http-302-frag.test @@ -1,126 +1,130 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "a1" : "10.0.0.1", - "a2" : "10.0.0.2", - "as1" : "AS0000 This is neat", - "as2" : "AS0001 Cool Beans!", - "by" : 2133, - "by1" : 859, - "by2" : 1274, - "db" : 1361, - "db1" : 419, - "db2" : 942, - "fb1" : "050100050100030e", - "fb2" : "0500050000010a00", + "dstASN" : "AS0001 Cool Beans!", + "dstBytes" : 1274, + "dstDataBytes" : 942, + "dstGEO" : "CA", + "dstIp" : "10.0.0.2", + "dstMac" : [ + "00:00:5e:00:01:01", + "00:26:88:df:17:c7" + ], + "dstMacCnt" : 2, + "dstOui" : [ + "ICANN, IANA Department", + "Juniper Networks" + ], + "dstOuiCnt" : 2, + "dstPackets" : 6, + "dstPayload8" : "0500050000010a00", + "dstPort" : 21477, + "dstRIR" : "TEST", + "fileId" : [], "firstPacket" : 1385474294492, - "fp" : 1385474294, - "fpd" : 1385474294492, - "fs" : [], - "g1" : "RUS", - "g2" : "CAN", - "hckey-term" : [ - "PREF" - ], - "hckey-term-cnt" : 1, - "hcval-term" : [ - "ID=xxxxxxxxxxxxxxxx:TM=xxxxxxxxxx:LM=xxxxxxxxxx:S=xxxxxxxxxxxx_6oz" - ], - "hcval-term-cnt" : 1, - "hdrs" : { - "hres-location" : [ - "http://www.google.de/?gws_rd=cr&ei=xxxxxxxxxxxxxxxxxxxxxx" - ] - }, - "hdver" : [ - "1.1" - ], - "hdvercnt" : 1, - "hh1" : [ - "http:header:accept", - "http:header:accept-encoding", - "http:header:accept-language", - "http:header:connection", - "http:header:cookie", - "http:header:host", - "http:header:user-agent" - ], - "hh1cnt" : 7, - "hh2" : [ - "http:header:alternate-protocol", - "http:header:cache-control", - "http:header:content-length", - "http:header:content-type", - "http:header:date", - "http:header:location", - "http:header:p3p", - "http:header:server", - "http:header:set-cookie", - "http:header:x-frame-options", - "http:header:x-xss-protection" - ], - "hh2cnt" : 11, - "hmd5" : [ - "222315d36e1313774cb1c2f0eb06864f" - ], - "hmd5cnt" : 1, - "ho" : [ - "www.google.com" - ], - "hocnt" : 1, - "hpath" : [ - "/" - ], - "hpathcnt" : 1, - "hsver" : [ - "1.1" - ], - "hsvercnt" : 1, "http" : { - "bodymagic-term" : [ + "bodyMagic" : [ "text/html" ], - "bodymagic-term-cnt" : 1, - "method-term" : [ + "bodyMagicCnt" : 1, + "clientVersion" : [ + "1.1" + ], + "clientVersionCnt" : 1, + "cookieKey" : [ + "PREF" + ], + "cookieKeyCnt" : 1, + "cookieValue" : [ + "ID=xxxxxxxxxxxxxxxx:TM=xxxxxxxxxx:LM=xxxxxxxxxx:S=xxxxxxxxxxxx_6oz" + ], + "cookieValueCnt" : 1, + "host" : [ + "www.google.com" + ], + "hostCnt" : 1, + "md5" : [ + "222315d36e1313774cb1c2f0eb06864f" + ], + "md5Cnt" : 1, + "method" : [ "GET" ], - "method-term-cnt" : 1, + "methodCnt" : 1, + "path" : [ + "/" + ], + "pathCnt" : 1, + "requestHeader" : [ + "accept", + "user-agent", + "accept-encoding", + "connection", + "host", + "cookie", + "accept-language" + ], + "requestHeaderCnt" : 7, + "response-location" : [ + "http://www.google.de/?gws_rd=cr&ei=xxxxxxxxxxxxxxxxxxxxxx" + ], + "responseHeader" : [ + "content-type", + "alternate-protocol", + "x-frame-options", + "cache-control", + "x-xss-protection", + "content-length", + "date", + "p3p", + "server", + "set-cookie", + "location" + ], + "responseHeaderCnt" : 11, + "serverVersion" : [ + "1.1" + ], + "serverVersionCnt" : 1, + "sha256" : [ + "892eea9b9c2f9ba779fe5c6deb3a5acf65ca3162aac8ad8d980608c669a47ad3" + ], + "sha256Cnt" : 1, "statuscode" : [ 302 ], - "statuscode-cnt" : 1 + "statuscodeCnt" : 1, + "uri" : [ + "www.google.com/" + ], + "uriCnt" : 1, + "useragent" : [ + "Mozilla/5.0 (Windows NT 5.1; rv:25.0) Gecko/20100101 Firefox/25.0" + ], + "useragentCnt" : 1 }, - "ipDst" : "10.0.0.2", - "ipSrc" : "10.0.0.1", + "ipProtocol" : 6, "lastPacket" : 1385474412431, - "lp" : 1385474412, - "lpd" : 1385474412431, - "mac1-term" : [ - "00:0b:45:b7:16:c0" - ], - "mac1-term-cnt" : 1, - "mac2-term" : [ - "00:26:88:df:17:c7", - "00:00:5e:00:01:01" - ], - "mac2-term-cnt" : 2, - "no" : "test", - "p1" : 1637, - "p2" : 21477, - "pa" : 14, - "pa1" : 8, - "pa2" : 6, - "portDst" : 21477, - "portSrc" : 1637, - "pr" : 6, - "prot-term" : [ - "http", - "socks", - "tcp" + "length" : 117939, + "node" : "test", + "packetLen" : [ + 78, + 78, + 70, + 73, + 72, + 91, + 80, + 465, + 1000, + 70, + 70, + 70, + 70, + 70 ], - "prot-term-cnt" : 3, - "ps" : [ + "packetPos" : [ 24, 102, 180, @@ -136,78 +140,77 @@ 5141, 5211 ], - "psl" : [ - 78, - 78, - 70, - 73, - 72, - 91, - 80, - 465, - 1000, - 70, - 70, - 70, - 70, - 70 + "protocol" : [ + "http", + "socks", + "tcp" + ], + "protocolCnt" : 3, + "segmentCnt" : 1, + "socks" : { + "host" : "www.google.com", + "port" : 80 + }, + "srcASN" : "AS0000 This is neat", + "srcBytes" : 859, + "srcDataBytes" : 419, + "srcGEO" : "RU", + "srcIp" : "10.0.0.1", + "srcMac" : [ + "00:0b:45:b7:16:c0" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Cisco Systems, Inc" ], - "rir2" : "TEST", - "sl" : 117939, - "socksho" : "www.google.com", - "sockspo" : 80, - "ss" : 1, - "ta" : [ + "srcOuiCnt" : 1, + "srcPackets" : 8, + "srcPayload8" : "050100050100030e", + "srcPort" : 1637, + "tags" : [ "dstip", "srcip" ], - "tacnt" : 2, - "tags-term" : [ - "srcip", - "dstip" - ], + "tagsCnt" : 2, "tcpflags" : { "ack" : 4, + "dstZero" : 0, "fin" : 2, "psh" : 6, "rst" : 0, + "srcZero" : 0, "syn" : 1, "syn-ack" : 1, "urg" : 0 }, "test" : { - "ip" : [ - 167772161 - ], - "ip-asn" : [ + "ASN" : [ "AS0000 This is neat" ], - "ip-geo" : [ - "RUS" + "GEO" : [ + "RU" ], - "ip-rir" : [ + "RIR" : [ "" ], + "ip" : [ + "10.0.0.1" + ], "number" : [ 33554442 ], - "string" : [ + "string.snow" : [ "16777226:1637,33554442:21477" ] }, "timestamp" : "SET", - "ua" : [ - "Mozilla/5.0 (Windows NT 5.1; rv:25.0) Gecko/20100101 Firefox/25.0" - ], - "uacnt" : 1, - "us" : [ - "//www.google.com/" - ], - "uscnt" : 1 + "totBytes" : 2133, + "totDataBytes" : 1361, + "totPackets" : 14 }, "header" : { "index" : { - "_index" : "tests_sessions-131126", + "_index" : "tests_sessions2-131126", "_type" : "session" } } diff --git a/tests/pcap/socks5-http-302.test b/tests/pcap/socks5-http-302.test index d96ecbd32a..54de8757d7 100644 --- a/tests/pcap/socks5-http-302.test +++ b/tests/pcap/socks5-http-302.test @@ -1,126 +1,130 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "a1" : "10.0.0.1", - "a2" : "10.0.0.2", - "as1" : "AS0000 This is neat", - "as2" : "AS0001 Cool Beans!", - "by" : 2176, - "by1" : 886, - "by2" : 1290, - "db" : 1361, - "db1" : 419, - "db2" : 942, - "fb1" : "050100050100030e", - "fb2" : "0500050000010a00", + "dstASN" : "AS0001 Cool Beans!", + "dstBytes" : 1290, + "dstDataBytes" : 942, + "dstGEO" : "CA", + "dstIp" : "10.0.0.2", + "dstMac" : [ + "00:00:5e:00:01:01", + "00:26:88:df:17:c7" + ], + "dstMacCnt" : 2, + "dstOui" : [ + "ICANN, IANA Department", + "Juniper Networks" + ], + "dstOuiCnt" : 2, + "dstPackets" : 6, + "dstPayload8" : "0500050000010a00", + "dstPort" : 21477, + "dstRIR" : "TEST", + "fileId" : [], "firstPacket" : 1385474294492, - "fp" : 1385474294, - "fpd" : 1385474294492, - "fs" : [], - "g1" : "RUS", - "g2" : "CAN", - "hckey-term" : [ - "PREF" - ], - "hckey-term-cnt" : 1, - "hcval-term" : [ - "ID=xxxxxxxxxxxxxxxx:TM=xxxxxxxxxx:LM=xxxxxxxxxx:S=xxxxxxxxxxxx_6oz" - ], - "hcval-term-cnt" : 1, - "hdrs" : { - "hres-location" : [ - "http://www.google.de/?gws_rd=cr&ei=xxxxxxxxxxxxxxxxxxxxxx" - ] - }, - "hdver" : [ - "1.1" - ], - "hdvercnt" : 1, - "hh1" : [ - "http:header:accept", - "http:header:accept-encoding", - "http:header:accept-language", - "http:header:connection", - "http:header:cookie", - "http:header:host", - "http:header:user-agent" - ], - "hh1cnt" : 7, - "hh2" : [ - "http:header:alternate-protocol", - "http:header:cache-control", - "http:header:content-length", - "http:header:content-type", - "http:header:date", - "http:header:location", - "http:header:p3p", - "http:header:server", - "http:header:set-cookie", - "http:header:x-frame-options", - "http:header:x-xss-protection" - ], - "hh2cnt" : 11, - "hmd5" : [ - "222315d36e1313774cb1c2f0eb06864f" - ], - "hmd5cnt" : 1, - "ho" : [ - "www.google.com" - ], - "hocnt" : 1, - "hpath" : [ - "/" - ], - "hpathcnt" : 1, - "hsver" : [ - "1.1" - ], - "hsvercnt" : 1, "http" : { - "bodymagic-term" : [ + "bodyMagic" : [ "text/html" ], - "bodymagic-term-cnt" : 1, - "method-term" : [ + "bodyMagicCnt" : 1, + "clientVersion" : [ + "1.1" + ], + "clientVersionCnt" : 1, + "cookieKey" : [ + "PREF" + ], + "cookieKeyCnt" : 1, + "cookieValue" : [ + "ID=xxxxxxxxxxxxxxxx:TM=xxxxxxxxxx:LM=xxxxxxxxxx:S=xxxxxxxxxxxx_6oz" + ], + "cookieValueCnt" : 1, + "host" : [ + "www.google.com" + ], + "hostCnt" : 1, + "md5" : [ + "222315d36e1313774cb1c2f0eb06864f" + ], + "md5Cnt" : 1, + "method" : [ "GET" ], - "method-term-cnt" : 1, + "methodCnt" : 1, + "path" : [ + "/" + ], + "pathCnt" : 1, + "requestHeader" : [ + "accept", + "user-agent", + "accept-encoding", + "connection", + "host", + "cookie", + "accept-language" + ], + "requestHeaderCnt" : 7, + "response-location" : [ + "http://www.google.de/?gws_rd=cr&ei=xxxxxxxxxxxxxxxxxxxxxx" + ], + "responseHeader" : [ + "content-type", + "alternate-protocol", + "x-frame-options", + "cache-control", + "x-xss-protection", + "content-length", + "date", + "p3p", + "server", + "set-cookie", + "location" + ], + "responseHeaderCnt" : 11, + "serverVersion" : [ + "1.1" + ], + "serverVersionCnt" : 1, + "sha256" : [ + "892eea9b9c2f9ba779fe5c6deb3a5acf65ca3162aac8ad8d980608c669a47ad3" + ], + "sha256Cnt" : 1, "statuscode" : [ 302 ], - "statuscode-cnt" : 1 + "statuscodeCnt" : 1, + "uri" : [ + "www.google.com/" + ], + "uriCnt" : 1, + "useragent" : [ + "Mozilla/5.0 (Windows NT 5.1; rv:25.0) Gecko/20100101 Firefox/25.0" + ], + "useragentCnt" : 1 }, - "ipDst" : "10.0.0.2", - "ipSrc" : "10.0.0.1", + "ipProtocol" : 6, "lastPacket" : 1385474412431, - "lp" : 1385474412, - "lpd" : 1385474412431, - "mac1-term" : [ - "00:0b:45:b7:16:c0" - ], - "mac1-term-cnt" : 1, - "mac2-term" : [ - "00:26:88:df:17:c7", - "00:00:5e:00:01:01" - ], - "mac2-term-cnt" : 2, - "no" : "test", - "p1" : 1637, - "p2" : 21477, - "pa" : 14, - "pa1" : 8, - "pa2" : 6, - "portDst" : 21477, - "portSrc" : 1637, - "pr" : 6, - "prot-term" : [ - "http", - "socks", - "tcp" + "length" : 117939, + "node" : "test", + "packetLen" : [ + 78, + 78, + 76, + 76, + 76, + 91, + 80, + 465, + 1000, + 76, + 76, + 76, + 76, + 76 ], - "prot-term-cnt" : 3, - "ps" : [ + "packetPos" : [ 24, 102, 180, @@ -136,78 +140,77 @@ 2272, 2348 ], - "psl" : [ - 78, - 78, - 76, - 76, - 76, - 91, - 80, - 465, - 1000, - 76, - 76, - 76, - 76, - 76 + "protocol" : [ + "http", + "socks", + "tcp" + ], + "protocolCnt" : 3, + "segmentCnt" : 1, + "socks" : { + "host" : "www.google.com", + "port" : 80 + }, + "srcASN" : "AS0000 This is neat", + "srcBytes" : 886, + "srcDataBytes" : 419, + "srcGEO" : "RU", + "srcIp" : "10.0.0.1", + "srcMac" : [ + "00:0b:45:b7:16:c0" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Cisco Systems, Inc" ], - "rir2" : "TEST", - "sl" : 117939, - "socksho" : "www.google.com", - "sockspo" : 80, - "ss" : 1, - "ta" : [ + "srcOuiCnt" : 1, + "srcPackets" : 8, + "srcPayload8" : "050100050100030e", + "srcPort" : 1637, + "tags" : [ "dstip", "srcip" ], - "tacnt" : 2, - "tags-term" : [ - "srcip", - "dstip" - ], + "tagsCnt" : 2, "tcpflags" : { "ack" : 4, + "dstZero" : 0, "fin" : 2, "psh" : 6, "rst" : 0, + "srcZero" : 0, "syn" : 1, "syn-ack" : 1, "urg" : 0 }, "test" : { - "ip" : [ - 167772161 - ], - "ip-asn" : [ + "ASN" : [ "AS0000 This is neat" ], - "ip-geo" : [ - "RUS" + "GEO" : [ + "RU" ], - "ip-rir" : [ + "RIR" : [ "" ], + "ip" : [ + "10.0.0.1" + ], "number" : [ 33554442 ], - "string" : [ + "string.snow" : [ "16777226:1637,33554442:21477" ] }, "timestamp" : "SET", - "ua" : [ - "Mozilla/5.0 (Windows NT 5.1; rv:25.0) Gecko/20100101 Firefox/25.0" - ], - "uacnt" : 1, - "us" : [ - "//www.google.com/" - ], - "uscnt" : 1 + "totBytes" : 2176, + "totDataBytes" : 1361, + "totPackets" : 14 }, "header" : { "index" : { - "_index" : "tests_sessions-131126", + "_index" : "tests_sessions2-131126", "_type" : "session" } } diff --git a/tests/pcap/socks5-rdp.test b/tests/pcap/socks5-rdp.test index 6e50155419..999d314d8e 100644 --- a/tests/pcap/socks5-rdp.test +++ b/tests/pcap/socks5-rdp.test @@ -1,56 +1,43 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "a1" : "10.0.0.3", - "a2" : "10.0.0.2", - "as1" : "AS0002 Hmm!@#$%^&*()", - "as2" : "AS0001 Cool Beans!", - "assocksip" : "AS0000 This is neat", - "by" : 668, - "by1" : 405, - "by2" : 263, - "db" : 85, - "db1" : 54, - "db2" : 31, - "fb1" : "050100050100010a", - "fb2" : "0500050000010000", - "firstPacket" : 1386644255859, - "fp" : 1386644255, - "fpd" : 1386644255859, - "fs" : [], - "g2" : "CAN", - "gsocksip" : "RUS", - "ipDst" : "10.0.0.2", - "ipSrc" : "10.0.0.3", - "lastPacket" : 1386644257101, - "lp" : 1386644257, - "lpd" : 1386644257101, - "mac1-term" : [ - "00:00:5e:00:01:01", - "80:71:1f:82:cf:c6" - ], - "mac1-term-cnt" : 2, - "mac2-term" : [ + "dstASN" : "AS0001 Cool Beans!", + "dstBytes" : 263, + "dstDataBytes" : 31, + "dstGEO" : "CA", + "dstIp" : "10.0.0.2", + "dstMac" : [ "00:0a:f3:31:94:00" ], - "mac2-term-cnt" : 1, - "no" : "test", - "p1" : 2276, - "p2" : 42356, - "pa" : 10, - "pa1" : 6, - "pa2" : 4, - "portDst" : 42356, - "portSrc" : 2276, - "pr" : 6, - "prot-term" : [ - "rdp", - "socks", - "tcp" + "dstMacCnt" : 1, + "dstOui" : [ + "Cisco Systems, Inc" + ], + "dstOuiCnt" : 1, + "dstPackets" : 4, + "dstPayload8" : "0500050000010000", + "dstPort" : 42356, + "dstRIR" : "TEST", + "fileId" : [], + "firstPacket" : 1386644255859, + "ipProtocol" : 6, + "lastPacket" : 1386644257101, + "length" : 1242, + "node" : "test", + "packetLen" : [ + 82, + 82, + 76, + 76, + 76, + 80, + 80, + 111, + 89, + 76 ], - "prot-term-cnt" : 3, - "ps" : [ + "packetPos" : [ 24, 106, 188, @@ -62,50 +49,64 @@ 687, 776 ], - "psl" : [ - 82, - 82, - 76, - 76, - 76, - 80, - 80, - 111, - 89, - 76 + "protocol" : [ + "rdp", + "socks", + "tcp" + ], + "protocolCnt" : 3, + "segmentCnt" : 1, + "socks" : { + "ASN" : "AS0000 This is neat", + "GEO" : "RU", + "ip" : "10.0.0.1", + "port" : 3389 + }, + "srcASN" : "AS0002 Hmm!@#$%^&*()", + "srcBytes" : 405, + "srcDataBytes" : 54, + "srcIp" : "10.0.0.3", + "srcMac" : [ + "00:00:5e:00:01:01", + "80:71:1f:82:cf:c6" ], - "rir2" : "TEST", - "sl" : 1242, - "socksip" : "10.0.0.1", - "sockspo" : 3389, - "ss" : 1, - "ta" : [ + "srcMacCnt" : 2, + "srcOui" : [ + "ICANN, IANA Department", + "Juniper Networks" + ], + "srcOuiCnt" : 2, + "srcPackets" : 6, + "srcPayload8" : "050100050100010a", + "srcPort" : 2276, + "tags" : [ "dstip", "srcip" ], - "tacnt" : 2, - "tags-term" : [ - "srcip", - "dstip" - ], + "tagsCnt" : 2, "tcpflags" : { "ack" : 2, + "dstZero" : 0, "fin" : 0, "psh" : 6, "rst" : 0, + "srcZero" : 0, "syn" : 1, "syn-ack" : 1, "urg" : 0 }, "timestamp" : "SET", + "totBytes" : 668, + "totDataBytes" : 85, + "totPackets" : 10, "user" : [ "xxx" ], - "usercnt" : 1 + "userCnt" : 1 }, "header" : { "index" : { - "_index" : "tests_sessions-131210", + "_index" : "tests_sessions2-131210", "_type" : "session" } } diff --git a/tests/pcap/socks5-reverse.test b/tests/pcap/socks5-reverse.test index c74c2cf3bb..18959ba34c 100644 --- a/tests/pcap/socks5-reverse.test +++ b/tests/pcap/socks5-reverse.test @@ -1,162 +1,204 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "a1" : "10.0.0.1", - "a2" : "10.0.0.2", - "as1" : "AS0000 This is neat", - "as2" : "AS0001 Cool Beans!", - "assocksip" : "AS15169 Google LLC", - "by" : 27311, - "by1" : 25112, - "by2" : 2199, - "db" : 24346, - "db1" : 23392, - "db2" : 954, - "fb1" : "4e5a8d08874e0500", - "fb2" : "050100050100014a", - "firstPacket" : 1386790367120, - "fp" : 1386790367, - "fpd" : 1386790367120, - "fs" : [], - "g1" : "RUS", - "g2" : "CAN", - "gsocksip" : "USA", - "hckey-term" : [ - "NID", - "PREF" - ], - "hckey-term-cnt" : 2, - "hcval-term" : [ - "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", - "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - ], - "hcval-term-cnt" : 2, - "hdrs" : { - "hreq-referer" : [ - "", - "http://www.google.com/search?client=firefox&rls=en&q=sheepskin%20boots&start=0&num=10&hl=en&gl=us&uule=w+CAIQICINVW5pdGVkIFN0YXRlcw" - ], - "hreq-referercnt" : 2, - "hres-location" : [ - "http://ipv4.google.com/sorry/IndexRedirect?continue=http://www.google.com/search?client=firefox&rls=en&q=sheepskin%20boots&start=10&num=10&hl=en&gl=us&uule=xxxxxxxxxxxxxxxxxxxxxxxxxxxx" - ] - }, - "hdver" : [ - "1.1" - ], - "hdvercnt" : 1, - "hh1" : [ - "http:header:accept", - "http:header:accept-encoding", - "http:header:cookie", - "http:header:host", - "http:header:referer", - "http:header:user-agent" - ], - "hh1cnt" : 6, - "hh2" : [ - "http:header:alternate-protocol", - "http:header:cache-control", - "http:header:content-encoding", - "http:header:content-length", - "http:header:content-type", - "http:header:date", - "http:header:expires", - "http:header:location", - "http:header:p3p", - "http:header:pragma", - "http:header:server", - "http:header:set-cookie", - "http:header:transfer-encoding", - "http:header:x-frame-options", - "http:header:x-xss-protection" - ], - "hh2cnt" : 15, - "hkey" : [ - "uule", - "hl", - "client", - "start", - "rls", - "q", - "num", - "gl" - ], - "hkeycnt" : 8, - "hmd5" : [ - "2069181ae704855f29caf964ca52ec49", - "b0cecae354b9eab1f04f70e46a612cb1" - ], - "hmd5cnt" : 2, - "ho" : [ - "www.google.com" - ], - "hocnt" : 1, - "hpath" : [ - "/search" + "dstASN" : "AS0001 Cool Beans!", + "dstBytes" : 2199, + "dstDataBytes" : 954, + "dstGEO" : "CA", + "dstIp" : "10.0.0.2", + "dstMac" : [ + "00:00:5e:00:01:01", + "80:71:1f:83:9f:c6" ], - "hpathcnt" : 1, - "hsver" : [ - "1.1" + "dstMacCnt" : 2, + "dstOui" : [ + "ICANN, IANA Department", + "Juniper Networks" ], - "hsvercnt" : 1, + "dstOuiCnt" : 2, + "dstPackets" : 21, + "dstPayload8" : "050100050100014a", + "dstPort" : 8855, + "dstRIR" : "TEST", + "fileId" : [], + "firstPacket" : 1386790367120, "http" : { - "bodymagic-term" : [ + "bodyMagic" : [ "application/x-gzip", "text/html" ], - "bodymagic-term-cnt" : 2, - "method-term" : [ + "bodyMagicCnt" : 2, + "clientVersion" : [ + "1.1" + ], + "clientVersionCnt" : 1, + "cookieKey" : [ + "NID", + "PREF" + ], + "cookieKeyCnt" : 2, + "cookieValue" : [ + "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + ], + "cookieValueCnt" : 2, + "host" : [ + "www.google.com" + ], + "hostCnt" : 1, + "key" : [ + "uule", + "hl", + "client", + "start", + "rls", + "q", + "num", + "gl" + ], + "keyCnt" : 8, + "md5" : [ + "2069181ae704855f29caf964ca52ec49", + "b0cecae354b9eab1f04f70e46a612cb1" + ], + "md5Cnt" : 2, + "method" : [ "GET" ], - "method-term-cnt" : 1, + "methodCnt" : 1, + "path" : [ + "/search" + ], + "pathCnt" : 1, + "request-referer" : [ + "", + "http://www.google.com/search?client=firefox&rls=en&q=sheepskin%20boots&start=0&num=10&hl=en&gl=us&uule=w+CAIQICINVW5pdGVkIFN0YXRlcw" + ], + "request-refererCnt" : 2, + "requestHeader" : [ + "accept", + "user-agent", + "referer", + "accept-encoding", + "host", + "cookie" + ], + "requestHeaderCnt" : 6, + "response-location" : [ + "http://ipv4.google.com/sorry/IndexRedirect?continue=http://www.google.com/search?client=firefox&rls=en&q=sheepskin%20boots&start=10&num=10&hl=en&gl=us&uule=xxxxxxxxxxxxxxxxxxxxxxxxxxxx" + ], + "responseHeader" : [ + "pragma", + "expires", + "content-type", + "alternate-protocol", + "transfer-encoding", + "x-frame-options", + "cache-control", + "x-xss-protection", + "content-length", + "date", + "content-encoding", + "p3p", + "server", + "set-cookie", + "location" + ], + "responseHeaderCnt" : 15, + "serverVersion" : [ + "1.1" + ], + "serverVersionCnt" : 1, + "sha256" : [ + "3de069d74ca3feb4d7a2fc381045e6d2f422d7b5de307cebbc61889d2724bc2d", + "6b4076081406f4529d876579ee60731ef737b1e99e42cf5e9058b19cbfa313b6" + ], + "sha256Cnt" : 2, "statuscode" : [ 302, 200 ], - "statuscode-cnt" : 2 + "statuscodeCnt" : 2, + "uri" : [ + "www.google.com/search?client=firefox&rls=en&q=Sheepskin%20Boots&start=0&num=10&hl=en&gl=us&uule=xxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "www.google.com/search?client=firefox&rls=en&q=Sheepskin%20Boots&start=10&num=10&hl=en&gl=us&uule=xxxxxxxxxxxxxxxxxxxxxxxxxxxx" + ], + "uriCnt" : 2, + "useragent" : [ + "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)" + ], + "useragentCnt" : 1, + "value" : [ + "firefox", + "xxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "en", + "sheepskin boots", + "10", + "0", + "us" + ], + "valueCnt" : 7 }, - "hval" : [ - "firefox", - "xxxxxxxxxxxxxxxxxxxxxxxxxxxx", - "en", - "sheepskin boots", - "10", - "0", - "us" - ], - "hvalcnt" : 7, - "ipDst" : "10.0.0.2", - "ipSrc" : "10.0.0.1", + "ipProtocol" : 6, "lastPacket" : 1386790404657, - "lp" : 1386790404, - "lpd" : 1386790404657, - "mac1-term" : [ - "00:0a:f3:31:94:00" - ], - "mac1-term-cnt" : 1, - "mac2-term" : [ - "00:00:5e:00:01:01", - "80:71:1f:83:9f:c6" - ], - "mac2-term-cnt" : 2, - "no" : "test", - "p1" : 54263, - "p2" : 8855, - "pa" : 52, - "pa1" : 31, - "pa2" : 21, - "portDst" : 8855, - "portSrc" : 54263, - "pr" : 6, - "prot-term" : [ - "http", - "socks", - "tcp" + "length" : 37537, + "node" : "test", + "packetLen" : [ + 82, + 82, + 76, + 76, + 76, + 80, + 76, + 76, + 76, + 80, + 363, + 76, + 1094, + 76, + 1430, + 424, + 76, + 1430, + 1430, + 76, + 1430, + 76, + 1430, + 76, + 1430, + 1430, + 154, + 76, + 76, + 1430, + 758, + 76, + 1430, + 758, + 76, + 1430, + 758, + 76, + 1430, + 76, + 758, + 1430, + 76, + 758, + 76, + 1238, + 718, + 76, + 1054, + 76, + 76, + 76 ], - "prot-term-cnt" : 3, - "ps" : [ + "packetPos" : [ 24, 106, 188, @@ -210,118 +252,81 @@ 28015, 28091 ], - "psl" : [ - 82, - 82, - 76, - 76, - 76, - 80, - 76, - 76, - 76, - 80, - 363, - 76, - 1094, - 76, - 1430, - 424, - 76, - 1430, - 1430, - 76, - 1430, - 76, - 1430, - 76, - 1430, - 1430, - 154, - 76, - 76, - 1430, - 758, - 76, - 1430, - 758, - 76, - 1430, - 758, - 76, - 1430, - 76, - 758, - 1430, - 76, - 758, - 76, - 1238, - 718, - 76, - 1054, - 76, - 76, - 76 + "protocol" : [ + "socksipset", + "http", + "socks", + "tcp" + ], + "protocolCnt" : 4, + "segmentCnt" : 1, + "socks" : { + "ASN" : "AS15169 Google LLC", + "GEO" : "US", + "RIR" : "ARIN", + "ip" : "74.125.131.103", + "port" : 80 + }, + "srcASN" : "AS0000 This is neat", + "srcBytes" : 25112, + "srcDataBytes" : 23392, + "srcGEO" : "RU", + "srcIp" : "10.0.0.1", + "srcMac" : [ + "00:0a:f3:31:94:00" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Cisco Systems, Inc" ], - "rir2" : "TEST", - "rirsocksip" : "ARIN", - "sl" : 37537, - "socksip" : "74.125.131.103", - "sockspo" : 80, - "ss" : 1, - "ta" : [ + "srcOuiCnt" : 1, + "srcPackets" : 31, + "srcPayload8" : "4e5a8d08874e0500", + "srcPort" : 54263, + "tags" : [ "dstip", "srcip" ], - "tacnt" : 2, - "tags-term" : [ - "srcip", - "dstip" - ], + "tagsCnt" : 2, "tcpflags" : { "ack" : 20, + "dstZero" : 0, "fin" : 1, "psh" : 29, "rst" : 0, + "srcZero" : 0, "syn" : 1, "syn-ack" : 1, "urg" : 0 }, "test" : { - "ip" : [ - 167772161 - ], - "ip-asn" : [ + "ASN" : [ "AS0000 This is neat" ], - "ip-geo" : [ - "RUS" + "GEO" : [ + "RU" ], - "ip-rir" : [ + "RIR" : [ "" ], + "ip" : [ + "10.0.0.1" + ], "number" : [ 33554442 ], - "string" : [ + "string.snow" : [ "16777226:54263,33554442:8855" ] }, "timestamp" : "SET", - "ua" : [ - "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)" - ], - "uacnt" : 1, - "us" : [ - "//www.google.com/search?client=firefox&rls=en&q=Sheepskin%20Boots&start=0&num=10&hl=en&gl=us&uule=xxxxxxxxxxxxxxxxxxxxxxxxxxxx", - "//www.google.com/search?client=firefox&rls=en&q=Sheepskin%20Boots&start=10&num=10&hl=en&gl=us&uule=xxxxxxxxxxxxxxxxxxxxxxxxxxxx" - ], - "uscnt" : 2 + "totBytes" : 27311, + "totDataBytes" : 24346, + "totPackets" : 52 }, "header" : { "index" : { - "_index" : "tests_sessions-131211", + "_index" : "tests_sessions2-131211", "_type" : "session" } } diff --git a/tests/pcap/socks5-smtp-503.test b/tests/pcap/socks5-smtp-503.test index 0cbc5632a7..0f5212a3b7 100644 --- a/tests/pcap/socks5-smtp-503.test +++ b/tests/pcap/socks5-smtp-503.test @@ -1,55 +1,54 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "a1" : "10.0.0.2", - "a2" : "10.0.0.1", - "as1" : "AS0001 Cool Beans!", - "as2" : "AS0000 This is neat", - "by" : 1492, - "by1" : 638, - "by2" : 854, - "db" : 327, - "db1" : 47, - "db2" : 280, - "fb1" : "050100050100030e", - "fb2" : "0500050000010a00", - "firstPacket" : 1385474626674, - "fp" : 1385474626, - "fpd" : 1385474626674, - "fs" : [], - "g1" : "CAN", - "g2" : "RUS", - "ipDst" : "10.0.0.1", - "ipSrc" : "10.0.0.2", - "lastPacket" : 1385474639455, - "lp" : 1385474639, - "lpd" : 1385474639455, - "mac1-term" : [ - "00:0a:f3:31:94:00" - ], - "mac1-term-cnt" : 1, - "mac2-term" : [ + "dstASN" : "AS0000 This is neat", + "dstBytes" : 854, + "dstDataBytes" : 280, + "dstGEO" : "RU", + "dstIp" : "10.0.0.1", + "dstMac" : [ "00:00:5e:00:01:01", "80:71:1f:82:cf:c6" ], - "mac2-term-cnt" : 2, - "no" : "test", - "p1" : 53709, - "p2" : 1080, - "pa" : 20, - "pa1" : 10, - "pa2" : 10, - "portDst" : 1080, - "portSrc" : 53709, - "pr" : 6, - "prot-term" : [ - "socks", - "smtp", - "tcp" + "dstMacCnt" : 2, + "dstOui" : [ + "ICANN, IANA Department", + "Juniper Networks" + ], + "dstOuiCnt" : 2, + "dstPackets" : 10, + "dstPayload8" : "0500050000010a00", + "dstPort" : 1080, + "fileId" : [], + "firstPacket" : 1385474626674, + "ipProtocol" : 6, + "lastPacket" : 1385474639455, + "length" : 12781, + "node" : "test", + "packetLen" : [ + 82, + 82, + 76, + 76, + 76, + 76, + 76, + 76, + 91, + 76, + 80, + 76, + 146, + 81, + 76, + 219, + 82, + 76, + 113, + 76 ], - "prot-term-cnt" : 3, - "ps" : [ + "packetPos" : [ 24, 106, 188, @@ -71,58 +70,59 @@ 1647, 1760 ], - "psl" : [ - 82, - 82, - 76, - 76, - 76, - 76, - 76, - 76, - 91, - 76, - 80, - 76, - 146, - 81, - 76, - 219, - 82, - 76, - 113, - 76 + "protocol" : [ + "socks", + "smtp", + "tcp" + ], + "protocolCnt" : 3, + "segmentCnt" : 1, + "socks" : { + "host" : "010.000.00.003", + "port" : 25 + }, + "srcASN" : "AS0001 Cool Beans!", + "srcBytes" : 638, + "srcDataBytes" : 47, + "srcGEO" : "CA", + "srcIp" : "10.0.0.2", + "srcMac" : [ + "00:0a:f3:31:94:00" ], - "rir1" : "TEST", - "sl" : 12781, - "socksho" : "010.000.00.003", - "sockspo" : 25, - "ss" : 1, - "ta" : [ + "srcMacCnt" : 1, + "srcOui" : [ + "Cisco Systems, Inc" + ], + "srcOuiCnt" : 1, + "srcPackets" : 10, + "srcPayload8" : "050100050100030e", + "srcPort" : 53709, + "srcRIR" : "TEST", + "tags" : [ "dstip", "smtp:authlogin", "srcip" ], - "tacnt" : 3, - "tags-term" : [ - "smtp:authlogin", - "srcip", - "dstip" - ], + "tagsCnt" : 3, "tcpflags" : { "ack" : 6, + "dstZero" : 0, "fin" : 0, "psh" : 10, "rst" : 1, + "srcZero" : 0, "syn" : 2, "syn-ack" : 1, "urg" : 0 }, - "timestamp" : "SET" + "timestamp" : "SET", + "totBytes" : 1492, + "totDataBytes" : 327, + "totPackets" : 20 }, "header" : { "index" : { - "_index" : "tests_sessions-131126", + "_index" : "tests_sessions2-131126", "_type" : "session" } } diff --git a/tests/pcap/ssh2-moloch-crash.test b/tests/pcap/ssh2-moloch-crash.test index d6f7d79d4c..21f9f812e7 100644 --- a/tests/pcap/ssh2-moloch-crash.test +++ b/tests/pcap/ssh2-moloch-crash.test @@ -1,79 +1,32 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "ipSrc" : "10.0.0.1", - "mac2-term-cnt" : 2, - "mac1-term-cnt" : 1, - "sshvercnt" : 2, - "pr" : 6, - "as2" : "AS0001 Cool Beans!", - "fp" : 1387565111, - "sl" : 106, - "db1" : 1109, - "lp" : 1387565112, - "by2" : 2037, - "mac2-term" : [ - "02:21:59:a1:83:b2", - "00:00:5e:00:01:01" + "dstASN" : "AS0001 Cool Beans!", + "dstBytes" : 2037, + "dstDataBytes" : 1407, + "dstGEO" : "CA", + "dstIp" : "10.0.0.2", + "dstMac" : [ + "00:00:5e:00:01:01", + "02:21:59:a1:83:b2" ], - "ipDst" : "10.0.0.2", - "tcpflags" : { - "fin" : 0, - "ack" : 7, - "psh" : 13, - "rst" : 0, - "syn" : 1, - "syn-ack" : 1, - "urg" : 0 - }, - "sshver" : [ - "ssh-1.99-openssh_3.9p1", - "ssh-2.0-openssh_5.3" + "dstMacCnt" : 2, + "dstOui" : [ + "ICANN, IANA Department" ], - "tacnt" : 2, - "test" : { - "ip-rir" : [ - "" - ], - "string" : [ - "16777226:61672,33554442:22" - ], - "ip" : [ - 167772161 - ], - "number" : [ - 33554442 - ], - "ip-geo" : [ - "RUS" - ], - "ip-asn" : [ - "AS0000 This is neat" - ] - }, - "rir2" : "TEST", + "dstOuiCnt" : 1, + "dstPackets" : 11, + "dstPayload8" : "5353482d312e3939", + "dstPort" : 22, + "dstRIR" : "TEST", + "fileId" : [], "firstPacket" : 1387565111946, - "prot-term" : [ - "ssh", - "tcp" - ], + "ipProtocol" : 6, "lastPacket" : 1387565112051, - "lpd" : 1387565112051, - "as1" : "AS0000 This is neat", - "p1" : 61672, - "db" : 2516, - "portSrc" : 61672, - "fb2" : "5353482d312e3939", - "a2" : "10.0.0.2", - "a1" : "10.0.0.1", - "pa2" : 11, - "timestamp" : "SET", - "pa1" : 11, - "p2" : 22, - "by" : 3752, - "by1" : 1715, - "psl" : [ + "length" : 106, + "node" : "test", + "packetLen" : [ 82, 82, 70, @@ -97,12 +50,7 @@ 150, 70 ], - "tags-term" : [ - "srcip", - "dstip" - ], - "g2" : "CAN", - "ps" : [ + "packetPos" : [ 24, 106, 188, @@ -126,28 +74,80 @@ 3908, 4058 ], - "db2" : 1407, - "fb1" : "5353482d322e302d", - "pa" : 22, - "g1" : "RUS", - "no" : "test", - "ta" : [ - "dstip", - "srcip" + "protocol" : [ + "ssh", + "tcp" ], - "fs" : [], - "portDst" : 22, - "prot-term-cnt" : 2, - "ss" : 1, - "mac1-term" : [ + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcASN" : "AS0000 This is neat", + "srcBytes" : 1715, + "srcDataBytes" : 1109, + "srcGEO" : "RU", + "srcIp" : "10.0.0.1", + "srcMac" : [ "00:0c:29:62:b6:75" ], - "fpd" : 1387565111946 + "srcMacCnt" : 1, + "srcOui" : [ + "VMware, Inc." + ], + "srcOuiCnt" : 1, + "srcPackets" : 11, + "srcPayload8" : "5353482d322e302d", + "srcPort" : 61672, + "ssh" : { + "version" : [ + "ssh-1.99-openssh_3.9p1", + "ssh-2.0-openssh_5.3" + ], + "versionCnt" : 2 + }, + "tags" : [ + "dstip", + "srcip" + ], + "tagsCnt" : 2, + "tcpflags" : { + "ack" : 7, + "dstZero" : 0, + "fin" : 0, + "psh" : 13, + "rst" : 0, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, + "test" : { + "ASN" : [ + "AS0000 This is neat" + ], + "GEO" : [ + "RU" + ], + "RIR" : [ + "" + ], + "ip" : [ + "10.0.0.1" + ], + "number" : [ + 33554442 + ], + "string.snow" : [ + "16777226:61672,33554442:22" + ] + }, + "timestamp" : "SET", + "totBytes" : 3752, + "totDataBytes" : 2516, + "totPackets" : 22 }, "header" : { "index" : { - "_type" : "session", - "_index" : "tests_sessions-131220" + "_index" : "tests_sessions2-131220", + "_type" : "session" } } } diff --git a/tests/pcap/ssh2.test b/tests/pcap/ssh2.test index 7ce1e5d15b..94dc5c8704 100644 --- a/tests/pcap/ssh2.test +++ b/tests/pcap/ssh2.test @@ -1,29 +1,32 @@ { - "sessions" : [ + "sessions2" : [ { "body" : { - "a2" : "10.0.0.2", - "fs" : [], - "ipDst" : "10.0.0.2", - "sshkey" : [ - "AAAAB3NzaC1yc2EAAAABeHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHg=" + "dstASN" : "AS0001 Cool Beans!", + "dstBytes" : 2037, + "dstDataBytes" : 1407, + "dstGEO" : "CA", + "dstIp" : "10.0.0.2", + "dstMac" : [ + "00:00:5e:00:01:01", + "02:21:59:a1:83:b2" ], - "by2" : 2037, - "by1" : 1715, - "pr" : 6, - "rir2" : "TEST", - "tacnt" : 2, - "p1" : 61672, - "pa1" : 11, - "p2" : 22, - "portSrc" : 61672, - "ta" : [ - "dstip", - "srcip" + "dstMacCnt" : 2, + "dstOui" : [ + "ICANN, IANA Department" ], - "pa" : 22, - "fb1" : "5353482d322e302d", - "psl" : [ + "dstOuiCnt" : 1, + "dstPackets" : 11, + "dstPayload8" : "5353482d312e3939", + "dstPort" : 22, + "dstRIR" : "TEST", + "fileId" : [], + "firstPacket" : 1387565111946, + "ipProtocol" : 6, + "lastPacket" : 1387565112051, + "length" : 106, + "node" : "test", + "packetLen" : [ 82, 82, 70, @@ -47,8 +50,7 @@ 150, 70 ], - "lastPacket" : 1387565112051, - "ps" : [ + "packetPos" : [ 24, 106, 188, @@ -72,86 +74,84 @@ 3908, 4058 ], - "as2" : "AS0001 Cool Beans!", - "g1" : "RUS", - "sshvercnt" : 2, - "firstPacket" : 1387565111946, - "prot-term-cnt" : 2, + "protocol" : [ + "ssh", + "tcp" + ], + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcASN" : "AS0000 This is neat", + "srcBytes" : 1715, + "srcDataBytes" : 1109, + "srcGEO" : "RU", + "srcIp" : "10.0.0.1", + "srcMac" : [ + "00:0c:29:62:b6:75" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "VMware, Inc." + ], + "srcOuiCnt" : 1, + "srcPackets" : 11, + "srcPayload8" : "5353482d322e302d", + "srcPort" : 61672, + "ssh" : { + "key" : [ + "AAAAB3NzaC1yc2EAAAABeHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHg=" + ], + "keyCnt" : 1, + "version" : [ + "ssh-1.99-openssh_3.9p1", + "ssh-2.0-openssh_5.3" + ], + "versionCnt" : 2 + }, + "tags" : [ + "dstip", + "srcip" + ], + "tagsCnt" : 2, + "tcpflags" : { + "ack" : 7, + "dstZero" : 0, + "fin" : 0, + "psh" : 13, + "rst" : 0, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, "test" : { - "ip-rir" : [ - "" + "ASN" : [ + "AS0000 This is neat" ], - "number" : [ - 33554442 + "GEO" : [ + "RU" + ], + "RIR" : [ + "" ], "ip" : [ - 167772161 + "10.0.0.1" ], - "ip-geo" : [ - "RUS" + "number" : [ + 33554442 ], - "string" : [ + "string.snow" : [ "16777226:61672,33554442:22" - ], - "ip-asn" : [ - "AS0000 This is neat" ] }, - "db" : 2516, - "as1" : "AS0000 This is neat", - "mac1-term" : [ - "00:0c:29:62:b6:75" - ], - "sl" : 106, - "pa2" : 11, - "sshver" : [ - "ssh-1.99-openssh_3.9p1", - "ssh-2.0-openssh_5.3" - ], - "fpd" : 1387565111946, - "db1" : 1109, - "tcpflags" : { - "syn" : 1, - "psh" : 13, - "syn-ack" : 1, - "rst" : 0, - "ack" : 7, - "urg" : 0, - "fin" : 0 - }, "timestamp" : "SET", - "fb2" : "5353482d312e3939", - "sshkeycnt" : 1, - "lp" : 1387565112, - "g2" : "CAN", - "ss" : 1, - "ipSrc" : "10.0.0.1", - "by" : 3752, - "db2" : 1407, - "mac2-term" : [ - "02:21:59:a1:83:b2", - "00:00:5e:00:01:01" - ], - "prot-term" : [ - "ssh", - "tcp" - ], - "fp" : 1387565111, - "lpd" : 1387565112051, - "no" : "test", - "a1" : "10.0.0.1", - "portDst" : 22, - "mac2-term-cnt" : 2, - "tags-term" : [ - "srcip", - "dstip" - ], - "mac1-term-cnt" : 1 + "totBytes" : 3752, + "totDataBytes" : 2516, + "totPackets" : 22 }, "header" : { "index" : { - "_type" : "session", - "_index" : "tests_sessions-131220" + "_index" : "tests_sessions2-131220", + "_type" : "session" } } } diff --git a/tests/pcap/ssl-selfsign.pcap b/tests/pcap/ssl-selfsign.pcap new file mode 100644 index 0000000000000000000000000000000000000000..64652d0f9a193509d8b3c18b025ce9131494b2ce GIT binary patch literal 8191 zcma)>2Q*#V*2nib2hr=%J0ZHGM(-`6Mu|=$(Oa}2+Rf5@s85&g7lSV@6bg}hQ6$Ge5C|#=gpL^vB?-HH zhJplKGSPAVlc_9vl}TW@)^=K^8I0ui78CxSdCC2YNr+?$A-{1MKXvl^@?T~C)Y*&V zhWO+2j4Gr7z+d_LGyufIbKSbRdAibN>36@0|?s!{5kVw8IMU2DY){sG6AXeF3E)TtX}XfC~^tnil{_ z03rYr-~)I8ZXgkOjTC^2JX&Ia5WoPS00DpwpaBrz0`%ue?Ec2>XT(*N|91P76X|y7 zcUhCe#{`a%HZS9cTktpc)PLZR@eJL%|DSF(kuR(MaNA|T*}Zo7jD)2(MyEsJ9DeA& zlC#8_LnZD)ut_(DZbUwQC!p2&&5>e2n@QXQ54;=n4+0-%8jLhRHqw69zg&y zhhSDL0L%n{82~UXEC5Ui3jhVZg@8d|Fa}WKE8iDW_IU<|gNDHtq7-t70|KF<0x&2K zAq*c4qJshdAkgNK9RaG;cPaMsEK>T_7L;-(m@xDug*VTbhs3eE%2b&nCL$oq3(OQw*FT^MP70)ul@t%)i(pc#sJs=dG&RXSHJPE ztKa9A^A-n1JLhl5@IPC=QVjwy*ss-ax%}Tj|5PB2cn!P-U(tMdCQ`~<@Hn9%tR<6w zWE486Em9`E?vod>&`x685}0aT>KR1U2P4#zYg}pv{@c4+xv=3anL8DGw zE9Z%=KMDjM4ugmMQX-#UdT3C|LhNAWY-^;+kDVQq@QYw~f0T#@OwAwqyA5gV8UOo* zLRQVc4(R`@8rlVT#&gSqEcW>Hclo=rEcj+fmVkWE$#r*mX@wP(i)0u0 z@DrseAHN8?eF&q4DIPiR8259BiFaF2C7W2%)G0D(zcqszVg$p8$kLlelS+;6%lYl} z5fR|g%V*w{FS|SkacnwU3O6pA3}~;NX}#-O5?6FUlkWMPXMy*bq2OGc=}wpKi%59$ zUirp;yM!HUv1meTEZL=y5Ig_~0Q~Kxt%mD0_6EF~G-X>mA_(1D_RRCwkH0G+R5{Y| z1oGt?`4p+4WzrXCmPLv~?FU#EAMcI|U~c7`g%9mVIDFh~sM1d=-K46H3v*O_P;;!R zr`F+Tcoq}xXiKrT z3X*Ee&}G@F)HlJen;s{R6WGyNv^L%k7mq~DhWRaL#ej!w0{bc~&6*+@{z?PTQL4|BXODhdPX2Iu;+my#NhlK$Q@yC$zn`q46< ze?zv+7^Jnv-&qr&#sp%Jwl3qRwPy%}f8dZIIj8@!7Im2-QsjBjX$XvDk?Z@4`WzcU z6K@k)q4RMC-oH86sdOtujO*;p{N*Ot@JH2S+=D_;QY{?6sQCI@w=)eKn&#Vb9PNEu z)KK%%x6RpHBC}KMd?^%!WhrUamrga13jgjD`{`AMKMygcmEnat(rM{0PA?TkU2^nk zFo70A;LrHsl(1pY`X4xCJf-^VSDZ%u>C{+@hhCib!XT$}Xyc&AFF&xs2UWp^$z)OD z@uz1ycNUPF2+cy$?l3Qgyk<7bxh6VP`|0DfDq*8;EVo^gg2siz5@o{NKJ-Jx@~K7h zD~nrA;A>ObDss?|dDDWE@#0xR$8APOheGu0PXd_m)z+taa_=*f$xB~X4LvkupCfjE zRI!ccfsTEvHm(1#v$ly;NrNMCCG*7z6ZeCgTImgn3c~oX@1g#W`*~X%Wy9eZAPuH$ z1_+ixM<90DOVcs%o?e%0w}{T|blBWVu^xG@J{W~^55X#&X>?O7qiA?@_TlYkkF1rJ z(3@E860o&b88w_6@f%l@eS=2hwoWsIhmJ=(OO_4McW6_aI-dLbX9qneiL;lW&8E2< zyK2gtGxF@?sP=>QHKrG~{0N~GG2+P@H-{%yPowsplW9>IrTUWF_T{eTjcj;e*k#A* z=yA@6eA9bOF6t{jGKp2^e0y`vKKP!Uf zX!39~aem!Oo2o!>Tui>jx5Ond3ME17pVY3G```Uu{@bOiemiWZWS)XEugL0QN-(t0OAaE(hJY66)L#L0K zG7R;}ADCz3OKF<5a22#`*Tjmd^)Hgj4B`pfkR+~4d&_K+60nrPO4Yu0s79FL!u_W| zplRV0wLjMA5pJ5vt?XU1O47sZ(^jk2Umv#=Za!|sqa{6WQl)a!5cYC%r#7z8^{+jZ zu=5yUh;K~ZfbQ>ql@FF7*E^7=TtUh=)xl_^nd=mcYXlLw&o@m|-Y+~{XA{HcvlFKAXP+rs z-K0ee)EvsG%Pb8nUbn*eu1FZn?B6l;a6PZmw_85@BHOF!7)HX>9#C_uqQY0ZHPAf{ zw?>@o%sl3kf;C#`wkyF5Hv*TY9$~zn<+yjP$jz(dV<{CC9F|a5aH6)fzl?WH7{M4g76U{*q!Lp`yf2N65AHq zT+H~cDJ`w-y|tV)`&U*^;;_ySM;k<(L?GpYmCk_@xA50sNvE+LrZX~QX3yfvOuj0- z;@aU?bcH+wE0{Wy(=6vG4a7$!3}hE>Y+3nYF48R9lTu8&HyCq(&e#mkC9|uAznoZ- zJaao9rS-k00~EC%e70F(U6<*QK6(*w(KVi??uXkw)!s8Q(1d-O{#f9QUQ5d)k!GAha;ZJ}Yp9MVVY zT>=Wudld9Km65e#@RWA9B%bkr)fycorB^>8a6xiHyv=4!u(Q|Nh;8*SiJ zGEx~U5K7`c*t#!Yu9Jp$vDQqcH3(=obd4bfMVMJl{lw#{?9z&5NrHZ0tS}$0}zk2N1_JPlq38w6M+ub})5? zzw-Q$SsasbQR=qQz?29f;P_q9Da5|VaZr}Uod35CwsN); z%2O4um;4ZWJ^U}lV?d*xl(m|2hnr(&92welIcJH#LlaczBB#S=>zS~mP%SsW-i9zo z?|mOfOKG!kK(JSPm#_=w*e?p1+uc_PhvW1N?#^WDb9mH9`^tQ%{D8X6z@c%i?~K|D zwkp5b<`-lnXd(P=01IbEIT5;Vmf{8Nxwx^prvfTxVzz(D44VMEB*F64aPHe}@4RJb z8^5T}HG(Y<)M{ax@k1R0ocxCO9Nn==nuCu#?L1v=h!8jL8%$MeSI;Zur?eY=X>J#E z6 z3De(ItPApm)5(m!)*WV7;7pvDNp#k#m;DR!I^%H;bK`750Q&vba8zLiA&7 zH*_n-%v33~09w;=y>uXqXvqCvBOjG7(=SHWyr z0n5| zj)UYAXbxn6Z|PVI`wIy@f1JpBGo=EK!&66A9KlS6SwG36Gj_eL+g!0~_3K?-&b>31 zdGF&Ji)QMhI1NMt5PcR30X-U_3?F3jYK!j8R;Hm+>3YOvq!36qckHRCrLj6HAjMBzSw4G(02{Oxzt;w%@8g^yxwu`aI z&FW&@;`qo(rbi6Gy?ldo$ugli4pmsItV#6*XIM)Z z#*x-uCsIF6(>CLAq4J$unCqNFK~wBZgtwY+?Ag^>efWxznc&ui zUq$%6jyYBXp+k<-P^) zGbUvcpyFy+Ax~0zWXM0VIuWN?Y)pDi>2Kz@q^uOR3nk<$Sf(fbA1(-*~o))b$Emrgb$1;R4(b^^`8juOpRQ zLN*e1ZVD~m%XMgCx$e6veqyv>kdE)WENAR7pz)$=&U439)w$bIyxzw9vrIaa4CH>z z*V|s#*NHkLsnMWSoa2#4pGAZLX;e*w4WnsCg%4xg+5||*}Y>> z(O9U?Akl5Cc^W!uDZaC1q`vD1rE&7{|tQmi8@W zaub!)VauqD!s1T#NNNvLKD6A%Kf6;f_M4iCcsJhq7iWCx6Ase>t__V~Vgp)+Pv6l^ zOli=BC#-q~XtJ@m#m{J(DLR`K3~ECsz?ywWFVUVBqslH@Yw7TZeUljRNqEUitd;cQ zl}Vv)-}Mu*iZyGT?lYK2^W(wZ8-{C%%oL`B4(qnbx+8dKY}WI_`i+Wr9>i_rLmVI8 zjax&PYwxFD*L?GZ5B>A=XvJKblvO@_O_QyY;`^V>;23vT=3NA+BBRFm*(BYwQ+xKZ z+aO{tBgHcWKxG?MJg@qUbZJAIX+c(4?M?bg3 z4Gm@V(>fozwx~hEILE54Ez452)eGtBRhr%fv&RzHC?N`VnVvV@|TL-ig2qqM3lH#ZorOp4~5z@JHdSaRM^?hq)6&PbxBlFrLMK4n{}7WfYm6<{GoQZFh@jFzn(339Y`r+ zbA*lbLj`jLcU%3NLODTA@13wNvOzHf;fGrH#arP=voZo*_&Aycx0mXD4QKkDZlq1p zXZ!iF1$8USVh0s^^Dg*mJwDMVGwR?a&A@KBef>%9y64xf_RJ{9Csku^UnBgR)^v_# zm$FOMSYd@u1mGv3f-8edb>Cd`!}%Q2VnWFeMcz{0)>Q%TO`d*n3{_zqm#nOIw-EmCXTb~QH~h~!X7?PAW_u#Hld8ZRXBhtT3R!}s{?9`VVZlw*r+ z(HDA@ysYL(5gC~Ggx^evuII+~$TN*j>;J>?ooy(p@X^x@s2;-NS|ok{u#h&mVLaLv zcU$V3yIC8pW!-o)0T=Uj75w-`f#=C4D8{8xB&HwwLQ+G4;ENz`LY$zkW4_*Neb=A?({TD@(QIK^ z1Dd1|KA`|<{!b?E3uN^pfo>er)QMb`eUojb%NRmd6n{Wi_butOXbVOKud{Xb-H0!j9G_(kW0jF38upZ#{1vVZz;m~;YG?P)t1E3L8tZ2_FTAR1RAyIO6YP_! zk|99RPEo8L=`Z11AUUFc-45%2w?6cg>&5>TLe zEkVR~{kvZfzl*?LV!tON~ktxz``3 zByXjC`~%gWW4gK}Sg?VEi+e;}!8KhebrKIhLes8>;o$i3lY$oUo?T1pk5)q(ir^~;HtCvu|o_sLkQ*VT#Ej~U<#_0uv-D&%C0?RS&0@ZWG^Hm-7h zOxP@`{+zH8{C>ig@*57d|5eT}dZ_=K9#lF^z!*7?yo{eSNqrs7f8dZIeYIbm)Dd6K zBz;;#{G|K5k72bKXQjQd5k^)dhekEbo>7gOjBf)Tp+aba9{-kBM@^jI5W5~nkX4F zp@k$!4x|Qb?63?;CPo%eO0oh5BRhvRkmT_*29kUNW-K6&!-5nNuqY%bl;^x;#g|^q z!RfUNqz#c?=kJB4S2+~pV4PtY%FIaV6_lnRfeTBuppd-wiWOgab;Fik@85=|S9nN* a%!UOdG}XhoL!AN1`K-87DkL4_Nv{BdC3Zjn literal 0 HcmV?d00001 diff --git a/tests/pcap/wireshark-dhcp.test b/tests/pcap/wireshark-dhcp.test new file mode 100644 index 0000000000..d2d87333ce --- /dev/null +++ b/tests/pcap/wireshark-dhcp.test @@ -0,0 +1,165 @@ +{ + "sessions2" : [ + { + "body" : { + "dhcp" : { + "id" : [ + "3d1d", + "3d1e" + ], + "idCnt" : 2, + "mac" : [ + "00:0b:82:01:fc:42" + ], + "macCnt" : 1, + "oui" : [ + "Grandstream Networks, Inc." + ], + "ouiCnt" : 1, + "type" : [ + "REQUEST", + "DISCOVER" + ], + "typeCnt" : 2 + }, + "dstBytes" : 0, + "dstDataBytes" : 0, + "dstIp" : "255.255.255.255", + "dstMac" : [ + "ff:ff:ff:ff:ff:ff" + ], + "dstMacCnt" : 1, + "dstPackets" : 0, + "dstPort" : 67, + "fileId" : [], + "firstPacket" : 1102274184317, + "ipProtocol" : 17, + "lastPacket" : 1102274184387, + "length" : 70, + "node" : "test", + "packetLen" : [ + 330, + 330 + ], + "packetPos" : [ + 24, + 712 + ], + "protocol" : [ + "udp", + "dhcp" + ], + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 628, + "srcDataBytes" : 612, + "srcIp" : "0.0.0.0", + "srcMac" : [ + "00:0b:82:01:fc:42" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Grandstream Networks, Inc." + ], + "srcOuiCnt" : 1, + "srcPackets" : 2, + "srcPayload8" : "0101060000003d1d", + "srcPort" : 68, + "timestamp" : "SET", + "totBytes" : 628, + "totDataBytes" : 612, + "totPackets" : 2 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-041205", + "_type" : "session" + } + } + }, + { + "body" : { + "dhcp" : { + "id" : [ + "3d1d", + "3d1e" + ], + "idCnt" : 2, + "mac" : [ + "00:0b:82:01:fc:42" + ], + "macCnt" : 1, + "oui" : [ + "Grandstream Networks, Inc." + ], + "ouiCnt" : 1, + "type" : [ + "ACK", + "OFFER" + ], + "typeCnt" : 2 + }, + "dstBytes" : 0, + "dstDataBytes" : 0, + "dstIp" : "192.168.0.10", + "dstMac" : [ + "00:0b:82:01:fc:42" + ], + "dstMacCnt" : 1, + "dstOui" : [ + "Grandstream Networks, Inc." + ], + "dstOuiCnt" : 1, + "dstPackets" : 0, + "dstPort" : 68, + "dstRIR" : "ARIN", + "fileId" : [], + "firstPacket" : 1102274184317, + "ipProtocol" : 17, + "lastPacket" : 1102274184387, + "length" : 70, + "node" : "test", + "packetLen" : [ + 358, + 358 + ], + "packetPos" : [ + 354, + 1042 + ], + "protocol" : [ + "udp", + "dhcp" + ], + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 684, + "srcDataBytes" : 668, + "srcIp" : "192.168.0.1", + "srcMac" : [ + "00:08:74:ad:f1:9b" + ], + "srcMacCnt" : 1, + "srcOui" : [ + "Dell Inc." + ], + "srcOuiCnt" : 1, + "srcPackets" : 2, + "srcPayload8" : "0201060000003d1d", + "srcPort" : 67, + "srcRIR" : "ARIN", + "timestamp" : "SET", + "totBytes" : 684, + "totDataBytes" : 668, + "totPackets" : 2 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-041205", + "_type" : "session" + } + } + } + ] +} + diff --git a/tests/plugins/test.c b/tests/plugins/test.c index 1763b06f7b..2889ea89fb 100644 --- a/tests/plugins/test.c +++ b/tests/plugins/test.c @@ -13,7 +13,7 @@ void test_plugin_pre_save(MolochSession_t *session, int UNUSED(final)) { if (MOLOCH_V6_TO_V4(session->addr1) == 0x0100000a) { char tmp[1000]; - moloch_field_int_add(test_ip, session, ((uint32_t *)session->addr1.s6_addr)[3]); + moloch_field_ip4_add(test_ip, session, ((uint32_t *)session->addr1.s6_addr)[3]); moloch_field_int_add(test_number, session, ((uint32_t *)session->addr2.s6_addr)[3]); sprintf(tmp, "%d:%d,%d:%d", ((uint32_t *)session->addr1.s6_addr)[3], session->port1, ((uint32_t *)session->addr2.s6_addr)[3], session->port2); moloch_field_string_add(test_string, session, tmp, -1, TRUE); @@ -32,7 +32,7 @@ void moloch_plugin_init() test_ip = moloch_field_define("test", "ip", "test.ip", "Test Ip", "test.ip", "Test IP", - MOLOCH_FIELD_TYPE_IP_HASH, 0, + MOLOCH_FIELD_TYPE_IP_GHASH, 0, NULL); test_string = moloch_field_define("test", "textfield", diff --git a/tests/postgresql.t b/tests/postgresql.t index 3d6ba95cf0..d05fa93b7f 100644 --- a/tests/postgresql.t +++ b/tests/postgresql.t @@ -4,7 +4,7 @@ use URI::Escape; use MolochTest; use strict; -my $pwd = getcwd() . "/pcap"; +my $pwd = "*/pcap"; my $files = "(file=$pwd/postgres-badpass.pcap||file=$pwd/postgres-good.pcap||file=$pwd/postgres-no-sslrequest.pcap)"; countTest(3, "date=-1&expression=" . uri_escape("$files&&protocols==postgresql")); diff --git a/tests/quic.t b/tests/quic.t new file mode 100644 index 0000000000..5385d4b243 --- /dev/null +++ b/tests/quic.t @@ -0,0 +1,21 @@ +use Test::More tests => 12; +use Cwd; +use URI::Escape; +use MolochTest; +use strict; + +my $pwd = "*/pcap"; +my $files = "(file=$pwd/quic24-wireshark.pcap||file=$pwd/fbzero-android.pcap)"; + +countTest(2, "date=-1&expression=" . uri_escape("$files&&protocols==quic")); + +# +countTest(1, "date=-1&expression=" . uri_escape("$files&&quic.host==graph.facebook.com")); +countTest(1, "date=-1&expression=" . uri_escape("$files&&quic.host==Graph.facebook.COM")); + +# +countTest(1, "date=-1&expression=" . uri_escape("$files&&quic.version==\"Q024\"")); +countTest(0, "date=-1&expression=" . uri_escape("$files&&quic.version==\"q024\"")); + +# +countTest(1, "date=-1&expression=" . uri_escape("$files&&quic.user-agent==\"canary Chrome/44.0.2375.0\"")); diff --git a/tests/rules.yaml b/tests/rules.yaml index 619620814b..2b4685a765 100644 --- a/tests/rules.yaml +++ b/tests/rules.yaml @@ -44,3 +44,23 @@ rules: # - 00:11:f5:13:d7:a3 ops: "protocols": "tlsrulestest" + + - name: "socks test" + when: "fieldSet" + fields: + ip.socks: + - 74.125.131.103 + ops: + "protocols": "socksipset" + + - name: "ip4 test" + when: "fieldSet" + fields: + ip.src: + - 10.44.100.0/24 + - 2001:06f8::/32 + port.dst: + - 443 + - 5353 + ops: + "protocols": "iprulztest" diff --git a/tests/sha256.wise b/tests/sha256.wise new file mode 100644 index 0000000000..2e4e97f88a --- /dev/null +++ b/tests/sha256.wise @@ -0,0 +1 @@ +61479904e443b354d4427a51b990c696f731e341e4c63328e07c1a92658ba591;tags=wisebysha2561;mysql.ver=wisebysha2561mysqlversion;test.ip=1::2 diff --git a/tests/smb.t b/tests/smb.t index 90178d6dcc..2615183051 100644 --- a/tests/smb.t +++ b/tests/smb.t @@ -4,7 +4,7 @@ use URI::Escape; use MolochTest; use strict; -my $pwd = getcwd() . "/pcap"; +my $pwd = "*/pcap"; my $files = "(file=$pwd/smb-port80.pcap||file=$pwd/smb-smbclient.pcap)"; countTest(2, "date=-1&expression=" . uri_escape("$files&&protocols==smb")); diff --git a/tests/socks.t b/tests/socks.t index fd758ef022..34491fca4e 100644 --- a/tests/socks.t +++ b/tests/socks.t @@ -4,7 +4,7 @@ use URI::Escape; use MolochTest; use strict; -my $pwd = getcwd() . "/pcap"; +my $pwd = "*/pcap"; my $files = "(file=$pwd/socks-http-example.pcap||file=$pwd/socks-http-pass.pcap||file=$pwd/socks-https-example.pcap||file=$pwd/socks5-http-302.pcap||file=$pwd/socks5-rdp.pcap||file=$pwd/socks5-reverse.pcap||file=$pwd/socks5-smtp-503.pcap)"; countTest(12, "date=-1&expression=" . uri_escape("$files&&protocols==socks")); @@ -18,19 +18,19 @@ countTest(12, "date=-1&expression=" . uri_escape("$files&&protocols==socks")); countTest(3, "date=-1&expression=" . uri_escape("$files&&ip.socks==93.184.216.119:80")); # country.socks - countTest(6, "date=-1&expression=" . uri_escape("$files&&country.socks==USA")); - countTest(6, "date=-1&expression=" . uri_escape("$files&&country.socks==usa")); + countTest(6, "date=-1&expression=" . uri_escape("$files&&country.socks==US")); + countTest(6, "date=-1&expression=" . uri_escape("$files&&country.socks==us")); countTest(0, "date=-1&expression=" . uri_escape("$files&&country.socks==EU")); countTest(0, "date=-1&expression=" . uri_escape("$files&&country.socks==eu")); -# rir.socks +# socks.rir countTest(5, "date=-1&expression=" . uri_escape("$files&&rir.socks==RIPE")); countTest(5, "date=-1&expression=" . uri_escape("$files&&rir.socks==ripe")); -# asn.socks +# socks.asn countTest(1, "date=-1&expression=" . uri_escape("$files&&asn.socks==\"AS0000 This is neat\"")); - countTest(1, "date=-1&expression=" . uri_escape("$files&&asn.socks==\"AS0000\"")); - countTest(1, "date=-1&expression=" . uri_escape("$files&&asn.socks==\"aS0000\"")); + countTest(1, "date=-1&expression=" . uri_escape("$files&&asn.socks==\"AS0000*\"")); + countTest(0, "date=-1&expression=" . uri_escape("$files&&asn.socks==\"aS0000*\"")); # socks.port countTest(6, "date=-1&expression=" . uri_escape("$files&&socks.port==80")); diff --git a/tests/ssh.t b/tests/ssh.t index 6852d59249..938f72f95e 100644 --- a/tests/ssh.t +++ b/tests/ssh.t @@ -4,7 +4,7 @@ use URI::Escape; use MolochTest; use strict; -my $pwd = getcwd() . "/pcap"; +my $pwd = "*/pcap"; my $files = "file=$pwd/ssh2.pcap"; countTest(1, "date=-1&expression=" . uri_escape("$files&&protocols==ssh")); diff --git a/tests/tagger.t b/tests/tagger.t index 0089ccfdb1..b54e4683d8 100644 --- a/tests/tagger.t +++ b/tests/tagger.t @@ -4,7 +4,7 @@ use URI::Escape; use MolochTest; use strict; -my $pwd = getcwd() . "/pcap"; +my $pwd = "*/pcap"; # tagger tests 1 countTest(4, "date=-1&expression=" . uri_escape("(file=$pwd/socks-https-example.pcap||file=$pwd/dns-mx.pcap)&&tags=hosttaggertest1")); diff --git a/tests/tests.pl b/tests/tests.pl index 12a68a867f..e30a7ec6c7 100755 --- a/tests/tests.pl +++ b/tests/tests.pl @@ -11,6 +11,7 @@ use URI::Escape; use TAP::Harness; use MolochTest; +use Socket6 qw(AF_INET6 inet_pton); $main::userAgent = LWP::UserAgent->new(timeout => 20); @@ -24,20 +25,16 @@ sub doGeo { system("wget --no-check-certificate https://www.iana.org/assignments/ipv4-address-space/ipv4-address-space.csv"); } - if (! -f "GeoIPASNum.dat") { - system("wget http://www.maxmind.com/download/geoip/database/asnum/GeoIPASNum.dat.gz; gunzip GeoIPASNum.dat.gz"); + if (! -f "oui.txt") { + system("wget -O oui.txt https://raw.githubusercontent.com/wireshark/wireshark/master/manuf"); } - if (! -f "GeoIPASNumv6.dat") { - system("wget http://download.maxmind.com/download/geoip/database/asnum/GeoIPASNumv6.dat.gz; gunzip GeoIPASNumv6.dat.gz"); + if (! -f "GeoLite2-Country.mmdb") { + system("wget -O GeoLite2-Country.mmdb.gz 'https://updates.maxmind.com/app/update_secure?edition_id=GeoLite2-Country'; gunzip GeoLite2-Country.mmdb.gz"); } - if (! -f "GeoIP.dat") { - system("wget http://www.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz; gunzip GeoIP.dat.gz"); - } - - if (! -f "GeoIPv6.dat") { - system("wget http://geolite.maxmind.com/download/geoip/database/GeoIPv6.dat.gz; gunzip GeoIPv6.dat.gz"); + if (! -f "GeoLite2-ASN.mmdb") { + system("wget -O GeoLite2-ASN.mmdb.gz 'https://updates.maxmind.com/app/update_secure?edition_id=GeoLite2-ASN'; gunzip GeoLite2-ASN.mmdb.gz"); } if (! -f "plugins/test.so" || (stat('../capture/moloch.h'))[9] > (stat('plugins/test.so'))[9]) { @@ -48,9 +45,9 @@ sub doGeo { sub sortJson { my ($json) = @_; - foreach my $session (@{$json->{sessions}}) { + foreach my $session (@{$json->{sessions2}}) { my $body = $session->{body}; - foreach my $i ("dnsip", "tags-term", "ta") { + foreach my $i ("tags", "srcMac", "dstMac", "srcOui", "dstOui") { if (exists $body->{$i}) { my @tmp = sort (@{$body->{$i}}); $body->{$i} = \@tmp; @@ -102,31 +99,34 @@ sub doFix { ################################################################################ sub fix { my ($json) = @_; - foreach my $session (@{$json->{sessions}}) { + my $json = sortJson($json); + foreach my $session (@{$json->{sessions2}}) { my $body = $session->{body}; delete $session->{header}->{index}->{_id}; - if (exists $body->{ro}) { - $body->{ro} = "SET"; + if (exists $body->{rootId}) { + $body->{rootId} = "SET"; } if (exists $body->{timestamp}) { $body->{timestamp} = "SET"; } - foreach my $field ("a1", "a2", "dnsip", "socksip", "eip") { - $body->{$field} = fixIp($body->{$field}) if (exists $body->{$field}); + + if ($body->{srcIp} =~ /:/) { + $body->{srcIp} = join ":", (unpack("H*", inet_pton(AF_INET6, $body->{srcIp})) =~ m/(....)/g ); + } + if ($body->{dstIp} =~ /:/) { + $body->{dstIp} = join ":", (unpack("H*", inet_pton(AF_INET6, $body->{dstIp})) =~ m/(....)/g ); } - if ($body->{radius}) { - foreach my $field ("eip", "fip") { - $body->{radius}->{$field} = fixIp($body->{radius}->{$field}) if (exists $body->{radius}->{$field}); + if (exists $body->{dns} && exists $body->{dns}->{ip}) { + for (my $i = 0; $i < @{$body->{dns}->{ip}}; $i++) { + if ($body->{dns}->{ip}[$i] =~ /:/) { + $body->{dns}->{ip}[$i] = join ":", (unpack("H*", inet_pton(AF_INET6, $body->{dns}->{ip}[$i])) =~ m/(....)/g ); + } } } - - foreach my $field ("ta", "hh1", "hh2") { - $body->{$field} = fixTags($json, $body->{$field}) if (exists $body->{$field}); - } } - @{$json->{sessions}} = sort {$a->{body}->{fpd} <=> $b->{body}->{fpd}} @{$json->{sessions}}; + @{$json->{sessions2}} = sort {$a->{body}->{firstPacket} <=> $b->{body}->{firstPacket}} @{$json->{sessions2}}; delete $json->{tags}; } @@ -211,12 +211,14 @@ sub doViewer { system("cd ../viewer ; node viewer.js -c ../tests/config.test.ini -n test --debug > /tmp/moloch.test &"); system("cd ../viewer ; node viewer.js -c ../tests/config.test.ini -n test2 --debug > /tmp/moloch.test2 &"); system("cd ../viewer ; node viewer.js -c ../tests/config.test.ini -n all --debug > /tmp/moloch.all &"); + system("cd ../parliament ; node parliament.js --regressionTests -c /dev/null --debug > /tmp/moloch.parliament 2>&1 &"); } else { system("cd ../capture/plugins/wiseService ; node wiseService.js -c ../../../tests/config.test.ini > /dev/null &"); system("cd ../viewer ; node multies.js -c ../tests/config.test.ini -n all > /dev/null &"); system("cd ../viewer ; node viewer.js -c ../tests/config.test.ini -n test > /dev/null &"); system("cd ../viewer ; node viewer.js -c ../tests/config.test.ini -n test2 > /dev/null &"); system("cd ../viewer ; node viewer.js -c ../tests/config.test.ini -n all > /dev/null &"); + system("cd ../parliament ; node parliament.js --regressionTests -c /dev/null > /dev/null 2>&1 &"); } sleep 1; sleep (10000) if ($cmd eq "--viewerhang"); @@ -241,7 +243,11 @@ sub doViewer { system("../capture/plugins/taggerUpload.pl $ELASTICSEARCH uri uri.tagger2.json uritaggertest2"); # Start Wise - system("cd ../capture/plugins/wiseService ; node wiseService.js -c ../../../tests/config.test.ini > /tmp/moloch.wise &"); + if ($main::debug) { + system("cd ../capture/plugins/wiseService ; node wiseService.js -c ../../../tests/config.test.ini > /tmp/moloch.wise &"); + } else { + system("cd ../capture/plugins/wiseService ; node wiseService.js -c ../../../tests/config.test.ini > /dev/null &"); + } sleep 1; $main::userAgent->get("$ELASTICSEARCH/_flush"); @@ -274,11 +280,13 @@ sub doViewer { system("cd ../viewer ; node viewer.js -c ../tests/config.test.ini -n test --debug > /tmp/moloch.test &"); system("cd ../viewer ; node viewer.js -c ../tests/config.test.ini -n test2 --debug > /tmp/moloch.test2 &"); system("cd ../viewer ; node viewer.js -c ../tests/config.test.ini -n all --debug > /tmp/moloch.all &"); + system("cd ../parliament ; node parliament.js --regressionTests -c /dev/null --debug > /tmp/moloch.parliament 2>&1 &"); } else { system("cd ../viewer ; node multies.js -c ../tests/config.test.ini -n all > /dev/null &"); system("cd ../viewer ; node viewer.js -c ../tests/config.test.ini -n test > /dev/null &"); system("cd ../viewer ; node viewer.js -c ../tests/config.test.ini -n test2 > /dev/null &"); system("cd ../viewer ; node viewer.js -c ../tests/config.test.ini -n all > /dev/null &"); + system("cd ../parliament ; node parliament.js --regressionTests -c /dev/null > /dev/null 2>&1 &"); } sleep 1; } @@ -300,6 +308,7 @@ sub doViewer { $main::userAgent->post("http://localhost:8125/shutdown"); $main::userAgent->post("http://localhost:8200/shutdown"); $main::userAgent->post("http://localhost:8081/shutdown"); + $main::userAgent->post("http://localhost:8008/shutdown"); } exit(1) if ( $parser->has_errors ); diff --git a/tests/tls.t b/tests/tls.t index 0ee55ccce0..e21d59b9e5 100644 --- a/tests/tls.t +++ b/tests/tls.t @@ -4,7 +4,7 @@ use URI::Escape; use MolochTest; use strict; -my $pwd = getcwd() . "/pcap"; +my $pwd = "*/pcap"; my $files = "(file=$pwd/openssl-ssl3.pcap||file=$pwd/openssl-tls1.pcap||file=$pwd/https3-301-get.pcap)"; countTest(3, "date=-1&expression=" . uri_escape("$files&&protocols==tls")); diff --git a/tests/wise.t b/tests/wise.t index 898943ac4c..ec2c9c56fc 100644 --- a/tests/wise.t +++ b/tests/wise.t @@ -1,5 +1,5 @@ # WISE tests -use Test::More tests => 40; +use Test::More tests => 44; use MolochTest; use Cwd; use URI::Escape; @@ -110,7 +110,7 @@ eq_or_diff($wise, 'Not found', "Zeus aol.com"); } -my $pwd = getcwd() . "/pcap"; +my $pwd = "*/pcap"; # wise tests 2 @@ -136,6 +136,9 @@ my $pwd = getcwd() . "/pcap"; countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/https-generalizedtime.pcap||file=$pwd/http-content-gzip.pcap)&&tags=ja3wise")); countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/https-generalizedtime.pcap||file=$pwd/http-content-gzip.pcap)&&tags=wisebyja31&&mysql.ver=wisebyja31mysqlversion&&test.ip=155.155.155.155")); + countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/smtp-zip.pcap)&&tags=sha256wise")); + countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/http-content-zip.pcap||file=$pwd/smtp-zip.pcap)&&tags=wisebysha2561&&mysql.ver=wisebysha2561mysqlversion&&test.ip=1::2")); + countTest(2, "date=-1&expression=" . uri_escape("(file=$pwd/smtp-data-250.pcap||file=$pwd/smtp-data-521.pcap)&&tags=emailwise")); countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/smtp-data-250.pcap||file=$pwd/smtp-data-521.pcap)&&tags=wisesrcmatch")); countTest(1, "date=-1&expression=" . uri_escape("(file=$pwd/smtp-data-250.pcap||file=$pwd/smtp-data-521.pcap)&&tags=wisedstmatch")); diff --git a/viewer/.jshintrc b/viewer/.jshintrc index 2b331a1842..19b3928a47 100644 --- a/viewer/.jshintrc +++ b/viewer/.jshintrc @@ -4,7 +4,7 @@ "curly": true, "eqeqeq": true, "immed": true, - "latedef": true, + "latedef": "nofunc", "newcap": true, "nonew": true, "undef": true, @@ -14,6 +14,6 @@ "predef": [ "angular", "document", "jasmine", "jQuery", "$", "window", "localStorage", "describe", "beforeEach", "afterEach", "inject", "it", "expect", "spyOn", - "sessionStorage", "alert", "d3", "cubism" + "sessionStorage", "alert", "d3", "cubism", "location" ] } diff --git a/viewer/Makefile.in b/viewer/Makefile.in index 897abd31e8..ce3f5246d6 100644 --- a/viewer/Makefile.in +++ b/viewer/Makefile.in @@ -8,14 +8,17 @@ CP = /bin/cp all: install: - @mkdir -p "$(VIEWERDIR)" + @mkdir -p "$(VIEWERDIR)" "$(VIEWERDIR)/vueapp" /bin/rm -f $(VIEWERDIR)/public/style.css $(INSTALL) *.js package.json $(VIEWERDIR) - npm update + npm install + (cd vueapp ; npm install) npm run bundle:min $(CP) -pr views public bundle $(VIEWERDIR) - (cd $(VIEWERDIR) ; npm update --production) + $(CP) -pr vueapp/dist "$(VIEWERDIR)/vueapp" + (cd $(VIEWERDIR) ; npm install --production) distclean realclean clean: rm -rf node_modules rm -rf bundle + (cd vueapp ; rm -rf node_modules ; rm -rf dist) diff --git a/viewer/README.md b/viewer/README.md index 7edc49ee73..9449653163 100644 --- a/viewer/README.md +++ b/viewer/README.md @@ -1,11 +1,10 @@ # Moloch Viewer -Moloch viewer is an [AngularJS][angularjs] web app. +Moloch viewer is an [AngularJS][angularjs] and [Vue.js][vuejs] web app. -Read the main Moloch README for more information on how to build and run the app -for demo or production. These instructions are for running in development mode out -of the source tree. +Read the main [Moloch README](../README.rst) for more information on how to build and run the app for demo or production. These instructions are for running in development mode out of the source tree. +--- ## Development @@ -13,13 +12,14 @@ The viewer uses a number of node.js tools for initialization and testing. You must have node.js and its package manager (npm) installed. You can get them from [http://nodejs.org/][node]. +**Currently, there are two separate web applications: an Angular application and a Vue application. The Angular application includes all pages except the Stats page. This will change as pages are implemented in Vue.** -### Install Dependencies +--- -The viewer mostly uses development dependencies that are all bundled using -[webpack][webpack]. +### Install Dependencies -* We get dependencies via `npm`, the [node package manager][npm]. +The viewer mostly uses development dependencies that are all bundled using [webpack][webpack]. +We get dependencies via `npm`, the [node package manager][npm]. In the viewer directory, execute: @@ -31,65 +31,97 @@ You should find that you have a new folder: * `node_modules` - contains the npm packages for the dependencies +_For now, the Vue app needs to be installed and built separately. This is because the Vue portion of the application is using different versions of some packages (e.g. [Bootstrap 4][bootstrap4] vs [Bootstrap 3][bootstrap3])._ + +To install dependencies for the Vue application, execute: + +``` +cd vueapp +npm install +``` + +--- ### Run the Application -The simplest way to start the web app for development and testing is: +**To run the web application, you must have an elasticsearch cluster running and already built and configured Moloch. Read the main [Moloch README](../README.rst) for more information.** + + +#### The simplest way to start the web app is: + ``` npm run start:test ``` -You must have an elasticsearch cluster running, have already built and -configured Moloch, and your `tests/config.test.ini` must be valid. +For this command to work, your `tests/config.test.ini` must be valid. -This command starts the node server and bundles all app files into -`viewer/bundles/app.bundle.js` and `viewer/bundles/vendor.bundle.js`. - -Webpack watches for changes to relevant files, and re-bundles the app after each save. +This command starts the node server, bundles all Angular app files into `viewer/bundles`, and bundles all Vue app files into `viewer/vueapp/dist`. Now browse to the app at `http://localhost:8123`. ---- -You can also start the app with an existing `config.ini` file: +#### To start the web app with a test admin user, run: ``` -npm start +npm run addtestuser +npm run start:testuser ``` -As above, you must have an elasticsearch cluster running, have already built and -configured Moloch, and your `config.ini` must be valid. +For this to work, your `tests/config.test.ini` must be valid. + +These first command adds an "admin" user. The second command starts the node server, bundles all Angular app files into `viewer/bundles`, and bundles all Vue app files into `viewer/vueapp/dist`. + +Now browse to the app at `http://localhost:8123` and login using username "admin" and password "admin". -This command starts the node server and bundles and minifies all app files into -`viewer/bundles/app.bundle.js` and `viewer/bundles/vendor.bundle.js`. + +#### To start the web app for **Vue** development and testing, run: + +``` +npm run start:vuewatch +``` + +For this command to work, your `tests/config.test.ini` must be valid. + +This command starts the node server, bundles all Vue app files into `viewer/vueapp/dist`, and bundles all Angular app files into `viewer/bundles`. + +Webpack watches for changes to relevant Vue files, and re-bundles the Vue app after each save. Now browse to the app at `http://localhost:8123`. ---- -Lastly, you can start the app without test data by creating `viewer/config.dev.ini`, -then executing: +#### To start the web app for **Angular** development and testing, run: ``` -npm run start:dev +npm run start:ngwatch ``` -As above, you must have an elasticsearch cluster running, have already built and -configured Moloch, and your `viewer/config.dev.ini` must be valid. +For this command to work, your `tests/config.test.ini` must be valid. + +This command starts the node server, bundles all Angular app files into `viewer/bundles`, and bundles all Vue app files into `viewer/vueapp/dist`. + +Webpack watches for changes to relevant Angular files, and re-bundles the Angular app after each save. + +Now browse to the app at `http://localhost:8123`. + + +#### You can also start the app with an existing config file: + +``` +npm start +``` -This command starts the node server and bundles all app files into -`viewer/bundles/app.bundle.js` and `viewer/bundles/vendor.bundle.js`. +For this command to work, your `config.ini` must be valid. -Webpack watches for changes to relevant files, and re-bundles the app after each save. +This command starts the node server, bundles and minifies all Angular app files into `viewer/bundles`, and bundles and minifies all Vue app files into `viewer/vueapp/dist`. Now browse to the app at `http://localhost:8123`. +--- ### Running Unit Tests -Moloch viewer includes many unit tests. These are written in [Jasmine][jasmine], -which are run with the [Karma Test Runner][karma]. +Moloch viewer includes many unit tests. These are written in [Jasmine][jasmine], which are run with the [Karma Test Runner][karma]. * the configuration is found at `viewer/karma.conf.js` * the unit tests are found near the code they are testing and are named as `*.test.js`. @@ -100,38 +132,13 @@ The easiest way to run the unit tests is to use the supplied npm script: npm test ``` -This script will start the Karma test runner to execute the unit tests. Before -running the test, the script makes sure that all JavaScript is linted. The tests -will not execute if the linter returns errors in the JavaScript. +This script will start the Karma test runner to execute the unit tests. Before running the test, the script makes sure that all JavaScript is linted. The tests will not execute if the linter returns errors in the JavaScript. +--- -### Directory Layout +### Contributing -``` -app/ --> all of the source files for the application - app.js --> main application module - app.css --> main stylesheet - imports other stylesheets - modules/ --> all app specific modules - index.js --> webpack entry file listing all components to be included in bundle - index.test.js --> webpack test entry file listing all tests to be included in testing bundle - module1/ --> module1 logic, views, styles, and tests - components/ --> logic controllers for views - services/ --> services to interact with the server - styles/ --> styles that pertain to this module's views - templates/ --> html views - tests/ --> jasmine test files - bundle/ --> where webpack stores the app bundles - app.bundle.js --> main app bundle - vendor.bundle.js --> bundled dependencies - app.bundle.js.map --> main app map file for debugging - vendor.bundle.js.map --> dependencies map file for debugging -karma.conf.js --> config file for running unit tests with Karma -webpack.config.js --> config file for webpack to bundle files -webpack.loaders.js --> config file for webpack to initialize loaders for different types of files -components/ --> reusable small components - index.js --> webpack entry file listing all components to be included in bundle - index.test.js --> webpack test entry file listing all tests to be included in testing bundle -``` +View the [contributing guide](../CONTRIBUTING.md) for more information. [angularjs]: http://angularjs.org/ [webpack]: https://webpack.github.io/ @@ -139,3 +146,6 @@ components/ --> reusable small components [karma]: https://karma-runner.github.io [node]: https://nodejs.org [npm]: https://www.npmjs.org/ +[vuejs]: https://vuejs.org/ +[bootstrap4]: https://getbootstrap.com/ +[bootstrap3]: https://getbootstrap.com/docs/3.3/ diff --git a/viewer/app/app.js b/viewer/app/app.js index fa8ed3495c..a32cd03839 100644 --- a/viewer/app/app.js +++ b/viewer/app/app.js @@ -88,11 +88,6 @@ template : '', reloadOnSearch: false }) - .when('/stats', { - title : 'Stats', - template : '', - reloadOnSearch: false - }) .when('/spiview', { title : 'SPI View', template : '', diff --git a/viewer/app/modules/connections/connections.component.js b/viewer/app/modules/connections/connections.component.js index 1ea01cce70..981508fc57 100644 --- a/viewer/app/modules/connections/connections.component.js +++ b/viewer/app/modules/connections/connections.component.js @@ -63,8 +63,8 @@ // load route params this.query = {}; this.query.length = this.$routeParams.connLength || 100; - this.query.srcField = this.$routeParams.srcField || 'a1'; - this.query.dstField = this.$routeParams.dstField || 'a2'; + this.query.srcField = this.$routeParams.srcField || 'srcIp'; + this.query.dstField = this.$routeParams.dstField || 'dstIp'; this.query.nodeDist = parseInt(this.$routeParams.nodeDist || '125'); this.query.minConn = parseInt(this.$routeParams.minConn || '1'); @@ -113,13 +113,13 @@ this.query.length = size; } - let srcField = current.params.srcField || 'a1'; + let srcField = current.params.srcField || 'srcIp'; if (srcField !== this.query.srcField) { change = true; this.query.srcField = srcField; } - let dstField = current.params.dstField || 'a2'; + let dstField = current.params.dstField || 'dstIp'; if (dstField !== this.query.dstField) { change = true; this.query.dstField = dstField; diff --git a/viewer/app/modules/help/help.html b/viewer/app/modules/help/help.html index c9d4f20e0b..4615bf3c46 100644 --- a/viewer/app/modules/help/help.html +++ b/viewer/app/modules/help/help.html @@ -319,17 +319,17 @@

Field Exists Search

Examples

- Find all the sessions involving Russia (RUS) or China (CHN) that are + Find all the sessions involving Russia (RU) or China (CN) that are using port 80 and also a hostname which contains "com": 

-          (country == RUS || country == CHN) && port == 80 && host == *com
+          (country == RU || country == CN) && port == 80 && host == *com

- Find all the sessions of type "text/plain", involving Canada (CAN), and + Find all the sessions of type "text/plain", involving Canada (CA), and containing less than 20 packets: 

-          tags == "http:content:text/plain" && country == CAN && packets < 20
+          tags == "http:content:text/plain" && country == CA && packets < 20

diff --git a/viewer/app/modules/search/components/expression.typeahead.js b/viewer/app/modules/search/components/expression.typeahead.js index 47fd026f1f..f9ec40493a 100644 --- a/viewer/app/modules/search/components/expression.typeahead.js +++ b/viewer/app/modules/search/components/expression.typeahead.js @@ -204,32 +204,6 @@ return; } - // autocomplete http.hasheader values after 1 char - if (lastToken.trim().length >= 1) { - if (/^(tags|http.hasheader)/.test(token)) { - this.loadingValues = true; - - this.promise = this.FieldService.getHasheaderValues({ - type:token, filter:lastToken - }); - - this.promise.then((result) => { - this.promise = null; - if (result) { - this.loadingValues = false; - this.results = result; - this.addExistsItem(lastToken, operatorToken); - } - }).catch((error) => { - this.promise = null; - this.loadingValues = false; - this.loadingError = error; - }); - - return; - } - } - // autocomplete other values after 2 chars if (lastToken.trim().length >= 2) { let params = { // build parameters for getting value(s) diff --git a/viewer/app/modules/search/services/field.service.js b/viewer/app/modules/search/services/field.service.js index 538728a831..57853fd732 100644 --- a/viewer/app/modules/search/services/field.service.js +++ b/viewer/app/modules/search/services/field.service.js @@ -85,42 +85,6 @@ return(promise); } - /** - * Gets hasheader field values from the server - * @param {Object} params The parameters to send with the query - * @returns {Promise} Promise A promise object that signals the completion - * or rejection of the request. - */ - getHasheaderValues(params) { - let deferred = this.$q.defer(); - - let request = this.$http({ - url : 'uniqueValue.json', - method : 'GET', - params : params, - timeout : deferred.promise - }); - - let promise = request - .then((response) => { - return(response.data); - }, (error) => { - return(this.$q.reject(error)); - }).catch(angular.noop); // handle abort - - promise.abort = () => { - deferred.resolve({error:'Request canceled.'}); - }; - - // cleanup - promise.finally(() => { - promise.abort = angular.noop; - deferred = request = promise = null; - }); - - return(promise); - } - /** * Gets the cached country code list * @returns {Promise} Promise A promise object that signals the completion diff --git a/viewer/app/modules/search/tests/field.service.test.js b/viewer/app/modules/search/tests/field.service.test.js index d5842ef65e..a3bf246007 100644 --- a/viewer/app/modules/search/tests/field.service.test.js +++ b/viewer/app/modules/search/tests/field.service.test.js @@ -56,12 +56,6 @@ $httpBackend.flush(); }); - it('should send a GET request for http.hasheader values (uniqueValue.json)', function() { - var result = FieldService.getHasheaderValues(); - $httpBackend.expectGET('uniqueValue.json'); - $httpBackend.flush(); - }); - }); }); diff --git a/viewer/app/modules/session/components/custom.columns.json b/viewer/app/modules/session/components/custom.columns.json index 01b804c82c..839aaf2a90 100644 --- a/viewer/app/modules/session/components/custom.columns.json +++ b/viewer/app/modules/session/components/custom.columns.json @@ -8,14 +8,14 @@ "help": "Information", "unsortable": true, "children": [ - "us", - "esrc", - "edst", - "esub", - "efn", - "dnsho", - "tls.alt", - "ircch" + "http.uri", + "email.src", + "email.dst", + "email.subject", + "email.filename", + "dns.host", + "cert.alt", + "irc.channel" ] }, "src": { @@ -25,10 +25,10 @@ "group": "general", "friendlyName": "Src IP / Country", "help": "Src IP & Country", - "sortBy": "a1", + "sortBy": "srcIp", "children": [ - "a1", - "g1" + "srcIp", + "srcGEO" ] }, "dst": { @@ -38,10 +38,10 @@ "group": "general", "friendlyName": "Dst IP / Country", "help": "Dst IP & Country", - "sortBy": "a2", + "sortBy": "dstIp", "children": [ - "a2", - "g2" + "dstIp", + "dstGEO" ] }, "dbby": { @@ -51,10 +51,10 @@ "group": "general", "friendlyName": "Databytes / Bytes", "help": "Data Bytes & Bytes", - "sortBy": "by", + "sortBy": "totBytes", "children": [ - "db", - "by" + "totDataBytes", + "totBytes" ] }, "pa1pa2": { @@ -64,10 +64,10 @@ "group": "general", "friendlyName": "Src Packets / Dst Packets", "help": "Src Packets & Dst Packets", - "sortBy": "by", + "sortBy": "totPackets", "children": [ - "pa1", - "pa2" + "srcPackets", + "dstPackets" ] }, "db1db2": { @@ -77,10 +77,10 @@ "group": "general", "friendlyName": "Src Databytes / Dst Databytes", "help": "Src Databytes & Dst Databytes", - "sortBy": "db1", + "sortBy": "totDataBytes", "children": [ - "db1", - "db2" + "srcDataBytes", + "dstDataBytes" ] }, "by1by2": { @@ -90,10 +90,10 @@ "group": "general", "friendlyName": "Src Bytes / Dst Bytes", "help": "Src Bytes & Dst Bytes", - "sortBy": "by1", + "sortBy": "totBytes", "children": [ - "by1", - "by2" + "srcBytes", + "dstBytes" ] }, "db1by1": { @@ -103,10 +103,10 @@ "group": "general", "friendlyName": "Src Databytes / Src Bytes", "help": "Src Databytes & Src Bytes", - "sortBy": "db1", + "sortBy": "srcDataBytes", "children": [ - "db1", - "by1" + "srcDataBytes", + "srcBytes" ] }, "db2by2": { @@ -116,10 +116,10 @@ "group": "general", "friendlyName": "Dst Databytes / Dst Bytes", "help": "Dst Databytes & Dst Bytes", - "sortBy": "db2", + "sortBy": "dstDataBytes", "children": [ - "db2", - "by2" + "dstDataBytes", + "dstBytes" ] } } diff --git a/viewer/app/modules/session/components/session.detail.component.js b/viewer/app/modules/session/components/session.detail.component.js index f3006348f7..7bdf88e209 100644 --- a/viewer/app/modules/session/components/session.detail.component.js +++ b/viewer/app/modules/session/components/session.detail.component.js @@ -181,7 +181,7 @@ getDetailData(message) { this.loading = true; - this.SessionService.getDetail(this.$scope.session.id, this.$scope.session.no) + this.SessionService.getDetail(this.$scope.session.id, this.$scope.session.node) .then((response) => { this.loading = false; this.$scope.detailHtml = this.$sce.trustAsHtml(response.data); @@ -217,7 +217,7 @@ } this.packetPromise = this.SessionService.getPackets(this.$scope.session.id, - this.$scope.session.no, this.$scope.params); + this.$scope.session.node, this.$scope.params); this.packetPromise.then((response) => { this.loadingPackets = false; @@ -442,8 +442,8 @@ scope.openPermalink = function() { $location.path('sessions') .search('expression', `id=${scope.session.id}`) - .search('startTime', scope.session.fp) - .search('stopTime', scope.session.lp) + .search('startTime', scope.session.firstPacket) + .search('stopTime', scope.session.lastPacket) .search('openAll', 1); }; diff --git a/viewer/app/modules/session/components/session.field.component.js b/viewer/app/modules/session/components/session.field.component.js index bebbe149c0..5f71779a7c 100644 --- a/viewer/app/modules/session/components/session.field.component.js +++ b/viewer/app/modules/session/components/session.field.component.js @@ -197,17 +197,6 @@ parseValue() { if (!this.field || !this.value) { return; } - // TODO: this goes away with ES5 - if (this.session && this.field.dbField === 'a1' && this.session['tipv61-term']) { - this.expr = 'tipv6.src'; - this.field = {dbField:'tipv61-term',exp:'tipv6.src',friendlyName:'IPv6 Src',group:'general',help:'Temporary IPv6 Source',portField:'p1',transform:'ipv6ToHex',type:'lotermfield'}; - this.value = this.session['tipv61-term']; - } else if (this.session && this.field.dbField === 'a2' && this.session['tipv62-term']) { - this.expr = 'tipv6.dst'; - this.field = {dbField:'tipv62-term',exp:'tipv6.dst',friendlyName:'IPv6 Dst',group:'general',help:'Temporary IPv6 Destination',portField:'p2',transform:'ipv6ToHex',type:'lotermfield'}; - this.value = this.session['tipv62-term']; - } - this.parsed = { queryVal: this.value, value : this.value @@ -224,7 +213,7 @@ case 'seconds': this.time = true; qVal = val; // save original value as the query value - val = this.$filter('timezoneDateString')(parseInt(val), this.timezone); + val = this.$filter('timezoneDateString')(parseInt(val)/1000, this.timezone); if (this.expr !== 'starttime' && this.expr !== 'stoptime') { // only starttime and stoptime fields are applied to time inputs this.time = false; @@ -232,7 +221,7 @@ } break; case 'ip': - val = this.$filter('extractIPString')(val); + val = val; qVal = val; // don't save original value (parsed val is query val) break; case 'lotermfield': @@ -240,7 +229,7 @@ val = this.$filter('extractIPv6String')(val); qVal = val; // don't save original value (parsed val is query val) } else if (this.field.transform === 'ipProtocolLookup') { - val = this.$filter('protocol')(val); + val = this.$filter('ipProtocol')(val); qVal = val; // don't save original value (parsed val is query val) } break; diff --git a/viewer/app/modules/session/components/session.list.component.js b/viewer/app/modules/session/components/session.list.component.js index d2feefe273..daf740c419 100644 --- a/viewer/app/modules/session/components/session.list.component.js +++ b/viewer/app/modules/session/components/session.list.component.js @@ -10,8 +10,8 @@ }; const defaultTableState = { - order : [['fp', 'asc']], - visibleHeaders: ['fp','lp','src','p1','dst','p2','pa','dbby','no','info'] + order : [['firstPacket', 'asc']], + visibleHeaders: ['firstPacket','lastPacket','src','srcPort','dst','dstPort','totPackets','dbby','node','info'] }; let customCols = require('./custom.columns.json'); @@ -201,9 +201,7 @@ this.stickySessions = []; // clear sticky sessions - // TODO: tipv6*-term goes away with ES5 - // clear fields to query for but always include protocols field - this.query.fields = ['pr','tipv61-term','tipv62-term']; + this.query.fields = ['ipProtocol']; this.mapHeadersToFields(); @@ -634,7 +632,7 @@ } // if there are no columns to sort by, sort by start time - if (!newSort) { newSort = 'fp'; } + if (!newSort) { newSort = 'firstPacket'; } this.tableState.order = [[newSort,'asc']]; } else { diff --git a/viewer/app/modules/session/templates/session.detail.html b/viewer/app/modules/session/templates/session.detail.html index d0adeb7941..3594913343 100644 --- a/viewer/app/modules/session/templates/session.detail.html +++ b/viewer/app/modules/session/templates/session.detail.html @@ -100,13 +100,13 @@