FreeBSD support, switch to chef/ubuntu boxes

* Tried to get FreeBSD working again.  wiseService will not run on
  FreeBSD but the plugin will.
* Switch to chef/ubuntu boxes for vagrant since they download faster
  and have swap turned on.  Goes against my swap is evil mantra,
  but is useful for VMs in limited memory hosted environments. :)

capture/Makefile.in

@@ -22,7 +22,7 @@ LIB_OTHER     = @NIDS_LIBS@ \
 	        thirdparty/http_parser.o \
 	        thirdparty/js0n.o \
 	        thirdparty/patricia.o \
-		-ldl -lpthread
+		@DL_LIB@ -lpthread
 
 C_FILES         = main.c db.c nids.c yara.c http.c config.c parsers.c plugins.c field.c trie.c
 O_FILES         = $(C_FILES:.c=.o)
@@ -38,9 +38,9 @@ all:thirdparty/js0n.o thirdparty/http_parser.o thirdparty/patricia.o
             @UNDEFINED_FLAGS@ \
 	    $(LIB_PCAP) \
 	    $(LIB_OTHER) \
-	    -lm -lresolv -lffi -lz
-	(cd parsers; make)
-	(cd plugins; make)
+	    -lm @RESOLV_LIB@ -lffi -lz
+	(cd parsers; $(MAKE))
+	(cd plugins; $(MAKE))
 
 snf:thirdparty/js0n.o thirdparty/http_parser.o thirdparty/patricia.o
 	gcc -ggdb -Wall -Wextra -D_GNU_SOURCE -c $(C_FILES) \
@@ -50,7 +50,7 @@ snf:thirdparty/js0n.o thirdparty/http_parser.o thirdparty/patricia.o
             @UNDEFINED_FLAGS@ \
 	    $(LIB_SNF) \
 	    $(LIB_OTHER) \
-	    -lrt -lm -lpcre -lresolv -luuid -lmagic -lffi -lz
+	    -lrt -lm -lpcre @RESOLV_LIB@ -luuid -lmagic -lffi -lz
 
 thirdparty/js0n.o:thirdparty/js0n.c
 	$(CC) -c thirdparty/js0n.c -o thirdparty/js0n.o
@@ -66,8 +66,8 @@ install: installdirs
 
 installdirs:
 	$(INSTALL) -d $(bindir)
-	(cd parsers; make install)
-	(cd plugins; make install)
+	(cd parsers; $(MAKE) install)
+	(cd plugins; $(MAKE) install)
 
 distclean realclean clean:
 	rm -f *.o moloch-capture

capture/moloch.h

@@ -16,6 +16,7 @@
  * limitations under the License.
  */
 
+#include <unistd.h>
 #include <netinet/in.h>
 #include <netinet/ip.h>
 #include <netinet/tcp.h>

capture/parsers/Makefile.in

@@ -4,6 +4,7 @@ INCLUDE_OTHER = -I.. -I../thirdparty \
                 @GLIB2_CFLAGS@ \
 	        @NIDS_CFLAGS@
 
+install_sh = @install_sh@
 mkdir_p = @mkdir_p@
 INSTALL = @INSTALL@
 PARSERSDIR = @prefix@/parsers

capture/plugins/Makefile.in

@@ -6,6 +6,7 @@ INCLUDE_OTHER = -I.. -I../thirdparty \
                 @GLIB2_CFLAGS@ \
 	        @NIDS_CFLAGS@
 
+install_sh = @install_sh@
 mkdir_p = @mkdir_p@
 INSTALL = @INSTALL@
 PLUGINDIR = @prefix@/plugins

capture/plugins/wiseService/wiseService.ini.sample

@@ -5,10 +5,13 @@
 # Configuration for the wiseService itself.
 [wiseService]
 port=8081
+# Exclude common DNSBL style lookups
 excludeDomains=*.bl.barracudabrts.com;*.zen.spamhaus.org;*.in-addr.arpa;*.avts.mcafee.com;*.avqs.mcafee.com;*.bl.barracuda.com;*.lbl8.mailshell.net;*.dnsbl.sorbs.net;*.s.sophosxl.net
 
 # OpenDNS Umbrella Integration - https://www.opendns.com/enterprise-security/
 [opendns]
+# Example, exclude office lookups to prevent DNS leakage
+# excludeDomains=*.office.example.com
 #key=
 
 # Threatstream OPTIC - http://threatstream.com/

configure

@@ -604,6 +604,8 @@ build_os
 build_vendor
 build_cpu
 build
+RESOLV_LIB
+DL_LIB
 am__fastdepCXX_FALSE
 am__fastdepCXX_TRUE
 CXXDEPMODE
@@ -3903,6 +3905,77 @@ _ACEOF
 fi
 
 
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for main in -ldl" >&5
+$as_echo_n "checking for main in -ldl... " >&6; }
+if ${ac_cv_lib_dl_main+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+  ac_check_lib_save_LIBS=$LIBS
+LIBS="-ldl  $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+
+int
+main ()
+{
+return main ();
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+  ac_cv_lib_dl_main=yes
+else
+  ac_cv_lib_dl_main=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+    conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_dl_main" >&5
+$as_echo "$ac_cv_lib_dl_main" >&6; }
+if test "x$ac_cv_lib_dl_main" = xyes; then :
+  DL_LIB=-ldl
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for main in -lresolv" >&5
+$as_echo_n "checking for main in -lresolv... " >&6; }
+if ${ac_cv_lib_resolv_main+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+  ac_check_lib_save_LIBS=$LIBS
+LIBS="-lresolv  $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+
+int
+main ()
+{
+return main ();
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+  ac_cv_lib_resolv_main=yes
+else
+  ac_cv_lib_resolv_main=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+    conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_resolv_main" >&5
+$as_echo "$ac_cv_lib_resolv_main" >&6; }
+if test "x$ac_cv_lib_resolv_main" = xyes; then :
+  RESOLV_LIB=-lresolv
+fi
+
+
+
 # Make sure we can run config.sub.
 $SHELL "$ac_aux_dir/config.sub" sun4 >/dev/null 2>&1 ||
   as_fn_error $? "cannot run $SHELL $ac_aux_dir/config.sub" "$LINENO" 5

configure.ac

@@ -21,6 +21,11 @@ AC_CHECK_LIB(magic, main,,AC_MSG_ERROR(please install magic library))
 AC_CHECK_LIB(rt, main,,)
 AC_CHECK_LIB(net, main,,)
 
+AC_CHECK_LIB(dl, main,DL_LIB=-ldl,)
+AC_SUBST(DL_LIB)
+AC_CHECK_LIB(resolv, main,RESOLV_LIB=-lresolv,)
+AC_SUBST(RESOLV_LIB)
+
 dnl OS Stuff
 AC_CANONICAL_HOST
 case $host_os in

db/Makefile.in

@@ -1,3 +1,4 @@
+install_sh = @install_sh@
 mkdir_p = @mkdir_p@
 INSTALL = @INSTALL@
 DBDIR = @prefix@/db

easybutton-build.sh

@@ -41,6 +41,8 @@ do
 done
 
 
+MAKE=make
+
 # Installing dependencies
 echo "MOLOCH: Installing Dependencies"
 if [ -f "/etc/redhat-release" ]; then
@@ -59,6 +61,11 @@ if [ -f "/etc/debian_version" ]; then
   fi
 fi
 
+if [ $(uname) == "FreeBSD" ]; then
+    pkg_add -Fr wget curl pcre flex bison gettext e2fsprogs-libuuid glib gmake
+    MAKE=gmake
+fi
+
 
 
 
@@ -69,15 +76,25 @@ fi
 cd thirdparty
 
 # glib
-if [ ! -f "glib-$GLIB.tar.xz" ]; then
-  wget http://ftp.gnome.org/pub/gnome/sources/glib/2.42/glib-$GLIB.tar.xz
-fi
-
-if [ ! -f "glib-$GLIB/gio/.libs/libgio-2.0.a" -o ! -f "glib-$GLIB/glib/.libs/libglib-2.0.a" ]; then
-  xzcat glib-$GLIB.tar.xz | tar xf -
-  (cd glib-$GLIB ; ./configure --disable-xattr --disable-shared --enable-static --disable-libelf --disable-selinux; make)
+if [ $(uname) == "FreeBSD" ]; then
+  #Screw it, use whatever the OS has
+  WITHGLIB=" "
 else
-  echo "MOLOCH: Not rebuilding glib"
+  WITHGLIB="--with-glib2=thirdparty/glib-$GLIB"
+  if [ ! -f "glib-$GLIB.tar.xz" ]; then
+    wget http://ftp.gnome.org/pub/gnome/sources/glib/2.42/glib-$GLIB.tar.xz
+  fi
+
+  if [ ! -f "glib-$GLIB/gio/.libs/libgio-2.0.a" -o ! -f "glib-$GLIB/glib/.libs/libglib-2.0.a" ]; then
+    xzcat glib-$GLIB.tar.xz | tar xf -
+    (cd glib-$GLIB ; ./configure --disable-xattr --disable-shared --enable-static --disable-libelf --disable-selinux; $MAKE)
+    if [ $? -ne 0 ]; then
+      echo "MOLOCH: $MAKE failed"
+      exit 1
+    fi
+  else
+    echo "MOLOCH: Not rebuilding glib"
+  fi
 fi
 
 # yara
@@ -87,7 +104,11 @@ fi
 
 if [ ! -f "yara-$YARA/libyara/.libs/libyara.a" ]; then
   tar zxf yara-$YARA.tar.gz
-  (cd yara-$YARA; ./configure --enable-static; make)
+  (cd yara-$YARA; ./configure --enable-static; $MAKE)
+  if [ $? -ne 0 ]; then
+    echo "MOLOCH: $MAKE failed"
+    exit 1
+  fi
 else
   echo "MOLOCH: Not rebuilding yara"
 fi
@@ -106,7 +127,11 @@ tar zxf GeoIP-$GEOIP.tar.gz
 #    (cd GeoIP-$GEOIP ; libtoolize -f)
 #  fi
 
-  (cd GeoIP-$GEOIP ; ./configure --enable-static; make)
+  (cd GeoIP-$GEOIP ; ./configure --enable-static; $MAKE)
+  if [ $? -ne 0 ]; then
+    echo "MOLOCH: $MAKE failed"
+    exit 1
+  fi
 else
   echo "MOLOCH: Not rebuilding libGeoIP"
 fi
@@ -118,7 +143,7 @@ if [ $DOPFRING -eq 1 ]; then
       wget -O PF_RING-$PFRING.tar.gz http://sourceforge.net/projects/ntop/files/PF_RING/PF_RING-$PFRING.tar.gz/download
     fi
     tar zxf PF_RING-$PFRING.tar.gz
-    (cd PF_RING-$PFRING; make)
+    (cd PF_RING-$PFRING; $MAKE)
     if [ $? -ne 0 ]; then
       echo "MOLOCH: pfring failed to build"
       exit 1
@@ -134,7 +159,11 @@ else
       wget http://www.tcpdump.org/release/libpcap-$PCAP.tar.gz
     fi
     tar zxf libpcap-$PCAP.tar.gz
-    (cd libpcap-$PCAP; ./configure --disable-dbus; make)
+    (cd libpcap-$PCAP; ./configure --disable-dbus; $MAKE)
+    if [ $? -ne 0 ]; then
+      echo "MOLOCH: $MAKE failed"
+      exit 1
+    fi
     PCAPDIR=`pwd`/libpcap-$PCAP
     PCAPBUILD="--with-libpcap=$PCAPDIR"
 fi
@@ -146,7 +175,14 @@ fi
 
 if [ ! -f "libnids-$NIDS/src/libnids.a" ]; then
   tar zxf libnids-$NIDS.tar.gz
-  ( cd libnids-$NIDS; ./configure --enable-static --disable-libnet --with-libpcap=$PCAPDIR --disable-libglib; make)
+  if [ $(uname) == "FreeBSD" ]; then
+    ( cd libnids-$NIDS; cp ../yara-$YARA/config.sub . ; touch src/alloca.h )
+  fi
+  ( cd libnids-$NIDS; ./configure --enable-static --disable-libnet --with-libpcap=$PCAPDIR --disable-libglib; $MAKE)
+  if [ $? -ne 0 ]; then
+    echo "MOLOCH: $MAKE failed"
+    exit 1
+  fi
 else 
   echo "MOLOCH: Not rebuilding libnids"
 fi
@@ -155,12 +191,12 @@ fi
 # Now build moloch
 echo "MOLOCH: Building capture"
 cd ..
-echo "./configure --prefix=$TDIR $PCAPBUILD --with-libnids=thirdparty/libnids-$NIDS --with-yara=thirdparty/yara-$YARA --with-GeoIP=thirdparty/GeoIP-$GEOIP --with-glib2=thirdparty/glib-$GLIB"
-./configure --prefix=$TDIR $PCAPBUILD --with-libnids=thirdparty/libnids-$NIDS --with-yara=thirdparty/yara-$YARA --with-GeoIP=thirdparty/GeoIP-$GEOIP --with-glib2=thirdparty/glib-$GLIB
+echo "./configure --prefix=$TDIR $PCAPBUILD --with-libnids=thirdparty/libnids-$NIDS --with-yara=thirdparty/yara-$YARA --with-GeoIP=thirdparty/GeoIP-$GEOIP $WITHGLIB"
+./configure --prefix=$TDIR $PCAPBUILD --with-libnids=thirdparty/libnids-$NIDS --with-yara=thirdparty/yara-$YARA --with-GeoIP=thirdparty/GeoIP-$GEOIP $WITHGLIB
 
-make
+$MAKE
 if [ $? -ne 0 ]; then
-  echo "MOLOCH: make failed"
+  echo "MOLOCH: $MAKE failed"
   exit 1
 fi
 

easybutton-config.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/sh
 # This script fills in the values in the etc/*.template files.
 # This script is auto run by easybutton-singlehost.sh
 

easybutton-singlehost.sh

@@ -84,6 +84,8 @@ if [ $JAVA_VAL -ne 0 ]; then
             echo "ERROR - 'yum install java-1.7.0-openjdk' failed"
             exit
         fi
+    elif [ $(uname) == "FreeBSD" ]; then
+        pkg_add -Fr openjdk7
     else
         echo "ERROR - Not sure how to install java for this OS, please install and run again"
         exit
@@ -157,6 +159,12 @@ cd elasticsearch-${ES}
 ./bin/plugin -install lukas-vlcek/bigdesk
 
 
+#make
+MAKE=make
+if [ $(uname) == "FreeBSD" ]; then
+    MAKE=gmake
+fi
+
 # NodeJS
 echo "MOLOCH: Downloading and installing node"
 cd ${INSTALL_DIR}/thirdparty
@@ -167,10 +175,10 @@ fi
 tar xfz node-v${NODEJS}.tar.gz
 cd node-v${NODEJS}
 ./configure 
-make
-make install
+$MAKE
+$MAKE install
 ./configure --prefix=${TDIR}
-make install
+$MAKE install
 
 if [ "x$http_proxy" != "x" ]; then
     ${TDIR}/bin/npm config set proxy $http_proxy
@@ -202,10 +210,10 @@ fi
 echo "MOLOCH: Installing"
 cd ${INSTALL_DIR}
 PATH=${TDIR}/bin:${PATH}
-make install
+$MAKE install
 
 if [ $? -ne 0 ]; then
-  echo "ERROR - 'make install' in moloch directory failed"
+  echo "ERROR - '$MAKE install' in moloch directory failed"
   exit 1
 fi