Merge branch 'master' into topic/jsiwek/review-rafael-bro-manual-changes

CHANGES

@@ -1,4 +1,37 @@
 
+2.2-105 | 2014-01-20 12:16:48 -0800
+
+  * Support GRE tunnel decapsulation, including enhanced GRE headers.
+    GRE tunnels are treated just like IP-in-IP tunnels by parsing past
+    the GRE header in between the delivery and payload IP packets.
+    Addresses BIT-867. (Jon Siwek)
+
+  * Simplify FragReassembler memory management. (Jon Siwek)
+
+2.2-102 | 2014-01-20 12:00:29 -0800
+
+  * Include file information (MIME type and description) into notice
+    emails if available. (Justin Azoff)
+
+2.2-100 | 2014-01-20 11:54:58 -0800
+
+  * Fix caching of recently validated SSL certifcates. (Justin Azoff)
+
+2.2-98 | 2014-01-20 11:50:32 -0800
+
+  * For notice suppresion, instead of storing the entire notice in
+    Notice::suppressing, just store the time the notice should be
+    suppressed until. This saves significant memory but can no longer
+    raise end_suppression, which has been removed. (Justin Azoff)
+
+2.2-96 | 2014-01-20 11:41:07 -0800
+
+  * Integrate libmagic 5.16. Bro now now always relies on
+    builtin/shipped magic library/database. (Jon Siwek)
+
+  * Bro now requires a CMake 2.8.x, but no longer a pre-installed
+    libmagic. (Jon Siwek)
+
 2.2-93 | 2014-01-13 09:16:51 -0800
 
   * Fixing compile problems with some versions of libc++. Reported by

CMakeLists.txt

@@ -1,5 +1,5 @@
 project(Bro C CXX)
-cmake_minimum_required(VERSION 2.6.3 FATAL_ERROR)
+cmake_minimum_required(VERSION 2.8.0 FATAL_ERROR)
 include(cmake/CommonCMakeConfig.cmake)
 
 ########################################################################
@@ -39,6 +39,32 @@ set(VERSION_MAJ_MIN "${VERSION_MAJOR}.${VERSION_MINOR}")
 ########################################################################
 ## Dependency Configuration
 
+include(ExternalProject)
+
+# LOG_* options to ExternalProject_Add appear in CMake 2.8.3.  If
+# available, using them hides external project configure/build output.
+if("${CMAKE_VERSION}" VERSION_GREATER 2.8.2)
+    set(EXTERNAL_PROJECT_LOG_OPTIONS
+        LOG_DOWNLOAD 1 LOG_UPDATE 1 LOG_CONFIGURE 1 LOG_BUILD 1 LOG_INSTALL 1)
+else()
+    set(EXTERNAL_PROJECT_LOG_OPTIONS)
+endif()
+
+set(LIBMAGIC_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/libmagic-prefix)
+set(LIBMAGIC_INCLUDE_DIR ${LIBMAGIC_PREFIX}/include)
+set(LIBMAGIC_LIB_DIR ${LIBMAGIC_PREFIX}/lib)
+set(LIBMAGIC_LIBRARY ${LIBMAGIC_LIB_DIR}/libmagic.a)
+ExternalProject_Add(libmagic
+    PREFIX ${LIBMAGIC_PREFIX}
+    URL ${CMAKE_CURRENT_SOURCE_DIR}/src/3rdparty/file-5.16.tar.gz
+    CONFIGURE_COMMAND ./configure --enable-static --disable-shared
+                                  --prefix=${LIBMAGIC_PREFIX}
+                                  --includedir=${LIBMAGIC_INCLUDE_DIR}
+                                  --libdir=${LIBMAGIC_LIB_DIR}
+    BUILD_IN_SOURCE 1
+    ${EXTERNAL_PROJECT_LOG_OPTIONS}
+)
+
 include(FindRequiredPackage)
 
 # Check cache value first to avoid displaying "Found sed" messages everytime
@@ -57,7 +83,6 @@ FindRequiredPackage(BISON)
 FindRequiredPackage(PCAP)
 FindRequiredPackage(OpenSSL)
 FindRequiredPackage(BIND)
-FindRequiredPackage(LibMagic)
 FindRequiredPackage(ZLIB)
 
 if (NOT BinPAC_ROOT_DIR AND
@@ -73,18 +98,12 @@ if (MISSING_PREREQS)
     message(FATAL_ERROR "Configuration aborted due to missing prerequisites")
 endif ()
 
-set(libmagic_req 5.04)
-if ( LibMagic_VERSION VERSION_LESS ${libmagic_req} )
-    message(FATAL_ERROR "libmagic of at least version ${libmagic_req} required "
-                        "(found ${LibMagic_VERSION})")
-endif ()
-
 include_directories(BEFORE
                     ${PCAP_INCLUDE_DIR}
                     ${OpenSSL_INCLUDE_DIR}
                     ${BIND_INCLUDE_DIR}
                     ${BinPAC_INCLUDE_DIR}
-                    ${LibMagic_INCLUDE_DIR}
+                    ${LIBMAGIC_INCLUDE_DIR}
                     ${ZLIB_INCLUDE_DIR}
 )
 
@@ -163,7 +182,7 @@ set(brodeps
     ${PCAP_LIBRARY}
     ${OpenSSL_LIBRARIES}
     ${BIND_LIBRARY}
-    ${LibMagic_LIBRARY}
+    ${LIBMAGIC_LIBRARY}
     ${ZLIB_LIBRARY}
     ${OPTLIBS}
 )

NEWS

@@ -9,9 +9,21 @@ Bro 2.3
 
 [In progress]
 
+Dependencies
+------------
+
+- Bro no longer requires a pre-installed libmagic (because it now
+  ships its own).
+
+- Compiling from source now needs a CMake version >= 2.8.0.
+
 New Functionality
 -----------------
 
+- Support for GRE tunnel decapsulation, including enhanced GRE
+  headers. GRE tunnels are treated just like IP-in-IP tunnels by
+  parsing past the GRE header in between the delivery and payload IP
+  packets.
 
 Changed Functionality
 ---------------------
@@ -22,6 +34,8 @@ Changed Functionality
 - ssl_client_hello() now receives a vector of ciphers, instead of a
   set, to preserve their order.
 
+- Notice::end_suppression() has been removed.
+
 Bro 2.2
 =======
 

VERSION

@@ -1 +1 @@
-2.2-93
+2.2-105

doc/install/install.rst

@@ -29,22 +29,20 @@ before you begin:
     * Libpcap                           (http://www.tcpdump.org)
     * OpenSSL libraries                 (http://www.openssl.org)
     * BIND8 library
-    * Libmagic 5.04 or greater
     * Libz
     * Bash (for BroControl)
     * Python (for BroControl)
 
 To build Bro from source, the following additional dependencies are required:
 
-    * CMake 2.6.3 or greater            (http://www.cmake.org)
+    * CMake 2.8.0 or greater            (http://www.cmake.org)
     * Make
     * C/C++ compiler
     * SWIG                              (http://www.swig.org)
     * Bison (GNU Parser Generator)
     * Flex  (Fast Lexical Analyzer)
     * Libpcap headers                   (http://www.tcpdump.org)
     * OpenSSL headers                   (http://www.openssl.org)
-    * libmagic headers
     * zlib headers
     * Perl
 
@@ -55,13 +53,13 @@ that ``bash`` and ``python`` are in your ``PATH``):
 
   .. console::
 
-     sudo yum install cmake make gcc gcc-c++ flex bison libpcap-devel openssl-devel python-devel swig zlib-devel file-devel
+     sudo yum install cmake make gcc gcc-c++ flex bison libpcap-devel openssl-devel python-devel swig zlib-devel
 
 * DEB/Debian-based Linux:
 
   .. console::
 
-     sudo apt-get install cmake make gcc g++ flex bison libpcap-dev libssl-dev python-dev swig zlib1g-dev libmagic-dev
+     sudo apt-get install cmake make gcc g++ flex bison libpcap-dev libssl-dev python-dev swig zlib1g-dev
 
 * FreeBSD:
 
@@ -78,15 +76,11 @@ that ``bash`` and ``python`` are in your ``PATH``):
   then going through its "Preferences..." -> "Downloads" menus to
   install the "Command Line Tools" component.
 
-  Lion (10.7) and Mountain Lion (10.8) come with all required
-  dependencies except for CMake_, SWIG_, and ``libmagic``.
-
+  OS X comes with all required dependencies except for CMake_ and SWIG_.
   Distributions of these dependencies can likely be obtained from your
   preferred Mac OS X package management system (e.g. MacPorts_, Fink_,
-  or Homebrew_).
-
-  Specifically for MacPorts, the ``cmake``, ``swig``,
-  ``swig-python`` and ``file`` packages provide the required dependencies.
+  or Homebrew_).  Specifically for MacPorts, the ``cmake``, ``swig``,
+  ``swig-python`` and packages provide the required dependencies.
 
 
 Optional Dependencies

scripts/base/frameworks/notice/cluster.bro

@@ -23,7 +23,8 @@ redef Cluster::worker2manager_events += /Notice::cluster_notice/;
 @if ( Cluster::local_node_type() != Cluster::MANAGER )
 event Notice::begin_suppression(n: Notice::Info)
 	{
-	suppressing[n$note, n$identifier] = n;
+	local suppress_until = n$ts + n$suppress_for;
+	suppressing[n$note, n$identifier] = suppress_until;
 	}
 @endif
 

scripts/base/frameworks/notice/main.bro

@@ -242,12 +242,6 @@ export {
 	##    being suppressed.
 	global suppressed: event(n: Notice::Info);
 
-	## This event is generated when a notice stops being suppressed.
-	##
-	## n: The record containing notice data regarding the notice type
-	##    that was being suppressed.
-	global end_suppression: event(n: Notice::Info);
-
 	## Call this function to send a notice in an email.  It is already used
 	## by default with the built in :bro:enum:`Notice::ACTION_EMAIL` and
 	## :bro:enum:`Notice::ACTION_PAGE` actions.
@@ -285,27 +279,22 @@ export {
 }
 
 # This is used as a hack to implement per-item expiration intervals.
-function per_notice_suppression_interval(t: table[Notice::Type, string] of Notice::Info, idx: any): interval
+function per_notice_suppression_interval(t: table[Notice::Type, string] of time, idx: any): interval
 	{
 	local n: Notice::Type;
 	local s: string;
 	[n,s] = idx;
 
-	local suppress_time = t[n,s]$suppress_for - (network_time() - t[n,s]$ts);
+	local suppress_time = t[n,s] - network_time();
 	if ( suppress_time < 0secs )
 		suppress_time = 0secs;
 
-	# If there is no more suppression time left, the notice needs to be sent
-	# to the end_suppression event.
-	if ( suppress_time == 0secs )
-		event Notice::end_suppression(t[n,s]);
-
 	return suppress_time;
 	}
 
 # This is the internally maintained notice suppression table.  It's
 # indexed on the Notice::Type and the $identifier field from the notice.
-global suppressing: table[Type, string] of Notice::Info = {}
+global suppressing: table[Type, string] of time = {}
 		&create_expire=0secs
 		&expire_func=per_notice_suppression_interval;
 
@@ -400,11 +389,22 @@ function email_notice_to(n: Notice::Info, dest: string, extend: bool)
 
 	# First off, finish the headers and include the human readable messages
 	# then leave a blank line after the message.
-	email_text = string_cat(email_text, "\nMessage: ", n$msg);
+	email_text = string_cat(email_text, "\nMessage: ", n$msg, "\n");
+
 	if ( n?$sub )
-		email_text = string_cat(email_text, "\nSub-message: ", n$sub);
+		email_text = string_cat(email_text, "Sub-message: ", n$sub, "\n");
+
+	email_text = string_cat(email_text, "\n");
+
+	# Add information about the file if it exists.
+	if ( n?$file_desc )
+		email_text = string_cat(email_text, "File Description: ", n$file_desc, "\n");
+
+	if ( n?$file_mime_type )
+		email_text = string_cat(email_text, "File MIME Type: ", n$file_mime_type, "\n");
 
-	email_text = string_cat(email_text, "\n\n");
+	if ( n?$file_desc || n?$file_mime_type )
+		email_text = string_cat(email_text, "\n");
 
 	# Next, add information about the connection if it exists.
 	if ( n?$id )
@@ -467,7 +467,8 @@ hook Notice::notice(n: Notice::Info) &priority=-5
 	     [n$note, n$identifier] !in suppressing &&
 	     n$suppress_for != 0secs )
 		{
-		suppressing[n$note, n$identifier] = n;
+		local suppress_until = n$ts + n$suppress_for;
+		suppressing[n$note, n$identifier] = suppress_until;
 		event Notice::begin_suppression(n);
 		}
 	}

scripts/base/init-bare.bro

@@ -3057,6 +3057,9 @@ export {
 	## Toggle whether to do GTPv1 decapsulation.
 	const enable_gtpv1 = T &redef;
 
+	## Toggle whether to do GRE decapsulation.
+	const enable_gre = T &redef;
+
 	## With this option set, the Teredo analysis will first check to see if
 	## other protocol analyzers have confirmed that they think they're
 	## parsing the right protocol and only continue with Teredo tunnel
@@ -3082,7 +3085,8 @@ export {
 	## may work better.
 	const delay_gtp_confirmation = F &redef;
 
-	## How often to cleanup internal state for inactive IP tunnels.
+	## How often to cleanup internal state for inactive IP tunnels
+	## (includes GRE tunnels).
 	const ip_tunnel_timeout = 24hrs &redef;
 } # end export
 module GLOBAL;

scripts/policy/protocols/ssl/validate-certs.bro

@@ -40,6 +40,7 @@ event ssl_established(c: connection) &priority=3
 		{
 		local result = x509_verify(c$ssl$cert, c$ssl$cert_chain, root_certs);
 		c$ssl$validation_status = x509_err2str(result);
+		recently_validated_certs[c$ssl$cert_hash] = c$ssl$validation_status;
 		}
 
 	if ( c$ssl$validation_status != "ok" )

src/CMakeLists.txt

@@ -388,6 +388,9 @@ install(TARGETS bro DESTINATION bin)
 set(BRO_EXE bro
     CACHE STRING "Bro executable binary" FORCE)
 
+# External libmagic project must be built before bro.
+add_dependencies(bro libmagic)
+
 # Target to create all the autogenerated files.
 add_custom_target(generate_outputs_stage1)
 add_dependencies(generate_outputs_stage1 ${bro_ALL_GENERATED_OUTPUTS})