Merge branch '0.10' into f2b-perfom-prepare-716-cs-0.10

ChangeLog

@@ -72,14 +72,33 @@ ver. 0.9.5 (2016/XX/XXX) - wanna-be-released
      - failregex of previous monit version merged as single expression.
    * filter.d/postfix.conf, filter.d/postfix-sasl.conf
      - extended failregex daemon part, matching also `postfix/smtps/smtpd` now (gh-1391)
+   * fixed a grave bug within tags substitutions because of incorrect detection of recursion 
+     in case of multiple inline substitutions of the same tag (affected actions: `bsd-ipfw`, etc). 
+     Now tracks the actual list of the already substituted tags (per tag instead of single list)
+   * filter.d/common.conf
+     - unexpected extra regex-space in generic `__prefix_line` (gh-1405)
+     - all optional spaces normalized in `common.conf`, test covered now
+     - generic `__prefix_line` extended with optional brackets for the date ambit (gh-1421),
+       added new parameter `__date_ambit`
+   * gentoo-initd fixed --pidfile bug: `--pidfile` is option of start-stop-daemon, 
+     not argument of fail2ban (see gh-1434)
+   * filter.d/asterisk.conf - fix security log support for PJSIP and Asterisk 13+
 
 - New Features:
    * New Actions:
      - action.d/firewallcmd-rich-rules and action.d/firewallcmd-rich-logging (gh-1367)
+
 - Enhancements:
+   * Extreme speedup of all sqlite database operations (gh-1436),
+     by using of following sqlite options:
+     - (synchronous = OFF) write data through OS without syncing
+     - (journal_mode = MEMORY) use memory for the transaction logging
+     - (temp_store = MEMORY) temporary tables and indices are kept in memory
    * journald journalmatch for pure-ftpd (gh-1362)
    * Add additional regex filter for dovecot ldap authentication failures (gh-1370)
-   * added additional regex filters for exim (gh-1371)
+   * filter.d/exim*conf
+     - added additional regexes (gh-1371)
+     - made port entry optional
 
 
 ver. 0.9.4 (2016/03/08) - for-you-ladies

MANIFEST

@@ -167,7 +167,6 @@ fail2ban/client/jailreader.py
 fail2ban/client/jailsreader.py
 fail2ban/exceptions.py
 fail2ban/helpers.py
-fail2ban/ipdns.py
 fail2ban/__init__.py
 fail2ban/protocol.py
 fail2ban/server/action.py
@@ -185,6 +184,7 @@ fail2ban/server/filter.py
 fail2ban/server/filterpyinotify.py
 fail2ban/server/filtersystemd.py
 fail2ban/server/__init__.py
+fail2ban/server/ipdns.py
 fail2ban/server/iso8601.py
 fail2ban/server/jail.py
 fail2ban/server/jails.py

bin/fail2ban-client

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
 # vi: set ft=python sts=4 ts=4 sw=4 noet :
 

bin/fail2ban-regex

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
 # vi: set ft=python sts=4 ts=4 sw=4 noet :
 #

bin/fail2ban-server

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
 # vi: set ft=python sts=4 ts=4 sw=4 noet :
 

bin/fail2ban-testcases

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
 # vi: set ft=python sts=4 ts=4 sw=4 noet :
 """Script to run Fail2Ban tests battery

config/action.d/badips.py

@@ -80,14 +80,17 @@ class BadIPsAction(ActionBase):
 		If invalid `category`, `score`, `banaction` or `updateperiod`.
 	"""
 
+	TIMEOUT = 10
 	_badips = "http://www.badips.com"
 	def _Request(self, url, **argv):
 		return Request(url, headers={'User-Agent': self.agent}, **argv)
 
 	def __init__(self, jail, name, category, score=3, age="24h", key=None,
-		banaction=None, bancategory=None, bankey=None, updateperiod=900, agent="Fail2Ban"):
+		banaction=None, bancategory=None, bankey=None, updateperiod=900, agent="Fail2Ban", 
+		timeout=TIMEOUT):
 		super(BadIPsAction, self).__init__(jail, name)
 
+		self.timeout = timeout
 		self.agent = agent
 		self.category = category
 		self.score = score
@@ -119,7 +122,7 @@ def getCategories(self, incParents=False):
 		"""
 		try:
 			response = urlopen(
-				self._Request("/".join([self._badips, "get", "categories"])), None, 3)
+				self._Request("/".join([self._badips, "get", "categories"])), timeout=self.timeout)
 		except HTTPError as response:
 			messages = json.loads(response.read().decode('utf-8'))
 			self._logSys.error(
@@ -173,7 +176,7 @@ def getList(self, category, score, age, key=None):
 				urlencode({'age': age})])
 			if key:
 				url = "&".join([url, urlencode({'key': key})])
-			response = urlopen(self._Request(url))
+			response = urlopen(self._Request(url), timeout=self.timeout)
 		except HTTPError as response:
 			messages = json.loads(response.read().decode('utf-8'))
 			self._logSys.error(
@@ -358,7 +361,7 @@ def ban(self, aInfo):
 			url = "/".join([self._badips, "add", self.category, aInfo['ip']])
 			if self.key:
 				url = "?".join([url, urlencode({'key': self.key})])
-			response = urlopen(self._Request(url))
+			response = urlopen(self._Request(url), timeout=self.timeout)
 		except HTTPError as response:
 			messages = json.loads(response.read().decode('utf-8'))
 			self._logSys.error(

config/action.d/firewallcmd-allports.conf

@@ -6,34 +6,26 @@
 
 [INCLUDES]
 
-before = iptables-common.conf
+before = firewallcmd-common.conf
 
 [Definition]
 
-actionstart = firewall-cmd --direct --add-chain ipv4 filter f2b-<name>
-              firewall-cmd --direct --add-rule ipv4 filter f2b-<name> 1000 -j RETURN
-              firewall-cmd --direct --add-rule ipv4 filter <chain> 0 -j f2b-<name>
+actionstart = firewall-cmd --direct --add-chain <family> filter f2b-<name>
+              firewall-cmd --direct --add-rule <family> filter f2b-<name> 1000 -j RETURN
+              firewall-cmd --direct --add-rule <family> filter <chain> 0 -j f2b-<name>
 
-actionstop = firewall-cmd --direct --remove-rule ipv4 filter <chain> 0 -j f2b-<name>
-             firewall-cmd --direct --remove-rules ipv4 filter f2b-<name>
-             firewall-cmd --direct --remove-chain ipv4 filter f2b-<name>
+actionstop = firewall-cmd --direct --remove-rule <family> filter <chain> 0 -j f2b-<name>
+             firewall-cmd --direct --remove-rules <family> filter f2b-<name>
+             firewall-cmd --direct --remove-chain <family> filter f2b-<name>
 
 
 # Example actioncheck: firewall-cmd --direct --get-chains ipv4 filter | sed -e 's, ,\n,g' | grep -q '^f2b-recidive$'
 
-actioncheck = firewall-cmd --direct --get-chains ipv4 filter | sed -e 's, ,\n,g' | grep -q '^f2b-<name>$'
+actioncheck = firewall-cmd --direct --get-chains <family> filter | sed -e 's, ,\n,g' | grep -q '^f2b-<name>$'
 
-actionban = firewall-cmd --direct --add-rule ipv4 filter f2b-<name> 0 -s <ip> -j <blocktype>
+actionban = firewall-cmd --direct --add-rule <family> filter f2b-<name> 0 -s <ip> -j <blocktype>
 
-actionunban = firewall-cmd --direct --remove-rule ipv4 filter f2b-<name> 0 -s <ip> -j <blocktype>
-
-[Init]
-
-# Default name of the chain
-#
-name = default
-
-chain = INPUT_direct
+actionunban = firewall-cmd --direct --remove-rule <family> filter f2b-<name> 0 -s <ip> -j <blocktype>
 
 # DEV NOTES:
 #

config/action.d/firewallcmd-common.conf

@@ -0,0 +1,76 @@
+# Fail2Ban configuration file
+#
+# Author: Donald Yandt
+#
+
+[Init]
+
+# Option:  name
+# Notes    Default name of the chain
+# Values:  STRING
+name = default
+
+# Option   port
+# Notes    Can also use port numbers separated by a comma and in rich-rules comma and/or space.
+# Value    STRING Default: 1:65535
+port = 1:65535
+
+# Option:  protocol
+# Notes    [ tcp | udp | icmp | all ]
+# Values:  STRING Default: tcp
+protocol = tcp
+
+# Option:  family(ipv4)
+# Notes    specifies the socket address family type
+# Values:  STRING  
+family = ipv4
+
+# Option:  chain
+# Notes    specifies the firewalld chain to which the Fail2Ban rules should be
+#          added
+# Values:  STRING  Default: INPUT_direct
+chain = INPUT_direct
+
+# Option:  zone
+# Notes    use command firewall-cmd --get-active-zones to see a list of all active zones. See firewalld man pages for more information on zones
+# Values:  STRING  Default: public
+zone = public
+
+# Option:  service
+# Notes    use command firewall-cmd --get-services to see a list of services available
+#          Examples services: amanda-client amanda-k5-client bacula bacula-client dhcp dhcpv6 dhcpv6-client dns freeipa-ldap freeipa-ldaps 
+#          freeipa-replication ftp high-availability http https imaps ipp ipp-client ipsec iscsi-target kadmin kerberos 
+#          kpasswd ldap ldaps libvirt libvirt-tls mdns mosh mountd ms-wbt mysql nfs ntp openvpn pmcd pmproxy pmwebapi pmwebapis pop3s 
+#          postgresql privoxy proxy-dhcp puppetmaster radius rpc-bind rsyncd samba samba-client sane smtp squid ssh synergy 
+#          telnet tftp tftp-client tinc tor-socks transmission-client vdsm vnc-server wbem-https xmpp-bosh xmpp-client xmpp-local xmpp-server
+# Values:  STRING Default: ssh 
+service = ssh
+
+# Option:  rejecttype (ipv4)
+# Notes    See iptables/firewalld man pages for ipv4 reject types. 
+# Values:  STRING
+rejecttype = icmp-port-unreachable
+
+# Option:  blocktype (ipv4/ipv6)
+# Notes    See iptables/firewalld man pages for jump targets. Common values are REJECT, 
+#          REJECT --reject-with icmp-port-unreachable, DROP
+# Values:  STRING
+blocktype = REJECT --reject-with <rejecttype>
+
+# Option:  rich-blocktype (ipv4/ipv6)
+# Notes    See firewalld man pages for jump targets. Common values are reject, 
+#          reject type="icmp-port-unreachable", drop
+# Values:  STRING
+rich-blocktype = reject type='<rejecttype>'
+
+[Init?family=inet6]
+
+# Option:  family(ipv6)
+# Notes    specifies the socket address family type
+# Values:  STRING  
+family = ipv6
+
+# Option:  rejecttype (ipv6)
+# Note:    See iptables/firewalld man pages for ipv6 reject types. 
+# Values:  STRING
+rejecttype = icmp6-port-unreachable

config/action.d/firewallcmd-ipset.conf

@@ -14,20 +14,20 @@
 
 [INCLUDES]
 
-before = iptables-common.conf
+before = firewallcmd-common.conf
 
 [Definition]
 
-actionstart = ipset create fail2ban-<name> hash:ip timeout <bantime>
-              firewall-cmd --direct --add-rule ipv4 filter <chain> 0 -p <protocol> -m multiport --dports <port> -m set --match-set fail2ban-<name> src -j <blocktype>
+actionstart = ipset create <ipmset> hash:ip timeout <bantime>
+              firewall-cmd --direct --add-rule <family> filter <chain> 0 -p <protocol> -m multiport --dports <port> -m set --match-set <ipmset> src -j <blocktype>
 
-actionstop = firewall-cmd --direct --remove-rule ipv4 filter <chain> 0 -p <protocol> -m multiport --dports <port> -m set --match-set fail2ban-<name> src -j <blocktype>
-             ipset flush fail2ban-<name>
-             ipset destroy fail2ban-<name>
+actionstop = firewall-cmd --direct --remove-rule <family> filter <chain> 0 -p <protocol> -m multiport --dports <port> -m set --match-set <ipmset> src -j <blocktype>
+             ipset flush <ipmset>
+             ipset destroy <ipmset>
 
-actionban = ipset add fail2ban-<name> <ip> timeout <bantime> -exist
+actionban = ipset add <ipmset> <ip> timeout <bantime> -exist
 
-actionunban = ipset del fail2ban-<name> <ip> -exist
+actionunban = ipset del <ipmset> <ip> -exist
 
 [Init]
 
@@ -44,6 +44,12 @@ chain = INPUT_direct
 
 bantime = 600
 
+ipmset = f2b-<name>
+
+[Init?family=inet6]
+
+ipmset = f2b-<name>6
+
 
 # DEV NOTES:
 #

config/action.d/firewallcmd-multiport.conf

@@ -5,59 +5,22 @@
 
 [INCLUDES]
 
-before = iptables-common.conf
+before = firewallcmd-common.conf
 
 [Definition]
 
-actionstart = firewall-cmd --direct --add-chain ipv4 filter f2b-<name>
-              firewall-cmd --direct --add-rule ipv4 filter f2b-<name> 1000 -j RETURN
-              firewall-cmd --direct --add-rule ipv4 filter <chain> 0 -m conntrack --ctstate NEW -p <protocol> -m multiport --dports <port> -j f2b-<name>
+actionstart = firewall-cmd --direct --add-chain <family> filter f2b-<name>
+              firewall-cmd --direct --add-rule <family> filter f2b-<name> 1000 -j RETURN
+              firewall-cmd --direct --add-rule <family> filter <chain> 0 -m conntrack --ctstate NEW -p <protocol> -m multiport --dports <port> -j f2b-<name>
 
-actionstop = firewall-cmd --direct --remove-rule ipv4 filter <chain> 0 -m conntrack --ctstate NEW -p <protocol> -m multiport --dports <port> -j f2b-<name>
-             firewall-cmd --direct --remove-rules ipv4 filter f2b-<name>
-             firewall-cmd --direct --remove-chain ipv4 filter f2b-<name>
+actionstop = firewall-cmd --direct --remove-rule <family> filter <chain> 0 -m conntrack --ctstate NEW -p <protocol> -m multiport --dports <port> -j f2b-<name>
+             firewall-cmd --direct --remove-rules <family> filter f2b-<name>
+             firewall-cmd --direct --remove-chain <family> filter f2b-<name>
 
 # Example actioncheck: firewall-cmd --direct --get-chains ipv4 filter | sed -e 's, ,\n,g' | grep -q '^f2b-apache-modsecurity$'
 
-actioncheck = firewall-cmd --direct --get-chains ipv4 filter | sed -e 's, ,\n,g' | grep -q '^f2b-<name>$'
+actioncheck = firewall-cmd --direct --get-chains <family> filter | sed -e 's, ,\n,g' | grep -q '^f2b-<name>$'
 
-actionban = firewall-cmd --direct --add-rule ipv4 filter f2b-<name> 0 -s <ip> -j <blocktype>
-
-actionunban = firewall-cmd --direct --remove-rule ipv4 filter f2b-<name> 0 -s <ip> -j <blocktype>
-
-[Init]
-
-# Default name of the chain
-name = default
-
-chain = INPUT_direct
-
-# Could also use port numbers separated by a comma. 
-port = 1:65535
-
-
-# Option:  protocol
-# Values:  [ tcp | udp | icmp | all ]
-
-protocol = tcp
-
-
-
-# DEV NOTES:
-#
-# Author: Donald Yandt 
-# Uses "FirewallD" instead of the "iptables daemon".
-#
-#
-# Output:
-# actionstart:
-# $ firewall-cmd --direct --add-chain ipv4 filter f2b-apache-modsecurity
-# success
-# $ firewall-cmd --direct --add-rule ipv4 filter f2b-apache-modsecurity 1000 -j RETURN
-# success
-# $ sudo firewall-cmd --direct --add-rule ipv4 filter INPUT_direct 0 -m state --state NEW -p tcp -m multiport --dports 80,443 -j f2b-apache-modsecurity
-# success
-# actioncheck:
-# $ firewall-cmd --direct --get-chains ipv4 filter f2b-apache-modsecurity | sed -e 's, ,\n,g' | grep -q '^f2b-apache-modsecurity$'
-# f2b-apache-modsecurity
+actionban = firewall-cmd --direct --add-rule <family> filter f2b-<name> 0 -s <ip> -j <blocktype>
 
+actionunban = firewall-cmd --direct --remove-rule <family> filter f2b-<name> 0 -s <ip> -j <blocktype>

config/action.d/firewallcmd-new.conf

@@ -4,32 +4,23 @@
 
 [INCLUDES]
 
-before = iptables-common.conf
+before = firewallcmd-common.conf
 
 [Definition]
 
-actionstart = firewall-cmd --direct --add-chain ipv4 filter f2b-<name>
-              firewall-cmd --direct --add-rule ipv4 filter f2b-<name> 1000 -j RETURN
-              firewall-cmd --direct --add-rule ipv4 filter <chain> 0 -m state --state NEW -p <protocol> -m multiport --dports <port> -j f2b-<name>
+actionstart = firewall-cmd --direct --add-chain <family> filter f2b-<name>
+              firewall-cmd --direct --add-rule <family> filter f2b-<name> 1000 -j RETURN
+              firewall-cmd --direct --add-rule <family> filter <chain> 0 -m state --state NEW -p <protocol> -m multiport --dports <port> -j f2b-<name>
 
-actionstop = firewall-cmd --direct --remove-rule ipv4 filter <chain> 0 -m state --state NEW -p <protocol> -m multiport --dports <port> -j f2b-<name>
-             firewall-cmd --direct --remove-rules ipv4 filter f2b-<name>
-             firewall-cmd --direct --remove-chain ipv4 filter f2b-<name>
+actionstop = firewall-cmd --direct --remove-rule <family> filter <chain> 0 -m state --state NEW -p <protocol> -m multiport --dports <port> -j f2b-<name>
+             firewall-cmd --direct --remove-rules <family> filter f2b-<name>
+             firewall-cmd --direct --remove-chain <family> filter f2b-<name>
 
-actioncheck = firewall-cmd --direct --get-chains ipv4 filter | grep -q 'f2b-<name>$'
+actioncheck = firewall-cmd --direct --get-chains <family> filter | grep -q 'f2b-<name>$'
 
-actionban = firewall-cmd --direct --add-rule ipv4 filter f2b-<name> 0 -s <ip> -j <blocktype>
+actionban = firewall-cmd --direct --add-rule <family> filter f2b-<name> 0 -s <ip> -j <blocktype>
 
-actionunban = firewall-cmd --direct --remove-rule ipv4 filter f2b-<name> 0 -s <ip> -j <blocktype>
-
-[Init]
-
-# Option:  chain
-# Notes    specifies the iptables chain to which the fail2ban rules should be
-#          added
-# Values:  [ STRING ]
-#
-chain = INPUT_direct
+actionunban = firewall-cmd --direct --remove-rule <family> filter f2b-<name> 0 -s <ip> -j <blocktype>
 
 # DEV NOTES:
 #