Support for excluding files

bakthat/__init__.py

@@ -8,8 +8,10 @@
 import hashlib
 import socket
 import re
+import fnmatch
 import mimetypes
 import calendar
+import functools
 from contextlib import closing  # for Python2.6 compatibility
 from gzip import GzipFile
 
@@ -20,7 +22,7 @@
 from byteformat import ByteFormatter
 
 from bakthat.backends import GlacierBackend, S3Backend, RotationConfig, SwiftBackend
-from bakthat.conf import config, load_config, DEFAULT_DESTINATION, DEFAULT_LOCATION, CONFIG_FILE
+from bakthat.conf import config, load_config, DEFAULT_DESTINATION, DEFAULT_LOCATION, CONFIG_FILE, EXCLUDE_FILES
 from bakthat.utils import _interval_string_to_seconds
 from bakthat.models import Backups
 from bakthat.sync import BakSyncer, bakmanager_hook, bakmanager_periodic_backups
@@ -169,6 +171,28 @@ def rotate_backups(filename, destination=None, profile="default", config=CONFIG_
     return deleted
 
 
+def _get_exclude(exclude_file):
+    """ Load a .gitignore like file to exclude files/dir from backups.
+
+    :type exclude_file: str
+    :param exclude_file: Path to the exclude file
+
+    :rtype: function
+    :return: A function ready to inject in tar.add(exlude=_exclude)
+
+    """
+    patterns = filter(None, open(exclude_file).read().split("\n"))
+
+    def _exclude(filename):
+        for pattern in patterns:
+            if re.search(fnmatch.translate(pattern), filename):
+                log.debug("{0} excluded".format(filename))
+                print "{0} excluded".format(filename)
+                return True
+        return False
+    return _exclude
+
+
 @app.cmd(help="Backup a file or a directory, backup the current directory if no arg is provided.")
 @app.cmd_arg('filename', type=str, default=os.getcwd(), nargs="?")
 @app.cmd_arg('-d', '--destination', type=str, help="s3|glacier|swift", default=None)
@@ -177,7 +201,8 @@ def rotate_backups(filename, destination=None, profile="default", config=CONFIG_
 @app.cmd_arg('-p', '--profile', type=str, default="default", help="profile name (default by default)")
 @app.cmd_arg('-c', '--config', type=str, default=CONFIG_FILE, help="path to config file")
 @app.cmd_arg('-k', '--key', type=str, default=None, help="Custom key for periodic backups (works only with BakManager.io hook.)")
-def backup(filename=os.getcwd(), destination=None, prompt="yes", tags=[], profile="default", config=CONFIG_FILE, key=None, **kwargs):
+@app.cmd_arg('--exclude-file', type=str, default=None)
+def backup(filename=os.getcwd(), destination=None, prompt="yes", tags=[], profile="default", config=CONFIG_FILE, key=None, exclude_file=None, **kwargs):
     """Perform backup.
 
     :type filename: str
@@ -219,6 +244,17 @@ def backup(filename=os.getcwd(), destination=None, prompt="yes", tags=[], profil
     if not compress:
         backup_file_fmt = "{0}.{1}"
 
+    if exclude_file and os.path.isfile(exclude_file):
+        EXCLUDE_FILES.insert(0, exclude_file)
+
+    _exclude = lambda filename: False
+    if os.path.isdir(filename):
+        join = functools.partial(os.path.join, filename)
+        for efile in EXCLUDE_FILES:
+            efile = join(efile)
+            if os.path.isfile(efile):
+                _exclude = _get_exclude(efile)
+
     log.info("Backing up " + filename)
     arcname = filename.strip('/').split('/')[-1]
     now = datetime.utcnow()
@@ -267,9 +303,10 @@ def backup(filename=os.getcwd(), destination=None, prompt="yes", tags=[], profil
     else:
         # If not we compress it
         log.info("Compressing...")
+
         with tempfile.NamedTemporaryFile(delete=False) as out:
             with closing(tarfile.open(fileobj=out, mode="w:gz")) as tar:
-                tar.add(filename, arcname=arcname)
+                tar.add(filename, arcname=arcname, exclude=_exclude)
             outname = out.name
             out.seek(0)
             backup_data["size"] = os.fstat(out.fileno()).st_size

bakthat/conf.py

@@ -11,6 +11,8 @@
 DEFAULT_LOCATION = "us-east-1"
 DEFAULT_DESTINATION = "s3"
 
+EXCLUDE_FILES = [".bakthatexclude", ".gitignore"]
+
 
 def load_config(config_file=CONFIG_FILE):
     """ Try to load a yaml config file. """

docs/user_guide.rst

@@ -94,13 +94,38 @@ If you don't specify a filename/dirname, bakthat will backup the current working
     $ bakthat backup myfile --prompt no
 
 
-.. note::
+Excluding files
+~~~~~~~~~~~~~~~
+
+.. versionadded:: 0.5.5
+
+Bakthat use a ".gitignore style" way to exclude files using Unix shell-style wildcards.
+
 
-    You can change the temp directory location by setting the TMPDIR, TEMP or TMP environment variables if the backup is too big to fit in the default temp directory.
+There is two way to exclude file:
+- by creating a **.bakthatexclude** file at the root of the directory you want to backup.
+- by specifying a file directly with the ``--exclude-file`` argument.
 
-    ::
+By default when performing a backup, if no exclude file is specified, it will look for either a **.bakthatexclude** file or a **.gitignore** file. So you backup a git repository, it will use the existing .gitignore if available. 
+
+Here is an example **.bakthatexclude** file, wich exlude all *.pyc and *.log files, and both tmp and cache directory.
+
+::
+
+    *.pyc
+    *.log
+    tmp
+    cache
+
+
+Temp directory
+~~~~~~~~~~~~~~
+
+You can change the temp directory location by setting the TMPDIR, TEMP or TMP environment variables if the backup is too big to fit in the default temp directory.
+
+::
 
-        $ export TMP=/home/thomas
+    $ export TMP=/home/thomas
 
 
 Restore