Skip to content

Commit

Permalink
Added ManifestStaticFilesStorage to staticfiles contrib app.
Browse files Browse the repository at this point in the history 
It uses a static manifest file that is created when running
collectstatic in the JSON format.
  • Loading branch information
@jezdez
jezdez committed Jan 20, 2014
1 parent ee25ea0 commit 8efd20f
Show file tree
Hide file tree
Showing 4 changed files with 304 additions and 152 deletions.
144 changes: 120 additions & 24 deletions django/contrib/staticfiles/storage.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import os
import posixpath
import re
import json

from django.conf import settings
from django.core.cache import (caches, InvalidCacheBackendError,
Expand Down Expand Up @@ -49,7 +50,7 @@ def path(self, name):
return super(StaticFilesStorage, self).path(name)


class CachedFilesMixin(object):
class HashedFilesMixin(object):
default_template = """url("%s")"""
patterns = (
("*.css", (
Expand All @@ -59,13 +60,9 @@ class CachedFilesMixin(object):
)

def __init__(self, *args, **kwargs):
super(CachedFilesMixin, self).__init__(*args, **kwargs)
try:
self.cache = caches['staticfiles']
except InvalidCacheBackendError:
# Use the default backend
self.cache = default_cache
super(HashedFilesMixin, self).__init__(*args, **kwargs)
self._patterns = OrderedDict()
self.hashed_files = {}
for extension, patterns in self.patterns:
for pattern in patterns:
if isinstance(pattern, (tuple, list)):
Expand Down Expand Up @@ -119,9 +116,6 @@ def hashed_name(self, name, content=None):
unparsed_name[2] += '?'
return urlunsplit(unparsed_name)

def cache_key(self, name):
return 'staticfiles:%s' % hashlib.md5(force_bytes(name)).hexdigest()

def url(self, name, force=False):
"""
Returns the real URL in DEBUG mode.
Expand All @@ -133,15 +127,9 @@ def url(self, name, force=False):
if urlsplit(clean_name).path.endswith('/'): # don't hash paths
hashed_name = name
else:
cache_key = self.cache_key(name)
hashed_name = self.cache.get(cache_key)
if hashed_name is None:
hashed_name = self.hashed_name(clean_name).replace('\\', '/')
# set the cache if there was a miss
# (e.g. if cache server goes down)
self.cache.set(cache_key, hashed_name)
hashed_name = self.stored_name(clean_name)

final_url = super(CachedFilesMixin, self).url(hashed_name)
final_url = super(HashedFilesMixin, self).url(hashed_name)

# Special casing for a @font-face hack, like url(myfont.eot?#iefix")
# http://www.fontspring.com/blog/the-new-bulletproof-font-face-syntax
Expand Down Expand Up @@ -220,7 +208,7 @@ def post_process(self, paths, dry_run=False, **options):
return

# where to store the new paths
hashed_paths = {}
hashed_files = OrderedDict()

# build a list of adjustable files
matches = lambda path: matches_patterns(path, self._patterns.keys())
Expand Down Expand Up @@ -261,22 +249,122 @@ def post_process(self, paths, dry_run=False, **options):
# then save the processed result
content_file = ContentFile(force_bytes(content))
saved_name = self._save(hashed_name, content_file)
hashed_name = force_text(saved_name.replace('\\', '/'))
hashed_name = force_text(self.clean_name(saved_name))
processed = True
else:
# or handle the case in which neither processing nor
# a change to the original file happened
if not hashed_file_exists:
processed = True
saved_name = self._save(hashed_name, original_file)
hashed_name = force_text(saved_name.replace('\\', '/'))
hashed_name = force_text(self.clean_name(saved_name))

# and then set the cache accordingly
hashed_paths[self.cache_key(name.replace('\\', '/'))] = hashed_name
hashed_files[self.hash_key(name)] = hashed_name
yield name, hashed_name, processed

# Finally set the cache
self.cache.set_many(hashed_paths)
# Finally store the processed paths
self.hashed_files.update(hashed_files)

def clean_name(self, name):
return name.replace('\\', '/')

def hash_key(self, name):
return name

def stored_name(self, name):
hash_key = self.hash_key(name)
cache_name = self.hashed_files.get(hash_key)
if cache_name is None:
cache_name = self.clean_name(self.hashed_name(name))
# store the hashed name if there was a miss, e.g.
# when the files are still processed
self.hashed_files[hash_key] = cache_name
return cache_name


class ManifestFilesMixin(HashedFilesMixin):
manifest_version = '1.0' # the manifest format standard
manifest_name = 'staticfiles.json'

def __init__(self, *args, **kwargs):
super(ManifestFilesMixin, self).__init__(*args, **kwargs)
self.hashed_files = self.load_manifest()

def read_manifest(self):
try:
with self.open(self.manifest_name) as manifest:
return manifest.read()
except IOError:
return None

def load_manifest(self):
content = self.read_manifest()
if content is None:
return OrderedDict()
try:
stored = json.loads(content, object_pairs_hook=OrderedDict)
except ValueError:
pass
else:
version = stored.get('version', None)
if version == '1.0':
return stored.get('paths', OrderedDict())
raise ValueError("Couldn't load manifest '%s' (version %s)" %
(self.manifest_name, self.manifest_version))

def post_process(self, *args, **kwargs):
all_post_processed = super(ManifestFilesMixin,
self).post_process(*args, **kwargs)
for post_processed in all_post_processed:
yield post_processed
payload = {'paths': self.hashed_files, 'version': self.manifest_version}
if self.exists(self.manifest_name):
self.delete(self.manifest_name)
self._save(self.manifest_name, ContentFile(json.dumps(payload)))


class _MappingCache(object):
"""
A small dict-like wrapper for a given cache backend instance.
"""
def __init__(self, cache):
self.cache = cache

def __setitem__(self, key, value):
self.cache.set(key, value)

def __getitem__(self, key):
value = self.cache.get(key, None)
if value is None:
raise KeyError("Couldn't find a file name '%s'" % key)
return value

def clear(self):
self.cache.clear()

def update(self, data):
self.cache.set_many(data)

def get(self, key, default=None):
try:
return self[key]
except KeyError:
return default


class CachedFilesMixin(HashedFilesMixin):
def __init__(self, *args, **kwargs):
super(CachedFilesMixin, self).__init__(*args, **kwargs)
try:
self.hashed_files = _MappingCache(caches['staticfiles'])
except InvalidCacheBackendError:
# Use the default backend
self.hashed_files = _MappingCache(default_cache)

def hash_key(self, name):
key = hashlib.md5(force_bytes(self.clean_name(name))).hexdigest()
return 'staticfiles:%s' % key


class CachedStaticFilesStorage(CachedFilesMixin, StaticFilesStorage):
Expand All @@ -287,6 +375,14 @@ class CachedStaticFilesStorage(CachedFilesMixin, StaticFilesStorage):
pass


class ManifestStaticFilesStorage(ManifestFilesMixin, StaticFilesStorage):
"""
A static file system storage backend which also saves
hashed copies of the files it saves.
"""
pass


class AppStaticStorage(FileSystemStorage):
"""
A file system storage backend that takes an app module and works
Expand Down
162 changes: 89 additions & 73 deletions docs/ref/contrib/staticfiles.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -210,91 +210,107 @@ StaticFilesStorage

.. class:: storage.StaticFilesStorage

A subclass of the :class:`~django.core.files.storage.FileSystemStorage`
storage backend that uses the :setting:`STATIC_ROOT` setting as the base
file system location and the :setting:`STATIC_URL` setting respectively
as the base URL.
A subclass of the :class:`~django.core.files.storage.FileSystemStorage`
storage backend that uses the :setting:`STATIC_ROOT` setting as the base
file system location and the :setting:`STATIC_URL` setting respectively
as the base URL.

.. method:: post_process(paths, **options)
.. method:: post_process(paths, **options)

This method is called by the :djadmin:`collectstatic` management command
after each run and gets passed the local storages and paths of found
files as a dictionary, as well as the command line options.
This method is called by the :djadmin:`collectstatic` management command
after each run and gets passed the local storages and paths of found
files as a dictionary, as well as the command line options.

The :class:`~django.contrib.staticfiles.storage.CachedStaticFilesStorage`
uses this behind the scenes to replace the paths with their hashed
counterparts and update the cache appropriately.
The :class:`~django.contrib.staticfiles.storage.CachedStaticFilesStorage`
uses this behind the scenes to replace the paths with their hashed
counterparts and update the cache appropriately.

CachedStaticFilesStorage
------------------------
ManifestStaticFilesStorage
--------------------------

.. class:: storage.CachedStaticFilesStorage
.. versionadded:: 1.7

.. class:: storage.ManifestStaticFilesStorage

A subclass of the :class:`~django.contrib.staticfiles.storage.StaticFilesStorage`
storage backend which stores the file names it handles by appending the MD5
hash of the file's content to the filename. For example, the file
``css/styles.css`` would also be saved as ``css/styles.55e7cbb9ba48.css``.

The purpose of this storage is to keep serving the old files in case some
pages still refer to those files, e.g. because they are cached by you or
a 3rd party proxy server. Additionally, it's very helpful if you want to
apply `far future Expires headers`_ to the deployed files to speed up the
load time for subsequent page visits.

The storage backend automatically replaces the paths found in the saved
files matching other saved files with the path of the cached copy (using
the :meth:`~django.contrib.staticfiles.storage.StaticFilesStorage.post_process`
method). The regular expressions used to find those paths
(``django.contrib.staticfiles.storage.HashedFilesMixin.patterns``)
by default covers the `@import`_ rule and `url()`_ statement of `Cascading
Style Sheets`_. For example, the ``'css/styles.css'`` file with the
content

.. code-block:: css+django

@import url("../admin/css/base.css");

would be replaced by calling the :meth:`~django.core.files.storage.Storage.url`
method of the ``ManifestStaticFilesStorage`` storage backend, ultimately
saving a ``'css/styles.55e7cbb9ba48.css'`` file with the following
content:

.. code-block:: css+django

A subclass of the :class:`~django.contrib.staticfiles.storage.StaticFilesStorage`
storage backend which caches the files it saves by appending the MD5 hash
of the file's content to the filename. For example, the file
``css/styles.css`` would also be saved as ``css/styles.55e7cbb9ba48.css``.

The purpose of this storage is to keep serving the old files in case some
pages still refer to those files, e.g. because they are cached by you or
a 3rd party proxy server. Additionally, it's very helpful if you want to
apply `far future Expires headers`_ to the deployed files to speed up the
load time for subsequent page visits.

The storage backend automatically replaces the paths found in the saved
files matching other saved files with the path of the cached copy (using
the :meth:`~django.contrib.staticfiles.storage.StaticFilesStorage.post_process`
method). The regular expressions used to find those paths
(``django.contrib.staticfiles.storage.CachedStaticFilesStorage.cached_patterns``)
by default cover the `@import`_ rule and `url()`_ statement of `Cascading
Style Sheets`_. For example, the ``'css/styles.css'`` file with the
content

.. code-block:: css+django

@import url("../admin/css/base.css");

would be replaced by calling the
:meth:`~django.core.files.storage.Storage.url`
method of the ``CachedStaticFilesStorage`` storage backend, ultimately
saving a ``'css/styles.55e7cbb9ba48.css'`` file with the following
content:

.. code-block:: css+django

@import url("../admin/css/base.27e20196a850.css");

To enable the ``CachedStaticFilesStorage`` you have to make sure the
following requirements are met:

* the :setting:`STATICFILES_STORAGE` setting is set to
``'django.contrib.staticfiles.storage.CachedStaticFilesStorage'``
* the :setting:`DEBUG` setting is set to ``False``
* you use the ``staticfiles`` :ttag:`static<staticfiles-static>` template
tag to refer to your static files in your templates
* you've collected all your static files by using the
:djadmin:`collectstatic` management command

Since creating the MD5 hash can be a performance burden to your website
during runtime, ``staticfiles`` will automatically try to cache the
hashed name for each file path using Django's :doc:`caching
framework</topics/cache>`. If you want to override certain options of the
cache backend the storage uses, simply specify a custom entry in the
:setting:`CACHES` setting named ``'staticfiles'``. It falls back to using
the ``'default'`` cache backend.

.. method:: file_hash(name, content=None)

The method that is used when creating the hashed name of a file.
Needs to return a hash for the given file name and content.
By default it calculates a MD5 hash from the content's chunks as
mentioned above.
@import url("../admin/css/base.27e20196a850.css");

To enable the ``ManifestStaticFilesStorage`` you have to make sure the
following requirements are met:

* the :setting:`STATICFILES_STORAGE` setting is set to
``'django.contrib.staticfiles.storage.ManifestStaticFilesStorage'``
* the :setting:`DEBUG` setting is set to ``False``
* you use the ``staticfiles`` :ttag:`static<staticfiles-static>` template
tag to refer to your static files in your templates
* you've collected all your static files by using the
:djadmin:`collectstatic` management command

Since creating the MD5 hash can be a performance burden to your website
during runtime, ``staticfiles`` will automatically store the mapping with
hashed names for all processed files in a file called ``staticfiles.json``.
This happens once when you run the :djadmin:`collectstatic` management
command.

.. method:: file_hash(name, content=None)

The method that is used when creating the hashed name of a file.
Needs to return a hash for the given file name and content.
By default it calculates a MD5 hash from the content's chunks as
mentioned above. Feel free to override this method to use your own
hashing algorithm.

.. _`far future Expires headers`: http://developer.yahoo.com/performance/rules.html#expires
.. _`@import`: http://www.w3.org/TR/CSS2/cascade.html#at-import
.. _`url()`: http://www.w3.org/TR/CSS2/syndata.html#uri
.. _`Cascading Style Sheets`: http://www.w3.org/Style/CSS/

CachedStaticFilesStorage
------------------------

.. class:: storage.CachedStaticFilesStorage

``CachedStaticFilesStorage`` is a similar class like the
:class:`~django.contrib.staticfiles.storage.ManifestStaticFilesStorage` class
but uses Django's :doc:`caching framework</topics/cache>` for storing the
hashed names of processed files instead of a static manifest file called
``staticfiles.json``. This is mostly useful for situations in which you don't
have accesss to the file system.

If you want to override certain options of the cache backend the storage uses,
simply specify a custom entry in the :setting:`CACHES` setting named
``'staticfiles'``. It falls back to using the ``'default'`` cache backend.

.. currentmodule:: django.contrib.staticfiles.templatetags.staticfiles

Template tags
Expand Down
Loading

0 comments on commit 8efd20f

Please sign in to comment.