replay: refactor user-agent exclusion, add .pt exclusion (for now)

William291234 · Sep 24, 2015 · 3d4b29d · 3d4b29d
1 parent 74547ce
commit 3d4b29d
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 2 deletions.
diff --git a/pywb/archivereplayview.py b/pywb/archivereplayview.py
@@ -20,6 +20,8 @@
 WBURL_RX = re.compile('(.*/)([0-9]{1,14})(\w{2}_)?(/https?://.*)')
 EXTRACT_ORIG_LINK = re.compile(r'<([^>]+)>;\s*rel=\"original\"')
 
+NO_GZIP_UAS = ['NCSA_Mosaic']
+
 
 #=============================================================================
 class ReplayHandler(WBHandler):
@@ -78,7 +80,7 @@ def _do_req(self, urls, host, env, skip_hosts):
 
         # disable gzip, as mosaic won't support it!
         # TODO: maybe ungzip later
-        if 'NCSA_Mosaic' in user_agent:
+        if any(exclude in user_agent for exclude in NO_GZIP_UAS):
             headers={'Accept-Encoding': 'identity'}
 
         for url in urls:

diff --git a/pywb/mementoquery.py b/pywb/mementoquery.py
@@ -19,7 +19,7 @@
 from redisclient import redisclient
 
 
-EXCLUDE_LIST = ('http://archive.today/', 'http://archive.is', 'https://archive.today/', 'https://archive.is/')
+EXCLUDE_LIST = ('http://archive.today/', 'http://archive.is', 'https://archive.today/', 'https://archive.is/', 'http://arquivo.pt')
 
 
 #=============================================================================