Add Fixtures.download_official_distribution to get binaries from apac…

…he.org

test/fixtures.py

@@ -1,9 +1,11 @@
 import logging
 import glob
 import os
+import os.path
 import shutil
 import subprocess
 import tempfile
+import urllib2
 import uuid
 
 from urlparse import urlparse
@@ -17,6 +19,43 @@ class Fixture(object):
     kafka_root = os.environ.get("KAFKA_ROOT", os.path.join(project_root, 'servers', kafka_version, "kafka-src"))
     ivy_root = os.environ.get('IVY_ROOT', os.path.expanduser("~/.ivy2/cache"))
 
+    @classmethod
+    def download_official_distribution(cls,
+                                       kafka_version=None,
+                                       scala_version=None,
+                                       output_dir=None):
+        if not kafka_version:
+            kafka_version = cls.kafka_version
+        if not scala_version:
+            scala_version = cls.scala_version
+        if not output_dir:
+            output_dir = os.path.join(cls.project_root, 'servers', 'dist')
+
+        distfile = 'kafka_%s-%s' % (scala_version, kafka_version,)
+        url_base = 'https://archive.apache.org/dist/kafka/%s/' % (kafka_version,)
+        output_file = os.path.join(output_dir, distfile + '.tgz')
+
+        if os.path.isfile(output_file):
+            logging.info("Found file already on disk: %s" % output_file)
+            return output_file
+
+        # New tarballs are .tgz, older ones are sometimes .tar.gz
+        try:
+            url = url_base + distfile + '.tgz'
+            logging.info("Attempting to download %s" % (url,))
+            response = urllib2.urlopen(url)
+        except urllib2.HTTPError:
+            logging.exception("HTTP Error")
+            url = url_base + distfile + '.tar.gz'
+            logging.info("Attempting to download %s" % (url,))
+            response = urllib2.urlopen(url)
+
+        logging.info("Saving distribution file to %s" % (output_file,))
+        with open(os.path.join(output_dir, distfile + '.tgz'), 'w') as f:
+            f.write(response.read())
+
+        return output_file
+
     @classmethod
     def test_resource(cls, filename):
         return os.path.join(cls.project_root, "servers", cls.kafka_version, "resources", filename)