py 2 legacy code cleanup

README.md

@@ -55,8 +55,6 @@ Download is as simple as creating an instance and launching it::
 Requirements
 ==============
 
- * Python 2.6 or greater.
- * Python 3.0 or greater.
- * For versions before Python 3.2, the `futures backport  <https://pypi.python.org/pypi/futures>`_ package is also required.
+ * Python 3.4 or greater.
 
-Copyright (C) 2014-2017 Itay Brandes.
+Copyright (C) 2014-2019 Itay Brandes.

pySmartDL/__init__.py

@@ -1,4 +1,4 @@
 from .pySmartDL import SmartDL, HashFailedException, CanceledException
-import utils
+from . import utils
 
 __version__ = pySmartDL.__version__

pySmartDL/pySmartDL.py

@@ -1,6 +1,6 @@
 import os
 import sys
-import urllib2
+import urllib.request, urllib.error, urllib.parse
 import copy
 import threading
 import time
@@ -9,17 +9,17 @@
 import base64
 import hashlib
 import logging
-from urlparse import urlparse
-from StringIO import StringIO
+from urllib.parse import urlparse
+from io import StringIO
 import multiprocessing.dummy as multiprocessing
 from ctypes import c_int
 
-import utils
+from . import utils
 
 __all__ = ['SmartDL', 'utils']
 __version_mjaor__ = 1
-__version_minor__ = 2
-__version_micro__ = 5
+__version_minor__ = 3
+__version_micro__ = 0
 __version__ = "%d.%d.%d" % (__version_mjaor__, __version_minor__, __version_micro__)
 DEFAULT_LOGGER_CREATED = False
 
@@ -30,9 +30,9 @@ def __init__(self, fn, calc_hash, needed_hash):
         self.calculated_hash = calc_hash
         self.needed_hash = needed_hash
     def __str__(self):
-        return 'HashFailedException(%s, got %s, expected %s)' % (self.filename, self.calculated_hash, self.needed_hash)
+        return 'HashFailedException({}, got {}, expected {})'.format(self.filename, self.calculated_hash, self.needed_hash)
     def __repr__(self):
-        return "<HashFailedException %s, got %s, expected %s>" % (self.filename, self.calculated_hash, self.needed_hash)
+        return '<HashFailedException {}, got {}, expected {}>'.format(self.filename, self.calculated_hash, self.needed_hash)
 
 class CanceledException(Exception):
     "Raised when the job is canceled."
@@ -75,14 +75,12 @@ class SmartDL:
     def __init__(self, urls, dest=None, progress_bar=True, fix_urls=True, threads=5, logger=None, connect_default_logger=False):
         global DEFAULT_LOGGER_CREATED
 
-        self.mirrors = [urls] if isinstance(urls, basestring) else urls
+        self.mirrors = [urls] if isinstance(urls, str) else urls
         if fix_urls:
             self.mirrors = [utils.url_fix(x) for x in self.mirrors]
         self.url = self.mirrors.pop(0)
 
-        fn = urllib2.unquote(os.path.basename(urlparse(self.url).path))
-        if sys.version_info < (3, 0):
-            fn = fn.decode('utf-8')  # required only on python 2
+        fn = urllib.parse.unquote(os.path.basename(urlparse(self.url).path))
         self.dest = dest or os.path.join(tempfile.gettempdir(), 'pySmartDL', fn)
         if self.dest[-1] == os.sep:
             if os.path.exists(self.dest[:-1]) and os.path.isfile(self.dest[:-1]):
@@ -198,7 +196,7 @@ def fetch_hash_sums(self):
         for filename in default_sums_filenames:
             try:
                 sums_url = "%s/%s" % (folder, filename)
-                obj = urllib2.urlopen(sums_url)
+                obj = urllib.request.urlopen(sums_url)
                 data = obj.read().split('\n')
                 obj.close()
 
@@ -210,7 +208,7 @@ def fetch_hash_sums(self):
                         self.add_hash_verification(algo, hash)
                         return
 
-            except urllib2.HTTPError:
+            except urllib.error.HTTPError:
                 continue
 
     def start(self, blocking=None):
@@ -245,18 +243,18 @@ def start(self, blocking=None):
                 return
 
         self.logger.info("Downloading '%s' to '%s'..." % (self.url, self.dest))
-        req = urllib2.Request(self.url, headers=self.headers)
+        req = urllib.request.Request(self.url, headers=self.headers)
         try:
-            urlObj = urllib2.urlopen(req, timeout=self.timeout)
-        except (urllib2.HTTPError, urllib2.URLError), e:
+            urlObj = urllib.request.urlopen(req, timeout=self.timeout)
+        except (urllib.error.HTTPError, urllib.error.URLError) as e:
             self.errors.append(e)
             if self.mirrors:
-                self.logger.info("%s. Trying next mirror..." % unicode(e))
+                self.logger.info("%s. Trying next mirror..." % str(e))
                 self.url = self.mirrors.pop(0)
                 self.start(blocking)
                 return
             else:
-                self.logger.warning(unicode(e))
+                self.logger.warning(str(e))
                 self.errors.append(e)
                 self._failed = True
                 self.status = "finished"
@@ -315,7 +313,7 @@ def retry(self, eStr=""):
             s = 'The maximum retry attempts reached'
             if eStr:
                 s += " (%s)" % eStr
-            self.errors.append(urllib2.HTTPError(self.url, "0", s, {}, StringIO()))
+            self.errors.append(urllib.error.HTTPError(self.url, "0", s, {}, StringIO()))
             self._failed = True
 
     def try_next_mirror(self, e=None):
@@ -413,7 +411,7 @@ def isSuccessful(self):
             n += 1
             time.sleep(0.1)
             if n >= 15:
-                raise RuntimeError("The download task must be finished in order to see if it's successful. (current status is %s)" % self.status)
+                raise RuntimeError("The download task must be finished in order to see if it's successful. (current status is {})".format(self.status))
 
         return not self._failed
 
@@ -580,7 +578,7 @@ def __init__(self, obj):
 
         self.dl_speed = 0
         self.eta = 0
-        self.lastBytesSamples = [] # list with last 50 Bytes Samples.
+        self.lastBytesSamples = []  # list with last 50 Bytes Samples.
         self.last_calculated_totalBytes = 0
         self.calcETA_queue = []
         self.calcETA_i = 0
@@ -604,7 +602,7 @@ def run(self):
                 else:
                     status = r"[*] %s / ??? MB @ %s/s   " % (utils.sizeof_human(self.shared_var.value), utils.sizeof_human(self.dl_speed))
                 status = status + chr(8)*(len(status)+1)
-                print status,
+                print(status, end=' ')
 
             time.sleep(0.1)
 
@@ -614,9 +612,9 @@ def run(self):
 
         if self.progress_bar:
             if self.obj.filesize:
-                print r"[*] %s / %s @ %s/s %s [100%%, 0s left]    " % (utils.sizeof_human(self.obj.filesize), utils.sizeof_human(self.obj.filesize), utils.sizeof_human(self.dl_speed), utils.progress_bar(1.0))
+                print(r"[*] %s / %s @ %s/s %s [100%%, 0s left]    " % (utils.sizeof_human(self.obj.filesize), utils.sizeof_human(self.obj.filesize), utils.sizeof_human(self.dl_speed), utils.progress_bar(1.0)))
             else:
-                print r"[*] %s / %s @ %s/s    " % (utils.sizeof_human(self.shared_var.value), utils.sizeof_human(self.shared_var.value), utils.sizeof_human(self.dl_speed))
+                print(r"[*] %s / %s @ %s/s    " % (utils.sizeof_human(self.shared_var.value), utils.sizeof_human(self.shared_var.value), utils.sizeof_human(self.dl_speed)))
 
         t2 = time.time()
         self.dl_time = float(t2-t1)
@@ -690,8 +688,8 @@ def post_threadpool_actions(pool, args, expected_filesize, SmartDL_obj):
         time.sleep(0.1)
 
     if pool.get_exceptions():
-        SmartDL_obj.logger.warning(unicode(pool.get_exceptions()[0]))
-        SmartDL_obj.retry(unicode(pool.get_exceptions()[0]))
+        SmartDL_obj.logger.warning(str(pool.get_exceptions()[0]))
+        SmartDL_obj.retry(str(pool.get_exceptions()[0]))
 
 
     if SmartDL_obj._killed:
@@ -754,10 +752,10 @@ def download(url, dest, startByte=0, endByte=None, headers=None, timeout=4, shar
         headers['Range'] = 'bytes=%d-%d' % (startByte, endByte)
 
     logger.info("Downloading '%s' to '%s'..." % (url, dest))
-    req = urllib2.Request(url, headers=headers)
+    req = urllib.request.Request(url, headers=headers)
     try:
-        urlObj = urllib2.urlopen(req, timeout=timeout)
-    except urllib2.HTTPError, e:
+        urlObj = urllib.request.urlopen(req, timeout=timeout)
+    except urllib.error.HTTPError as e:
         if e.code == 416:
             '''
             HTTP 416 Error: Requested Range Not Satisfiable. Happens when we ask
@@ -811,8 +809,8 @@ def download(url, dest, startByte=0, endByte=None, headers=None, timeout=4, shar
 
             try:
                 buff = urlObj.read(block_sz)
-            except Exception, e:
-                logger.error(unicode(e))
+            except Exception as e:
+                logger.error(str(e))
                 if shared_var:
                     shared_var.value -= filesize_dl
                 raise

pySmartDL/utils.py

@@ -5,9 +5,9 @@
 
 import os
 import sys
-import urlparse
-import urllib
-import urllib2
+import urllib.parse
+import urllib.request, urllib.parse, urllib.error
+import urllib.request, urllib.error, urllib.parse
 import random
 import logging
 import re
@@ -58,12 +58,10 @@ def url_fix(s, charset='utf-8'):
                     
     (taken from `werkzeug.utils <http://werkzeug.pocoo.org/docs/utils/>`_)
     '''
-    if sys.version_info < (3, 0) and isinstance(s, unicode):
-        s = s.encode(charset, 'ignore')
-    scheme, netloc, path, qs, anchor = urlparse.urlsplit(s)
-    path = urllib.quote(path, '/%')
-    qs = urllib.quote_plus(qs, ':&=')
-    return urlparse.urlunsplit((scheme, netloc, path, qs, anchor))
+    scheme, netloc, path, qs, anchor = urllib.parse.urlsplit(s)
+    path = urllib.parse.quote(path, '/%')
+    qs = urllib.parse.quote_plus(qs, ':&=')
+    return urllib.parse.urlunsplit((scheme, netloc, path, qs, anchor))
 
 def progress_bar(progress, length=20):
     '''
@@ -103,8 +101,8 @@ def is_HTTPRange_supported(url, timeout=15):
         return False
 
     headers = {'Range': 'bytes=0-3'}
-    req = urllib2.Request(url, headers=headers)
-    urlObj = urllib2.urlopen(req, timeout=timeout)
+    req = urllib.request.Request(url, headers=headers)
+    urlObj = urllib.request.urlopen(req, timeout=timeout)
     filesize = int(urlObj.headers["Content-Length"])
 
     urlObj.close()
@@ -123,8 +121,8 @@ def get_filesize(url, timeout=15):
     '''
     # url = url_fix(url)
     try:
-        urlObj = urllib2.urlopen(url, timeout=timeout)
-    except (urllib2.HTTPError, urllib2.URLError) as e:
+        urlObj = urllib.request.urlopen(url, timeout=timeout)
+    except (urllib.error.HTTPError, urllib.error.URLError) as e:
         return 0
     try:
         file_size = int(urlObj.headers["Content-Length"])
@@ -172,7 +170,7 @@ def sizeof_human(num):
     
     :rtype: string
     '''
-    unit_list = zip(['B', 'kB', 'MB', 'GB', 'TB', 'PB'], [0, 0, 1, 2, 2, 2])
+    unit_list = list(zip(['B', 'kB', 'MB', 'GB', 'TB', 'PB'], [0, 0, 1, 2, 2, 2]))
 
     if num > 1:
         exponent = min(int(log(num, 1024)), len(unit_list) - 1)

test/test_pySmartDL.py

@@ -16,20 +16,20 @@
 #
 
 class TestSmartDL(unittest.TestCase):
-    def test_2to3(self):
-        assert True
     def setUp(self):
         self.dl_dir = os.path.join(os.getenv('tmp'), "".join([random.choice(string.ascii_letters+string.digits) for i in range(8)]), '')
         while os.path.exists(self.dl_dir):
             self.dl_dir = os.path.join(os.getenv('tmp'), "".join([random.choice(string.ascii_letters+string.digits) for i in range(8)]), '')
 
-        self.default_7za920_mirrors = [ "http://mirror.ufs.ac.za/7zip/9.20/7za920.zip",
-                                        "http://www.bevc.net/dl/7za920.zip",
-                                        "http://ftp.jaist.ac.jp/pub/sourceforge/s/project/se/sevenzip/7-Zip/9.20/7za920.zip",
-                                        "http://www.mirrorservice.org/sites/downloads.sourceforge.net/s/se/sevenzip/7-Zip/9.20/7za920.zip"]
-    def test_dependencies(self):
-        from concurrent import futures
-        self.assertTrue(sys.version_info >= (2,6))
+        self.default_7za920_mirrors = [
+            "http://mirror.ufs.ac.za/7zip/9.20/7za920.zip",
+            "http://www.bevc.net/dl/7za920.zip",
+            "http://ftp.jaist.ac.jp/pub/sourceforge/s/project/se/sevenzip/7-Zip/9.20/7za920.zip",
+            "http://www.mirrorservice.org/sites/downloads.sourceforge.net/s/se/sevenzip/7-Zip/9.20/7za920.zip"
+        ]
+
+    def set_dependencies(self):
+        self.assertTrue(sys.version_info >= (3, 4))
 
     def test_download(self):
         obj = pySmartDL.SmartDL(self.default_7za920_mirrors, dest=self.dl_dir, progress_bar=False)
@@ -43,7 +43,7 @@ def test_download(self):
             self.assertEqual(data, 'PK')
 
     def test_mirrors(self):
-        urls = ["http://totally_fake_website/7za.zip" ,"http://mirror.ufs.ac.za/7zip/9.20/7za920.zip"]
+        urls = ["http://totally_fake_website/7za.zip", "http://mirror.ufs.ac.za/7zip/9.20/7za920.zip"]
         obj = pySmartDL.SmartDL(urls, dest=self.dl_dir, progress_bar=False)
         obj.start()
 
@@ -96,7 +96,7 @@ def test_basic_auth(self):
         self.assertTrue(json.loads(data)['authenticated'])
 
     def test_unicode(self):
-        url = u"http://he.wikipedia.org/wiki/ג'חנון"
+        url = "http://he.wikipedia.org/wiki/ג'חנון"
         obj = pySmartDL.SmartDL(url, progress_bar=False)
         obj.start()
 
@@ -105,7 +105,4 @@ def test_suite():
     return suite
 
 if __name__ == '__main__':
-    if sys.version_info < (2,7):
-        unittest.main()
-    else:
-        unittest.main(verbosity=2)
+    unittest.main(verbosity=2)