Merge pull request pypa#1395 from dstufft/decode-egg-info-1.5.X

Decode egg info 1.5.x

pip/req.py

@@ -1,6 +1,7 @@
 from email.parser import FeedParser
 import os
 import imp
+import locale
 import pkg_resources
 import re
 import sys
@@ -35,6 +36,30 @@
 from pip.wheel import move_wheel_files, Wheel, wheel_ext
 
 
+def read_text_file(filename):
+    """Return the contents of *filename*.
+
+    Try to decode the file contents with utf-8, the preffered system encoding
+    (e.g., cp1252 on some Windows machines) and latin1, in that order. Decoding
+    a byte string with latin1 will never raise an error. In the worst case, the
+    returned string will contain some garbage characters.
+
+    """
+    with open(filename, 'rb') as fp:
+        data = fp.read()
+
+    encodings = ['utf-8', locale.getpreferredencoding(False), 'latin1']
+    for enc in encodings:
+        try:
+            data = data.decode(enc)
+        except UnicodeDecodeError:
+            continue
+        break
+
+    assert type(data) != bytes  # Latin1 should have worked.
+    return data
+
+
 class InstallRequirement(object):
 
     def __init__(self, req, comes_from, source_dir=None, editable=False,
@@ -322,9 +347,7 @@ def egg_info_data(self, filename):
         filename = self.egg_info_path(filename)
         if not os.path.exists(filename):
             return None
-        fp = open(filename, 'r')
-        data = fp.read()
-        fp.close()
+        data = read_text_file(filename)
         return data
 
     def egg_info_path(self, filename):

tests/unit/test_req.py

@@ -7,12 +7,12 @@
 import pip.wheel
 
 from pkg_resources import Distribution
-from mock import Mock, patch
+from mock import Mock, patch, mock_open
 from pip.exceptions import PreviousBuildDirError, InvalidWheelFilename, UnsupportedWheel
 from pip.index import PackageFinder
 from pip.log import logger
-from pip.req import (InstallRequirement, RequirementSet, parse_editable,
-                     Requirements, parse_requirements)
+from pip.req import (read_text_file, InstallRequirement, RequirementSet,
+                     parse_editable, Requirements, parse_requirements)
 from tests.lib import assert_raises_regexp
 
 
@@ -53,6 +53,21 @@ def test_no_reuse_existing_build_dir(self, data):
             )
 
 
+@pytest.mark.parametrize(('file_contents', 'expected'), [
+    (b'\xf6\x80', b'\xc3\xb6\xe2\x82\xac'),  # cp1252
+    (b'\xc3\xb6\xe2\x82\xac', b'\xc3\xb6\xe2\x82\xac'),  # utf-8
+    (b'\xc3\xb6\xe2', b'\xc3\x83\xc2\xb6\xc3\xa2'),  # Garbage
+])
+def test_egg_info_data(file_contents, expected):
+    om = mock_open(read_data=file_contents)
+    em = Mock()
+    em.return_value = 'cp1252'
+    with patch('pip.req.open', om, create=True):
+        with patch('locale.getpreferredencoding', em):
+            ret = read_text_file('foo')
+    assert ret == expected.decode('utf-8')
+
+
 class TestInstallRequirement(object):
 
     def test_url_with_query(self):