sha1: full core is not necessary to emulate bug

rarfile.py

@@ -2534,117 +2534,58 @@ class Rar3Sha1(object):
 
     _BLK = struct.Struct(b'>16L')
     _BLKx = struct.Struct(b'<16L')
-    _STATE = struct.Struct(b'>5L')
-    _BITS = struct.Struct(b'>Q')
-    _PAD = b'\x80' + (b'\x00' * (block_size - 1))
-    _ZEROS = b'\x00' * block_size
 
-    __slots__ = ('_nbytes', '_state', '_buf', '_rarbug', '_workspace')
+    __slots__ = ('_nbytes', '_md', '_rarbug', '_workspace')
 
     def __init__(self, data=None, rarbug=False):
+        self._md = sha1()
         self._nbytes = 0
-        self._state = [0x67452301, 0xefcdab89, 0x98badcfe, 0x10325476, 0xc3d2e1f0]
-        self._buf = bytearray(self._ZEROS)
         self._rarbug = rarbug
-        self._workspace = [0] * 80
+        self._workspace = [0] * 16
         self.update(data)
 
     def copy(self):
         """Return new instance with same state."""
         s = Rar3Sha1()
+        s._md = self._md.copy()
         s._nbytes = self._nbytes
-        s._state[:] = self._state
-        s._buf[:] = self._buf
         return s
 
     def update(self, data):
         """Process more data."""
         if not data:
             return
 
-        mdata = memoryview(data)
+        self._md.update(data)
 
-        dpos = 0
         bufpos = self._nbytes & 63
+        self._nbytes += len(data)
 
-        # first block must always go via buf to skip bug
-        n = 64 - bufpos
-        if n > len(data):
-            self._buf[bufpos : bufpos + len(data)] = data
-            self._nbytes += len(data)
-            return
-        self._buf[bufpos : 64] = mdata[:n]
-        self._transform(self._buf)
-        dpos = n
-
+        # first block must not trigger bug
+        dpos = self.block_size - bufpos
         while dpos + self.block_size <= len(data):
-            workbuf = self._transform(mdata[dpos:dpos + self.block_size])
             if self._rarbug:
-                self._BLKx.pack_into(data, dpos, *workbuf[-16:])
+                self._corrupt(data, dpos)
             dpos += self.block_size
 
-        if dpos < len(data):
-            self._buf[:len(data) - dpos] = mdata[dpos:]
-
-        self._nbytes += len(data)
-
     def digest(self):
         """Return final state."""
-        bufpos = self._nbytes & 63
-        pad_len = self.block_size - 8 - bufpos
-        if pad_len <= 0:
-            pad_len += self.block_size
-        s = self.copy()
-        s.update(self._PAD[:pad_len] + self._BITS.pack(self._nbytes * 8))
-        return self._STATE.pack(*s._state)
+        return self._md.digest()
 
     def hexdigest(self):
         """Return final state as hex string."""
-        return tohex(self.digest())
+        return self._md.hexdigest()
 
-    def _transform(self, data):
-        """SHA1 core."""
+    def _corrupt(self, data, dpos):
+        """Corruption from SHA1 core."""
         ws = self._workspace
-        ws[:16] = self._BLK.unpack(data)
-        a, b, c, d, e = self._state
-        t = 0
-
-        while t < 20:
-            if t >= 16:
-                tmp = ws[t - 3] ^ ws[t - 8] ^ ws[t - 14] ^ ws[t - 16]
-                ws[t] = ((tmp << 1) | (tmp >> (32 - 1))) & 0xFFFFFFFF
-            tmp = ((((a << 5) | (a >> (32 - 5))) & 0xFFFFFFFF) + (d ^ (b & (c ^ d))) + e + ws[t] + 0x5a827999) & 0xFFFFFFFF
-            e = d; d = c; c = ((b << 30) | (b >> (32 - 30))) & 0xFFFFFFFF; b = a; a = tmp
-            t += 1
-
-        while t < 40:
-            tmp = ws[t - 3] ^ ws[t - 8] ^ ws[t - 14] ^ ws[t - 16]
-            ws[t] = ((tmp << 1) | (tmp >> (32 - 1))) & 0xFFFFFFFF
-            tmp = ((((a << 5) | (a >> (32 - 5))) & 0xFFFFFFFF) + (b ^ c ^ d) + e + ws[t] + 0x6ed9eba1) & 0xFFFFFFFF
-            e = d; d = c; c = ((b << 30) | (b >> (32 - 30))) & 0xFFFFFFFF; b = a; a = tmp
-            t += 1
-
-        while t < 60:
-            tmp = ws[t - 3] ^ ws[t - 8] ^ ws[t - 14] ^ ws[t - 16]
-            ws[t] = ((tmp << 1) | (tmp >> (32 - 1))) & 0xFFFFFFFF
-            tmp = ((((a << 5) | (a >> (32 - 5))) & 0xFFFFFFFF) + ((b & c) | (b & d) | (c & d)) + e + ws[t] + 0x8f1bbcdc) & 0xFFFFFFFF
-            e = d; d = c; c = ((b << 30) | (b >> (32 - 30))) & 0xFFFFFFFF; b = a; a = tmp
-            t += 1
-
+        ws[:] = self._BLK.unpack_from(data, dpos)
+        t = 16
         while t < 80:
-            tmp = ws[t - 3] ^ ws[t - 8] ^ ws[t - 14] ^ ws[t - 16]
-            ws[t] = ((tmp << 1) | (tmp >> (32 - 1))) & 0xFFFFFFFF
-            tmp = ((((a << 5) | (a >> (32 - 5))) & 0xFFFFFFFF) + (b ^ c ^ d) + e + ws[t] + 0xca62c1d6) & 0xFFFFFFFF
-            e = d; d = c; c = ((b << 30) | (b >> (32 - 30))) & 0xFFFFFFFF; b = a; a = tmp
+            tmp = ws[(t - 3) & 15] ^ ws[(t - 8) & 15] ^ ws[(t - 14) & 15] ^ ws[(t - 16) & 15]
+            ws[t & 15] = ((tmp << 1) | (tmp >> (32 - 1))) & 0xFFFFFFFF
             t += 1
-
-        self._state[0] = (self._state[0] + a) & 0xFFFFFFFF
-        self._state[1] = (self._state[1] + b) & 0xFFFFFFFF
-        self._state[2] = (self._state[2] + c) & 0xFFFFFFFF
-        self._state[3] = (self._state[3] + d) & 0xFFFFFFFF
-        self._state[4] = (self._state[4] + e) & 0xFFFFFFFF
-
-        return ws
+        self._BLKx.pack_into(data, dpos, *ws)
 
 
 ##

test/test_hashing.py

@@ -73,6 +73,20 @@ def test_rar3_sha1():
         h2 = Rar3Sha1(data).hexdigest()
         eq_(h1, h2)
 
+    data = bytearray([(i & 255) for i in range(2000)])
+    x1 = hashlib.sha1()
+    x2 = Rar3Sha1()
+    for step in (3, 17, 67, 128, 157):
+        pos = 0
+        while pos < len(data):
+            pos2 = pos + step
+            if pos2 > len(data):
+                pos2 = len(data)
+            x1.update(data[pos:pos2])
+            x2.update(data[pos:pos2])
+            eq_(x1.hexdigest(), x2.hexdigest())
+            pos = pos2
+
 def test_rar3_s2k():
     exp = ('a160cb31cb262e9231c0b6fc984fbb0d', 'aa54a659fb0c359b30f353a6343fb11d')
     key, iv = rarfile.rar3_s2k(b'password', unhexlify('00FF00'))