Merge pull request RustPython#926 from youknowone/number-hashes

cthulahoops · web-flow · commit 2c747168decf · 2019-05-13T09:02:19.000+01:00
Fix number hashes for small numbers
diff --git a/tests/snippets/dict.py b/tests/snippets/dict.py
@@ -198,3 +198,5 @@ def __eq__(self, other):
 
 w = {1: 1, **x, 2: 2, **y, 3: 3, **z, 4: 4}
 assert w == {1: 1, 'a': 1, 'b': 2, 'c': 3, 2: 2, 'd': 3, 3: 3, 'e': 3, 4: 4}
+
+assert str({True: True, 1.0: 1.0}) == str({True: 1.0})
diff --git a/tests/snippets/floats.py b/tests/snippets/floats.py
@@ -98,6 +98,21 @@
 assert_raises(ValueError, lambda: float('foo'))
 assert_raises(OverflowError, lambda: float(2**10000))
 
+# check eq and hash for small numbers
+
+assert 1.0 == 1
+assert 1.0 == True
+assert 0.0 == 0
+assert 0.0 == False
+assert hash(1.0) == hash(1)
+assert hash(1.0) == hash(True)
+assert hash(0.0) == hash(0)
+assert hash(0.0) == hash(False)
+assert hash(1.0) != hash(1.0000000001)
+
+assert 5.0 in {3, 4, 5}
+assert {-1: 2}[-1.0] == 2
+
 # check that magic methods are implemented for ints and floats
 
 assert 1.0.__add__(1.0) == 2.0
diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs
@@ -303,8 +303,7 @@ fn builtin_hasattr(obj: PyObjectRef, attr: PyStringRef, vm: &VirtualMachine) ->
 
 fn builtin_hash(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult {
     arg_check!(vm, args, required = [(obj, None)]);
-
-    vm.call_method(obj, "__hash__", vec![])
+    vm._hash(obj).and_then(|v| Ok(vm.new_int(v)))
 }
 
 // builtin_help
diff --git a/vm/src/dictdatatype.rs b/vm/src/dictdatatype.rs
@@ -1,8 +1,6 @@
 use crate::obj::objbool;
-use crate::obj::objint;
 use crate::pyobject::{IdProtocol, PyObjectRef, PyResult};
 use crate::vm::VirtualMachine;
-use num_traits::ToPrimitive;
 /// Ordered dictionary implementation.
 /// Inspired by: https://morepypy.blogspot.com/2015/01/faster-more-memory-efficient-and-more.html
 /// And: https://www.youtube.com/watch?v=p33CVV29OG8
@@ -218,8 +216,7 @@ enum LookupResult {
 }
 
 fn calc_hash(vm: &VirtualMachine, key: &PyObjectRef) -> PyResult<usize> {
-    let hash = vm.call_method(key, "__hash__", vec![])?;
-    Ok(objint::get_value(&hash).to_usize().unwrap())
+    Ok(vm._hash(key)? as usize)
 }
 
 /// Invoke __eq__ on two keys
diff --git a/vm/src/lib.rs b/vm/src/lib.rs
@@ -55,6 +55,7 @@ pub mod frame;
 pub mod function;
 pub mod import;
 pub mod obj;
+mod pyhash;
 pub mod pyobject;
 pub mod stdlib;
 mod symboltable;
diff --git a/vm/src/obj/objbytearray.rs b/vm/src/obj/objbytearray.rs
@@ -135,7 +135,7 @@ impl PyByteArrayRef {
     }
 
     #[pymethod(name = "__hash__")]
-    fn hash(self, vm: &VirtualMachine) -> PyResult {
+    fn hash(self, vm: &VirtualMachine) -> PyResult<()> {
         Err(vm.new_type_error("unhashable type: bytearray".to_string()))
     }
 
diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs
@@ -2,6 +2,7 @@ use crate::obj::objint::PyIntRef;
 use crate::obj::objnone::PyNoneRef;
 use crate::obj::objslice::PySliceRef;
 use crate::obj::objtuple::PyTupleRef;
+use crate::pyhash;
 use crate::pyobject::Either;
 use crate::pyobject::PyRef;
 use crate::pyobject::PyValue;
@@ -12,17 +13,12 @@ use core::ops::Range;
 use num_bigint::BigInt;
 
 use crate::function::OptionalArg;
-
-use crate::vm::VirtualMachine;
-
 use crate::pyobject::{PyResult, TypeProtocol};
-
-use crate::obj::objstr::{PyString, PyStringRef};
-use std::collections::hash_map::DefaultHasher;
-use std::hash::{Hash, Hasher};
+use crate::vm::VirtualMachine;
 
 use super::objint;
 use super::objsequence::{is_valid_slice_arg, PySliceableSequence};
+use super::objstr::{PyString, PyStringRef};
 
 use crate::obj::objint::PyInt;
 use num_integer::Integer;
@@ -379,10 +375,8 @@ impl PyByteInner {
         }
     }
 
-    pub fn hash(&self) -> usize {
-        let mut hasher = DefaultHasher::new();
-        self.elements.hash(&mut hasher);
-        hasher.finish() as usize
+    pub fn hash(&self) -> pyhash::PyHash {
+        pyhash::hash_value(&self.elements)
     }
 
     pub fn add(&self, other: PyByteInner) -> Vec<u8> {
diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs
@@ -1,8 +1,8 @@
 use crate::obj::objint::PyIntRef;
-
 use crate::obj::objslice::PySliceRef;
 use crate::obj::objstr::PyStringRef;
 use crate::obj::objtuple::PyTupleRef;
+use crate::pyhash;
 
 use crate::pyobject::Either;
 use crate::vm::VirtualMachine;
@@ -125,7 +125,7 @@ impl PyBytesRef {
     }
 
     #[pymethod(name = "__hash__")]
-    fn hash(self, _vm: &VirtualMachine) -> usize {
+    fn hash(self, _vm: &VirtualMachine) -> pyhash::PyHash {
         self.inner.hash()
     }
 
diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs
@@ -298,7 +298,7 @@ impl PyDictRef {
         Ok(PyDict { entries }.into_ref(vm))
     }
 
-    fn hash(self, vm: &VirtualMachine) -> PyResult {
+    fn hash(self, vm: &VirtualMachine) -> PyResult<()> {
         Err(vm.new_type_error("unhashable type".to_string()))
     }
 
diff --git a/vm/src/obj/objfloat.rs b/vm/src/obj/objfloat.rs
@@ -4,6 +4,7 @@ use super::objstr;
 use super::objtype;
 use crate::function::OptionalArg;
 use crate::obj::objtype::PyClassRef;
+use crate::pyhash;
 use crate::pyobject::{
     IdProtocol, IntoPyObject, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue,
     TypeProtocol,
@@ -429,6 +430,11 @@ impl PyFloat {
         zelf
     }
 
+    #[pymethod(name = "__hash__")]
+    fn hash(&self, _vm: &VirtualMachine) -> pyhash::PyHash {
+        pyhash::hash_float(self.value)
+    }
+
     #[pyproperty(name = "real")]
     fn real(zelf: PyRef<Self>, _vm: &VirtualMachine) -> PyFloatRef {
         zelf
diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs
@@ -1,12 +1,12 @@
 use std::fmt;
-use std::hash::{Hash, Hasher};
 
 use num_bigint::{BigInt, Sign};
 use num_integer::Integer;
 use num_traits::{One, Pow, Signed, ToPrimitive, Zero};
 
 use crate::format::FormatSpec;
 use crate::function::{KwArgs, OptionalArg, PyFuncArgs};
+use crate::pyhash;
 use crate::pyobject::{
     IntoPyObject, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject,
     TypeProtocol,
@@ -400,10 +400,11 @@ impl PyInt {
     }
 
     #[pymethod(name = "__hash__")]
-    fn hash(&self, _vm: &VirtualMachine) -> u64 {
-        let mut hasher = std::collections::hash_map::DefaultHasher::new();
-        self.value.hash(&mut hasher);
-        hasher.finish()
+    pub fn hash(&self, _vm: &VirtualMachine) -> pyhash::PyHash {
+        match self.value.to_i64() {
+            Some(value) => (value % pyhash::MODULUS as i64),
+            None => (&self.value % pyhash::MODULUS).to_i64().unwrap(),
+        }
     }
 
     #[pymethod(name = "__abs__")]
diff --git a/vm/src/obj/objlist.rs b/vm/src/obj/objlist.rs
@@ -387,7 +387,7 @@ impl PyListRef {
         Ok(s)
     }
 
-    fn hash(self, vm: &VirtualMachine) -> PyResult {
+    fn hash(self, vm: &VirtualMachine) -> PyResult<()> {
         Err(vm.new_type_error("unhashable type".to_string()))
     }
 
diff --git a/vm/src/obj/objobject.rs b/vm/src/obj/objobject.rs
@@ -5,6 +5,7 @@ use super::objtype;
 use crate::function::PyFuncArgs;
 use crate::obj::objproperty::PropertyBuilder;
 use crate::obj::objtype::PyClassRef;
+use crate::pyhash;
 use crate::pyobject::{
     IdProtocol, ItemProtocol, PyAttributes, PyContext, PyObject, PyObjectRef, PyResult, PyValue,
     TryFromObject, TypeProtocol,
@@ -55,8 +56,8 @@ fn object_ge(_zelf: PyObjectRef, _other: PyObjectRef, vm: &VirtualMachine) -> Py
     vm.ctx.not_implemented()
 }
 
-fn object_hash(zelf: PyObjectRef, _vm: &VirtualMachine) -> u64 {
-    zelf.get_id() as u64
+fn object_hash(zelf: PyObjectRef, _vm: &VirtualMachine) -> pyhash::PyHash {
+    zelf.get_id() as pyhash::PyHash
 }
 
 fn object_setattr(
diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs
@@ -759,7 +759,7 @@ impl TryFromObject for SetIterable {
     }
 }
 
-fn set_hash(_zelf: PySetRef, vm: &VirtualMachine) -> PyResult {
+fn set_hash(_zelf: PySetRef, vm: &VirtualMachine) -> PyResult<()> {
     Err(vm.new_type_error("unhashable type".to_string()))
 }
 
diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs
@@ -1,7 +1,6 @@
 extern crate unicode_xid;
 
 use std::fmt;
-use std::hash::{Hash, Hasher};
 use std::ops::Range;
 use std::str::FromStr;
 use std::string::ToString;
@@ -13,6 +12,7 @@ use unicode_xid::UnicodeXID;
 
 use crate::format::{FormatParseError, FormatPart, FormatString};
 use crate::function::{single_or_tuple_any, OptionalArg, PyFuncArgs};
+use crate::pyhash;
 use crate::pyobject::{
     IdProtocol, IntoPyObject, ItemProtocol, PyClassImpl, PyContext, PyIterable, PyObjectRef, PyRef,
     PyResult, PyValue, TryFromObject, TryIntoRef, TypeProtocol,
@@ -172,10 +172,8 @@ impl PyString {
     }
 
     #[pymethod(name = "__hash__")]
-    fn hash(&self, _vm: &VirtualMachine) -> usize {
-        let mut hasher = std::collections::hash_map::DefaultHasher::new();
-        self.value.hash(&mut hasher);
-        hasher.finish() as usize
+    fn hash(&self, _vm: &VirtualMachine) -> pyhash::PyHash {
+        pyhash::hash_value(&self.value)
     }
 
     #[pymethod(name = "__len__")]
diff --git a/vm/src/obj/objtuple.rs b/vm/src/obj/objtuple.rs
@@ -1,13 +1,12 @@
 use std::cell::{Cell, RefCell};
 use std::fmt;
-use std::hash::{Hash, Hasher};
 
 use crate::function::OptionalArg;
+use crate::pyhash;
 use crate::pyobject::{IdProtocol, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue};
 use crate::vm::{ReprGuard, VirtualMachine};
 
 use super::objbool;
-use super::objint;
 use super::objiter;
 use super::objsequence::{
     get_elements, get_item, seq_equal, seq_ge, seq_gt, seq_le, seq_lt, seq_mul,
@@ -129,14 +128,8 @@ impl PyTupleRef {
         }
     }
 
-    fn hash(self, vm: &VirtualMachine) -> PyResult<u64> {
-        let mut hasher = std::collections::hash_map::DefaultHasher::new();
-        for element in self.elements.borrow().iter() {
-            let hash_result = vm.call_method(element, "__hash__", vec![])?;
-            let element_hash = objint::get_value(&hash_result);
-            element_hash.hash(&mut hasher);
-        }
-        Ok(hasher.finish())
+    fn hash(self, vm: &VirtualMachine) -> PyResult<pyhash::PyHash> {
+        pyhash::hash_iter(self.elements.borrow().iter(), vm)
     }
 
     fn iter(self, _vm: &VirtualMachine) -> PyTupleIterator {
diff --git a/vm/src/pyhash.rs b/vm/src/pyhash.rs
@@ -0,0 +1,85 @@
+use std::hash::{Hash, Hasher};
+
+use crate::pyobject::PyObjectRef;
+use crate::pyobject::PyResult;
+use crate::vm::VirtualMachine;
+
+pub type PyHash = i64;
+pub type PyUHash = u64;
+
+pub const BITS: usize = 61;
+pub const MODULUS: PyUHash = (1 << BITS) - 1;
+// pub const CUTOFF: usize = 7;
+
+pub const INF: PyHash = 314159;
+pub const NAN: PyHash = 0;
+
+pub fn hash_float(value: f64) -> PyHash {
+    // cpython _Py_HashDouble
+    if !value.is_finite() {
+        return if value.is_infinite() {
+            if value > 0.0 {
+                INF
+            } else {
+                -INF
+            }
+        } else {
+            NAN
+        };
+    }
+
+    let frexp = if 0.0 == value {
+        (value, 0i32)
+    } else {
+        let bits = value.to_bits();
+        let exponent: i32 = ((bits >> 52) & 0x7ff) as i32 - 1022;
+        let mantissa_bits = bits & (0x000fffffffffffff) | (1022 << 52);
+        (f64::from_bits(mantissa_bits), exponent)
+    };
+
+    // process 28 bits at a time;  this should work well both for binary
+    // and hexadecimal floating point.
+    let mut m = frexp.0;
+    let mut e = frexp.1;
+    let mut x: PyUHash = 0;
+    while m != 0.0 {
+        x = ((x << 28) & MODULUS) | x >> (BITS - 28);
+        m *= 268435456.0; // 2**28
+        e -= 28;
+        let y = m as PyUHash; // pull out integer part
+        m -= y as f64;
+        x += y;
+        if x >= MODULUS {
+            x -= MODULUS;
+        }
+    }
+
+    // adjust for the exponent;  first reduce it modulo BITS
+    const BITS32: i32 = BITS as i32;
+    e = if e >= 0 {
+        e % BITS32
+    } else {
+        BITS32 - 1 - ((-1 - e) % BITS32)
+    };
+    x = ((x << e) & MODULUS) | x >> (BITS32 - e);
+
+    x as PyHash * value.signum() as PyHash
+}
+
+pub fn hash_value<T: Hash>(data: &T) -> PyHash {
+    let mut hasher = std::collections::hash_map::DefaultHasher::new();
+    data.hash(&mut hasher);
+    hasher.finish() as PyHash
+}
+
+pub fn hash_iter<'a, I: std::iter::Iterator<Item = &'a PyObjectRef>>(
+    iter: I,
+    vm: &VirtualMachine,
+) -> PyResult<PyHash> {
+    let mut hasher = std::collections::hash_map::DefaultHasher::new();
+    for element in iter {
+        let item_hash = vm._hash(&element)?;
+        item_hash.hash(&mut hasher);
+    }
+    Ok(hasher.finish() as PyHash)
+}
diff --git a/vm/src/vm.rs b/vm/src/vm.rs

Original file line number	Diff line number	Diff line change
`@@ -303,8 +303,7 @@ fn builtin_hasattr(obj: PyObjectRef, attr: PyStringRef, vm: &VirtualMachine) ->`
`303`	`303`
`304`	`304`	`fn builtin_hash(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult {`
`305`	`305`	`arg_check!(vm, args, required = [(obj, None)]);`
`306`		`-`
`307`		`- vm.call_method(obj, "__hash__", vec![])`
	`306`	`+ vm._hash(obj).and_then(\|v\| Ok(vm.new_int(v)))`
`308`	`307`	`}`
`309`	`308`
`310`	`309`	`// builtin_help`
Original file line number	Diff line number	Diff line change
`@@ -135,7 +135,7 @@ impl PyByteArrayRef {`
`135`	`135`	`}`
`136`	`136`
`137`	`137`	`#[pymethod(name = "__hash__")]`
`138`		`- fn hash(self, vm: &VirtualMachine) -> PyResult {`
	`138`	`+ fn hash(self, vm: &VirtualMachine) -> PyResult<()> {`
`139`	`139`	`Err(vm.new_type_error("unhashable type: bytearray".to_string()))`
`140`	`140`	`}`
`141`	`141`
Original file line number	Diff line number	Diff line change
`@@ -298,7 +298,7 @@ impl PyDictRef {`
`298`	`298`	`Ok(PyDict { entries }.into_ref(vm))`
`299`	`299`	`}`
`300`	`300`
`301`		`- fn hash(self, vm: &VirtualMachine) -> PyResult {`
	`301`	`+ fn hash(self, vm: &VirtualMachine) -> PyResult<()> {`
`302`	`302`	`Err(vm.new_type_error("unhashable type".to_string()))`
`303`	`303`	`}`
`304`	`304`