Fix bytes decode/encoding tests

Author: coolreader18

Lib/_codecs.py

@@ -915,7 +915,7 @@ def PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder='native', final=Tru
     p = []
     if byteorder == 'native':
         if (size >= 2):
-            bom = (ord(s[ihi]) << 8) | ord(s[ilo])
+            bom = (s[ihi] << 8) | s[ilo]
 #ifdef BYTEORDER_IS_LITTLE_ENDIAN
             if sys.byteorder == 'little':
                 if (bom == 0xFEFF):
@@ -962,11 +962,11 @@ def PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder='native', final=Tru
 #           /* The remaining input chars are ignored if the callback
 ##             chooses to skip the input */
 
-        ch = (ord(s[q+ihi]) << 8) | ord(s[q+ilo])
+        ch = (s[q+ihi] << 8) | s[q+ilo]
         q += 2
 
         if (ch < 0xD800 or ch > 0xDFFF):
-            p += chr(ch)
+            p.append(chr(ch))
             continue
 
         #/* UTF-16 code pair: */
@@ -977,15 +977,14 @@ def PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder='native', final=Tru
             unicode_call_errorhandler(errors, 'utf-16', errmsg, s, startinpos, endinpos, True)
 
         if (0xD800 <= ch and ch <= 0xDBFF):
-            ch2 = (ord(s[q+ihi]) << 8) | ord(s[q+ilo])
+            ch2 = (s[q+ihi] << 8) | s[q+ilo]
             q += 2
             if (0xDC00 <= ch2 and ch2 <= 0xDFFF):
     #ifndef Py_UNICODE_WIDE
                 if sys.maxunicode < 65536:
-                    p += chr(ch)
-                    p += chr(ch2)
+                    p += [chr(ch), chr(ch2)]
                 else:
-                    p += chr((((ch & 0x3FF)<<10) | (ch2 & 0x3FF)) + 0x10000)
+                    p.append(chr((((ch & 0x3FF)<<10) | (ch2 & 0x3FF)) + 0x10000))
     #endif
                 continue
 

Lib/test/test_bytes.py

@@ -300,8 +300,6 @@ def test_extended_getslice(self):
                 for step in indices[1:]:
                     self.assertEqual(b[start:stop:step], self.type2test(L[start:stop:step]))
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_encoding(self):
         sample = "Hello world\n\u1234\u5678\u9abc"
         for enc in ("utf-8", "utf-16"):
@@ -311,8 +309,6 @@ def test_encoding(self):
         b = self.type2test(sample, "latin-1", "ignore")
         self.assertEqual(b, self.type2test(sample[:-3], "utf-8"))
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_decode(self):
         sample = "Hello world\n\u1234\u5678\u9abc"
         for enc in ("utf-8", "utf-16"):

vm/src/bytesinner.rs

@@ -1290,3 +1290,29 @@ impl<'s> PyCommonString<'s, u8> for [u8] {
         splited
     }
 }
+
+#[derive(FromArgs)]
+pub struct DecodeArgs {
+    #[pyarg(positional_or_keyword, default = "None")]
+    encoding: Option<PyStringRef>,
+    #[pyarg(positional_or_keyword, default = "None")]
+    errors: Option<PyStringRef>,
+}
+
+pub fn bytes_decode(
+    zelf: PyObjectRef,
+    args: DecodeArgs,
+    vm: &VirtualMachine,
+) -> PyResult<PyStringRef> {
+    let DecodeArgs { encoding, errors } = args;
+    vm.decode(zelf, encoding.clone(), errors)?
+        .downcast::<PyString>()
+        .map_err(|obj| {
+            vm.new_type_error(format!(
+                "'{}' decoder returned '{}' instead of 'str'; use codecs.encode() to \
+                     encode arbitrary types",
+                encoding.as_ref().map_or("utf-8", |s| s.borrow_value()),
+                obj.lease_class().name,
+            ))
+        })
+}

vm/src/obj/objbytearray.rs

@@ -7,17 +7,17 @@ use std::mem::size_of;
 use super::objint::PyIntRef;
 use super::objiter;
 use super::objsequence::SequenceIndex;
-use super::objstr::{PyString, PyStringRef};
+use super::objstr::PyStringRef;
 use super::objtype::PyClassRef;
 use crate::bytesinner::{
-    ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, ByteInnerSplitOptions,
-    ByteInnerTranslateOptions, ByteOr, PyBytesInner,
+    bytes_decode, ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions,
+    ByteInnerSplitOptions, ByteInnerTranslateOptions, ByteOr, DecodeArgs, PyBytesInner,
 };
 use crate::common::cell::{PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard};
 use crate::function::{OptionalArg, OptionalOption};
 use crate::pyobject::{
     BorrowValue, Either, PyClassImpl, PyComparisonValue, PyContext, PyIterable, PyObjectRef, PyRef,
-    PyResult, PyValue, TryFromObject, TypeProtocol,
+    PyResult, PyValue, TryFromObject,
 };
 use crate::pystr::{self, PyCommonString};
 use crate::vm::VirtualMachine;
@@ -582,23 +582,8 @@ impl PyByteArray {
     }
 
     #[pymethod]
-    fn decode(
-        zelf: PyRef<Self>,
-        encoding: OptionalArg<PyStringRef>,
-        errors: OptionalArg<PyStringRef>,
-        vm: &VirtualMachine,
-    ) -> PyResult<PyStringRef> {
-        let encoding = encoding.into_option();
-        vm.decode(zelf.into_object(), encoding.clone(), errors.into_option())?
-            .downcast::<PyString>()
-            .map_err(|obj| {
-                vm.new_type_error(format!(
-                    "'{}' decoder returned '{}' instead of 'str'; use codecs.encode() to \
-                     encode arbitrary types",
-                    encoding.as_ref().map_or("utf-8", |s| s.borrow_value()),
-                    obj.lease_class().name,
-                ))
-            })
+    fn decode(zelf: PyRef<Self>, args: DecodeArgs, vm: &VirtualMachine) -> PyResult<PyStringRef> {
+        bytes_decode(zelf.into_object(), args, vm)
     }
 }
 

vm/src/obj/objbytes.rs

@@ -6,18 +6,18 @@ use std::ops::Deref;
 use super::objint::PyIntRef;
 use super::objiter;
 use super::objsequence::SequenceIndex;
-use super::objstr::{PyString, PyStringRef};
+use super::objstr::PyStringRef;
 use super::objtype::PyClassRef;
 use crate::bytesinner::{
-    ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, ByteInnerSplitOptions,
-    ByteInnerTranslateOptions, PyBytesInner,
+    bytes_decode, ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions,
+    ByteInnerSplitOptions, ByteInnerTranslateOptions, DecodeArgs, PyBytesInner,
 };
 use crate::function::{OptionalArg, OptionalOption};
 use crate::pyobject::{
     BorrowValue, Either, IntoPyObject,
     PyArithmaticValue::{self, *},
     PyClassImpl, PyComparisonValue, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue,
-    TryFromObject, TypeProtocol,
+    TryFromObject,
 };
 use crate::pystr::{self, PyCommonString};
 use crate::vm::VirtualMachine;
@@ -467,24 +467,9 @@ impl PyBytes {
     /// For a list of possible encodings,
     /// see https://docs.python.org/3/library/codecs.html#standard-encodings
     /// currently, only 'utf-8' and 'ascii' emplemented
-    #[pymethod(name = "decode")]
-    fn decode(
-        zelf: PyRef<Self>,
-        encoding: OptionalArg<PyStringRef>,
-        errors: OptionalArg<PyStringRef>,
-        vm: &VirtualMachine,
-    ) -> PyResult<PyStringRef> {
-        let encoding = encoding.into_option();
-        vm.decode(zelf.into_object(), encoding.clone(), errors.into_option())?
-            .downcast::<PyString>()
-            .map_err(|obj| {
-                vm.new_type_error(format!(
-                    "'{}' decoder returned '{}' instead of 'str'; use codecs.encode() to \
-                     encode arbitrary types",
-                    encoding.as_ref().map_or("utf-8", |s| s.borrow_value()),
-                    obj.lease_class().name,
-                ))
-            })
+    #[pymethod]
+    fn decode(zelf: PyRef<Self>, args: DecodeArgs, vm: &VirtualMachine) -> PyResult<PyStringRef> {
+        bytes_decode(zelf.into_object(), args, vm)
     }
 }