Skip to content

Commit 0449b2a

Browse files
coolreader18coolreader18
authored
Merge pull request RustPython#1593 from RustPython/coolreader18/raw-byte-lit-fix
Fix raw byte literals
2 parents 3b2cea5 + 583b5ff commit 0449b2a

File tree

2 files changed

+53
-37
lines changed

2 files changed

+53
-37
lines changed

‎parser/src/lexer.rs

Lines changed: 47 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -630,12 +630,15 @@ where
630630

631631
let tok = if is_bytes {
632632
if string_content.is_ascii() {
633-
Tok::Bytes {
634-
value: lex_byte(string_content).map_err(|error| LexicalError {
633+
let value = if is_raw {
634+
string_content.into_bytes()
635+
} else {
636+
lex_byte(string_content).map_err(|error| LexicalError {
635637
error,
636638
location: self.get_pos(),
637-
})?,
638-
}
639+
})?
640+
};
641+
Tok::Bytes { value }
639642
} else {
640643
return Err(LexicalError {
641644
error: LexicalErrorType::StringError,
@@ -1330,16 +1333,14 @@ fn lex_byte(s: String) -> Result<Vec<u8>, LexicalErrorType> {
13301333
mod tests {
13311334
use super::{make_tokenizer, NewlineHandler, Tok};
13321335
use num_bigint::BigInt;
1333-
use std::iter::FromIterator;
1334-
use std::iter::Iterator;
13351336

13361337
const WINDOWS_EOL: &str = "\r\n";
13371338
const MAC_EOL: &str = "\r";
13381339
const UNIX_EOL: &str = "\n";
13391340

1340-
pub fn lex_source(source: &String) -> Vec<Tok> {
1341+
pub fn lex_source(source: &str) -> Vec<Tok> {
13411342
let lexer = make_tokenizer(source);
1342-
Vec::from_iter(lexer.map(|x| x.unwrap().1))
1343+
lexer.map(|x| x.unwrap().1).collect()
13431344
}
13441345

13451346
#[test]
@@ -1354,8 +1355,8 @@ mod tests {
13541355

13551356
#[test]
13561357
fn test_raw_string() {
1357-
let source = String::from("r\"\\\\\" \"\\\\\"");
1358-
let tokens = lex_source(&source);
1358+
let source = "r\"\\\\\" \"\\\\\"";
1359+
let tokens = lex_source(source);
13591360
assert_eq!(
13601361
tokens,
13611362
vec![
@@ -1374,8 +1375,8 @@ mod tests {
13741375

13751376
#[test]
13761377
fn test_numbers() {
1377-
let source = String::from("0x2f 0b1101 0 123 0.2 2j 2.2j");
1378-
let tokens = lex_source(&source);
1378+
let source = "0x2f 0b1101 0 123 0.2 2j 2.2j";
1379+
let tokens = lex_source(source);
13791380
assert_eq!(
13801381
tokens,
13811382
vec![
@@ -1410,7 +1411,7 @@ mod tests {
14101411
$(
14111412
#[test]
14121413
fn $name() {
1413-
let source = String::from(format!(r"99232 # {}", $eol));
1414+
let source = format!(r"99232 # {}", $eol);
14141415
let tokens = lex_source(&source);
14151416
assert_eq!(tokens, vec![Tok::Int { value: BigInt::from(99232) }, Tok::Newline]);
14161417
}
@@ -1430,7 +1431,7 @@ mod tests {
14301431
$(
14311432
#[test]
14321433
fn $name() {
1433-
let source = String::from(format!("123 # Foo{}456", $eol));
1434+
let source = format!("123 # Foo{}456", $eol);
14341435
let tokens = lex_source(&source);
14351436
assert_eq!(
14361437
tokens,
@@ -1454,8 +1455,8 @@ mod tests {
14541455

14551456
#[test]
14561457
fn test_assignment() {
1457-
let source = String::from(r"avariable = 99 + 2-0");
1458-
let tokens = lex_source(&source);
1458+
let source = r"avariable = 99 + 2-0";
1459+
let tokens = lex_source(source);
14591460
assert_eq!(
14601461
tokens,
14611462
vec![
@@ -1484,7 +1485,7 @@ mod tests {
14841485
$(
14851486
#[test]
14861487
fn $name() {
1487-
let source = String::from(format!("def foo():{} return 99{}{}", $eol, $eol, $eol));
1488+
let source = format!("def foo():{} return 99{}{}", $eol, $eol, $eol);
14881489
let tokens = lex_source(&source);
14891490
assert_eq!(
14901491
tokens,
@@ -1520,7 +1521,7 @@ mod tests {
15201521
$(
15211522
#[test]
15221523
fn $name() {
1523-
let source = String::from(format!("def foo():{} if x:{}{} return 99{}{}", $eol, $eol, $eol, $eol, $eol));
1524+
let source = format!("def foo():{} if x:{}{} return 99{}{}", $eol, $eol, $eol, $eol, $eol);
15241525
let tokens = lex_source(&source);
15251526
assert_eq!(
15261527
tokens,
@@ -1558,7 +1559,7 @@ mod tests {
15581559
$(
15591560
#[test]
15601561
fn $name() {
1561-
let source = String::from(format!("def foo():{}\tif x:{}{}\t return 99{}{}", $eol, $eol, $eol, $eol, $eol));
1562+
let source = format!("def foo():{}\tif x:{}{}\t return 99{}{}", $eol, $eol, $eol, $eol, $eol);
15621563
let tokens = lex_source(&source);
15631564
assert_eq!(
15641565
tokens,
@@ -1608,7 +1609,7 @@ mod tests {
16081609
$(
16091610
#[test]
16101611
fn $name() {
1611-
let source = String::from(format!("x = [{} 1,2{}]{}", $eol, $eol, $eol));
1612+
let source = format!("x = [{} 1,2{}]{}", $eol, $eol, $eol);
16121613
let tokens = lex_source(&source);
16131614
assert_eq!(
16141615
tokens,
@@ -1638,8 +1639,8 @@ mod tests {
16381639

16391640
#[test]
16401641
fn test_operators() {
1641-
let source = String::from("//////=/ /");
1642-
let tokens = lex_source(&source);
1642+
let source = "//////=/ /";
1643+
let tokens = lex_source(source);
16431644
assert_eq!(
16441645
tokens,
16451646
vec![
@@ -1655,8 +1656,8 @@ mod tests {
16551656

16561657
#[test]
16571658
fn test_string() {
1658-
let source = String::from(r#""double" 'single' 'can\'t' "\\\"" '\t\r\n' '\g' r'raw\''"#);
1659-
let tokens = lex_source(&source);
1659+
let source = r#""double" 'single' 'can\'t' "\\\"" '\t\r\n' '\g' r'raw\''"#;
1660+
let tokens = lex_source(source);
16601661
assert_eq!(
16611662
tokens,
16621663
vec![
@@ -1698,7 +1699,7 @@ mod tests {
16981699
$(
16991700
#[test]
17001701
fn $name() {
1701-
let source = String::from(format!("\"abc\\{}def\"", $eol));
1702+
let source = format!("\"abc\\{}def\"", $eol);
17021703
let tokens = lex_source(&source);
17031704
assert_eq!(
17041705
tokens,
@@ -1724,30 +1725,42 @@ mod tests {
17241725
#[test]
17251726
fn test_single_quoted_byte() {
17261727
// single quote
1727-
let all = r##"b'\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff'"##;
1728-
let source = String::from(all);
1729-
let tokens = lex_source(&source);
1728+
let source = r##"b'\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff'"##;
1729+
let tokens = lex_source(source);
17301730
let res = (0..=255).collect::<Vec<u8>>();
17311731
assert_eq!(tokens, vec![Tok::Bytes { value: res }, Tok::Newline]);
17321732
}
17331733

17341734
#[test]
17351735
fn test_double_quoted_byte() {
17361736
// double quote
1737-
let all = r##"b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff""##;
1738-
let source = String::from(all);
1739-
let tokens = lex_source(&source);
1737+
let source = r##"b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff""##;
1738+
let tokens = lex_source(source);
17401739
let res = (0..=255).collect::<Vec<u8>>();
17411740
assert_eq!(tokens, vec![Tok::Bytes { value: res }, Tok::Newline]);
17421741
}
17431742

17441743
#[test]
17451744
fn test_escape_char_in_byte_literal() {
17461745
// backslash doesnt escape
1747-
let all = r##"b"omkmok\Xaa""##;
1748-
let source = String::from(all);
1749-
let tokens = lex_source(&source);
1746+
let source = r##"b"omkmok\Xaa""##;
1747+
let tokens = lex_source(source);
17501748
let res = vec![111, 109, 107, 109, 111, 107, 92, 88, 97, 97];
17511749
assert_eq!(tokens, vec![Tok::Bytes { value: res }, Tok::Newline]);
17521750
}
1751+
1752+
#[test]
1753+
fn test_raw_byte_literal() {
1754+
let source = r"rb'\x1z'";
1755+
let tokens = lex_source(source);
1756+
assert_eq!(
1757+
tokens,
1758+
vec![
1759+
Tok::Bytes {
1760+
value: b"\\x1z".to_vec()
1761+
},
1762+
Tok::Newline
1763+
]
1764+
)
1765+
}
17531766
}

‎tests/snippets/stdlib_select.py

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
from testutils import assert_raises, assert_equal
1+
from testutils import assert_raises
22

33
import select
44
import sys
@@ -36,6 +36,9 @@ def fileno(self):
3636
sendr.connect(("127.0.0.1", 9988))
3737
sendr.send(b"aaaa")
3838

39-
res = select.select([recvr], [sendr], [])
39+
rres, wres, xres = select.select([recvr], [sendr], [])
4040

41-
assert_equal(res, ([recvr], [sendr], []))
41+
if "win" not in sys.platform:
42+
assert recvr in rres
43+
44+
assert sendr in wres

0 commit comments

Comments
 (0)