Implement CPython-like implicit string concatenation in parser (RustPython#4097)

charliermarsh · web-flow · commit bea97cd25419 · 2022-08-21T13:49:00.000+09:00
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/parser/Cargo.toml b/parser/Cargo.toml
@@ -9,22 +9,23 @@ license = "MIT"
 edition = "2021"
 
 [build-dependencies]
-tiny-keccak = { version = "2", features = ["sha3"] }
-phf_codegen = "0.10"
-lalrpop = { version = "0.19.8", optional = true }
 anyhow = "1.0.45"
+lalrpop = { version = "0.19.8", optional = true }
+phf_codegen = "0.10"
+tiny-keccak = { version = "2", features = ["sha3"] }
 
 [dependencies]
-rustpython-ast = { path = "../ast" }
+ahash = "0.7.6"
+itertools = "0.10.3"
 lalrpop-util = "0.19.8"
 log = "0.4.16"
 num-bigint = "0.4.3"
 num-traits = "0.2.14"
+phf = "0.10.1"
+rustpython-ast = { path = "../ast" }
 unic-emoji-char = "0.9.0"
 unic-ucd-ident  = "0.9.0"
 unicode_names2 = "0.5.0"
-phf = "0.10.1"
-ahash = "0.7.6"
 
 [dev-dependencies]
 insta = "1.14.0"
diff --git a/parser/python.lalrpop b/parser/python.lalrpop
@@ -6,9 +6,9 @@
 use crate::{
     ast,
     error::{LexicalError, LexicalErrorType},
-    fstring::parse_located_fstring,
     function::{ArgumentList, parse_args, parse_params},
     lexer,
+    string::parse_strings,
     token::StringKind
 };
 use num_bigint::BigInt;
@@ -961,26 +961,7 @@ SliceOp: Option<ast::Expr> = {
 }
 
 Atom: ast::Expr = {
-    <location:@L> <s:(@L string)+> =>? {
-        let values = s.into_iter().map(|(loc, (value, kind))| {
-            if let StringKind::F = kind {
-                parse_located_fstring(&value, loc)
-            } else {
-                let kind = (kind == StringKind::U).then(|| "u".to_owned());
-                Ok(ast::Expr::new(
-                    loc,
-                    ast::ExprKind::Constant { value: value.into(), kind },
-                ))
-            }
-        });
-        let values = values.collect::<Result<Vec<_>, _>>()?;
-
-        Ok(if values.len() > 1 {
-            ast::Expr::new(location, ast::ExprKind::JoinedStr { values })
-        } else {
-            values.into_iter().next().unwrap()
-        })
-    },
+    <location:@L> <s:(@L string)+> =>? parse_strings(s).map_err(|e| e.into()),
     <location:@L> <value:Constant> => ast::Expr {
         location,
         custom: (),
diff --git a/parser/src/lib.rs b/parser/src/lib.rs
@@ -30,4 +30,5 @@ pub mod mode;
 pub mod parser;
 #[rustfmt::skip]
 mod python;
+mod string;
 pub mod token;
diff --git a/parser/src/parser.rs b/parser/src/parser.rs
@@ -91,6 +91,20 @@ mod tests {
         insta::assert_debug_snapshot!(parse_ast);
     }
 
+    #[test]
+    fn test_parse_string() {
+        let source = String::from("'Hello world'");
+        let parse_ast = parse_program(&source).unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_parse_f_string() {
+        let source = String::from("f'Hello world'");
+        let parse_ast = parse_program(&source).unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
     #[test]
     fn test_parse_print_hello() {
         let source = String::from("print('Hello world')");
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_f_string.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_f_string.snap
@@ -0,0 +1,39 @@
+---
+source: parser/src/parser.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 3,
+        },
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 3,
+                },
+                custom: (),
+                node: JoinedStr {
+                    values: [
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 3,
+                            },
+                            custom: (),
+                            node: Constant {
+                                value: Str(
+                                    "Hello world",
+                                ),
+                                kind: None,
+                            },
+                        },
+                    ],
+                },
+            },
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_string.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_string.snap
@@ -0,0 +1,28 @@
+---
+source: parser/src/parser.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 2,
+        },
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 2,
+                },
+                custom: (),
+                node: Constant {
+                    value: Str(
+                        "Hello world",
+                    ),
+                    kind: None,
+                },
+            },
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_1.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_1.snap
@@ -0,0 +1,39 @@
+---
+source: parser/src/string.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 2,
+        },
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 2,
+                },
+                custom: (),
+                node: JoinedStr {
+                    values: [
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 2,
+                            },
+                            custom: (),
+                            node: Constant {
+                                value: Str(
+                                    "Hello world",
+                                ),
+                                kind: None,
+                            },
+                        },
+                    ],
+                },
+            },
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_2.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_2.snap
@@ -0,0 +1,39 @@
+---
+source: parser/src/string.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 2,
+        },
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 2,
+                },
+                custom: (),
+                node: JoinedStr {
+                    values: [
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 2,
+                            },
+                            custom: (),
+                            node: Constant {
+                                value: Str(
+                                    "Hello world",
+                                ),
+                                kind: None,
+                            },
+                        },
+                    ],
+                },
+            },
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_3.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_3.snap
@@ -0,0 +1,63 @@
+---
+source: parser/src/string.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 2,
+        },
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 2,
+                },
+                custom: (),
+                node: JoinedStr {
+                    values: [
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 2,
+                            },
+                            custom: (),
+                            node: Constant {
+                                value: Str(
+                                    "Hello world",
+                                ),
+                                kind: None,
+                            },
+                        },
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 12,
+                            },
+                            custom: (),
+                            node: FormattedValue {
+                                value: Located {
+                                    location: Location {
+                                        row: 1,
+                                        column: 3,
+                                    },
+                                    custom: (),
+                                    node: Constant {
+                                        value: Str(
+                                            "!",
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                conversion: 0,
+                                format_spec: None,
+                            },
+                        },
+                    ],
+                },
+            },
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_string_concat.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_string_concat.snap
@@ -0,0 +1,28 @@
+---
+source: parser/src/string.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 2,
+        },
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 2,
+                },
+                custom: (),
+                node: Constant {
+                    value: Str(
+                        "Hello world",
+                    ),
+                    kind: None,
+                },
+            },
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_1.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_1.snap
@@ -0,0 +1,41 @@
+---
+source: parser/src/string.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 3,
+        },
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 3,
+                },
+                custom: (),
+                node: JoinedStr {
+                    values: [
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 3,
+                            },
+                            custom: (),
+                            node: Constant {
+                                value: Str(
+                                    "Hello world",
+                                ),
+                                kind: Some(
+                                    "u",
+                                ),
+                            },
+                        },
+                    ],
+                },
+            },
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_2.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_2.snap
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_1.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_1.snap
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_2.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_2.snap
diff --git a/parser/src/string.rs b/parser/src/string.rs
